#!/bin/ksh # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # # # Licensed Materials - Property of IBM # # (C) COPYRIGHT International Business Machines Corp. 2000,2007 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#)11 1.59.1.3 src/avs/fs/mmfs/ts/admin/mmchcluster.sh, mmfs, avs_rgpfs24, rgpfs24s009a 12/19/06 13:10:44 ############################################################################### # # Usage: # mmchcluster {[-p PrimaryServer] [-s SecondaryServer]} # or # mmchcluster -p LATEST # or # mmchcluster {[-r RemoteShellCommand] [-R RemoteFileCopyCommand]} # or # mmchcluster -C ClusterName # or # mmchcluster -N {NodeDesc[,NodeDesc...] | NodeFile} # # where: # # -p PrimaryServer specifies the node to be used as the primary server # of the GPFS sdrfs data for this cluster. # # LATEST requests a check to be made that all currently # available nodes point to the correct primary and # backup server. # # -s SecondaryServer specifies the node to be used as the backup server # of the GPFS sdrfs data for this cluster (optional). # To remove a backup server, specify -s "". # # -r RemoteShellCommand specifies the fully qualified pathname for # the remote shell program to be used by GPFS. # The default is /usr/bin/rsh. # # -R RemoteFileCopyCommand specifies the fully qualified pathname for # the remote file copy program to be used by GPFS. # The default is /usr/bin/rcp. # # -C ClusterName specifies a new name for the cluster. If the name # contains dots it is assumed to be a fully qualified # domain name. Otherwise, the domain will default # to the domain of the primary configuration server. # # -N NodeDesc,NodeDesc,... specifies a comma-separated list of node # descriptors that specify the admin node # interfaces to be used in the cluster. # The node descriptors have the format: # daemonNodeName:nodeRoles:adminNodeName: # The nodeRoles field is currently just a place-holder # and is ignored. # # -N NodeFile specifies a file of node descriptors that specify # the admin node interfaces to be used in the cluster. # The lines in the input file have the format: # daemonNodeName:nodeRoles:adminNodeName: # The nodeRoles field is currently just a place-holder # and is ignored. # # Note: When used with the -p or -s options, this command will most # likely be needed when the current primary server is not available # and it will be impossible to obtain the sdr lock and protect # against concurrent execution of some other mm command. # Under such conditions, the user must assure that no other mm # command is run until the completion of the mmchcluster command # and that as many of the remaining nodes as possible are available. # ############################################################################### # Include global declarations and service routines. . /usr/lpp/mmfs/bin/mmglobfuncs . /usr/lpp/mmfs/bin/mmsdrfsdef sourceFile="mmchcluster.sh" [[ -n $DEBUG || -n $DEBUGmmchcluster ]] && set -x $mmTRACE_ENTER "$*" # Local work files. Names should be of the form: # fn=${tmpDir}fn.${mmcmd}.$$ allNodes=${tmpDir}allNodes.${mmcmd}.$$ clientNodes=${tmpDir}clientNodes.${mmcmd}.$$ inputNodes=${tmpDir}inputNodes.${mmcmd}.$$ processedNodes=${tmpDir}processedNodes.${mmcmd}.$$ initErrors=${tmpDir}initErrors.${mmcmd}.$$ # Note: Do not include initErrors in LOCAL_FILES yet; we'll do it later. LOCAL_FILES=" $allNodes $clientNodes $inputNodes $processedNodes " # Local declarations usageMsg=359 newNodeNumbers="" backupServer="" rshPath="" rcpPath="" integer nodeCount integer n rc=0 Cflag="" Nflag="" pflag="" rflag="" Rflag="" sflag="" Carg="" parg="" rarg="" Rarg="" sarg="" otherOpt="" # Local functions ########################################################################## # # Function: Specify the admin network for the GPFS cluster. # # Input: $1 - file or list of node descriptors containing the # adapter information as follows: # daemonNodeName:nodeRoles:adminNodeName: # # Returns: 0 - no errors encountered # non-zero - unexpected error # ########################################################################## function specifyAdminNetwork # { typeset sourceFile="mmchcluster.sh" [[ -n $DEBUG || -n $DEBUGspecifyAdminNetwork ]] && set -x $mmTRACE_ENTER "$*" typeset networkInfo="$1" typeset failedNodes sdrfsLine mmcommonOutput typeset nodeLine nodeName nodeName2 nodeStatus # typeset nodeRoles typeset hostResult nodeNumber adminNodeName adminIpa typeset nodeError newPrimaryName newBackupName commitOptions typeset rc=0 typeset changeMade="" typeset fatalError="" typeset sharedSdrservPort="" # The input parameter may be either a list or a file. Which is it? if [[ -f $networkInfo ]] then # It is a file; verify its existence and create our own copy. checkUserFile $networkInfo $inputNodes [[ $? -ne 0 ]] && cleanupAndExit else # It is not a file, so it must be a list. # Convert the input node list into a file. $rm -f $inputNodes IFS=',' for nodeDesc in $networkInfo do print -- "$nodeDesc" >> $inputNodes checkForErrors "writing to $inputNodes" $? done IFS="$IFS_sv" # Restore the default IFS setting. fi # Check the input data for correctness. # We check all the records rather than stop on the first error. $rm -f $processedNodes $touch $processedNodes # Ensure the tmp file exists even if empty. IFS=":" # Change the field separator to ':'. exec 3<&- exec 3< $inputNodes while read -u3 nodeLine do # Parse the line. set -f ; set -- $nodeLine ; set +f nodeName=$1 # nodeRoles=$2 nodeName2=$3 IFS="$IFS_sv" # Restore the default IFS setting. # Make sure neither node name is specified more than once. $grep -qw $nodeName $processedNodes > /dev/null 2>&1 if [[ $? -eq 0 ]] then # The node name is specified twice. printErrorMsg 347 $mmcmd $nodeName fatalError=yes fi # Check the admin node name if it was specified. if [[ -n $nodeName2 && $nodeName2 != $nodeName ]] then $grep -qw $nodeName2 $processedNodes > /dev/null 2>&1 if [[ $? -eq 0 ]] then # The node is specified twice. printErrorMsg 347 $mmcmd $nodeName2 fatalError=yes fi fi # end of if [[ -n $nodeName2 && $nodeName2 != $nodeName ]] # Add the node names to the list of processed nodes. print -- "${nodeName}:${nodeName2}" >> $processedNodes checkForErrors "Writing to file $processedNodes" $? IFS=":" # Change the separator back to ":" for the next iteration. done # end of while read -u3 nodeLine IFS="$IFS_sv" # Restore the default IFS settings. # Return to the caller if we encountered an error. [[ -n $fatalError ]] && return 1 # Ensure that the local copy of the mmsdrfs is up-to-date. # Set up trap exception handling and obtain the lock. trap pretrap HUP INT QUIT KILL gpfsInitOutput=$(gpfsInit $lockId) setGlobalVar $? $gpfsInitOutput # Stop here if the admin network support has not been activated yet. if [[ $sdrfsFormatLevel -eq 0 ]] then print -u2 "$mmcmd: The separate administration network support has not been enabled yet." print -u2 " Run \"mmchconfig release=LATEST\" to activate the new function." cleanupAndExit fi # Determine the lookup order for resolving host names. [[ $osName != AIX ]] && resolveOrder=$(setHostResolveOrder) # Go through the current mmsdrfs file. Increment the generation # number and build the node name list that will be needed later. # Remove all admin network related information. $rm -f $newsdrfs $nodefile newPrimaryName="" newBackupName="" IFS=":" # Change the field separator to ':'. exec 3<&- exec 3< $mmsdrfsFile while read -u3 sdrfsLine do # Parse the line. set -f ; set -A v -- - $sdrfsLine ; set +f IFS="$IFS_sv" # Restore the default IFS settings. printLine=true # Assume the line will be printed. case ${v[$LINE_TYPE_Field]} in $VERSION_LINE ) # This is the global header line. # Save the version line for updating later. versionLine=$(print_newLine) printLine=false ;; $NODESET_HDR ) # If the daemon and the mmsdrserv tcp ports are shared, # it will be necessary to ensure that the daemon is down # on the config server nodes if there names will change. if [[ -z ${v[$GETOBJECT_PORT_Field]} || ${v[$TCP_PORT_Field]} = ${v[$GETOBJECT_PORT_Field]} ]] then sharedSdrservPort=yes fi ;; $MEMBER_NODE ) # This line describes a node. # Add the reliable node name to nodefile. print -- "${v[$REL_HOSTNAME_Field]}" >> $nodefile checkForErrors "writing to file $nodefile" $? # Reset the node error flag. nodeError="" # Obtain the data for this node from the node file. nodeLine=$($awk -F: ' \ $1 == "'${v[$DAEMON_NODENAME_Field]}'" || \ $1 == "'${v[$REL_HOSTNAME_Field]}'" || \ $1 == "'${v[$NODE_NAME_Field]}'" || \ $1 == "'${v[$ADMIN_SHORTNAME_Field]}'" || \ $1 == "'${v[$NODE_NUMBER_Field]}'" || \ $1 == "'${v[$IPA_Field]}'" { \ { print $0 } \ { exit } \ } \ ' $inputNodes) if [[ -n $nodeLine ]] then # We found data for this node. Parse the input. IFS=":" # Change the field separator to ':'. set -f ; set -- $nodeLine ; set +f nodeName=$1 nodeName2=$3 IFS="$IFS_sv" # Restore the default IFS setting. # Determine the daemon node name. if [[ -n ${v[$DAEMON_NODENAME_Field]} ]] then daemonNodeName=${v[$DAEMON_NODENAME_Field]} else daemonNodeName=${v[$REL_HOSTNAME_Field]} fi # Did the user reset or specify the admin node name? if [[ -z $nodeName2 ]] then # The admin node name was null, indicating "reset"; # set the admin node name to the daemon node name value. adminNodeName=$daemonNodeName adminShortName=${v[$NODE_NAME_Field]} else # The admin node name was not null, indicating "specify"; # Determine the IP address for the specified admin node name. hostResult=$($host $nodeName2) set -f ; set -- $hostResult ; set +f adminNodeName=$1 adminShortName=${1%% *|.*} # Exclude everything after the first dot. adminIpa=${3%%,*} # Check that the admin node name has a valid IP address. if [[ -z $adminIpa ]] then # An invalid node name was specified. printErrorMsg 54 $mmcmd $nodeName2 fatalError=yes break fi # Invoke the checkAdapter function to ensure that # the specified adapter interface exists on the node. mmcommonOutput=$($mmcommon on1 ${v[$REL_HOSTNAME_Field]} \ checkAdapter $adminIpa 2> $errMsg) rc=$? set -f ; set -- $mmcommonOutput ; set +f nodeStatus=$1 if [[ $rc != 0 || $nodeStatus != success ]] then # The checkAdapter call failed. # We will not define a new admin node name for this node # but we will continue to process the remaining nodes. # Tell the world what went wrong with this node. if [[ $nodeStatus = ipa_alias ]] then # IP address aliasing is not supported. printErrorMsg 476 $mmcmd $nodeName2 elif [[ $nodeStatus = ipa_missing ]] then # The admin IP address is not known on the node. printErrorMsg 154 $mmcmd $nodeName2 ${v[$REL_HOSTNAME_Field]} elif [[ $rc = $MM_HostDown || $rc = $MM_ConnectTimeout ]] then # The node cannot be reached. printErrorMsg 340 $mmcmd ${v[$REL_HOSTNAME_Field]} else # Unexpected error. Display all possible error messages. [[ -s $errMsg ]] && $cat $errMsg 1>&2 [[ $rc -eq 0 ]] && rc=1 checkForErrors "checkAdapter ${v[$REL_HOSTNAME_Field]}" $rc fi # Append the node name to the list of failed nodes and # set a flag to indicate the node name did not check out. failedNodes="${failedNodes}\n\t${nodeName}" nodeError=yes fi # end of if [[ $rc != 0 || $nodeStatus != success ]] fi # end of if [[ -z $nodeName2 ]] # Update the member line if there was no error. if [[ -z $nodeError ]] then # Remember the new primary or backup server name for updating # the version line later if this is one of those servers. [[ ${v[$REL_HOSTNAME_Field]} = $primaryServer ]] && \ newPrimaryName=$adminNodeName [[ ${v[$REL_HOSTNAME_Field]} = $backupServer ]] && \ newBackupName=$adminNodeName # Things checked out ok. Set the node name fields. v[$DAEMON_NODENAME_Field]=$daemonNodeName v[$REL_HOSTNAME_Field]=$adminNodeName v[$ADMIN_SHORTNAME_Field]=$adminShortName changeMade=yes fi $rm -f $errMsg fi # end of if [[ -n $nodeLine ]] ;; * ) # We are not interested in any other lines. ;; esac # end of case ${v[$LINE_TYPE_Field]} in # Unless suppressed, write the line to the new mmsdrfs file. if [[ $printLine = true ]] then print_newLine >> $newsdrfs checkForErrors "writing to file $newsdrfs" $? fi IFS=":" # Change the separator back to ":" for the next iteration. done # end of while read -u3 IFS="$IFS_sv" # Restore the default IFS settings. # Go through the mmsdrfs file to update the NSD servers admin node names. $rm -f $tmpsdrfs IFS=":" exec 3<&- exec 3< $newsdrfs while read -u3 sdrfsLine do # Parse the line. set -f ; set -A v -- - $sdrfsLine ; set +f IFS="$IFS_sv" # Change some of the fields depending on the type of line. case ${v[$LINE_TYPE_Field]} in $SG_DISKS ) # This is the line for some disk. # If this disk is an NSD with a valid PVID value, # make sure the daemon nsd server names are recorded. if [[ ${v[$DISK_TYPE_Field]} = nsd && -n ${v[$PVID_Field]} ]] then # If a server node was specified, check that it is valid and # convert it to get the potentially new admin adapter name. # We determine whether a server was specified by checking for an # admin nsd server name, but we do not use that name for finding # the node information, since the old admin node name may # no longer exist as a result of the update we just did. # We use the daemon node name to find the node instead, # since mmchcluster -N does not change daemon node names. if [[ -n ${v[$NSD_PRIMARY_NODE_Field]} ]] then # If no daemon node name has yet been recorded for the # primary NSD server, determine and store it now. server=${v[$DAEMON_NSD_PRIMARY_Field]} if [[ -z $server ]] then server=$(checkAndConvertNodeValue \ ${v[$NSD_PRIMARY_NODE_Field]} $DAEMON_NODENAME_Field) checkForErrors "checkAndConvertNodeValue" $? v[$DAEMON_NSD_PRIMARY_Field]=$server fi # Use the primary server's daemon node name to obtain # the primary server's admin node name. v[$NSD_PRIMARY_NODE_Field]=$(checkAndConvertNodeValue \ $server $REL_HOSTNAME_Field $newsdrfs) checkForErrors "checkAndConvertNodeValue $server" $? fi if [[ -n ${v[$NSD_BACKUP_NODE_Field]} ]] then # If no daemon node name has yet been recorded for the # backup NSD server, determine and store it now. backup=${v[$DAEMON_NSD_BACKUP_Field]} if [[ -z $backup ]] then backup=$(checkAndConvertNodeValue \ ${v[$NSD_BACKUP_NODE_Field]} $DAEMON_NODENAME_Field) checkForErrors "checkAndConvertNodeValue" $? v[$DAEMON_NSD_BACKUP_Field]=$backup fi # Use the backup server's daemon node name to obtain # the backup server's admin node name. v[$NSD_BACKUP_NODE_Field]=$(checkAndConvertNodeValue \ $backup $REL_HOSTNAME_Field $newsdrfs) checkForErrors "checkAndConvertNodeValue $backup" $? fi fi # end of if (v[$DISK_TYPE_Field] == "nsd" && -n v[$PVID_Field]) ;; * ) # We are not interested in any other lines. ;; esac # end Change some of the fields # Build and write the line to the temp version of the mmsdrfs file. print_newLine >> $tmpsdrfs checkForErrors "writing to file $tmpsdrfs" $? IFS=":" # Change the separator back to ":" for the next iteration. done # end while read -u3 sdrfsLine IFS="$IFS_sv" # Restore the default IFS settings. # If a fatal error occurred, or if no changes were made, # release the lock, report any failed nodes, and return. if [[ -n $fatalError || -z $changeMade ]] then freeLockOnServer $primaryServer $ourNodeNumber >/dev/null if [[ -n $failedNodes ]] then # Administrative node names were not defined for nodes ... printErrorMsg 174 $mmcmd $failedNodes fi if [[ -n $fatalError ]] then printErrorMsg 389 $mmcmd # The command failed. else printErrorMsg 387 $mmcmd $mmcmd # Command quitting due to no valid nodes. fi return 1 fi # Create the updated version line and add it to the new sdrfs file. # The generation number is incremented and the server names may change. IFS=":" # Change the field separator to ':'. set -f ; set -A v -- - $versionLine ; set +f IFS="$IFS_sv" # Restore the default IFS setting. newGenNumber=${v[$SDRFS_GENNUM_Field]}+1 v[$SDRFS_GENNUM_Field]=$newGenNumber [[ -n $newPrimaryName ]] && v[$PRIMARY_SERVER_Field]=$newPrimaryName [[ -n $newBackupName ]] && v[$BACKUP_SERVER_Field]=$newBackupName print_newLine >> $tmpsdrfs checkForErrors "writing to file $tmpsdrfs" $? # If the GPFS and mmsdrserv daemons share the same tcp port number, # and the names of the primary or backup configuration servers are # changing, it is necessary to ensure that the GPFS daemon is down # on the server nodes and the mmsdrserv daemon is restarted. # Otherwise, the server nodes will continue giving (stale) Gpfs object # or return ESDR_NOT_SERVER errors. if [[ -n $sharedSdrservPort && ( -n $newPrimaryName || -n $newBackupName ) ]] then # Get the names of the config servers. print -- "${v[$PRIMARY_SERVER_Field]}\n${v[$BACKUP_SERVER_Field]}" > $tmpNodes checkForErrors "writing to file $tmpNodes" $? # Verify the daemon is down; do not lock the Gpfs object. printInfoMsg 453 verifyDaemonInactive $tmpNodes [[ $? -ne 0 ]] && return 1 commitOptions="initLocalNodeData,KILLSDRSERV" else commitOptions="initLocalNodeData" fi # end of if [[ -n $sharedSdrservPort ]] # Make sure the new sdrfs file is properly sorted. LC_ALL=C $SORT_MMSDRFS $tmpsdrfs -o $newsdrfs # Put the new mmsdrfs file into the sdr. This will make the newly-added # admin nodes visible to the rest of the nodes in the cluster. trap "" HUP INT QUIT KILL gpfsObjectInfo=$(commitChanges $nsId $nsId \ $gpfsObjectInfo $newGenNumber $newsdrfs $primaryServer $commitOptions) rc=$? if [[ $rc -ne 0 ]] then # We were unable to replace the file in the sdr. printErrorMsg 381 $mmcmd return 1 fi # Unlock the sdr. freeLockOnServer $primaryServer $ourNodeNumber >/dev/null trap posttrap HUP INT QUIT KILL # Propagate the new mmsdrfs file to all nodes in the cluster. # This process is asynchronous. propagateSdrfsFile async $nodefile $newsdrfs $newGenNumber initLocalNodeData # Report any nodes that did not check successfully. if [[ -n $failedNodes ]] then # Administrative node names were not defined for nodes ... printErrorMsg 174 $mmcmd $failedNodes fi return 0 } #----- end of function specifyAdminNetwork ------------------- ################################################################### # This function is called if there is an interrupt after the new # mmsdrfs file was committed on the new primary and backup servers # but before the change was propagated to the rest of the nodes. ################################################################### function localPosttrap { $mmTRACE_ENTER "$*" # Tell the guy which nodes must be up and which command to run. printErrorMsg 350 $mmcmd "\n\t$newPrimaryServer\t$newBackupServer" printErrorMsg 344 $mmcmd "mmchcluster -p LATEST" cleanupAndExit 2 } #----- end of function localPosttrap ------------------------ ###################### # Mainline processing ###################### ################################################### # Process the command arguments. ################################################### [[ $arg1 = '-?' || $arg1 = '-h' || $arg1 = '--help' || $arg1 = '--' ]] && \ syntaxError "help" $usageMsg [[ $argc -lt 2 ]] && \ syntaxError "missingArgs" $usageMsg while getopts :C:N:p:r:R:s: OPT do case $OPT in C) # cluster name [[ -n $Cflag ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Cflag="-$OPT" Carg=$OPTARG ;; N) # define/replace secondary network [[ -n $Nflag ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Nflag="-$OPT" Narg=$OPTARG ;; p) # primary server [[ -n $pflag ]] && syntaxError "multiple" $noUsageMsg "-$OPT" pflag="-$OPT" parg=$OPTARG otherOpt="-$OPT" ;; r) # remote shell command [[ -n $rflag ]] && syntaxError "multiple" $noUsageMsg "-$OPT" rflag="-$OPT" rarg=$OPTARG [[ $rarg = ${rarg#/} ]] && \ syntaxError "absolutePath_2" $noUsageMsg "-$OPT" "$rarg" otherOpt="-$OPT" ;; R) # remote file copy command [[ -n $Rflag ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Rflag="-$OPT" Rarg=$OPTARG [[ $Rarg = ${Rarg#/} ]] && \ syntaxError "absolutePath_2" $noUsageMsg "-$OPT" "$Rarg" otherOpt="-$OPT" ;; s) # secondary server [[ -n $sflag ]] && syntaxError "multiple" $noUsageMsg "-$OPT" sflag="-$OPT" sarg=$OPTARG otherOpt="-$OPT" ;; +[CNprRs]) # Invalid option syntaxError "invalidOption" $usageMsg $OPT ;; :) # Missing argument syntaxError "missingValue" $usageMsg $OPTARG ;; *) # Invalid option syntaxError "invalidOption" $usageMsg $OPTARG ;; esac done shift OPTIND-1 [[ $# != 0 ]] && syntaxError "extraArg" $usageMsg $1 [[ -n $sflag && $parg = LATEST ]] && \ syntaxError "invalidCombination" $usageMsg "-s" "-p LATEST" [[ -n $rflag && -n $pflag ]] && \ syntaxError "invalidCombination" $usageMsg "-r" "-p" [[ -n $rflag && -n $sflag ]] && \ syntaxError "invalidCombination" $usageMsg "-r" "-s" [[ -n $Rflag && -n $pflag ]] && \ syntaxError "invalidCombination" $usageMsg "-R" "-p" [[ -n $Rflag && -n $sflag ]] && \ syntaxError "invalidCombination" $usageMsg "-R" "-s" # The primary GPFS cluster configuration server cannot be removed. [[ -n $pflag && $parg = "" ]] && \ syntaxError "missingValue" $usageMsg "-p" [[ -n $Nflag && -n $otherOpt ]] && \ syntaxError "invalidCombination" $usageMsg "-N" "$otherOpt" [[ -n $Cflag && -n $otherOpt ]] && \ syntaxError "invalidCombination" $usageMsg "-C" "$otherOpt" ############################################################################# # If the request is to change a remote command, invoke the mmsetrcmd script. # Keep in mind that rarg and Rarg may include options for the respective # commands and, therefore, must always be quoted. ############################################################################# if [[ -n $rflag || -n $Rflag ]] then if [[ -z $Rflag ]] then $mmsetrcmd "$rflag" "$rarg" rc=$? elif [[ -z $rflag ]] then $mmsetrcmd "$Rflag" "$Rarg" rc=$? else $mmsetrcmd "$rflag" "$rarg" "$Rflag" "$Rarg" rc=$? fi cleanupAndExit $rc fi ############################################################# # If the request is to specify changes to the admin network, # invoke the function to do the work and exit. ############################################################# if [[ -n $Nflag ]] then specifyAdminNetwork "$Narg" cleanupAndExit $? fi ######################################################## # If the request is to change the cluster name, # invoke the mmsetrcmd script. ######################################################## if [[ -n $Cflag ]] then $mmsetrcmd "$Cflag" "$Carg" cleanupAndExit $? fi ################################################################# # Set up trap exception handling and call the gpfsInit function. # It will attempt to ensure that the local copy of the mmsdrfs # and the rest of the GPFS system files are up-to-date. # Try to get the lock but do not fail if this is not possible. ################################################################# trap pretrap HUP INT QUIT KILL if [[ $parg = LATEST ]] then # The LATEST keyword was specified. Try to obtain the # most recent mmsdrfs file (i.e., the mmsdrfs file with the # highest gen number) among all the nodes in the cluster. # To do that, use the local mmsdrfs file as a starting point. getNodeList $REL_HOSTNAME_Field $HOME_CLUSTER $mmsdrfsFile > $allNodes gpfsInitOutput=$(gpfsInitFromNonServer $allNodes $mmsdrfsFile) rc=$? else # The LATEST keyword was not specified. Try to obtain # the mmsdrfs file from one of the servers with locking. gpfsInitOutput=$(gpfsInit $lockId 2> $initErrors) rc=$? LOCAL_FILES="$LOCAL_FILES $initErrors " if [[ $rc -ne 0 ]] then # We failed to get the sdrfs file with a lock. Check whether # some other mm command currently holds the lock. If yes, give up. $grep -e "Timed out waiting for lock: Try again later." \ -e "6027-1229" $initErrors > /dev/null 2>&1 ec=$? if [[ $ec -eq 0 ]] then # Display the messages from gpfsInit. $cat $initErrors | \ $grep -v -e "6027-1227" -e "file is locked. Retrying..." 1>&2 cleanupAndExit fi # We failed to get the sdrfs file with a lock. Display any messages. $cat $initErrors 1>&2 # Processing continues. printErrorMsg 437 $mmcmd # Now try the gpfsInit again, but this time do not ask for a lock. # If there is a backup server, and if it is available, # we should be able to get the latest GPFS system files from there. gpfsInitOutput=$(gpfsInit nolock 2>/dev/null) rc=$? if [[ $rc -ne 0 ]] then # We also failed to get the sdrfs file without locking. Now try # to obtain the most recent mmsdrfs file (i.e., the mmsdrfs file # with the highest gen number) among all the nodes in the cluster. # To do that, use the local mmsdrfs file as a starting point. getNodeList $REL_HOSTNAME_Field $HOME_CLUSTER $mmsdrfsFile > $allNodes gpfsInitOutput=$(gpfsInitFromNonServer $allNodes $mmsdrfsFile) rc=$? fi fi fi # end of if [[ $parg = LATEST ]] # Check whether we succeeded in obtaining the desired mmsdrfs file. if [[ $rc -ne 0 ]] then # Not enough nodes are available. printErrorMsg 378 $mmcmd cleanupAndExit fi # Parse the output from the init function. setGlobalVar $rc $gpfsInitOutput if [[ $MMMODE = single ]] then # Command currently not valid for cluster type single. printErrorMsg 376 $mmcmd single cleanupAndExit fi if [[ $MMMODE != lc ]] then # Unknown GPFS nodeset type printErrorMsg 338 $mmcmd $MMMODE cleanupAndExit fi ####################################################### # Determine the reliable hostnames of the new servers. ####################################################### if [[ -n $pflag && $parg != LATEST ]] then # Find the name of the primary server. newPrimaryServer=$(checkAndConvertNodeValue $parg $REL_HOSTNAME_Field) if [[ $? -ne 0 ]] then printErrorMsg 352 $mmcmd $parg cleanupAndExit fi else # If -p not specified, the primary server remains the same. newPrimaryServer=$primaryServer fi # end of if [[ -n $parg && $parg != LATEST ]] if [[ -n $sflag ]] then if [[ -n $sarg ]] then # Find the name of the secondary server. newBackupServer=$(checkAndConvertNodeValue $sarg $REL_HOSTNAME_Field) if [[ $? -ne 0 ]] then printErrorMsg 352 $mmcmd $sarg cleanupAndExit fi else # We are deleting the backup server (-s "" was specified). newBackupServer="" fi else # If -s not specified, the backup server remains the same. newBackupServer=$backupServer fi # end of if [[ -n $sarg ]] # Cross check the two server names. if [[ $newBackupServer = $newPrimaryServer ]] then # The same node was specified as primary and backup server. printErrorMsg 346 $mmcmd cleanupAndExit fi # Check whether anything needs to be done at all. [[ $newPrimaryServer = $primaryServer && \ $newBackupServer = $backupServer && \ $parg != LATEST ]] && \ cleanupAndExit 0 # Servers are already as desired. ################################################################# # Go through the current mmsdrfs file. Increment the generation # number and change the server names. Create a file with the # reliable hostnames of all nodes in the cluster. ################################################################# $rm -f $newsdrfs $allNodes $clientNodes IFS=":" # Change the field separator to ':'. exec 3<&- exec 3< $mmsdrfsFile while read -u3 sdrfsLine do # Parse the line. set -f ; set -A v -- - $sdrfsLine ; set +f IFS="$IFS_sv" # Restore the default IFS settings. # Change some of the fields depending on the type of line. case ${v[$LINE_TYPE_Field]} in $VERSION_LINE ) # Increment the generation number. newGenNumber=${v[$SDRFS_GENNUM_Field]}+1 v[$SDRFS_GENNUM_Field]=$newGenNumber v[$PRIMARY_SERVER_Field]=$newPrimaryServer v[$BACKUP_SERVER_Field]=$newBackupServer ;; $NODESET_HDR ) # If the daemon and the mmsdrserv tcp ports are shared, # it will be necessary to ensure that the daemon is down # on the old and new config server nodes. if [[ -z ${v[$GETOBJECT_PORT_Field]} || ${v[$TCP_PORT_Field]} = ${v[$GETOBJECT_PORT_Field]} ]] then daemonMustBeDown=yes fi ;; $MEMBER_NODE ) # If this is our node, save the reliable name. [[ ${v[$NODE_NUMBER_Field]} = $ourNodeNumber ]] && \ ourNodeName=${v[$REL_HOSTNAME_Field]} # All nodes will go in the allNodes file. print -- "${v[$REL_HOSTNAME_Field]}" >> $allNodes checkForErrors "writing to file $allNodes" $? # The server nodes and the local node will # not go in the clientNodes file. if [[ ${v[$REL_HOSTNAME_Field]} != $newPrimaryServer && ${v[$REL_HOSTNAME_Field]} != $newBackupServer && ${v[$REL_HOSTNAME_Field]} != $ourNodeName ]] then print -- "${v[$REL_HOSTNAME_Field]}" >> $clientNodes checkForErrors "writing to file $clientNodes" $? fi ;; * ) # Pass all other lines without change. ;; esac # end Change some of the fields # Build and write the line to the new mmsdrfs file. print_newLine >> $newsdrfs checkForErrors "writing to file $newsdrfs" $? IFS=":" # Change the separator back to ":" for the next iteration. done # end of while read -u3 sdrfsLine IFS="$IFS_sv" # Restore the default IFS settings. ####################################################################### # If the GPFS and mmsdrserv daemons share the same tcp port number, # it is necessary to ensure that the GPFS daemon is down on the old # and new configuration server nodes. Otherwise, the old server nodes # will continue giving (stale) Gpfs object information, while the new # servers will not be able to respond to requests because the GPFS # daemon cannot assume mmsdrserv duties if it is already running. ####################################################################### if [[ -n $daemonMustBeDown && $parg != LATEST ]] then # Put the old and new server names in a file. print -- "$primaryServer\n$backupServer\n" \ "$newPrimaryServer\n$newBackupServer" > $tmpNodes checkForErrors "writing to file $tmpNodes" $? # Eliminate duplicate names. $sort -u $tmpNodes -o $tmpNodes checkForErrors "sort $tmpNodes" $? # Verify the daemon is down; do not lock the Gpfs object. printInfoMsg 453 verifyDaemonInactive $tmpNodes [[ $? -ne 0 ]] && cleanupAndExit fi # end of if [[ -n $daemonMustBeDown ]] ###################################################### # First, put the new mmsdrfs file on the two servers. # This must succeed no matter what. ###################################################### trap "" HUP INT QUIT KILL gpfsObjectInfo=$(commitChanges \ $nsId $nsId $gpfsObjectInfo $newGenNumber $newsdrfs \ $newPrimaryServer FORCE $newBackupServer) rc=$? if [[ $rc -ne 0 ]] then # Cannot replace file in the sdr. printErrorMsg 381 $mmcmd # The mmchcluster failed - get out. # Tell the guy which nodes must be up and which command to run. printErrorMsg 350 $mmcmd "\n\t$newPrimaryServer\t$newBackupServer" printErrorMsg 344 $mmcmd "mmchcluster" cleanupAndExit fi # Restore interrupts. trap localPosttrap HUP INT QUIT KILL ################################################# # Propagate the changes to the non-server nodes. ################################################# if [[ $ourNodeName != $newPrimaryServer && $ourNodeName != $newBackupServer ]] then $cp $newsdrfs $mmsdrfsFile checkForErrors "writing to file $mmsdrfsFile" $? fi if [[ -s $clientNodes ]] then # Calculate the checksum of the new mmsdrfs file. sumOutput=$($sum $newsdrfs) checkForErrors "sum $newsdrfs" $? set -f ; set -- $sumOutput ; set +f newSum=$1 #esjxx See if this can be replaced with pushSdr # Tell all client nodes to copy the file from us. $mmcommon onall $clientNodes $unreachedNodes copyRemoteFile \ $ourNodeName $mmsdrfsFile $mmsdrfsFile $newSum > $tmpfile 2>&1 rc=$? # Make a list of the nodes that were successfully updated. For each # such node there will be a line in tmpfile that looks like this: # nodename: copyRemoteFile:0 updatedNodes=$($awk -F: ' { \ if (($2 ~ "copyRemoteFile") && ($3 == "0")) { \ { print $1 } \ } \ } ' $tmpfile) checkForErrors awk $? # Determine the nodes that did not get the new data. exec 3<&- exec 3< $clientNodes while read -u3 nodeName do for goodNode in $updatedNodes do [[ $nodeName = $goodNode ]] && \ break done [[ $nodeName != $goodNode ]] && \ failedNodes="${failedNodes}\n\t${nodeName}" done # If any nodes failed, put out as much information as possible. if [[ -n $failedNodes ]] then # Collect error messages, if any, in file tmpfile2. $grep -v "copyRemoteFile:" $tmpfile > $tmpfile2 [[ -s $tmpfile2 ]] && \ $cat $tmpfile2 1>&2 # Tell the user which nodes failed. printErrorMsg 377 $mmcmd "$failedNodes" # Tell the guy which nodes must be up and which command to run. printErrorMsg 350 $mmcmd "\n\t$newPrimaryServer\t$newBackupServer" printErrorMsg 344 $mmcmd "mmchcluster -p LATEST" cleanupAndExit fi # end if [[ -n $failedNodes ]] fi # end if [[ ! -s $clientNodes ]] ############################## # Unlock the sdr. ############################## [[ $sdrLocked = yes ]] && \ freeLockOnServer $primaryServer $ourNodeNumber > /dev/null sdrLocked=no trap posttrap HUP INT QUIT KILL # Issue "command was successful" message. printErrorMsg 272 $mmcmd cleanupAndExit 0