#!/bin/ksh # IBM_PROLOG_BEGIN_TAG # This is an automatically generated prolog. # # # # Licensed Materials - Property of IBM # # (C) COPYRIGHT International Business Machines Corp. 2000,2007 # All Rights Reserved # # US Government Users Restricted Rights - Use, duplication or # disclosure restricted by GSA ADP Schedule Contract with IBM Corp. # # IBM_PROLOG_END_TAG # @(#)99 1.131.1.9 src/avs/fs/mmfs/ts/admin/mmcrcluster.sh, mmfs, avs_rgpfs24, rgpfs24s012a 4/2/07 12:58:08 ############################################################################### # # Usage: # mmcrcluster -N {NodeDesc[,NodeDesc...] | NodeFile} # -p PrimaryServer [-s SecondaryServer] # [-r remoteShellCommand] [-R remoteFileCopyCommand] # [-C ClusterName] [-U DomainName] [-A] [-c ConfigFile] # # where: # # -N NodeDesc,NodeDesc,... specifies a comma-separated list of node # descriptors that detail the node interfaces # to be added to the cluster. The nodes must not # presently belong to the cluster. # # -N NodeFile specifies a file of node descriptors that detail # the node interfaces to be added to the cluster. # The nodes must not presently belong to the cluster. # # -p PrimaryServer specifies the node to be used as the primary server # of the GPFS sdrfs data for this cluster # # -s SecondaryServer specifies the node to be used as the secondary server # of the GPFS sdrfs data for this cluster (optional) # # -r remoteShellCommand specifies the fully-qualified pathname for # the remote shell program to be used by GPFS. # The default is /usr/bin/rsh. # # -R remoteFileCopyCommand specifies the fully-qualified pathname for # the remote file copy program to be used by GPFS. # The default is /usr/bin/rcp. # # -C ClusterName user-specified name for the cluster. If the name # contains dots it is assumed to be a fully-qualified # domain name. Otherwise, the domain will default # to the domain of the primary configuration server. # If not specified, or if single dot is specified, # ClusterName will default to the fully-qualified name # of the primary config server (-p parameter). # # -U DomainName UID domain name # # -A specifies automatic bringing up of mmfs daemons at # boot time. Default is no autoload. # # -c ConfigFile file with mmfs.cfg parameters; see mmfs.cfg.sample # in /usr/lpp/mmfs/samples # # # Obsolete but still supported options: # # -n NodeFile specifies a file of node descriptors that detail the # node interfaces making up the GPFS cluster # # Each node descriptor has the format: # # nodeName:nodeRoles:adminNodeName: # # where: # # nodeName is either a short or fully-qualified hostname, # or an IP address for the primary GPFS network # to be used for daemon to daemon communications # # nodeRoles is a '-' separated list of node roles. Unless changed # on the mmconfig or mmaddnode commands, these roles will # be associated with the node when it becomes a member of # a nodeset. # # adminNodeName is either a short or fully-qualified hostname, or an IP # address to be used by the admin scripts to communicate # between nodes. This is an optional parameter; if it # is not specified, the nodeName value is used. # # Undocumented options: # # -t ClusterType specifies the cluster type for this cluster # (supported values are "lc" and "single"). # See note below for additional information. # # The format of the -t parameter is ClusterType[/EnvironmentType] # where "ClusterType" is the GPFS cluster type as defined above. # The optional "EnvironmentType" designates the environment within # which the GPFS cluster is being created. The following table # shows the allowed values. # # ClusterType EnvironmentType Remarks # # lc lc2 true loose cluster; no RSCT # # lc lc old style Linux only loose cluster; # RSCT subsystem is controlled by GPFS; # this combination is no longer allowed # # single single single node environment # # Notes: # # - Nodes are designated as quorum or non-quorum nodes when they # are added to the cluster (mmcrcluster, mmaddnode). # # - The quorum designation can be changed with mmchconfig. # # - It is not allowed to change (or delete from the cluster) # the last quorum node. # # - If VSD-based disks will be used, all AIX nodes must belong to # an RSCT peer domain. VSD disks will not be visible to nodes # that do not belong to the RSCT peer domain. This rule cannot # be enforced by mmcrcluster or mmaddnode. # ############################################################################### # Include global declarations and service routines. . /usr/lpp/mmfs/bin/mmglobfuncs . /usr/lpp/mmfs/bin/mmsdrfsdef sourceFile="mmcrcluster.sh" [[ -n $DEBUG || -n $DEBUGmmcrcluster ]] && set -x $mmTRACE_ENTER "$*" # Local work files. Names should be of the form: # fn=${tmpDir}fn.${mmcmd}.$$ inputNodes=${tmpDir}inputNodes.${mmcmd}.$$ processedNodes=${tmpDir}processedNodes.${mmcmd}.$$ goodnodes=${tmpDir}goodnodes.${mmcmd}.$$ singleNode=${tmpDir}singleNode.${mmcmd}.$$ LOCAL_FILES=" $inputNodes $processedNodes $goodnodes $singleNode " # Local declarations usageMsg=343 backupServer="" rshPath="" rcpPath="" nodesetId=$HOME_CLUSTER integer nodeNumber=0 integer nodeCount=0 integer totalNodeCount=0 integer quorumNodeCount=0 rc=0 fatalError="" primaryInCluster="" backupInCluster="" lowestVersion=100000 highestVersion=0 osEnvironment=$osName commProtocol=TCP tcpPort=$defaultTcpPort newKeyGenNumber=0 committedKeyGenNumber=0 secLevel=0 keyfileFormatLevel=0 quorumDefault=$nonQuorumNode eventsPort="" mmsdrservPort="" lapiWindow=$noWindow # obsolete switchNumber=$noSwitch # obsolete Aarg=no carg="" Carg="" narg="" Narg="" parg="" rarg="" Rarg="" sarg="" Uarg="" typeset -l targ="" typeset -l role # Local routines ##################################################################### # # Function: Verifies that a command executed successfully. If the # return code from the command is not zero, the function # issues a message and sets a failure flag. It returns # with the same return code that was specified on input. # # Input: $1 - name of the command to check # $2 - return code from the execution of the command # ##################################################################### function checkForErrorsAndReturn { if [ $2 != "0" ] then fatalError=yes # Unexpected error printErrorMsg 171 "$mmcmd" "$1" $2 fi return $2 } #---------- end of function checkForErrorsAndReturn ----------- ######################################################################## # # Function: Verifies the user provided file with mmfs.cfg parameters # and filters out all blank and comment lines. Lines with # configuration parameters that are invalid, or require # additional processing, or have already been established # by the mmcrcluster processing, are filtered out as well. # # Input: $1 - name of file with the user-specified mmfs.cfg values. # $2 - name of output file with verified mmfs.cfg values. # # Output: None explicit. # # Returns: Always zero. # ######################################################################## function processUserConfigFile # { typeset sourceFile="mmcrcluster.sh" [[ -n $DEBUG || -n $DEBUGprocessUserConfigFile ]] && set -x $mmTRACE_ENTER "$*" typeset cfgFile=$1 typeset verifiedCfgFile=$2 typeset -l cfgParm_lc typeset cfgLine cfgParm errCode parmOut typeset nodeOverrideInEffect=false # Issue a progress message (processing user config data . . . ). printInfoMsg 143 "$(date)" $mmcmd $cfgFile $rm -f $verifiedCfgFile exec 4<&- exec 4< $cfgFile while read -u4 cfgLine do # Skip empty and comment lines. [[ $cfgLine = *([$BLANKchar$TABchar]) ]] && continue [[ $cfgLine = *([$BLANKchar$TABchar])#* ]] && continue # Parse the input line. # # Each line is expected to consist of a config parameter and # a corresponding configuration value. If a line contains a # node name, or a list of comma separated names, enclosed in # square brackets, the line introduces a node-override section. # A node-override section is terminated by a "[common]" line. set -f ; set -- $cfgLine ; set +f cfgParm=$1 # Convert the parameter to lower case characters only. cfgParm_lc="$cfgParm" # See if this is a node-override delimiter line. if [[ $cfgParm = "[common]" ]] then # This is the terminating line of a node-override section. nodeOverrideInEffect=false errCode=0 elif [[ $cfgParm = \[* ]] then # This is the begining of a node-override section. nodeOverrideInEffect=true errCode=0 else # This is a line with a config paramter/value pair. # Examine the parameter to see if it should be suppressed. case $cfgParm_lc in # The following parameters are always set by mmcrcluster. autoload ) errCode=1 ;; clusterid ) errCode=1 ;; clustername ) errCode=1 ;; clustertype ) errCode=1 ;; maxfeaturelevelallowed ) errCode=1 ;; usedisklease ) errCode=1 ;; uiddomain ) [[ -n $Uarg ]] && errCode=1 ;; # The following parameters must always be set with mmchconfig. cipherlist ) errCode=2 ;; eventsexportertcpport ) errCode=2 ;; gpfseventsport ) errCode=2 ;; mmsdrservport ) errCode=2 ;; mmsdrservtcpport ) errCode=2 ;; tcpport ) errCode=2 ;; tiebreakerdisk ) errCode=2 ;; tsctcpport ) errCode=2 ;; # The following parameters are obsolete. comm_protocol ) errCode=3 ;; corequorum ) errCode=3 ;; diskquorum ) errCode=3 ;; dynmemsize ) errCode=3 ;; gpfsobjectport ) errCode=3 ;; group ) errCode=3 ;; importancefactor ) errCode=3 ;; locktableversion ) errCode=3 ;; logdir ) errCode=3 ;; mallocsize ) errCode=3 ;; maxpagepool ) errCode=3 ;; memrebalanceinterval ) errCode=3 ;; mmgetobjdport ) errCode=3 ;; minpagepool ) errCode=3 ;; multinode ) errCode=3 ;; nodelist ) errCode=3 ;; nodeprefs ) errCode=3 ;; recgroup ) errCode=3 ;; singlenodequorum ) errCode=3 ;; spsecworkerthreads ) errCode=3 ;; useauthentication ) errCode=3 ;; usesinglenodequorum ) errCode=3 ;; usespsecurity ) errCode=3 ;; worker2threads ) errCode=3 ;; # Everything else looks acceptable for now. * ) errCode=0 ;; esac # end of case $cfgParm_lc in # If no error was detected so far, see if the parameter # is allowed to appear in a node-override section. if [[ $errCode -eq 0 && $nodeOverrideInEffect = true ]] then case $cfgParm_lc in # The following parameters cannot appear in a node-override section. allowdummyconnections ) errCode=4 ;; allowremoteconnections ) errCode=4 ;; automountdir ) errCode=4 ;; distributedtokenserver ) errCode=4 ;; leasedmstimeout ) errCode=4 ;; leaseduration ) errCode=4 ;; leaserecoverywait ) errCode=4 ;; listenonallinterfaces ) errCode=4 ;; maxmissedpingtimeout ) errCode=4 ;; maxtokenservers ) errCode=4 ;; minmissedpingtimeout ) errCode=4 ;; minquorumnodes ) errCode=4 ;; multitmmountthreshold ) errCode=4 ;; pindaemon ) errCode=4 ;; pingperiod ) errCode=4 ;; res ) errCode=4 ;; setctimeonattrchange ) errCode=4 ;; totalpingtimeout ) errCode=4 ;; tscprimary ) errCode=4 ;; uiddomain ) errCode=4 ;; writealldescreplicas ) errCode=4 ;; # Everything else is acceptable. * ) errCode=0 ;; esac # end of case $cfgParm_lc in fi # end of if [[ -z $errCode && -n $nodeOverrideInEffect ]] fi # end of if [[ $cfgParm = "[common]" ]] # If an error was detected, issue an appropriate message and # skip the line. Otherwise, add the line to the verified file. if [[ $errCode -eq 1 ]] then # Parameter is set by mmcrcluster. printErrorMsg 144 $mmcmd $cfgParm "$cfgLine" elif [[ $errCode -eq 2 ]] then # Parameter must be set using mmchconfig. printErrorMsg 145 $mmcmd $cfgParm mmchconfig "$cfgLine" elif [[ $errCode -eq 3 ]] then # Parameter is obsolete. printErrorMsg 146 $mmcmd $cfgParm "$cfgLine" elif [[ $errCode -eq 4 ]] then # Parameter cannot appear in a node-override section. printErrorMsg 147 $mmcmd $cfgParm "$cfgLine" else # The line looks OK. Ensure the parameter is spelled correctly. # Print the line to the verified file. case $cfgParm_lc in allowdeleteaclonchmod ) parmOut=allowDeleteAclOnChmod ;; allowdummyconnections ) parmOut=allowDummyConnections ;; allowremoteconnections ) parmOut=allowRemoteConnections ;; allowsynchronousfcntlretries ) parmOut=allowSynchronousFcntlRetries ;; assertonstructureerror ) parmOut=assertOnStructureError ;; asyncsocketnotify ) parmOut=asyncSocketNotify ;; automountdir ) parmOut=automountDir ;; autosgloadbalance ) parmOut=autoSgLoadBalance ;; crashdump ) parmOut=crashdump ;; datastructuredump ) parmOut=dataStructureDump ;; datastructuredumponsgpanic ) parmOut=dataStructureDumpOnSGPanic ;; distributedtokenserver ) parmOut=distributedTokenServer ;; dmapienable ) parmOut=dmapiEnable ;; dmapieventbuffers ) parmOut=dmapiEventBuffers ;; dmapieventtimeout ) parmOut=dmapiEventTimeout ;; dmapimounttimeout ) parmOut=dmapiMountTimeout ;; dmapisessionfailuretimeout ) parmOut=dmapiSessionFailureTimeout ;; dmapiworkerthreads ) parmOut=dmapiWorkerThreads ;; eewatchdoghungthreadcutoff ) parmOut=eeWatchDogHungThreadCutoff ;; eewatchdoginterval ) parmOut=eeWatchDogInterval ;; enablenfscluster ) parmOut=enableNFSCluster ;; enableuidremap ) parmOut=enableUIDremap ;; enablestatuidremap ) parmOut=enableStatUIDremap ;; enabletreebasedquotas ) parmOut=enableTreeBasedQuotas ;; envvar ) parmOut=envVar ;; flusheddatatarget ) parmOut=flushedDataTarget ;; flushedinodetarget ) parmOut=flushedInodeTarget ;; healthcheckinterval ) parmOut=healthCheckInterval ;; hotlistpct ) parmOut=hotlistPct ;; ignorereplicaspaceonstat ) parmOut=IgnoreReplicaSpaceOnStat ;; iohistorysize ) parmOut=ioHistorySize ;; leasedmstimeout ) parmOut=leaseDMSTimeout ;; leaseduration ) parmOut=leaseDuration ;; leaserecoverywait ) parmOut=leaseRecoveryWait ;; license ) parmOut=LICENSE ;; listenonallinterfaces ) parmOut=listenOnAllInterfaces ;; mmapkprocs ) parmOut=mmapKprocs ;; maxallocpcttocache ) parmOut=maxAllocPctToCache ;; maxbackgrounddeletionthreads ) parmOut=maxBackgroundDeletionThreads ;; maxblocksize ) parmOut=maxblocksize ;; maxbufferdescs ) parmOut=maxBufferDescs ;; maxdatashippoolsize ) parmOut=maxDataShipPoolSize ;; maxdiskaddrbuffs ) parmOut=maxDiskAddrBuffs ;; maxfcntlrangesperfile ) parmOut=maxFcntlRangesPerFile ;; maxfilecleaners ) parmOut=maxFileCleaners ;; maxfilestocache ) parmOut=maxFilesToCache ;; maxinodedeallochistory ) parmOut=maxInodeDeallocHistory ;; maxinodedeallocprefetch ) parmOut=maxInodeDeallocPrefetch ;; maxmbps ) parmOut=maxMBpS ;; maxmissedpingtimeout ) parmOut=maxMissedPingTimeout ;; maxnfsdelegationtimeout ) parmOut=maxNFSDelegationTimeout ;; maxreceiverthreads ) parmOut=maxReceiverThreads ;; maxsgdesciobufsize ) parmOut=maxSGDescIOBufSize ;; maxstatcache ) parmOut=maxStatCache ;; maxtokenservers ) parmOut=maxTokenServers ;; minmissedpingtimeout ) parmOut=minMissedPingTimeout ;; minquorumnodes ) parmOut=minQuorumNodes ;; mmsdrservtimeout ) parmOut=mmsdrservTimeout ;; mmsdrservworkerpool ) parmOut=mmsdrservWorkerPool ;; multitmmountthreshold ) parmOut=multiTMMountThreshold ;; nfsprefetchstrategy ) parmOut=nfsPrefetchStrategy ;; nsdbufspace ) parmOut=nsdbufspace ;; nsdmaxworkerthreads ) parmOut=nsdMaxWorkerThreads ;; nsdminworkerthreads ) parmOut=nsdMinWorkerThreads ;; nsdservercheckingintervalformount ) parmOut=nsdServerCheckingIntervalForMount ;; nsdserverwaitconfig ) parmOut=nsdServerWaitConfig ;; nsdserverwaittimeformount ) parmOut=nsdServerWaitTimeForMount ;; nsdserverwaittimewindowonmount ) parmOut=nsdServerWaitTimeWindowOnMount ;; nsdthreadsperdisk ) parmOut=nsdThreadsPerDisk ;; openssllibname ) parmOut=openssllibname ;; pagepool ) parmOut=pagepool ;; panicondiskfail ) parmOut=unmountOnDiskFail ;; pcttokenmgrstorageuse ) parmOut=pctTokenMgrStorageUse ;; pindaemon ) parmOut=pindaemon ;; pingperiod ) parmOut=pingPeriod ;; pinmaster ) parmOut=pinmaster ;; prefetchpct ) parmOut=prefetchPct ;; prefetchthreads ) parmOut=prefetchThreads ;; prefetchtimeout ) parmOut=prefetchTimeout ;; priority ) parmOut=priority ;; readreplicapolicy ) parmOut=readReplicaPolicy ;; res ) parmOut=res ;; retryfcntltokenthreshold ) parmOut=retryFcntlTokenThreshold ;; seqdiscardthreshhold ) parmOut=seqDiscardThreshhold ;; setctimeonattrchange ) parmOut=setCtimeOnAttrChange ;; sharedmemlimit ) parmOut=sharedMemLimit ;; socketrcvbuffersize ) parmOut=socketRcvBufferSize ;; socketsndbuffersize ) parmOut=socketSndBufferSize ;; statcachedirpct ) parmOut=statCacheDirPct ;; subnets ) parmOut=subnets ;; syncinterval ) parmOut=syncInterval ;; takeovertimeout ) parmOut=takeovertimeout ;; tokenmemlimit ) parmOut=tokenMemLimit ;; totalpingtimeout ) parmOut=totalPingTimeout ;; trace ) parmOut=trace ;; tscprimary ) parmOut=tscPrimary ;; tscworkerpool ) parmOut=tscWorkerPool ;; uiddomain ) parmOut=uidDomain ;; uidexpiration ) parmOut=uidExpiration ;; unmountondiskfail ) parmOut=unmountOnDiskFail ;; wait4rvsd ) parmOut=wait4RVSD ;; waitforvsd ) parmOut=wait4RVSD ;; watchdogtimeout ) parmOut=watchdogtimeout ;; worker1threads ) parmOut=worker1Threads ;; worker3threads ) parmOut=worker3Threads ;; writealldescreplicas ) parmOut=writeAllDescReplicas ;; writebehindthreshhold ) parmOut=writebehindThreshhold ;; \[* ) parmOut="" ;; * ) parmOut="$cfgParm" ;; esac # end of case $cfgParm_lc in # Print the line to the verified file. print -- "$parmOut${cfgLine#$cfgParm}" >> $verifiedCfgFile checkForErrors "writing to file $verifiedCfgFile" $? fi done # end of while read -u4 cfgLine return 0 } #---------- end of function processUserConfigFile ----------- ###################### # Mainline processing ###################### ################################# # Process the command arguments. ################################# [[ $arg1 = '-?' || $arg1 = '-h' || $arg1 = '--help' || $arg1 = '--' ]] && \ syntaxError "help" $usageMsg while getopts :Ac:C:n:N:p:r:R:s:t:U: OPT do case $OPT in A) # autoload option [[ $Aarg = yes ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Aarg=yes ;; c) # config file name [[ -n $carg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" carg=$OPTARG ;; C) # cluster name [[ -n $Carg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Carg=$OPTARG checkName clusterName 255 "$Carg" [[ $? -ne 0 ]] && cleanupAndExit ;; n) # node descriptors file [[ -n $narg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" narg=$OPTARG ;; N) # node descriptors list or file [[ -n $Narg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Narg=$OPTARG ;; p) # primary server [[ -n $parg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" parg=$OPTARG ;; r) # remote shell command [[ -n $rarg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" rarg=$OPTARG [[ $rarg = ${rarg#/} ]] && \ syntaxError "absolutePath_2" $noUsageMsg "-$OPT" "$rarg" ;; R) # remote file copy command [[ -n $Rarg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Rarg=$OPTARG [[ $Rarg = ${Rarg#/} ]] && \ syntaxError "absolutePath_2" $noUsageMsg "-$OPT" "$Rarg" ;; s) # secondary server [[ -n $sarg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" sarg=$OPTARG ;; t) # cluster type [[ -n $targ ]] && syntaxError "multiple" $noUsageMsg "-$OPT" targ=$OPTARG ;; U) # UID domain name [[ -n $Uarg ]] && syntaxError "multiple" $noUsageMsg "-$OPT" Uarg=$OPTARG ;; +[AcCnNprRstU]) # Invalid option syntaxError "invalidOption" $usageMsg $OPT ;; :) # Missing argument syntaxError "missingValue" $usageMsg $OPTARG ;; *) # Invalid option syntaxError "invalidOption" $usageMsg $OPTARG ;; esac done # end of while getopts :Ac:C:n:N:p:r:R:s:t:U: OPT do shift OPTIND-1 [[ $# -ne 0 ]] && syntaxError "extraArg" $usageMsg $1 [[ -z $targ ]] && targ="lc/lc2" [[ $targ != single* && ( -z $parg || ( -z $Narg && -z $narg ) ) ]] && \ syntaxError "missingArgs" $usageMsg [[ -n $narg && -n $Narg ]] && \ syntaxError "invalidCombination" $usageMsg "-n" "-N" ############################################################## # If the cluster type is single, simulate missing parameters. ############################################################## if [[ $targ = single* ]] then if [[ -z $Narg && -z $narg ]] then print -- "$($hostname)" > $singleNode Narg=$singleNode fi [[ -z $parg ]] && \ parg=$($hostname) quorumDefault=$quorumNode mmsdrservPort=0 fi ########################################################### # Check the value of the mandatory cluster type parameter. ########################################################### IFS='/' set -f ; set -- $targ ; set +f clusterType=$1 environmentType=$2 IFS="$IFS_sv" case $clusterType in lc ) [[ -z $environmentType ]] && \ environmentType=lc2 [[ $environmentType != lc2 ]] && \ invalidClusterType=yes ;; single ) [[ -z $environmentType ]] && \ environmentType=single [[ $environmentType != single ]] && \ invalidClusterType=yes ;; * ) invalidClusterType=yes ;; esac if [[ $invalidClusterType = yes ]] then # Invalid value for cluster type. printErrorMsg 153 $mmcmd "-t" cleanupAndExit fi clusterSubtype=$environmentType ######################################################## # Check whether the prerequisite software is installed. ######################################################## checkPrereqs $environmentType if [[ $? -ne 0 ]] then # The prerequisite software is not installed. printErrorMsg 349 $mmcmd $($hostname) cleanupAndExit fi ########################################################################## # Set global variables. These are normally determined automatically, # but since a cluster does not exist yet, they need to be set explicitly. ########################################################################## export MMMODE=$clusterType export environmentType=$environmentType ourNodeName=$(hostname) ################################################## # Check the mandatory node descriptors parameter. ################################################## if [[ -n $Narg ]] then # The -N parameter may be either a list or a file. Which is it? if [[ -f $Narg ]] then # It is a file; verify its existence and create our own copy. checkUserFile $Narg $inputNodes [[ $? -ne 0 ]] && cleanupAndExit else # It is not a file, so it must be a list. # Convert the input node list into a file. $rm -f $inputNodes IFS=',' for nodeDesc in $Narg do print -- "$nodeDesc" >> $inputNodes checkForErrors "writing to $inputNodes" $? done IFS="$IFS_sv" # Restore the default IFS setting. fi # end of if [[ -f $Narg ]] else # Since -N was not specified, -n must have been. # Check the node names file parameter and create our own copy. checkUserFile $narg $inputNodes [[ $? -ne 0 ]] && cleanupAndExit fi # end of if [[ -n $Narg ]] # Make sure that at least one node is designated as quorum, # and count the number of quorum nodes. # If there are more than 7 quorum nodes, issue a warning; # If there are more than 128 quorum nodes, issue an error and quit. if [[ $clusterType = lc ]] then quorumNodeCount=$($awk -F: ' \ BEGIN { nqnodes = 0 } \ $1 ~ /^#/ || NF <= 1 { next } \ $2 ~ "quorum" && $2 !~ "nonquorum" { \ { nqnodes += 1 } \ } \ END { print nqnodes } \ ' $inputNodes) if [[ $quorumNodeCount -eq 0 ]] then # At least one quorum node must be defined. printErrorMsg 53 $mmcmd cleanupAndExit fi if [[ $quorumNodeCount -gt $maxQuorumNodes ]] then # Error: The number of quorum nodes exceeds the maximum allowed. printErrorMsg 393 $mmcmd $maxQuorumNodes cleanupAndExit fi if [[ $quorumNodeCount -gt $maxRecQuorumNodes ]] then # Warning: The number of quorum nodes exceeds the recommended maximum. printErrorMsg 394 $mmcmd $maxRecQuorumNodes fi fi # end of if [[ $clusterType = lc ]] ################################################### # Process the optional remote commands parameters. ################################################### if [[ -n $rarg ]] then if [[ ! -x $rarg ]] then # Path is not an executable. printErrorMsg 422 $mmcmd ${rarg%% *} cleanupAndExit fi rshPath="$rarg" rsh="$rshPath" export GPFS_rshPath="$rshPath" else rshPath="_DEFAULT_" fi if [[ -n $Rarg ]] then if [[ ! -x $Rarg ]] then # Path is not an executable. printErrorMsg 423 $mmcmd ${Rarg%% *} cleanupAndExit fi rcpPath="$Rarg" rcp="$rcpPath" export GPFS_rcpPath="$rcpPath" else rcpPath="_DEFAULT_" fi ################################################# # Check the optional -c ConfigFile parameter. ################################################# if [[ -n $carg && ( ! -f $carg || ! -r $carg ) ]] then # Can't read the file. printErrorMsg 43 $mmcmd $carg cleanupAndExit fi #################################################### # Fail the command if an sdrfs file already exists. # This should never happen, as mmcrcluster should # only be run once before the sdrfs file exists. #################################################### if [[ -f $mmsdrfsFile ]] then printErrorMsg 348 $mmcmd $($hostname) cleanupAndExit fi ################################## # Set up trap exception handling. ################################## trap pretrap2 HUP INT QUIT KILL ####################################################### # Determine the lookup order for resolving host names. ####################################################### [[ $osName != AIX ]] && resolveOrder=$(setHostResolveOrder) ######################################## # Process the primary server parameter. ######################################## # Find the name and IP address of the primary server. hostResult=$($host $parg) set -f ; set -- $hostResult ; set +f primaryServer=$1 primaryIPaddr=${3%%,*} # Exclude everything after the first comma. if [[ -z $primaryIPaddr ]] then # Invalid node name specified. printErrorMsg 54 $mmcmd $parg cleanupAndExit fi ##################################################### # Process the backup server parameter, if specified. ##################################################### if [[ -n $sarg ]] then # Find the name and IP address of the secondary server. hostResult=$($host $sarg) set -f ; set -- $hostResult ; set +f backupServer=$1 backupIPaddr=${3%%,*} # Exclude everything after the first comma. if [[ -z $backupIPaddr ]] then # An invalid node name was specified. printErrorMsg 54 $mmcmd $sarg cleanupAndExit fi # Verify that the two servers are different. if [[ $primaryServer = $backupServer ]] then printErrorMsg 346 $mmcmd $primaryServer cleanupAndExit fi else backupServer="_NOSECONDARY_" fi # end of if [[ -n $sarg ]] ###################################### # Process the cluster name parameter. ###################################### if [[ -z $Carg || $Carg = "." ]] then clusterName=$primaryServer else if [[ $Carg = *.* ]] then clusterName=$Carg else clusterName=${Carg}.${primaryServer#*.} fi fi ########################################## # Generate a unique cluster identifier. ########################################## clusterId=$($mmcrclusterid $primaryIPaddr) # Create the overloaded clType and clusterIdAndSt parameters # that will be passed to the checkNewClusterNode routine. if [[ $clusterType = $environmentType ]] then clType=$clusterType else clType="${clusterType}/${environmentType}" fi clusterIdAndSt="${clusterId}:${clusterSubtype}" ###################################################################### # Generate the node information for the mmsdrfs file. # # Loop through the nodes to be added to the sdrfs file, checking # as we go. When the loop is done we know which nodes can be added # to the sdrfs file, which ones can't, and whether the primary and # backup servers belong to the cluster. A MEMBER_NODE line will be # generated for each node that can be added to the new GPFS cluster. ###################################################################### $rm -f $tmpsdrfs $goodnodes $processedNodes $touch $tmpsdrfs $goodnodes $processedNodes exec 3<&- exec 3< $inputNodes while read -u3 nodeDesc do # Skip empty and comment lines. [[ $nodeDesc = *([$BLANKchar$TABchar]) ]] && continue [[ $nodeDesc = *([$BLANKchar$TABchar])#* ]] && continue # Keep track of the total number of nodes specified by the user. totalNodeCount=$totalNodeCount+1 # Parse the node descriptor. IFS=':' set -f ; set -- $nodeDesc ; set +f nodeName=$1 nodeRoles=$2 nodeName2=$3 IFS="$IFS_sv" # Assume default values for the node role fields. designation=$CLIENT quorumField=$quorumDefault # Process the node roles list. if [[ -n $nodeRoles ]] then IFS="-" set -f ; set -- $nodeRoles ; set +f IFS="$IFS_sv" while [[ -n $1 ]] do role=$1 # Convert the node's role to lower case only. case $role in $CLIENT ) designation=$CLIENT ;; $MANAGER ) designation=$MANAGER ;; $QUORUM ) quorumField=$quorumNode ;; $NONQUORUM ) quorumField=$nonQuorumNode ;; * ) # Invalid node designations specified. printErrorMsg 293 $mmcmd "$nodeDesc" fatalError=yes break 2 ;; esac # Move to the next field. shift done # end while [[ -n $1 ]] fi # end if [[ -n $nodeRoles ]] # At this point, nodeName can be any one of the following: # a fully-qualified adapter port name, a short name, or an IP address. # Determine the values for all three. We will designate the fully # qualified hostname as the reliable hostname for the node. hostResult=$($host $nodeName) set -f ; set -- $hostResult ; set +f daemonNodeName=$1 shortName=${1%% *|.*} # Exclude everything after the first dot. ipa=${3%%,*} # Exclude everything after the first comma. if [[ -z $ipa ]] then # An invalid node name was specified. printErrorMsg 54 $mmcmd $nodeName fatalError=yes break fi # At this point, if it was specified, nodeName2 could be a # fully-qualified adapter port name, a short name, or an IP address. # Convert it to a fully-qualified node name in case it is not one already. if [[ -n $nodeName2 ]] then hostResult=$($host $nodeName2) set -f ; set -- $hostResult ; set +f adminNodeName=$1 adminShortName=${1%% *|.*} # Exclude everything after the first dot. adminIpa=${3%%,*} # Exclude everything after the first comma. if [[ -z $adminIpa ]] then # An invalid admin node name was specified. printErrorMsg 54 $mmcmd $nodeName2 fatalError=yes break fi else # The user did not set a distinct admin node name, so set the # admin node names to be the same as the daemon node names. adminNodeName=$daemonNodeName adminShortName=$shortName fi # Make sure neither node name (admin or daemon) is specified more than once. $grep -qw $daemonNodeName $processedNodes > /dev/null 2>&1 if [[ $? -eq 0 ]] then # The node is specified twice. printErrorMsg 347 $mmcmd $nodeName fatalError=yes break fi if [[ $adminNodeName != $daemonNodeName ]] then $grep -qw $adminNodeName $processedNodes > /dev/null 2>&1 if [[ $? -eq 0 ]] then # The node is specified twice. printErrorMsg 347 $mmcmd $nodeName2 fatalError=yes break fi fi # Assign a node number to the node. nodeNumber=$nodeNumber+1 gsNodeNumber=$nodeNumber adapterType="" if [[ $environmentType = single ]] then adapterType="lo" if [[ $nodeNumber -gt 1 ]] then # Invalid node name specified. printErrorMsg 54 $mmcmd $nodeName fatalError=yes break fi fi # end of if [[ $environmentType = single ]] # Add the daemon and admin node names to the list of processed nodes. print -- "${daemonNodeName}:${adminNodeName}" >> $processedNodes checkForErrorsAndReturn "writing to file $processedNodes" $? # Build a line with the local node data for the node. The full-blown # MEMBER_NODE line will be created further down when we have all of the # information that we need. sdrfsLine="$nodesetId:$MEMBER_NODE::0:$nodeNumber:$shortName:$ipa" sdrfsLine="$sdrfsLine:$adminNodeName:$designation:$adapterType:$lapiWindow" sdrfsLine="$sdrfsLine:$switchNumber:$OLD_NODE:$adapterType:$daemonNodeName" sdrfsLine="$sdrfsLine:$adminShortName::::$quorumField:$gsNodeNumber:" # Invoke the checkNewClusterNode function to assure that the # node is new and that its level of GPFS supports clusters. # If it passes these tests, a skeleton sdrfs file is stored on the node. printInfoMsg 416 "$(date)" $mmcmd $daemonNodeName runOutput=$(run on1 $adminNodeName checkNewClusterNode \ $clType $primaryServer $backupServer "$sdrfsLine" \ "$rshPath" "$rcpPath" "$clusterIdAndSt" 2> $errMsg) rc=$? IFS=':' set -f ; set -- $runOutput ; set +f IFS="$IFS_sv" keyword=$1 nodeStatus=$2 adapterType=$3 installedDaemonVersion=$4 installedProductVersion=$5 installedOsName=$6 if [[ $rc = 0 && $nodeStatus = success && $keyword = checkNewClusterNode ]] then # The checkNewClusterNode call succeeded. # Build the line that will represent this node in the mmsdrfs file. nodeCount=$nodeCount+1 sdrfsLine="$nodesetId:$MEMBER_NODE::$nodeCount:$nodeNumber:$shortName:$ipa" sdrfsLine="$sdrfsLine:$adminNodeName:$designation:$adapterType:$lapiWindow" sdrfsLine="$sdrfsLine:$switchNumber:$OLD_NODE:$adapterType:$daemonNodeName" sdrfsLine="$sdrfsLine:$adminShortName:$installedDaemonVersion:$installedProductVersion" sdrfsLine="$sdrfsLine:$installedOsName:$quorumField:$gsNodeNumber:" # Add the MEMBER_NODE line to the other lines # that will go in the mmsdrfs file. print -- "$sdrfsLine" >> $tmpsdrfs checkForErrorsAndReturn "Writing to file $tmpsdrfs" $? [[ $? -ne 0 ]] && break # Add the node name to the list of successful nodes. print -- "$adminNodeName" >> $goodnodes checkForErrorsAndReturn "Writing to file $goodnodes" $? [[ $? -ne 0 ]] && break # If this node is one of the server nodes, set a flag to indicate that # the server was found, and set the server name to the admin node name. if [[ $adminNodeName = $primaryServer || $daemonNodeName = $primaryServer ]] then primaryServer=$adminNodeName primaryInCluster=yes fi if [[ $adminNodeName = $backupServer || $daemonNodeName = $backupServer ]] then backupServer=$adminNodeName backupInCluster=yes fi # Is this a quorum node? [[ $quorumField != $nonQuorumNode ]] && \ quorumNodeCount=$quorumNodeCount+1 # Keep track of the daemon version and os installed. [[ $installedDaemonVersion -lt $lowestVersion ]] && \ lowestVersion=$installedDaemonVersion [[ $installedDaemonVersion -gt $highestVersion ]] && \ highestVersion=$installedDaemonVersion [[ -n $installedOsName && $installedOsName != $osEnvironment ]] && \ osEnvironment="mixed" else # The checkNewClusterNode call failed. # Not all errors are considered terminal. # If an individual node fails for a known reason, # we will not include it in the cluster but will # continue with the rest of the nodes. # Tell the world what went wrong. if [[ $nodeStatus = not_new ]] then # The node already belongs to a cluster. printErrorMsg 348 $mmcmd $adminNodeName elif [[ $nodeStatus = not_supported ]] then # Wrong GPFS code level. printErrorMsg 349 $mmcmd $adminNodeName elif [[ $nodeStatus = ipa_alias ]] then # IP address aliasing is not supported. printErrorMsg 476 $mmcmd $nodeName elif [[ $nodeStatus = ipa_missing ]] then # The daemon node adapter was not found on the admin node. printErrorMsg 175 $mmcmd $nodeName $nodeName2 elif [[ $rc = $MM_HostDown || $rc = $MM_ConnectTimeout ]] then # The node cannot be reached. printErrorMsg 340 $mmcmd $adminNodeName else # Unexpected error. Display error messages. [[ -s $errMsg ]] && $cat $errMsg 1>&2 [[ $rc -eq 0 ]] && rc=1 checkForErrorsAndReturn "checkNewClusterNode $adminNodeName" $rc run on1 $adminNodeName removeFromCluster > /dev/null 2>&1 & # break fi # If any of the server nodes fails, give up. if [[ $adminNodeName = $primaryServer || $adminNodeName = $backupServer ]] then fatalError=yes break fi # Append the node name to the list of failed nodes. failedNodes="$failedNodes\n\t\t$adminNodeName" # Adjust the node number for the next iteration. nodeNumber=$nodeNumber-1 fi # end of if [[ $rc = 0 && $nodeStatus = success ]] $rm -f $errMsg done # end of while read -u3 nodeDesc (Loop through the nodes to be added) ##################################################### # There is no point continuing if not even one node # will be added to the cluster. ##################################################### if [[ $totalNodeCount -eq 0 ]] then # The user provided input file appears to be empty. # Note that we can have either -N or -n specified but not both. printErrorMsg 329 $mmcmd $Narg $narg cleanupAndExit fi if [[ ! -s $goodnodes ]] then # The command failed. printErrorMsg 389 $mmcmd cleanupAndExit fi ###################################################### # If no severe errors were detected so far, # ensure certain other requirements are met as well. ###################################################### if [[ -z $fatalError ]] then # Ensure that the server nodes are members of the cluster # and were successfully initialized (checkNewClusterNode). if [[ -z $primaryInCluster ]] then printErrorMsg 352 $mmcmd "$primaryServer" fatalError=yes fi if [[ -z $backupInCluster && $backupServer != "_NOSECONDARY_" ]] then printErrorMsg 352 $mmcmd "$backupServer" fatalError=yes fi # Ensure that there is at least one quorum node in the cluster. if [[ $quorumNodeCount -eq 0 ]] then printErrorMsg 53 $mmcmd fatalError=yes fi # Ensure the node on which mmcrcluster runs (this node) is # going to be a member of the cluster. if [[ ! -f $mmfsNodeData ]] then # Run the command from a node that is part of the cluster. printErrorMsg 322 $mmcmd fatalError=yes fi fi # end of if [[ -z $fatalError ]] ######################################################### # If no severe errors were detected so far, # build the new mmsdrfs file. ######################################################### if [[ -z $fatalError ]] then #------------------------------------------------- # Create a mmsdrfs file with gen number 2 since # the nodes already have files with gen number 1. #------------------------------------------------- # Create the version line. newGenNumber=2 # gen number 2, since the nodes already have gen number 1 [[ $backupServer = "_NOSECONDARY_" ]] && backupServer="" [[ $rshPath = "_DEFAULT_" ]] && rshPath="" [[ $rcpPath = "_DEFAULT_" ]] && rcpPath="" timeStamp=$($perl -e 'print time') sdrfsLine="$GLOBAL_ID:$VERSION_LINE::$CURRENT_SDRFS_FORMAT:$CURRENT_SDRFS_VERSION" sdrfsLine="$sdrfsLine:$newGenNumber::$MMMODE:$primaryServer:$backupServer" sdrfsLine="$sdrfsLine::$rshPath:$rcpPath:$clusterId:$clusterSubtype" sdrfsLine="$sdrfsLine:$timeStamp:$availableField:$clusterName" sdrfsLine="$sdrfsLine:$newKeyGenNumber:$secLevel:$committedKeyGenNumber" sdrfsLine="$sdrfsLine:$keyfileFormatLevel:" print -- "$sdrfsLine" > $newsdrfs # Add threatening comment lines - DO NOT MESS WITH THIS FILE sdrfsLine="$GLOBAL_ID:$COMMENT_LINE::1:" print -- "$sdrfsLine" >> $newsdrfs sdrfsLine="$GLOBAL_ID:$COMMENT_LINE::2:$warningText" print -- "$sdrfsLine" >> $newsdrfs sdrfsLine="$GLOBAL_ID:$COMMENT_LINE::3:" print -- "$sdrfsLine" >> $newsdrfs # Add a header line for the nodeset. sdrfsLine="$nodesetId:$NODESET_HDR:::$nodeCount:$commProtocol:$cipherList" sdrfsLine="$sdrfsLine:$tcpPort:$obsoleteField:$obsoleteField:$availableField" sdrfsLine="$sdrfsLine:$lowestVersion:$highestVersion:$osEnvironment" sdrfsLine="$sdrfsLine:$availableField:$eventsPort:$mmsdrservPort:" print -- "$sdrfsLine" >> $newsdrfs checkForErrors "writing to file $newsdrfs" $? # Copy the lines describing the individual nodes. $cat $tmpsdrfs >> $newsdrfs checkForErrorsAndReturn "Writing to file $newsdrfs" $? #-------------------------- # Build the mmfs.cfg file. #-------------------------- # Start the mmfs.cfg file with the warning text. print -- "# " > $newcfg print -- "# WARNING: This is a machine generated file. Do not edit! " >> $newcfg print -- "# Use the mmchconfig command to change configuration parameters. " >> $newcfg print -- "# " >> $newcfg # The following parameters are always specified # regardless of any value already existing in the config file. print -- "clusterName $clusterName" >> $newcfg print -- "clusterId $clusterId" >> $newcfg print -- "clusterType $MMMODE" >> $newcfg print -- "autoload $Aarg" >> $newcfg [[ $MMMODE = lc ]] && print -- "useDiskLease yes" >> $newcfg [[ -n $Uarg ]] && print -- "uidDomain $Uarg" >> $newcfg [[ -z $lowestVersion ]] && lowestVersion="$initialDaemonVersion" print -- "maxFeatureLevelAllowed $lowestVersion" >> $newcfg checkForErrorsAndReturn "Writing to file $newcfg" $? # If a configuration file is specified (-c option), # verify its correctness and append it to our mmfs.cfg file. if [[ -n $carg ]] then processUserConfigFile $carg $tmpCfg if [[ -s $tmpCfg ]] then $cat $tmpCfg >> $newcfg checkForErrorsAndReturn "cat $tmpCfg >> $newcfg" $? fi fi #-------------------------------------------- # Add the mmfs.cfg data to the mmsdrfs file. #-------------------------------------------- appendCfgFile $nodesetId $newcfg $newsdrfs checkForErrorsAndReturn "appendCfgFile" $? #-------------------------------------------- # Sort the new version of the mmsdrfs file. #-------------------------------------------- LC_ALL=C $SORT_MMSDRFS $newsdrfs -o $newsdrfs checkForErrorsAndReturn "Sorting $newsdrfs" $? fi # end of if [[ -z $fatalError ]] ############################################################ # If anything failed, remove the skeleton sdrfs files from # the non-server nodes, issue an error message, and die. ############################################################ if [[ -n $fatalError ]] then # Remove already-installed system files. printErrorMsg 351 $mmcmd $ln $goodnodes ${goodnodes}async $mmcommon onall_async ${goodnodes}async removeFromClusterCR & # The command failed. printErrorMsg 389 $mmcmd cleanupAndExit fi ################################################################## # Put the new mmsdrfs file on the primary and secondary servers. ################################################################## # Set some values that are expected by commitChanges. getLocalNodeData nsId=0 gpfsObjectInfo="0:1:" # Commit the changes. trap "" HUP INT QUIT KILL gpfsObjectInfo=$(commitChanges \ $nsId $nsId $gpfsObjectInfo $newGenNumber $newsdrfs \ $primaryServer FORCE $backupServer) rc=$? if [[ $rc -ne 0 ]] then # We cannot replace the file in the sdr. printErrorMsg 381 $mmcmd # Remove already-installed system files. printErrorMsg 351 $mmcmd $ln $goodnodes ${goodnodes}async $mmcommon onall_async ${goodnodes}async removeFromClusterCR & cleanupAndExit fi # Restore interrupts. trap posttrap HUP INT QUIT KILL ####################################################################### # At this point, skeleton sdrfs files have been put on the non-server # nodes in the success list and the real sdrfs file containing the # nodes from the cluster has been created and put on the server nodes. # If there are any nodes in the failure list, we issue a message # telling the user to use the mmaddnode command to add them to the # sdrfs file once they become reachable. ####################################################################### # Report any nodes that could not be added. [[ -n $failedNodes ]] && \ printErrorMsg 353 $mmcmd "$failedNodes" # If not all of the nodes were added to the cluster, # tell the user how many made it through. [[ $nodeCount -lt $totalNodeCount ]] && \ printErrorMsg 12 $mmcmd $nodeCount $totalNodeCount # Issue "command was successful" message. printErrorMsg 272 $mmcmd ####################################################################### # Update the mmsdrfs file on all nodes that were successfully added # to the GPFS cluster. This process is asynchronous. ####################################################################### propagateSdrfsFile async $goodnodes $newsdrfs $newGenNumber cleanupAndExit $rc