source: gpfs_3.1_ker2.6.20/lpp/mmfs/bin/mmaddnode @ 146

Last change on this file since 146 was 16, checked in by rock, 17 years ago
  • Property svn:executable set to *
File size: 30.3 KB
Line 
1#!/bin/ksh
2# IBM_PROLOG_BEGIN_TAG
3# This is an automatically generated prolog.
4# 
5# 
6# 
7# Licensed Materials - Property of IBM
8# 
9# (C) COPYRIGHT International Business Machines Corp. 1997,2006
10# All Rights Reserved
11# 
12# US Government Users Restricted Rights - Use, duplication or
13# disclosure restricted by GSA ADP Schedule Contract with IBM Corp.
14# 
15# IBM_PROLOG_END_TAG
16# @(#)15 1.136.1.3 src/avs/fs/mmfs/ts/admin/mmaddnode.sh, mmfs, avs_rgpfs24, rgpfs24s005a 6/14/06 14:39:38
17###############################################################################
18#
19# Usage:
20#
21#   mmaddnode -N {NodeDesc[,NodeDesc...] | NodeFile}
22#
23# where:
24#
25#   -N NodeDesc,NodeDesc,...  specifies a comma-separated list of node
26#                             descriptors that detail the node interfaces
27#                             to be added to the cluster.  The nodes must not
28#                             presently belong to the cluster.
29#
30#   -N NodeFile    specifies a file of node descriptors that detail
31#                  the node interfaces to be added to the cluster.
32#                  The nodes must not presently belong to the cluster.
33#
34# Obsolete but still supported options:
35#
36#   NodeDesc,NodeDesc,...   specifies a comma-separated list of node
37#                           descriptors that detail the node interfaces
38#                           to be added to the cluster.  The nodes must not
39#                           presently belong to the cluster.
40#
41#   -n NodeFile    specifies a file of node descriptors that detail
42#                  the node interfaces to be added to the cluster.
43#                  The nodes must not presently belong to the cluster.
44#
45# Each node descriptor has the format:
46#
47#   nodeName:nodeRoles:adminNodeName:
48#
49# where
50#
51#   nodeName        is either a short or fully-qualified hostname,
52#                   or an IP address for the primary GPFS network
53#                   to be used for daemon-to-daemon communications.
54#
55#   nodeRoles       is a '-' separated list of node roles.  Unless changed
56#                   on the mmconfig or mmaddnode commands, these roles will
57#                   be associated with the node when it becomes a member of
58#                   a nodeset.
59#
60#   adminNodeName   is either a short or fully-qualified hostname, or an IP
61#                   address to be used by the admin scripts to communicate
62#                   between nodes.  This is an optional parameter; if it
63#                   is not specified, the nodeName value is used.
64#
65###############################################################################
66
67# Include global declarations and service routines.
68. /usr/lpp/mmfs/bin/mmglobfuncs
69. /usr/lpp/mmfs/bin/mmsdrfsdef
70. /usr/lpp/mmfs/bin/mmfsfuncs
71
72sourceFile="mmaddnode.sh"
73[[ -n $DEBUG || -n $DEBUGmmaddnode ]] && set -x
74$mmTRACE_ENTER "$*"
75
76
77# Local work files.  Names should be of the form:
78#   fn=${tmpDir}fn.${mmcmd}.$$
79allnodes=${tmpDir}allnodes.${mmcmd}.$$
80allnodenames=${tmpDir}allnodenames.${mmcmd}.$$
81goodnodes=${tmpDir}goodnodes.${mmcmd}.$$
82processedNodes=${tmpDir}processedNodes.${mmcmd}.$$
83inputNodes=${tmpDir}inputNodes.${mmcmd}.$$
84oldcfgFile=${tmpDir}oldcfgFile.${mmcmd}.$$
85diskLines=${tmpDir}diskLines.${mmcmd}.$$
86
87LOCAL_FILES=" $allnodenames $allnodes $goodnodes $processedNodes $inputNodes $oldcfgFile $diskLines "
88
89
90# Local declarations
91
92usageMsg=355
93integer nodeCount=0
94integer nodeNumber=0
95integer totalNodeCount=0
96integer quorumNodeCount=0
97integer nodesAddedToCluster=0
98integer highestNodeNumber=0
99narg=""
100lapiWindow=$noWindow      # obsolete
101switchNumber=$noSwitch    # obsolete
102quorumDefault=$nonQuorumNode
103fatalError=""
104rc=0
105typeset -l role
106typeset -l adapter_lc
107
108
109# Local routines
110
111
112#####################################################################
113#
114# Function:  Verifies that a command executed successfully.
115#            If the return code from the command is not zero,
116#            the function issues a message and sets a failure flag.
117#            It returns with the same return code that was specified
118#            on input.
119#
120# Input:     $1 - name of the command to check
121#            $2 - return code from the execution of the command
122#
123#####################################################################
124function checkForErrorsAndReturn
125{
126  typeset sourceFile="mmaddnode.sh"
127  $mmTRACE_ENTER "$*"
128  if [ $2 != "0" ]
129  then
130    fatalError=yes
131    # Unexpected error
132    printErrorMsg 171 "$mmcmd" "$1" $2
133  fi
134
135  return $2
136
137}  #------ end of function checkForErrorsAndReturn --------------
138
139
140
141#######################
142# Mainline processing
143#######################
144
145
146#################################
147# Process the command arguments.
148#################################
149[[ $arg1 = '-?' || $arg1 = '-h' || $arg1 = '--help' || $arg1 = '--' ]] &&  \
150  syntaxError "help" $usageMsg
151
152while getopts :n:N: OPT
153do
154  case $OPT in
155
156    n) # node descriptors file
157       [[ -n $narg ]] && syntaxError "multiple" $noUsageMsg "-$OPT"
158       narg=$OPTARG
159       argc=argc-2
160       ;;
161
162    N) # node descriptors list or file
163       [[ -n $Narg ]] && syntaxError "multiple" $noUsageMsg "-$OPT"
164       Narg=$OPTARG
165       argc=argc-2
166       ;;
167
168    +[nN]) # Invalid option
169       syntaxError "invalidOption" $usageMsg $OPT
170       ;;
171
172    :) # Missing argument
173       syntaxError "missingValue" $usageMsg $OPTARG
174       ;;
175
176    *) # Invalid option
177       syntaxError "invalidOption" $usageMsg $OPTARG
178       ;;
179
180  esac
181
182done  # end of while getopts :n:N: OPT do
183
184# Drop the processed options from the parameter list.
185shift OPTIND-1
186
187
188# Complete the parameter checking.
189[[ -n $narg && -n $Narg ]] &&  \
190  syntaxError "invalidCombination" $usageMsg "-n" "-N"
191
192if [[ -n $Narg ]]
193then
194  # The -N parameter may be either a list or a file.  Which is it?
195  if [[ -f $Narg ]]
196  then
197    # It is a file; verify its existence and create our own copy.
198    checkUserFile $Narg $inputNodes
199    [[ $? -ne 0 ]] && cleanupAndExit
200  else
201    # It is not a file, so it must be a list.
202    # Convert the input node list into a file.
203    $rm -f $inputNodes
204    IFS=','
205    for nodeDesc in $Narg
206    do
207      print -- "$nodeDesc" >> $inputNodes
208      checkForErrors "writing to $inputNodes" $?
209    done
210    IFS="$IFS_sv"    # Restore the default IFS setting.
211  fi  # end of if [[ -f $Narg ]]
212
213elif [[ -z $narg ]]
214then
215  # If neither the -N nor the -n option is not used,
216  # a list of node names is required.
217  if [[ $argc -eq 1 ]]
218  then
219    # If there is exactly one string left,
220    # it is assumed to be the list of nodes to add.
221    arglist=$1
222  elif [[ $argc -gt 1 ]]
223  then
224    # If more than one string is left,
225    # we have a syntax error.
226    syntaxError "extraArg" $usageMsg "$2"
227  else
228    # If there are no more parameters,
229    # a required parameter is missing.
230    syntaxError "missingArgs" $usageMsg
231  fi
232
233  # Convert the input node list into a file.
234  $rm -f $inputNodes
235  IFS=','
236  for nodeDesc in $arglist
237  do
238    print -- "$nodeDesc" >> $inputNodes
239    checkForErrors "writing to $inputNodes" $?
240  done
241  IFS="$IFS_sv"    # Restore the default IFS setting.
242
243else
244  # If -n is specified, there should be no other parms.
245  [[ $argc -gt 0 ]] &&  \
246    syntaxError "extraArg" $usageMsg "$1"
247
248  # Check the node names file parameter and create our own copy.
249  checkUserFile $narg $inputNodes
250  [[ $? -ne 0 ]] && cleanupAndExit
251fi   # end of if [[ -z $narg ]]
252
253
254######################################################################
255# Set up trap exception handling and call the gpfsInit function.
256# It will ensure that the local copy of the mmsdrfs and the rest of
257# the GPFS system files are up-to-date and will obtain the sdr lock.
258######################################################################
259trap pretrap HUP INT QUIT KILL
260gpfsInitOutput=$(gpfsInit $lockId)
261setGlobalVar $? $gpfsInitOutput
262
263if [[ $MMMODE != lc ]]
264then
265  # Command is not valid in the user's environment.
266  printErrorMsg 376 $mmcmd $MMMODE
267  cleanupAndExit
268fi
269
270# Determine the lookup order for resolving host names.
271[[ $osName != AIX ]] && resolveOrder=$(setHostResolveOrder)
272
273
274####################################################
275# Find out if disk-based quorum is in effect.
276####################################################
277diskQuorumInEffect=$(showCfgValue tiebreakerDisks)
278[[ $diskQuorumInEffect = no ]] && diskQuorumInEffect=""
279
280
281############################################################
282# Go through the current mmsdrfs file.
283# Increment the gen number and collect needed information.
284############################################################
285$rm -f $newsdrfs $allnodenames $allnodes $diskLines $oldcfgFile
286[[ -z $backupServer ]] && backupServer="_NOSECONDARY_"
287
288IFS=":"         # Change the field separator to ':'.
289exec 3<&-
290exec 3< $mmsdrfsFile
291while read -u3 sdrfsLine
292do
293  # Parse the line.
294  set -f ; set -A v -- - $sdrfsLine ; set +f
295  IFS="$IFS_sv"    # Restore the default IFS settings.
296  printLine=true   # Assume the line will be printed.
297
298  # Change some of the fields depending on the type of line.
299  case ${v[$LINE_TYPE_Field]} in
300
301    $VERSION_LINE )  # This is the global header line.
302      # Increment the generation number
303      newGenNumber=${v[$SDRFS_GENNUM_Field]}+1
304      v[$SDRFS_GENNUM_Field]=$newGenNumber
305
306      # Create the overloaded clType and clusterIdAndSt parameters
307      # that will be passed to the checkNewClusterNode routine.
308      clType="${MMMODE}/${environmentType}"
309      clusterIdAndSt="${v[$CLUSTERID_Field]}:${v[$CLUSTER_SUBTYPE_Field]}"
310      ;;
311
312    $NODESET_HDR )    # This is the nodeset header line.
313      # Skip over this line.  It will be rebuilt after
314      # we have the new value for node count.
315      nodesetHdr=$sdrfsLine
316      nodeCount=${v[$NODE_COUNT_Field]}
317      lowestVersion=${v[$MIN_DAEMON_VERSION_Field]}
318      highestVersion=${v[$MAX_DAEMON_VERSION_Field]}
319      osEnvironment=${v[$OS_ENVIRONMENT_Field]}
320      [[ -z $highestVersion ]] && highestVersion=0
321      [[ -z $osEnvironment ]] && osEnvironment=$osName
322      printLine=false
323      ;;
324
325    $MEMBER_NODE )  # This line describes a node.
326      # Keep track of the highest node number assigned to a node.
327      [[ $highestNodeNumber -lt ${v[$NODE_NUMBER_Field]} ]] &&  \
328        highestNodeNumber=${v[$NODE_NUMBER_Field]}
329
330      # Add the node's admin name to the list of admin names in the cluster.
331      print -- "${v[$REL_HOSTNAME_Field]}" >> $allnodes
332      checkForErrors "writing to file $allnodes" $?
333
334      # Add the daemon node name and the admin node name to the
335      # list of all daemon and admin node names in the cluster.
336      print -- "${v[$DAEMON_NODENAME_Field]}:${v[$REL_HOSTNAME_Field]}" >> $allnodenames
337      checkForErrors "writing to file $allnodenames" $?
338
339      # Keep track of the overall number of quorum nodes.
340      # If disk quorum is in effect, collect information
341      # about the quorum nodes in the cluster.
342      if [[ ${v[$CORE_QUORUM_Field]} = $quorumNode ]]
343      then
344        (( quorumNodeCount += 1 ))
345        if [[ -n $diskQuorumInEffect ]]
346        then
347          if [[ -z $quorumNodeNames ]]
348          then
349            # This is the first node to add to the lists.
350            quorumNodeNumbers="${v[$NODE_NUMBER_Field]}"
351            quorumNodeNames="${v[$REL_HOSTNAME_Field]}"
352          else
353            if [[ ${v[$REL_HOSTNAME_Field]} = $ourNodeName ]]
354            then
355              # This is the local node; add it at the front of the lists
356              # so it will be the first quorum node used.
357              quorumNodeNumbers="${v[$NODE_NUMBER_Field]},${quorumNodeNumbers}"
358              quorumNodeNames="${v[$REL_HOSTNAME_Field]},${quorumNodeNames}"
359            else
360              # This is not the local node; add it at the end of the lists.
361              quorumNodeNumbers="${quorumNodeNumbers},${v[$NODE_NUMBER_Field]}"
362              quorumNodeNames="${quorumNodeNames},${v[$REL_HOSTNAME_Field]}"
363            fi
364          fi  # end of if [[ -z $quorumNodeNames ]]
365        fi  # end of if [[ -n $diskQuorumInEffect ]]
366      fi  # end of if [[ ${v[$CORE_QUORUM_Field]} = $quorumNode ]]
367      ;;
368
369    $SG_DISKS )  # This line describes a disk.
370      # Collect the lines that represent the quorum disks.
371      if [[ -n $diskQuorumInEffect && ${v[$PAXOS_Field]} = $PaxosDisk ]]
372      then
373        print_newLine >> $diskLines
374        checkForErrors "writing to file $diskLines" $?
375      fi  # end if [[ -n $diskQuorumInEffect && ...
376      ;;
377
378    $MMFSCFG )     # This line contains mmfs.cfg information.
379      # Extract the mmfs.cfg information.
380      # It is everything after the first 4 fields.
381      cfgLine="${v[5]}:${v[6]}:${v[7]}:${v[8]}:${v[9]}:${v[10]}:${v[11]}"
382      cfgLine="$cfgLine:${v[12]}:${v[13]}:${v[14]}:${v[15]}:${v[16]}"
383      cfgLine="$cfgLine:${v[17]}:${v[18]}:${v[19]}:${v[20]}:${v[21]}:${v[22]}"
384
385      # To preserve tabs, temporarily set IFS to new line only.
386      IFS="
387"
388      # Strip trailing colons and write the line to the file.
389      print "${cfgLine%%+(:)}" >> $oldcfgFile
390      checkForErrors "writing to file $oldcfgFile" $?
391      IFS="$IFS_sv"
392      printLine=false
393
394      # Retrieve the value of maxFeatureLevelAllowed.
395      set -f ; set -- $cfgLine ; set +f
396      attribute=$1
397      value=$2
398      [[ $attribute = maxFeatureLevelAllowed ]] &&  \
399        maxFeatureLevelAllowed=${value%%+(:)}
400      ;;
401
402    * )  # No need to look at any of the other lines.
403      ;;
404
405  esac  # end Change some of the fields
406
407  # Build and write the line to the new mmsdrfs file.
408  if [[ $printLine = true ]]
409  then
410    print_newLine >> $newsdrfs
411    checkForErrors "writing to file $newsdrfs" $?
412  fi
413
414  IFS=":"  # Change the separator back to ":" for the next iteration.
415
416done  # end while read -u3 sdrfsLine
417
418IFS="$IFS_sv"  # Restore the default IFS settings.
419
420# Save the highest node number found.
421nodeNumber=$highestNodeNumber
422
423
424#######################################################################
425# Generate the node information for the mmsdrfs file.
426#
427# Loop through the nodes to be added to the sdrfs file, checking
428# as we go.  When the loop is done we know which nodes can be added
429# to the sdrfs file and which ones can't.  A MEMBER_NODE line will be
430# generated for each node that can be added to the new GPFS cluster.
431#######################################################################
432# Loop through the nodes to be added.
433$rm -f $tmpsdrfs $goodnodes $processedNodes
434$touch $tmpsdrfs $goodnodes $processedNodes
435exec 3<&-
436exec 3< $inputNodes
437while read -u3 nodeDesc
438do
439  # Skip empty and comment lines
440  [[ $nodeDesc = *([$BLANKchar$TABchar])   ]] && continue
441  [[ $nodeDesc = *([$BLANKchar$TABchar])#* ]] && continue
442
443  # Keep track of the total number of nodes specified by the user.
444  totalNodeCount=$totalNodeCount+1
445
446  IFS=':'
447  set -f ; set -- $nodeDesc ; set +f
448  nodeName=$1
449  nodeRoles=$2
450  nodeName2=$3
451  IFS="$IFS_sv"
452
453  [[ $nodeName = "localhost" ]] && continue
454
455  # Process the node roles list.
456  designation=$CLIENT
457  quorumField=$quorumDefault
458  if [[ -n $nodeRoles ]]
459  then
460    IFS="-"
461    set -f ; set -- $nodeRoles ; set +f
462    IFS="$IFS_sv"
463    while [[ -n $1 ]]
464    do
465      role=$1  # Convert the node's role to lower case only.
466      case $role in
467        $CLIENT )
468          designation=$CLIENT
469          ;;
470
471        $MANAGER )
472          designation=$MANAGER
473          ;;
474
475        $QUORUM )
476          quorumField=$quorumNode
477          ;;
478
479        $NONQUORUM ) quorumField=$nonQuorumNode
480          ;;
481
482        * )
483          # Invalid node designations specified.
484          printErrorMsg 293 $mmcmd "$nodeDesc"
485          fatalError=yes
486          break 2
487          ;;
488      esac
489
490      # Move to the next field.
491      shift
492    done  # end while [[ -n $1 ]]
493  fi  # end if [[ -n $nodeRoles ]]
494
495  # Determine the value of addNodeState.
496  if [[ $quorumField = $nonQuorumNode ]]
497  then
498    addNodeState=$OLD_NODE
499  else
500    addNodeState=$NEW_NODE
501  fi
502
503  # At this point, the daemon node name could be a fully-qualified
504  # adapter port name, a short name, or an IP address.  Determine the
505  # fully-qualified hostname, the short name, and the IP address for
506  # the specified node interface.
507  hostResult=$($host $nodeName)
508  set -f ; set -- $hostResult ; set +f
509  daemonNodeName=$1
510  shortName=${1%% *|.*}    # Exclude everything after the first dot.
511  ipa=${3%%,*}             # Exclude everything after the first comma.
512  if [[ -z $ipa ]]
513  then
514    # Invalid node name specified.
515    printErrorMsg 54 $mmcmd $nodeName
516    fatalError=yes
517    break
518  fi
519
520  # At this point, if it was specified, the admin node name could be a
521  # fully-qualified adapter port name, a short name, or an IP address.
522  # Determine the fully-qualified hostname and the IP address for the
523  # specified node interface.
524  if [[ -n $nodeName2 ]]
525  then
526    # Stop here if the admin network support has not been activated yet.
527    if [[ $sdrfsFormatLevel -eq 0 ]]
528    then
529      print -u2 "$mmcmd:  The separate administration network support has not been enabled yet."
530      print -u2 "    Run \"mmchconfig release=LATEST\" to activate the new function."
531      fatalError=yes
532      break
533    fi
534
535    hostResult=$($host $nodeName2)
536    set -f ; set -- $hostResult ; set +f
537    adminNodeName=$1
538    adminShortName=${1%% *|.*}  # Exclude everything after the first dot.
539    adminIpa=${3%%,*}           # Exclude everything after the first comma.
540    if [[ -z $adminIpa ]]
541    then
542      # An invalid admin node name was specified.
543      printErrorMsg 54 $mmcmd $nodeName2
544      fatalError=yes
545      break
546    fi
547  else
548    # The user did not set a distinct admin node name, so set the
549    # admin node names to be the same as the daemon node names.
550    adminNodeName=$daemonNodeName
551    adminShortName=$shortName
552  fi
553
554  # Assign a node number to the node.
555  nodeNumber=$nodeNumber+1
556  gsNodeNumber=$nodeNumber
557  adapterType=""
558
559  # Ensure that the node interfaces do not already exist in the cluster.
560  $grep -qw $daemonNodeName $allnodenames > /dev/null 2>&1
561  if [[ $? -eq 0 ]]
562  then
563    # The node already belongs to the cluster.
564    printErrorMsg 152 $mmcmd $nodeName
565    fatalError=yes
566    break
567  fi
568  if [[ $adminNodeName != $daemonNodeName ]]
569  then
570    $grep -qw $adminNodeName $allnodenames > /dev/null 2>&1
571    if [[ $? -eq 0 ]]
572    then
573      # The node already belongs to the cluster.
574      printErrorMsg 152 $mmcmd $nodeName2
575      fatalError=yes
576      break
577    fi
578  fi
579
580  # Make sure neither node name (admin or daemon) is specified more than once.
581  $grep -qw $daemonNodeName $processedNodes > /dev/null 2>&1
582  if [[ $? -eq 0 ]]
583  then
584    # The node is specified twice.
585    printErrorMsg 347 $mmcmd $nodeName
586    fatalError=yes
587    break
588  fi
589  if [[ $adminNodeName != $daemonNodeName ]]
590  then
591    $grep -qw $adminNodeName $processedNodes > /dev/null 2>&1
592    if [[ $? -eq 0 ]]
593    then
594      # The node is specified twice.
595      printErrorMsg 347 $mmcmd $nodeName2
596      fatalError=yes
597      break
598    fi
599  fi
600
601  # If disk-based quorum is in effect, make sure that adding
602  # this node will not exceed the total limit of quorum nodes.
603  if [[ -n $diskQuorumInEffect     &&
604        $quorumNodeCount -eq 8     &&
605        $quorumField = $quorumNode ]]
606  then
607    # There are more than eight quorum nodes while tiebreaker disks are in use.
608    printErrorMsg 131 $mmcmd
609    # Adding the node to the cluster will exceed the quorum node limit.
610    printErrorMsg 134 $mmcmd $nodeName
611    fatalError=yes
612    break
613  fi
614
615  # Add the daemon and admin node names to the list of processed nodes.
616  print -- "${daemonNodeName}:${adminNodeName}" >> $processedNodes
617  checkForErrorsAndReturn "writing to file $processedNodes" $?
618  [[ $? -ne 0 ]] && break
619
620  # Build a line with the local node data for the node.  The full-blown
621  # MEMBER_NODE line will be created further down when we have all of the
622  # information that we need.
623  sdrfsLine="$nodesetId:$MEMBER_NODE::$nodeCount:$nodeNumber:$shortName:$ipa"
624  sdrfsLine="$sdrfsLine:$adminNodeName:$designation:$adapterType:$lapiWindow"
625  sdrfsLine="$sdrfsLine:$switchNumber:$addNodeState:$adapterType:$daemonNodeName"
626  sdrfsLine="$sdrfsLine:$adminShortName::::$quorumField:$gsNodeNumber:"
627
628  # Invoke the checkNewClusterNode function to ensure that
629  # the node is new and that its level of GPFS supports clusters.
630  # If it passes these tests, a skeleton sdrfs file is stored on the node.
631  printInfoMsg 416 "$(date)" $mmcmd $daemonNodeName
632  runOutput=$(run on1 $adminNodeName checkNewClusterNode  \
633     $clType $primaryServer $backupServer "$sdrfsLine"    \
634     "$rsh" "$rcp" "$clusterIdAndSt" 2> $errMsg)
635  rc=$?
636  IFS=':'
637  set -f ; set -- $runOutput ; set +f
638  IFS="$IFS_sv"
639  keyword=$1
640  nodeStatus=$2
641  adapterType=$3
642  installedDaemonVersion=$4
643  installedProductVersion=$5
644  installedOsName=$6
645
646  if [[ $rc = 0 && $nodeStatus = success && $keyword = checkNewClusterNode ]]
647  then
648    # The checkNewClusterNode call succeeded.
649    # Build the line that will represent this node in the mmsdrfs file.
650    nodeCount=$nodeCount+1
651    sdrfsLine="$nodesetId:$MEMBER_NODE::$nodeCount:$nodeNumber:$shortName"
652    sdrfsLine="$sdrfsLine:$ipa:$adminNodeName:$designation:$adapterType"
653    sdrfsLine="$sdrfsLine:$lapiWindow:$switchNumber:$addNodeState:$adapterType"
654    sdrfsLine="$sdrfsLine:$daemonNodeName:$adminShortName"
655    sdrfsLine="$sdrfsLine:$installedDaemonVersion:$installedProductVersion"
656    sdrfsLine="$sdrfsLine:$installedOsName:$quorumField:$gsNodeNumber:"
657
658    # Add the MEMBER_NODE line to the other lines
659    # that will go in the mmsdrfs file.
660    print -- "$sdrfsLine" >> $tmpsdrfs
661    checkForErrorsAndReturn "Writing to file $tmpsdrfs" $?
662    [[ $? -ne 0 ]] && break
663
664    # Add the node name to the list of successful nodes.
665    print -- "$adminNodeName" >> $goodnodes
666    checkForErrorsAndReturn "Writing to file $goodnodes" $?
667    [[ $? -ne 0 ]] && break
668    nodesAddedToCluster=$nodesAddedToCluster+1
669
670    if [[ $quorumField = $quorumNode ]]
671    then
672      (( quorumNodeCount += 1 ))
673      if [[ -n $diskQuorumInEffect ]]
674      then
675        newQuorumNodes=yes
676        if [[ -z $quorumNodeNames ]]
677        then
678          # This is the first node to add to the lists.
679          quorumNodeNumbers="${v[$NODE_NUMBER_Field]}"
680          quorumNodeNames="${v[$REL_HOSTNAME_Field]}"
681        else
682          if [[ ${v[$REL_HOSTNAME_Field]} = $ourNodeName ]]
683          then
684            # This is the local node; add it at the front of the lists
685            # so it will be the first quorum node used.
686            quorumNodeNumbers="${v[$NODE_NUMBER_Field]},${quorumNodeNumbers}"
687            quorumNodeNames="${v[$REL_HOSTNAME_Field]},${quorumNodeNames}"
688          else
689            # This is not the local node; add it at the end of the lists.
690            quorumNodeNumbers="${quorumNodeNumbers},${v[$NODE_NUMBER_Field]}"
691            quorumNodeNames="${quorumNodeNames},${v[$REL_HOSTNAME_Field]}"
692          fi
693        fi  # end of if [[ -z $quorumNodeNames ]]
694      fi  # end of if [[ -n $diskQuorumInEffect ]]
695    fi  # end of if [[ $quorumField = $quorumNode ]]
696
697    # Ensure that we will not exceed the maximum number of quorum nodes.
698    if [[ $quorumNodeCount -gt $maxQuorumNodes && -z $fatalError ]]
699    then
700      # Error:  The number of quorum nodes exceeds the maximum allowed.
701      printErrorMsg 393 $mmcmd $maxQuorumNodes
702      fatalError=yes
703      break
704    fi
705
706  else
707    # The checkNewClusterNode call failed.
708    # Not all errors are considered terminal.
709    # If an individual node fails for a known reason,
710    # we will not include it in the cluster but will
711    # continue with the rest of the nodes.
712
713    # Tell the world what went wrong.
714    if [[ $nodeStatus = not_new ]]
715    then
716      # Node already belongs to a cluster.
717      printErrorMsg 348 $mmcmd $adminNodeName
718    elif [[ $nodeStatus = not_supported ]]
719    then
720      # Wrong GPFS code level.
721      printErrorMsg 349 $mmcmd $adminNodeName
722    elif [[ $nodeStatus = ipa_alias ]]
723    then
724      # IP address aliasing is not supported.
725      printErrorMsg 476 $mmcmd $nodeName
726    elif [[ $nodeStatus = ipa_missing ]]
727    then
728      # The daemon node adapter was not found on the admin node.
729      printErrorMsg 175 $mmcmd $nodeName $nodeName2
730    elif [[ $rc = $MM_HostDown || $rc = $MM_ConnectTimeout ]]
731    then
732      # The node cannot be reached.
733      printErrorMsg 340 $mmcmd $adminNodeName
734    else
735      # Unexpected error.  Display all possible error messages.
736      [[ -s $errMsg ]] && $cat $errMsg 1>&2
737      [[ $rc -eq 0 ]] && rc=1
738      checkForErrorsAndReturn "checkNewClusterNode $adminNodeName" $rc
739      run on1 $adminNodeName removeFromCluster > /dev/null 2>&1 &
740#     break
741    fi
742
743    # Append the node name to the list of failed nodes.
744    failedNodes="$failedNodes\n\t\t$adminNodeName"
745
746    # Adjust the node number for the next iteration.
747    nodeNumber=$nodeNumber-1
748
749  fi  # end of if [[ $rc = 0 && $nodeStatus = success ]]
750
751  $rm -f $errMsg
752
753done  # end of while read -u3 nodeDesc (Loop through the nodes to be added)
754
755
756#########################################################
757# If we have no nodes to add, issue a message and quit.
758#########################################################
759if [[ ! -s $goodnodes ]]
760then
761  # Command is quitting due to no valid nodes.
762  printErrorMsg 387 $mmcmd $mmcmd
763  cleanupAndExit
764fi
765
766
767#######################################################################
768# At this point, we have successfully processed all of the new nodes.
769# Next, build the nodeset header line and add it to the mmsdrfs file.
770#######################################################################
771# Parse the old nodeset header line.
772IFS=":"
773set -f ; set -A v -- - $nodesetHdr ; set +f
774IFS="$IFS_sv"
775
776v[$NODE_COUNT_Field]=$nodeCount
777[[ $highestVersion = 0 ]] && highestVersion=""
778v[$MAX_DAEMON_VERSION_Field]=$highestVersion
779v[$OS_ENVIRONMENT_Field]=$osEnvironment
780
781# Add the nodeset header line to the mmsdrfs file.
782print_newLine >> $newsdrfs
783checkForErrors "writing to file $newsdrfs" $?
784
785
786############################################################
787# Add the lines for the new nodes to the new mmsdrfs file.
788############################################################
789$cat $tmpsdrfs >> $newsdrfs
790if [[ $? -ne 0 ]]
791then
792  printErrorMsg 171 "$mmcmd" "copying $tmpsdrfs to $newsdrfs" $?
793  fatalError=yes
794fi
795
796
797#######################################################
798# Put the cfg information back into the mmsdrfs file.
799#######################################################
800if [[ $mmfscfgModified = yes ]]
801then
802  appendCfgFile $nodesetId $tmpCfg $newsdrfs
803else
804  appendCfgFile $nodesetId $oldcfgFile $newsdrfs
805fi
806checkForErrors "appendCfgFile" $?
807mmfscfgModified=no
808
809
810#############################################
811# Sort the new version of the mmsdrfs file.
812#############################################
813LC_ALL=C $SORT_MMSDRFS $newsdrfs -o $newsdrfs
814if [[ $? -ne 0 ]]
815then
816  printErrorMsg 171 "$mmcmd" "sorting $newsdrfs" $?
817  fatalError=yes
818fi
819
820
821#####################################################################
822# If disk-based quorum is in effect, and new quorum nodes are being
823# added to the cluster, the quorum disks must be reformatted.
824#####################################################################
825if [[ -n $diskQuorumInEffect && -n $newQuorumNodes && -z $fatalError ]]
826then
827  formatPaxosDisks $diskLines $quorumNodeNumbers $quorumNodeNames mmaddnode
828  if [[ $? -ne 0 ]]
829  then
830    # GPFS failed to initialize the tiebreaker disks.
831    printErrorMsg 132 $mmcmd
832    fatalError=yes
833  fi
834fi
835
836
837####################################################################
838# Issue a warning if we will exceed the maximum recommended number
839# of quorum nodes.
840####################################################################
841if [[ $quorumNodeCount -gt $maxRecQuorumNodes && -z $fatalError ]]
842then
843  # Warning:  The number of quorum nodes exceeds the recommended maximum.
844  printErrorMsg 394 $mmcmd $maxRecQuorumNodes
845fi
846
847
848############################################################
849# If anything failed, remove the skeleton sdrfs files from
850# the new nodes, issue an error message, and die.
851############################################################
852if [[ -n $fatalError ]]
853then
854  # Remove the already-installed system files.
855  printErrorMsg 351 $mmcmd
856  $ln $goodnodes ${goodnodes}async
857  $mmcommon onall_async ${goodnodes}async removeFromCluster &
858
859  # The command failed.
860  printErrorMsg 389 $mmcmd
861  cleanupAndExit
862fi
863
864
865##############################################################
866# Lock the Gpfs object to prevent the daemon from coming up
867# until all changes are successfully committed.
868##############################################################
869[[ $getCredCalled = no ]] && getCred
870setRunningCommand "$mmcmd" $primaryServer
871checkForErrors setRunningCommand $?
872gpfsLocked=yes
873
874
875##########################################################################
876# Put the new mmsdrfs file into the sdr.  This will make the newly-added
877# nodes visible to the rest of the nodes in the cluster.
878##########################################################################
879trap "" HUP INT QUIT KILL
880gpfsObjectInfo=$(commitChanges  \
881   $HOME_CLUSTER $nsId $gpfsObjectInfo $newGenNumber $newsdrfs $primaryServer)
882rc=$?
883if [[ $rc -ne 0 ]]
884then
885  # We cannot replace the file in the sdr.
886  printErrorMsg 381 $mmcmd
887
888  # Remove the already-installed system files.
889  printErrorMsg 351 $mmcmd
890  $ln $goodnodes ${goodnodes}async
891  $mmcommon onall_async ${goodnodes}async removeFromCluster &
892  cleanupAndExit
893fi
894
895
896#####################################################################
897# Unlock the sdr.
898#####################################################################
899[[ $sdrLocked = yes ]] &&  \
900  freeLockOnServer $primaryServer $ourNodeNumber > /dev/null
901sdrLocked=no
902trap posttrap HUP INT QUIT KILL
903
904
905#####################################################################
906# At this point, skeleton sdrfs files have been put on the nodes
907# in the success list.  If there are any nodes in the failure list,
908# we issue a message telling the user to use the mmaddnode command
909# to add them to the sdrfs file once they become reachable.
910#####################################################################
911# Report any nodes that could not be added to the user.
912[[ -n $failedNodes ]] &&  \
913  printErrorMsg 353 $mmcmd "$failedNodes"
914
915# If not all nodes were added to the cluster,
916# tell the user how many made it through.
917[[ $nodesAddedToCluster -lt $totalNodeCount ]] &&  \
918  printErrorMsg 12 $mmcmd $nodesAddedToCluster $totalNodeCount
919
920# Issue "command was successful" message
921printErrorMsg 272 $mmcmd
922
923
924###################################################################
925# Asynchronously propagate the changes to all nodes, new and old.
926###################################################################
927$cat $goodnodes >> $allnodes
928checkForErrors "appending $goodnodes to $allnodes" $?
929propagateSdrfsFile async $allnodes $newsdrfs $newGenNumber rereadNodeList
930
931cleanupAndExit 0
932
Note: See TracBrowser for help on using the repository browser.