#!/bin/ksh
# IBM_PROLOG_BEGIN_TAG 
# This is an automatically generated prolog. 
#  
#  
#  
# Licensed Materials - Property of IBM 
#  
# (C) COPYRIGHT International Business Machines Corp. 2006,2007 
# All Rights Reserved 
#  
# US Government Users Restricted Rights - Use, duplication or 
# disclosure restricted by GSA ADP Schedule Contract with IBM Corp. 
#  
# IBM_PROLOG_END_TAG 
# @(#)28	1.29.1.1  src/avs/fs/mmfs/ts/admin/mmtrace.sh, mmfs, avs_rgpfs24, rgpfs24s010a 2/1/07 12:58:58
###########################################################################
#
# Script to gather GPFS traces for debugging purposes.
#
# Syntax: mmtrace [start | stop] [noformat | formatall] [tail | head | cont] [dispatch]
#                 [trace={io | all | def | same}]
#                 [trace="trace_class level [trace_class level ...]"]
#
# Normal use is to issue "mmtrace" to start trace, rerun failure scenario,
# then issue "mmtrace stop" when failure event occurs.  If the problem
# is a daemon failure then "mmtrace stop" can be called from the runmmfs
# script to capture the event and stop trace before the daemon starts up
# again.  If the problem is a daemon failure during startup, use the -T
# option on the mmstartup command.
#
# "mmtrace" with no trace arguments can be used to start/restart the trace
# without modifing the existing trace level settings. "mmfsadm showtrace"
# will show the current level settings. A set of trace levels can be set
# permanently using
#   "mmchconfig trace="trace_class level [trace_class level ...]"
# These will be the default trace levels whenever the GPFS daemon starts,
# but will not actually produce traces until mmtrace is run to start the
# AIX trace or the Linux lxtrace command.
#
# "mmtrace io" can be used to trace a minimal set of operations that
# show application requests to GPFS and the resulting disk IO.
#
# "mmtrace all" can be used to trace all GPFS traces at level 9,
# plus all AIX traces (if running on AIX).
#
# "mmtrace def" can be used to trace a generic set of GPFS trace levels.
#
# "mmtrace same" will not change the GPFS trace levels.
#
# See "Modifiable trace levels" at the end of the script if the default
# traces need to be changed.
#
# The "noformat" option can be used to just keep the raw unformatted
# trcfiles and not spend CPU cycles formatting them into trcrpts while
# doing some other tests.
#
# The "formatall" option can be used to have this script format all the raw
# trcfiles it finds in the TRCDIR directory (presumably left there by
# previous mmtrace calls that had the "noformat" option specified.)
# "formatall" implies "stop" if no other parameter specified.
#
# Note that a new trace report is produced each time this script is run.
# Also, in the AIX environment, system trace is stopped and started
# regardless of whether it was originally initiated by this script.
# As a result, this script should be used carefully and generally only
# at the direction of IBM service for the purposes of debugging GPFS.
#
# Trace output is captured to the directory specified by TRCDIR below
# which by default is /tmp/mmfs.
#
###########################################################################
#
# Include global declarations and service routines
. /usr/lpp/mmfs/bin/mmglobfuncs
. /usr/lpp/mmfs/bin/mmsdrfsdef

sourceFile="mmtrace.sh"
[[ -n $DEBUG || -n $DEBUGmmtrace ]] && set -x

# Local routines

####################################################################
#
# Function:  Locate the active dataStructureDump mmfs.cfg entry 
#            for the local node, specified by its short name.
#
#
# Input:     nodename -- Local host short name
# Output:    Value of active dataStructureDump, empty if no
#            stanza located.
#
####################################################################
function getDataDumpDir
{
  value=$(perl -e '
    # Open mmfs.cfg for processing
    open(MMFSCFG, $ARGV[0]) or exit;

    # Host short name
    $hosts = $ARGV[1];

    # Obtain interface info for specified node
    ($name, $aliases, $addrtype, $length, @addrs) = gethostbyname $hosts;

    # Combine short/long hostnames and any aliases
    $hosts = join(" ", $hosts, $name, $aliases);

    # Empty dataDumpDir
    undef $dataDumpDir;

    # Process each line of file (common stanzas first)
    $this_node = true;
    LINE: while ($line = <MMFSCFG>)
    {
      # Skip comment/blank lines
      next LINE if $line =~ m/^ *#/;
      next LINE if $line =~ m/^ *$/;

      chomp($line);
      if ($line =~ /^ *\[(.*)\]/)
      {
        # Node specific stanzas --> search node list (brackets stripped)
        $this_node = false;
        foreach $node (split(",", $1))
        {
          if ($node eq "common")
          {
            # Common stanza --> applies to all nodes
            $this_node = true;
            last;
          }
          else
          {
            # Node(s) specific stanza
            foreach $host (split(" ", $hosts))
            {
              if ($node eq $host)
              {
                # Node stanza located
                $this_node = true;
                last;
              }
            }
          }
        }
      }
      elsif (($line =~ s/^ *dataStructureDump *//) && ($this_node eq true))
      {
        # Node specific dataStructureDump attribute --> set value
        $dataDumpDir = $line;
      }
    }

    # Close mmfs.cfg and return result via print
    close(MMFSCFG);
    print $dataDumpDir;
  ' $mmfscfgFile $1)

  # Return the result
  print -- "$value"
  return 0
} #------------- end of function getDataDumpDir ------------------

####################################################################
#
# Function:  Stop the currently running trace, if any.
#            If trace was running then move the raw trace file
#            to the output directory and produce a trace report.
#            The report is run in the background to avoid delays
#            in starting GPFS.
#
####################################################################
function stopTrace
{
  typeset sourceFile="mmtrace.sh"
  [[ -n $DEBUG || -n $DEBUGstopTrace ]] && set -x

  if [[ $osName = AIX ]]
  then
    # Stop the current trace
    $trcstop >/dev/null 2>&1
    if [[ $? -eq 0 ]]
    then
      $mv $RUNTRCFILE $TRCFILE
      if [[ $? = 0 ]]
      then
        print -- "$mmcmd: move $RUNTRCFILE $TRCFILE"
      else
        # tracing may have been going to default trcfile
        $mv /var/adm/ras/trcfile $TRCFILE
        if [[ $? = 0 ]]
        then
          print -- "$mmcmd: move /var/adm/ras/trcfile $TRCFILE"
        else
          # mv command failed
          printErrorMsg 104 "$mmcmd" "mv $RUNTRCFILE $TRCFILE"
          [[ $trace_format != "formatall" ]] && cleanupAndExit
        fi
      fi
      sync
    fi
    if [[ $trace_format = "formatall" ]]
    then
      for TRCFILE in $TRCPREF.*
      do
        TRCRPTFILE=$TRCRPTPREF.${TRCFILE#$TRCPREF.}
         # Start formating trcfiles, one at a time
        print -- "$mmcmd: formatting $TRCFILE to $TRCRPTFILE"
        $trcrpt -O tid=on${trace_dispatch} -t $TRCRPTFMTAIX -o $TRCRPTFILE $TRCFILE && \
          sync && $rm $TRCFILE
      done
    elif [[ -f $TRCFILE && $trace_format != "noformat" ]]
    then
      # Start formating the file in the background
      TRCRPTFILE=$TRCRPTPREF.${TRCFILE#$TRCPREF.}
      print -- "$mmcmd: formatting $TRCFILE to $TRCRPTFILE"
      ($trcrpt -O tid=on${trace_dispatch} -t $TRCRPTFMTAIX -o $TRCRPTFILE $TRCFILE && \
         sync && $rm $TRCFILE; ) &
    fi

  elif [[ $osName = Linux ]]
  then
    # Stop the current trace
    $lxtrace off >/dev/null 2>&1
    if [[ $? -eq 0 ]]
    then
      $mv $RUNTRCFILE $TRCFILE
      if [[ $? = 0 ]]
      then
        print -- "$mmcmd: move $RUNTRCFILE $TRCFILE"
      else
        # tracing may have been going to default trcfile
        $mv /tmp/lxtrace.trc $TRCFILE
        if [[ $? = 0 ]]
        then
          print -- "$mmcmd: move /tmp/lxtrace.trc $TRCFILE"
        else
          # mv command failed
          printErrorMsg 104 "$mmcmd" "mv $RUNTRCFILE $TRCFILE"
          [[ $trace_format != "formatall" ]] && cleanupAndExit
        fi
      fi
      sync
    fi

    if [[ ! -s $RUNTRCERRFILE ]]
    then
      # Tracing err file empty --> remove it
      rm -f $RUNTRCERRFILE
    elif [[ -f $TRCFILE ]]
    then
      # Tracing errors occured --> pair it with tracing file
      $mv $RUNTRCERRFILE $TRCERRFILE
      if [[ $? -ne 0 ]]
      then
        # mv command failed
        printErrorMsg 104 "$mmcmd" "mv $RUNTRCERRFILE $TRCERRFILE"
        [[ $trace_format != "formatall" ]] && cleanupAndExit
      fi 
    fi

    if [[ $trace_format = "formatall"  ]]
    then
      for TRCFILE in $TRCPREF.*
      do
        TRCRPTFILE=$TRCRPTPREF.${TRCFILE#$TRCPREF.}
        TRCRPTERRFILE=$TRCRPTERRPREF.${TRCERRFILE#$TRCERRPREF.}
        print -- "$mmcmd: formatting $TRCFILE to $TRCRPTFILE"
        # Start formating trcfiles, one at a time
        $lxtrace format -t $TRCRPTFMTLINUX -o $TRCRPTFILE $TRCFILE && \
          sync && $rm $TRCFILE
        [[ -f $TRCERRFILE ]] && $mv $TRCERRFILE $TRCRPTERRFILE && sync
      done
    elif [[ -f $TRCFILE && $trace_format != "noformat" ]]
    then
      # Start formating the file in the background
      TRCRPTFILE=$TRCRPTPREF.${TRCFILE#$TRCPREF.}
      TRCRPTERRFILE=$TRCRPTERRPREF.${TRCERRFILE#$TRCERRPREF.}
      print -- "$mmcmd: formatting $TRCFILE to $TRCRPTFILE"
      ($lxtrace format -t $TRCRPTFMTLINUX -o $TRCRPTFILE $TRCFILE && \
         sync && $rm $TRCFILE; ) &
      ([[ -f $TRCERRFILE ]] && $mv $TRCERRFILE $TRCRPTERRFILE && sync) &
    fi

  else
    checkForErrors "Unknown operating system $osName" 1
  fi

  return 0
} #------------- end of function stopTrace ------------------

####################################################################
#
# Function:  Turn tracing on.
#
####################################################################
function startTrace
{
  typeset sourceFile="mmtrace.sh"
  [[ -n $DEBUG || -n $DEBUGstartTrace ]] && set -x

  if [[ $osName = AIX ]]
  then
    hooks=""
    if [[ $trace_settings != "all" ]]; then
      hooks="-j 005,006,00A,306,307,308,309"
      [[ -n $trace_dispatch ]] && hooks=$hooks",100,101,102,103,106,200"
    fi
    # The trace options specify a default 16MB logfile, a 8MB memory buffer,
    # and logging the last of the wrapping memory buffer of trace data.
    # -L value must be twice the setting of -T
    $trace -a $traceht -L $TRCFILESIZE -T $TRCBUFSIZE $hooks -o $RUNTRCFILE

  elif [[ $osName = Linux ]]
  then
    # Use the default 16MB file and 64K buffer. Redirect stderr to a file to
    # capture tracing errors. Redirect stdin/stdout to /dev/null to prevent
    # remote shell invocataions of mmtrace from waiting on open pipes.
    $lxtrace on $RUNTRCFILE $TRCFILESIZE $TRCBUFSIZE 0</dev/null 1>/dev/null \
      2>$RUNTRCERRFILE

  else
    checkForErrors "Unknown operating system $osName" 1
  fi

  return 0
} #------------- end of function startTrace ------------------


############################
# Mainline processing
############################

#################################
# Process the command arguments.
#################################

trace_action=""
trace_format=""
trace_settings=""
trace_help=""
trace_headtail=""
trace_dispatch=""
traceht="-l"

while [ "$1" != "" ]
do
  case $1 in
    start)
      # Start tracing
      [[ -n $trace_action ]] && trace_help="true"
      trace_action=$1
      ;;
    stop)
      # Stop tracing
      [[ -n $trace_action ]] && trace_help="true"
      trace_action=$1
      ;;
    noformat)
      # Do not format traces
      [[ -n $trace_format ]] && trace_help="true"
      trace_format=$1
      ;;
    formatall)
      # Format traces
      [[ -n $trace_format ]] && trace_help="true"
      trace_format=$1
      ;;
    head)
      # Head of trace
      [[ -n $trace_headtail ]] && trace_help="true"
      trace_headtail=$1; traceht="-f"
      ;;
    cont)
      # Continuous trace to file
      [[ -n $trace_headtail ]] && trace_help="true"
      trace_headtail=$1; traceht=""
      ;;
    tail)
      # Tail of trace
      [[ -n $trace_headtail ]] && trace_help="true"
      trace_headtail=$1; traceht="-l"
      ;;
    dispatch)
      # Add AIX dispatch tracehooks
      [[ -n $trace_dispatch ]] && trace_help="true"
      trace_dispatch=",cpuid=on"
      ;;
    trace*)
      # Trace settings
      [[ -n $trace_settings ]] && trace_help="true"
      trace_settings=$1
      ;;
    *)
      # Unknown option/action --> Display usage
      trace_help="true"
  esac
  shift
done

# Check for help requests
if [[ $trace_help = "true" ]]
then
  print -u2 -- "Usage: mmtrace [start | stop] [noformat | formatall] [tail | head | cont] [dispatch]"
  print -u2 -- "               [trace={io | all | def | same}]"
  print -u2 -- "               [trace=\"trace_class level [trace_class level ...]\"]"
  print -u2 -- ""
  print -u2 -- "  If \"formatall\" is specified without a trace action, then"
  print -u2 -- "  \"stop\" is the default. All other command invocations"
  print -u2 -- "  where neither \"start\" nor \"stop\" are specified default"
  print -u2 -- "  to \"start\"."
  print -u2 -- ""
  print -u2 -- "  Specifying \"trace=all\", \"trace=io\", or \"trace=def\""
  print -u2 -- "  enables predefined trace settings."
  print -u2 -- ""
  print -u2 -- "  Specifying \"trace=same\" does not modify the current settings."
  print -u2 -- ""
  print -u2 -- "  If no trace= argument given and no trace levels are"
  print -u2 -- "  currently set, the default trace settings will be used."
  print -u2 -- ""
  print -u2 -- "  A set of trace levels can be set permanently using"
  print -u2 -- "    "mmchconfig trace="trace_class level [trace_class level ...]"
  print -u2 -- "  These will be the default trace levels whenever the GPFS daemon"
  print -u2 -- "  starts, but will not actually produce traces until mmtrace"
  print -u2 -- "  is run to start the AIX trace or the Linux lxtrace command."
  print -u2 -- ""
  print -u2 -- "  Default trace output directory is /tmp/mmfs. Override by"
  print -u2 -- "  setting TRCDIR environment variable."
  print -u2 -- ""
  print -u2 -- "  AIX only: (Linux trace only implements \"cont\" function)"
  print -u2 -- "    If \"tail\" is specified, the trace buffer wraps and is"
  print -u2 -- "      written to the trace file when \"mmtrace stop\" is issued."
  print -u2 -- "      (This is the default)"
  print -u2 -- "    If \"head\" is specified, the trace buffer fills only once"
  print -u2 -- "      and is written to the trace file when \"mmtrace stop\" is issued."
  print -u2 -- "    If \"cont\" is specified, the trace file is continuously"
  print -u2 -- "      written to as the trace buffer fills up. This file will wrap."
  print -u2 -- "      The tracefile size defaults to 16M and can be overridden"
  print -u2 -- "      by setting the TRCFILESIZE environment variable."
  print -u2 -- "    If \"dispatch\" is specified, AIX dispatching tracehooks"
  print -u2 -- "      will also be enabled."
  return 1
fi

# If a trace action has not been specified, default to "start" in all cases
# except when "formatall" has been requested.
if [[ -z $trace_action && $trace_format = "formatall" ]]
then
  trace_action="stop"
elif [[ -z $trace_action ]]
then
  trace_action="start"
fi

if [[ -n $trace_settings ]]
then
  # Retrieve tha actual trace classes/levels
  trace_settings=${trace_settings#*=}
fi

# Ensure the GPFS system data is up to date.
gpfsInitOutput=$(gpfsInit nolock)
setGlobalVar $? $gpfsInitOutput

# Define the short hostname
SHORTHOST=$($hostname -s)

# Define the trace directory. The default value may be overridden with the
# following priority utilized:
#     1) TRCDIR environment variable
#     2) dataStructureDump setting in mmfs.cfg
#     3) /tmp/mmfs default
[[ -z $TRCDIR ]] && TRCDIR=$(getDataDumpDir $SHORTHOST)

# Default TRCDIR if overrides NULL or not fully qualified
if [[ -z $TRCDIR || ($TRCDIR = ${TRCDIR#/}) ]]
then
  TRCDIR=/tmp/mmfs
fi

# Define the trace report directory and formatting templates
[[ -z $TRCRPTDIR || ($TRCRPTDIR = ${TRCRPTDIR#/}) ]] && TRCRPTDIR=$TRCDIR
[[ -z $TRCRPTFMTAIX ]] && TRCRPTFMTAIX=/etc/trcfmt
[[ -z $TRCRPTFMTLINUX ]] && TRCRPTFMTLINUX=/usr/lpp/mmfs/mmfs.trcfmt

#get env value for TRCDISPATCH for AIX
[[ -n $TRCDISPATCH && $TRCDISPATCH != off && $TRCDISPATCH != no && $TRCDISPATCH != 0 ]] && trace_dispatch=",cpuid=on"

# Define the trace file name, trace report prefix, and
# file size and trace buffer sizes.
TRCPREF=$TRCDIR/trcfile
TRCERRPREF=$TRCDIR/trcerrfile
TRCRPTPREF=$TRCRPTDIR/trcrpt
TRCRPTERRPREF=$TRCRPTDIR/trcerr
if [[ $osName = AIX ]]
then
  RUNTRCFILE=$TRCDIR/trcfile.$SHORTHOST
  [[ -z $TRCFILESIZE ]] && TRCFILESIZE=16000000
  [[ -z $TRCBUFSIZE ]] && TRCBUFSIZE=8000000

  # Verify file/buffer sizes
  if [ $TRCFILESIZE -lt 2*$TRCBUFSIZE ]
  then
    print -u2 -- "$osName trace file must be twice the trace buffer size"
    return 1
  fi
elif [[ $osName = Linux ]]
then
  RUNTRCFILE=$TRCDIR/lxtrace.trc.$SHORTHOST
  RUNTRCERRFILE=$TRCDIR/lxtrace.trcerr.$SHORTHOST
  [[ -z $TRCFILESIZE ]] && TRCFILESIZE=16777216
  [[ -z $TRCBUFSIZE ]] && TRCBUFSIZE=65536

  # Verify file/buffer sizes
  if [ $TRCFILESIZE -lt $TRCBUFSIZE ]
  then
    print -u2 -- "$osName trace file must be larger than the trace buffer size"
    return 1
  fi
fi

# Define the name of the trace file being formatted this time
DATEHMS=$($date +"%y%m%d.%H.%M.%S")
TRCFILE=$TRCPREF.$DATEHMS.$SHORTHOST
TRCERRFILE=$TRCERRPREF.$DATEHMS.$SHORTHOST

# Ensure the trace files/reports directories exist
$mkdir -p $TRCDIR $TRCRPTDIR

# Stop the current trace, producing a trace report if requested.
stopTrace

# If option is not "stop" then start trace and enable GPFS trace levels.
if [[ $trace_action = "start" ]]
then
  # Turn tracing on.
  startTrace

  # Check if gpfs is up If not, we rely on trace levels in mmfs.cfg.
  $tsstatus -1 >/dev/null 2>&1
  rc=$?
  if [[ $rc -eq 0 || $rc -eq 2 ]]
  then

    if [[ -z $trace_settings ]]
    then
      # Check whether any trace levels have been set. If not use default set
      anytrace=$($mmfsadm showtrace 2>/dev/null |
                 $awk '{if ($2 == ":" && $3 != "0") {print "1";exit}}')
      [[ -z $anytrace ]] && trace_settings="def"
    fi
    if [[ $trace_settings = "io" ]]
    then
      # Minimal trace levels for tracking application requests and
      # disk IO that results from those requests
      $mmfsadm trace all 0 io 1 vnop 1 vnode 1
    elif [[ $trace_settings = "all" ]]
    then
      # Maximum trace levels for GPFS
      $mmfsadm trace all 9
    elif [[ $trace_settings = "def" ]]
    then
      # A general set of trace levels for GPFS
      $mmfsadm trace all 4 tm 2 thread 1 mutex 1 vnode 2 ksvfs 3 klockl 2 io 3 pgalloc 1 mb 1 lock 2 fsck 3
    elif [[ $trace_settings = "same" ]]
    then
      : # nop
    elif [[ -n $trace_settings ]]
    then
      # User specified trace settings
      $mmfsadm trace $trace_settings
    fi
  fi
fi

cleanupAndExit 0