#! /bin/awk -f # fsstruct will decode the AIX errpt fsstruct entries to be more readable. # extract error messages with errpt -a # setting all=1 will print out all error entries, all=0 will just print out # GPFS related error entries. ############################################################################# # # Usage: fsstruct.awk [] # ############################################################################# BEGIN { if (ARGC<2) { print "Usage: fsstruct.awk [] " terminate=1 exit 1 } wide=1 #set to 0 for pre-gpfs2.3 versions that do not use wide disk addresses all=0 decode=1 name[101]="FSErrInconsistentInode "; sens[101]=5; fmt[101]="4 8 8 2 v" name[102]="FSErrIndirectBlock "; sens[102]=5; fmt[102]="f 4 4 4 v" name[104]="FSErrBadReplicationCounts"; sens[104]=5; fmt[104]="4 v" name[105]="FSErrBadPtrReplications "; sens[105]=5; fmt[105]="4 2 2 2 v" name[106]="FSErrIndirectionLevel "; sens[106]=5; fmt[106]="4 2 v" name[107]="FSErrDeallocBlock "; sens[107]=1; fmt[107]="4 4 4 4 4 4 4 4" #wide alt#name[107]="FSErrDeallocBlock "; sens[107]=1; fmt[107]="4 4 8 4 4 4 4" name[108]="FSErrValidate "; sens[108]=5; fmt[108]="2 d 2 r v" name[109]="FSErrBadDiskAddrIndex "; sens[109]=1; fmt[109]="d 2" name[110]="FSErrBadDiskAddrSector "; sens[110]=1; fmt[110]="d 2 2" name[111]="FSErrBadDirBlock "; sens[111]=2; fmt[111]="4 4 4 4 4 4 4 4 4 4" name[112]="FSErrBadInodeStatus "; sens[112]=5; fmt[112]="4 v" name[113]="FSErrBadInodeOrGen "; sens[113]=5; fmt[113]="4 4 4 v" name[114]="FSErrAllocBlock "; sens[114]=1; fmt[114]="4 4 4 4" name[115]="FSErrBadDittoAddr "; sens[115]=2; fmt[115]="4 4 4 4 4 4 4 4 8" name[116]="FSErrSnapInodeModified "; sens[116]=5; fmt[116]="4 4 4 4 v" # subcode of 111 FSErrBadDirBlock name[201]="DirErrCheckHeaderFailed "; sens[201]=2; fmt[201]="4 4 4 4 4 4 4 4 4 4" name[202]="DirErrDotDotNotFound "; sens[202]=2; fmt[202]="4 4 4 4 4 4" name[203]="DirErrGenNumMismatch "; sens[203]=2; fmt[203]="4 4 4 4 4 4" name[204]="DirErrInodeNumMismatch "; sens[204]=2; fmt[204]="4 4 4 4 4 4" name[205]="DirErrInodeCorrupted "; sens[205]=2; fmt[205]="4 4 4 4 4 4" name[206]="DirErrDirectCorrupted "; sens[206]=2; fmt[206]="4 4 4 4 4 4" valtype[1]="inode" valtype[2]="indBlock" valtype[3]="dataAllocmap" valtype[4]="inodeAllocmap" valtype[5]="dirBlock" valtype[6]="bufDataAllocMap" buftype[0]="PBT_UNKNOWN" buftype[1]="PBT_FREE" buftype[2]="PBT_METADATA" buftype[3]="PBT_TEMP" buftype[4]="PBT_DISKDESC" buftype[5]="PBT_SGDESC" buftype[6]="PBT_DATA" buftype[7]="PBT_IND" buftype[8]="PBT_INODE" buftype[9]="PBT_LLDATA" buftype[10]="PBT_LLINDIRECT" buftype[11]="PBT_LLINODE" buftype[12]="PBT_ALLOC" buftype[13]="PBT_IALLOC" buftype[14]="PBT_LOG" buftype[15]="PBT_LOGDESC" buftype[16]="PBT_RECOV" buftype[17]="PBT_LOG_WRAP" buftype[18]="PBT_ISCAN" allerrtype["FFFF"]="NoBlock" allerrtype["FFFE"]="LinkListFirst" allerrtype["FFFD"]="NoConsecutive" allerrtype["FFFC"]="LinkListNth" allerrtype["FFFB"]="LinkListNew" monthof["Jan"]="01" monthof["Feb"]="02" monthof["Mar"]="03" monthof["Apr"]="04" monthof["May"]="05" monthof["Jun"]="06" monthof["Jul"]="07" monthof["Aug"]="08" monthof["Sep"]="09" monthof["Oct"]="10" monthof["Nov"]="11" monthof["Dec"]="12" } function hexToDecimal(hex, d, ch, i, val) { d = 0 for (i=1 ; i<=length(hex) ; i++) { ch = substr(hex,i,1) val = index("0123456789ABCDEF", ch) if (val == 0) val = index("0123456789abcdef", ch) d = d*16 + val - 1 } return d } function finishGPFSrec() { if (ingpfs) { ingpfs=0 printed=0 prest="" if (ingeneric) {ingeneric=0; prest="rc="rcode" reason="rscode" lvl="lvl" line "lineno" "module} else if (inshutdown) {inshutdown=0; prest="rc="rcode" reason="rscode" lvl="lvl" line "lineno" "module} else if (inlongdiskio) {inlongdiskio=0} else if (inunmount) {inunmount=0; prest="rc="scode" "vol} else if (indiskfail) {indiskfail=0; prest="rc="rcode" "vol" disk "physvol} else if (intraceback) {intraceback=0} else if (inmoreinfo) {inmoreinfo=0; prest="failedassert="assert} else if (inphoenix) {inphoenix=0; prest=phoemsg} else if (inquota) {inquota=0} else if (inenviron) {inenviron=0; prest="rc="rcode" "phoemsg} else if (inrecovery) {inrecovery=0} else if (inwarning) {inwarning=0} else if (infsstruct) {infsstruct=0; printed=1} if (!printed) printf("%s %s %s %s %s\n", date, node, label, id, prest) } else if (all && havestuff) printf("%s %s %s %s %s\n", date, node, label, id, otherstuff) havestuff=0 } /LABEL:/ {label=$NF; havestuff=1; otherstuff=""} /IDENTIFIER:/ {id=$NF} /MMFS_GENERIC/ {ingpfs=1; ingeneric=1} /MMFS_ABNORMAL_SHUTD/ {ingpfs=1; inshutdown=1} /MMFS_LONGDISKIO/ {ingpfs=1; inlongdiskio=1} /MMFS_SYSTEM_UNMOUNT/ {ingpfs=1; inunmount=1} /MMFS_DISKFAIL/ {ingpfs=1; indiskfail=1} /MMFS_TRACEBACK/ {ingpfs=1; intraceback=1} /MMFS_MOREINFO/ {ingpfs=1; inmoreinfo=1} /MMFS_PHOENIX/ {ingpfs=1; inphoenix=1} /MMFS_QUOTA/ {ingpfs=1; inquota=1} /MMFS_ENVIRON/ {ingpfs=1; inenviron=1} /MMFS_RECOVERY/ {ingpfs=1; inrecovery=1} /MMFS_SYSTEM_WARNING/ {ingpfs=1; inwarning=1} /MMFS_FSSTRUCT/ {ingpfs=1; infsstruct=1} /Date.Time:/ { pos=index($0,"Date/Time:") $0=substr($0,pos) day=$4; if (day < 10) day="0"day mon=monthof[$3]; if (mon == "") mon=$3 date=$NF"/"mon"/"day"@"$5 } /Node Id:/ {node=$NF} /STATUS CODE/ {getline; scode=$NF} /RETURN CODE/ { getline; rcode=$NF if (label == "VSD_EXT2_ER") {otherstuff=otherstuff" vsderr="$NF} } /REASON CODE/ {getline; if (NF == 2) rscode=$(NF-1) $NF; else rscode=$NF} /ERROR CODE/ {getline; code=$NF; if (infsstruct) count[code]++} /LINE/ {if (NF <= 2) {getline; lineno=$NF}} /MAINTENANCE LEVEL/ {getline; lvl=$NF} /REQUESTER/ {getline; assert=$0} /ERROR DATA/ {inphoedata=NF; phoemsg=""} /SIGNAL NUMBER/ {getline; otherstuff=otherstuff" sig="$NF} /CORE FILE NAME/ {getline; otherstuff=otherstuff" core="$NF} /PROGRAM NAME/ {getline; otherstuff=otherstuff" prog="$NF} /heartbeat is late/ {getline; otherstuff=otherstuff" lateheartbeat="$NF} /process was blocked/ {getline; otherstuff=otherstuff" stucktopologyservice="$NF} /Interface name/ {getline; otherstuff=otherstuff" interface="$NF} /DIAGNOSTIC EXPLANATION/ { if (index($0, "Refer to the") == 0) if (label == "VSD_INT_ER" || label == "RVSDD_ER") {getline; otherstuff=otherstuff" vsdexpl="$0} } /INVALID VALUE/ { if (label == "VSD_INT_ER") {getline; otherstuff=otherstuff" vsderr="$NF} } /MAJOR NUMBER/ { if (label == "VSD_EXT2_ER") {getline; otherstuff=otherstuff" major="$NF} } /MINOR NUMBER/ { if (label == "VSD_EXT2_ER") {getline; otherstuff=otherstuff" minor="$NF} } /FUNCTION/ { if (label == "VSD_EXT2_ER") { getline; otherstuff=otherstuff" function="$NF getline; getline; otherstuff=otherstuff" extfunc="$NF } else if (label == "VSD_STOPVSD_ST") {getline; otherstuff=otherstuff" function="$NF} } /FAILING MODULE/ { if (label == "SRC_RSTRT" || label == "SRC_TRYX") {getline; otherstuff=otherstuff" restart="$NF} } /Adapter interface name/ { if (label == "TS_MISCFG_ER") {getline; otherstuff=otherstuff" interface="$NF} } /Adapter offset/ { if (label == "TS_MISCFG_ER") {getline; otherstuff=otherstuff" offset="$NF} } /Adapter expected IP address/ { if (label == "TS_MISCFG_ER") {getline; otherstuff=otherstuff" expected="$NF} } /FILE SYSTEM DEVICE AND MOUNT POINT/ { if (index(label,"FS_FULL") > 0) {getline; otherstuff=otherstuff" "$(NF-1)" "$NF} } /JFS2 MAJOR/ { if (index(label,"FS_FULL") > 0) {getline; otherstuff=otherstuff" "$(NF-1)" "$NF} } /SOFT IPL 1/ { if (label == "REBOOT_ID") {getline; if ($NF == "0") otherstuff=otherstuff" SOFT IPL" else if ($NF == "1") otherstuff=otherstuff" HALT" else if ($NF == "2") otherstuff=otherstuff" TIME REBOOT" else otherstuff=otherstuff" boottype="$NF } } /TIME/ { if (label == "DUMP_STATS") {getline; otherstuff=otherstuff" "$0} } /Type:/ { if (substr(label,1,2) == "FC" || substr(label,1,7) == "SC_DISK") { otherstuff=otherstuff" "$NF} } /Resource Name:/ { if (substr(label,1,2) == "FC" || substr(label,1,7) == "SC_DISK") { otherstuff=otherstuff" "$NF} } /Resource Class:/ { if (substr(label,1,2) == "FC" || substr(label,1,7) == "SC_DISK") { otherstuff=otherstuff" "$NF} } /Resource Type:/ { if (substr(label,1,2) == "FC" || substr(label,1,7) == "SC_DISK") { otherstuff=otherstuff" "$NF} } /Location:/ { if (substr(label,1,2) == "FC" || substr(label,1,7) == "SC_DISK") { otherstuff=otherstuff" "$NF} } /PHYSICAL VOLUME DEVICE MAJOR/ { if (label == "LVM_IO_FAIL") { getline; if (NF > 3) otherstuff=otherstuff" physicalvol="$(NF-3)$(NF-2)$(NF-1)$NF else otherstuff=otherstuff" physicalvol="$(NF-1)$NF } next } /LOGICAL VOLUME DEVICE MAJOR/ { if (label == "LVM_IO_FAIL") { getline; if (NF > 3) otherstuff=otherstuff" logicalvol="$(NF-3)$(NF-2)$(NF-1)$NF else otherstuff=otherstuff" logicalvol="$(NF-1)$NF } next } /PHYSICAL VOLUME/ {getline; physvol=$NF; next} /VOLUME/ { getline; vol=$NF if (label == "VSD_STOPVSD_ST") {otherstuff=otherstuff" vg="$NF} } /PHYSICAL BUFFER TRANSACTION TIME/ { if (label == "LVM_IO_FAIL") {getline; otherstuff=otherstuff" iotime="$NF} next } /BLOCK NUMBER/ { if (label == "LVM_IO_FAIL") {getline; otherstuff=otherstuff" err="code" sector="$NF} next } /PANIC STRING/ { if (label == "KERNEL_PANIC") {getline; otherstuff=otherstuff" panic="$0} } /ASSERT STRING/ { if (label == "KERNEL_PANIC") {getline; otherstuff=otherstuff" assert="$0} } /DETECTING MODULE/ { if (ingpfs) { getline; module=$NF pos=index(module,"/") while (pos > 0) { module=substr(module,pos+1) pos=index(module,"/") } } else {getline; otherstuff=otherstuff" detectedby="$0} } /SENSE DATA/ { if (infsstruct) { if (NF == 3) indsh=1 else indsh=0 sl=sens[code]; if (sl == 0) sl=5 sense="" for (i=1; i <= sl; i++) { getline; if (indsh) $1="" sense = sense " " $0 } $0 = sense NF = split($0, wds) wd=2 if (code == 111) # FSErrBadDirBlock { code=hexToDecimal($(wd+1)); if (code >= 201 && code <= 206) wd+=2 else code=111 } formt=fmt[code] if (!decode || formt == "") printf("%s %s %s %s %s %s %s\n", date, node, "MMFS_FSSTRUCT", vol, code, name[code], sense) else { tokens=0 for (f=1; f <= length(formt); f++) { ch = substr(formt,f,1) if (ch != " ") { tokens++ if (ch == "4") { tok[tokens] = $wd $(wd+1); wd+=2 } else if (ch == "2") { tok[tokens] = $wd; wd+=1 } else if (ch == "8") { tok[tokens] = $wd $(wd+1) $(wd+2) $(wd+3); wd+=4 } else if (ch == "d") { if (wide) { tok[tokens] = $wd $(wd+1)":"$(wd+2) $(wd+3) $(wd+4) $(wd+5)"("hexToDecimal($wd $(wd+1))":"hexToDecimal($(wd+2) $(wd+3) $(wd+4) $(wd+5))")"; wd+=6 } else { tok[tokens] = $wd":"$(wd+1) $(wd+2)"("hexToDecimal($wd)":"hexToDecimal($(wd+1) $(wd+2))")"; wd+=3 } } else if (ch == "r") { nval=$wd; wd++ if (nval == "FFFF") tok[tokens]="[broken]" else { nval+=0 tok[tokens] = "[nVal="nval while (nval > 0) { if (wide) { tok[tokens] = tok[tokens]" "$wd $(wd+1)":"$(wd+2) $(wd+3) $(wd+4) $(wd+5)"("hexToDecimal($wd $(wd+1))":"hexToDecimal($(wd+2) $(wd+3) $(wd+4) $(wd+5))")"; wd+=6 } else { tok[tokens] = tok[tokens]" "$wd":"$(wd+1) $(wd+2)"("hexToDecimal($wd)":"hexToDecimal($(wd+1) $(wd+2))")"; wd+=3 } nval-- } tok[tokens] = tok[tokens]"]" } } else if (ch == "f") { nval=$wd; wd++ if (nval == "FFFF") tok[tokens]="[broken" else { nval+=0 tok[tokens] = "[nVal="nval while (nval > 0) { if (wide) { tok[tokens] = tok[tokens]" "$wd $(wd+1)":"$(wd+2) $(wd+3) $(wd+4) $(wd+5)"("hexToDecimal($wd $(wd+1))":"hexToDecimal($(wd+2) $(wd+3) $(wd+4) $(wd+5))")"; wd+=6 } else { tok[tokens] = tok[tokens]" "$wd":"$(wd+1) $(wd+2)"("hexToDecimal($wd)":"hexToDecimal($(wd+1) $(wd+2))")"; wd+=3 } nval-- } } tok[tokens] = tok[tokens]" subl="$wd"]"; wd++ } else if (ch == "v") { tok[tokens] = "(len="$wd $(wd+1)")"; wd+=2 while (NF >= wd && $NF == "0000") NF-- while (wd <= NF) { tok[tokens] = tok[tokens]" "$wd; wd++ } } } } rest="" while (NF >= wd && $NF == "0000") NF-- while (wd <= NF) { rest = rest" "$wd; wd++ } if (code == 101) # FSErrInconsistentInode { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="allocBlks="tok[2] tok[3]="lastBlock="tok[3] tok[4]="fragSubbl="tok[4] tok[5]="inode="tok[5] } else if (code == 102) # FSErrIndirectBlock { tok[1]="da="tok[1] tok[2]="gen="tok[2] tok[3]="inodeNum="tok[3]"("hexToDecimal(tok[3])")" tok[4]="expectBlkNum="tok[4] tok[5]="indBlock="tok[5] } else if (code == 104) # FSErrBadReplicationCounts { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="inode="tok[2] } else if (code == 105) # FSErrBadPtrReplications { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="dataPtrsPerInode="tok[2] tok[3]="indPtrsPerInode="tok[3] tok[4]="dataPtrsPerIndBlk="tok[4] tok[5]="inode="tok[5] } else if (code == 106) # FSErrIndirectionLevel { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="invalIndLvl="tok[2] tok[3]="inode="tok[3] } else if (code == 107) # FSErrDeallocBlock { tok[1]="errno="tok[1] tok[2]="diskNum="tok[2] if (wide || tok[4] > 32) # subblocks too big, assume 8 byte sector number { tok[3]="sector="tok[3] tok[4] tok[4]="nSubblocks="tok[5] tok[5]="region="tok[6] tok[6]="segment="tok[7] tok[7]="subBlock="tok[8] } else # assume 4 byte sector number { tok[3]="sector="tok[3] tok[4]="nSubblocks="tok[4] tok[5]="region="tok[5] tok[6]="segment="tok[6] tok[7]="subBlock="tok[7] } } else if (code == 108) # FSErrValidate { type=valtype[tok[1]+0] if (type == "") type=tok[1] tok[1]="type="type tok[2]="da="tok[2] tok[3]="sectors="tok[3] tok[4]="repda="tok[4] tok[5]="data="tok[5] } else if (code == 109) # FSErrBadDiskAddrIndex { tok[1]="da="tok[1] type=buftype[tok[2]+0] if (type == "") type=tok[2] tok[2]="type="type } else if (code == 110) # FSErrBadDiskAddrSector { tok[1]="da="tok[1] tok[2]="sectors="tok[2] type=buftype[hexToDecimal(tok[3])] if (type == "") type=tok[3] tok[3]="type="type } else if (code == 111) # FSErrBadDirBlock { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="blockNum="tok[2] tok[3]="errorcode="tok[3] tok[4]="errno="tok[4] tok[5]="dirhdr="tok[5] # and pick up tok[6 7 8 9] } else if (code == 112) # FSErrBadInodeStatus { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="inode="tok[2] } else if (code == 113) # FSErrBadInodeOrGen { tok[1]="fileInodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="dirInodeNum="tok[2]"("hexToDecimal(tok[2])")" tok[3]="gen="tok[3] tok[4]="filename="tok[4] } else if (code == 114) # FSErrAllocBlock { type=allerrtype[tok[1]] if (type == "") type=tok[1] tok[1]="type="type tok[2]="diskIndex="tok[2] tok[3]="recordNo="tok[3] tok[4]="nSubblocks="tok[4] } else if (code == 115) # FSErrBadDittoAddr { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="gen="tok[2] tok[3]="snapId="tok[3] tok[4]="inodeStatus="tok[4] tok[5]="InodeNumPrevSnap="tok[5]"("hexToDecimal(tok[5])")" tok[6]="genPrevSnap="tok[6] tok[7]="snapIdPrevSnap="tok[7] tok[8]="inodeStatusPrevSnap="tok[8] tok[9]="blockNum="tok[9] } else if (code == 116) # FSErrSnapInodeModified { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="snapId="tok[2] tok[3]="nextSnapId="tok[3] tok[4]="fileModifiedSnapId="tok[4] tok[5]="inode="tok[5] } else if (code == 201) # DirErrCheckHeaderFailed { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="snapId="tok[2] tok[3]="blockNum="tok[3] tok[4]="errorcode="tok[4] tok[5]="rc="tok[5] tok[6]="dirhdr="tok[6] # and pick up tok[7 8 9 10 11] } else if (code == 202) # DirErrDotDotNotFound { tok[1]="errorcode="tok[1] tok[2]="rc="tok[2] tok[3]="srcInodeNum="tok[3]"("hexToDecimal(tok[3])")" tok[4]="srcGen="tok[4] } else if (code == 203) # DirErrGenNumMismatch { tok[1]="errorcode="tok[1] tok[2]="rc="tok[2] tok[3]="srcInodeNum="tok[3]"("hexToDecimal(tok[3])")" tok[4]="srcGen="tok[4] tok[5]="entryGen="tok[5] } else if (code == 204) # DirErrInodeNumMismatch { tok[1]="errorcode="tok[1] tok[2]="rc="tok[2] tok[3]="srcInodeNum="tok[3]"("hexToDecimal(tok[3])")" tok[4]="entryInodeNum="tok[4]"("hexToDecimal(tok[4])")" } else if (code == 205) # DirErrInodeCorrupted { tok[3]="inodeNum="tok[3]"("hexToDecimal(tok[3])")" tok[4]="srcGen="tok[4] tok[5]="entryGen="tok[5] tok[6]="entryInodeStatus="tok[6] } else if (code == 206) # DirErrDirectCorrupted { tok[1]="inodeNum="tok[1]"("hexToDecimal(tok[1])")" tok[2]="blockNum="tok[2] tok[3]="errorcode="tok[3] tok[4]="errno="tok[4] tok[5]="dirhdr="tok[5] # and pick up tok[6 7 8 9] } printf("%s %s %s %s %s %s", date, node, "MMFS_FSSTRUCT", vol, code, name[code]) for (i = 1; i <= tokens; i++) printf(" %s", tok[i]) if (rest != "") printf(" rest =%s\n", rest) else printf("\n") } } } /---------------------/ {inphoedata=0; finishGPFSrec()} { if (inphoedata > 0) { if (inphoedata > 2) $1="" phoemsg=phoemsg" "$0 } } END { if (terminate) exit 1 finishGPFSrec() for (code in count) printf("+++code: %4d %s %9d times\n", code, name[code], count[code]) }