#! /usr/bin/awk -f
# /usr/lpp/mmfs/samples/util/tsinode $mountpoint > $fsname.files
#   (the output file will be about 170*#ofFiles,
#    so be sure you have plenty of space, or pipe it into tsfilehist.awk)
#
# Usage: tsfilehist.awk [variable-overrides] $fsname.files > $fsname.filesummary
#   nbuckets=#buckets-for-histogram          (default 100)
#   blocksize=[16 64 256 384 512 1024 4096]  (default 256)
#   availablespace=size-of-filesystem   (bytes, or with suffix {s|k|m|g})
#     (if specified, the AvSpace%ile column will show % of available space)
#   migrated=1    (if files could be migrated out by HSM)
BEGIN {
  if (ARGC<1)
  {
    print "Usage: tsfilehist.awk [variable-overrides] $fsname.files > $fsname.filesummary"
    print "       nbuckets=#buckets-for-histogram          (default 100)"
    print "       blocksize=[16 64 256 384 512 1024 4096]  (default 256)"
    print "       availablespace=size-of-filesystem   (bytes, or with suffix {s|k|m|g})"
    print "       migrated=1    (if files could be migrated out by HSM)"
    terminate=1
    exit 1
  }

  nbuckets=100
  availablespace=0 # unknown at start
  migrated=0
#  SIZ = logical size BKS = blocksize of filesystem SUB = sub-block size
  SIZ[4096]="4M";  BKS[4096]=4194304; SUB[4096]=131072
  SIZ[1024]="1M";  BKS[1024]=1048576; SUB[1024]=32768
  SIZ[512]="512K"; BKS[512]=524288;   SUB[512]=16384
  SIZ[384]="384K"; BKS[384]=393216;   SUB[384]=12288
  SIZ[256]="256K"; BKS[256]=262144;   SUB[256]=8192
  SIZ[64]="64K";   BKS[64]=65536;     SUB[64]=2048
  SIZ[16]="16K";   BKS[16]=16384;     SUB[16]=512
}
{
#  sample output:
#    inode     gen    uid    gid       size      space     mode nlink      atime               mtime               ctime        flags
#    30721   65536    100    100    8037722    8044544 -rwxr-xr-x 1 1085616373.647071599 1085609949.810665286 1085609949.810665286 replmeta ind=2
#
# parse input
  if ($1 == "inode") # first line
  {
    if (blocksize == 0) blocksize = 256
    else if (SIZ[blocksize] == "")
    {
      print "Unexpected blocksize", blocksize
      exit 1
    }
  }
  else  if ($1 == "" || index($0,"logfile") != 0 || index($0,"reserved") != 0)
    ; #throw away lines
  else if ((substr($7,1,1) == "-" || substr($7,1,1) == "l" || substr($7,1,1) == "d"))
  {
    entries += 1;

    if (substr($7,1,1) == "d")  # directories
    {
      directories += 1;
      space=$6
      if (availablespace != 0)
        totsize+=space;
    }
    else    # files
    {
      files++
      size=$5
      space=$6
      if (index($0,"repldata") != 0) space /= 2
      if (size > maxsize) maxsize = size
      if (size == 0)
        zerolen++
      else if (migrated && space == 0)  # Presumed to be a migrated file
      {
        migratedfiles++
        migratedsize+=size
        blks=int(size/BKS[blocksize]);
        frag=size%BKS[blocksize];
        subb=int(frag/SUB[blocksize]);
        subf=size%SUB[blocksize];
        if (subf > 0)
          subb += 1;
        if (blks == 0)
          msubblarray[subb] += 1;
        else if (blks == 1 && frag == 0)
          msubblarray[32] += 1;
        else if (blks <= nbuckets)
          mdiskarray[blks] += 1;
        else
          mdiskarray[nbuckets+1] += 1;
      }
      else # if (space > 0)
      {
        if (space > size)
          totalactualwaste += space-size
        totsize+=size;
        blks=int(size/BKS[blocksize]);
        frag=size%BKS[blocksize];
        subb=int(frag/SUB[blocksize]);
        subf=size%SUB[blocksize];
        if (subf > 0) 
        {
          totwaste += SUB[blocksize]-subf;
          subb += 1;
        }
        if (blks == 0)
          subblarray[subb] += 1;
        else if (blks == 1 && frag == 0)
          subblarray[32] += 1;
        else if (blks <= nbuckets)
          diskarray[blks] += 1;
        else
          diskarray[nbuckets+1] += 1;
        if (blks > 67) ind64++
        if (blks > 32) ind32++
      }
    }
  }
  else
    entries += 1;
}

END {
  if (terminate) exit 1
  unit[1]="GB";
  unit[2]="TB";
  unit[3]="PB";
  printf ("\nAll: Files       = %14d\n",files)
  if (availablespace != 0)
  {
    lastav=substr(availablespace,length(availablespace))
    if (lastav == "s" || lastav == "S") availablespace *= 512
    else if (lastav == "k" || lastav == "K") availablespace *= 1024
    else if (lastav == "m" || lastav == "M") availablespace *= 1024*1024
    else if (lastav == "g" || lastav == "G") availablespace *= 1024*1024*1024
    printf ("Available space  = %14d",availablespace)
    tot=int((availablespace+500000000)/1000000000)
    for (i=1; (tot > 0) && (i <= 3); i++)
    {
      printf ("%12d %s",tot, unit[i])
      tot=int((tot+500)/1000)
    }
    printf ("\n")
  }
  printf ("Total Size       = %14d",totsize)
  tot=int((totsize+500000000)/1000000000)
  for (i=1; (tot > 0) && (i <= 3); i++)
  {
    printf ("%12d %s",tot, unit[i])
    tot=int((tot+500)/1000)
  }

  printf ("\nLargest File     = %14d\n",maxsize);
  printf ("Average Size     = %14d\n",totsize/files);
  printf ("\nNon-zero: Files  = %14d\n",files-zerolen);
  printf ("Average NZ size  = %14d\n",totsize/(files-zerolen));
  if (migratedsize > 0)
  {
    printf ("\nMigrated: Files  = %14d\n",migratedfiles);
    printf ("Total Mig size   = %14d",migratedsize);
    tot=int((migratedsize+500000000)/1000000000);
    for (i=1; (tot > 0) && (i <= 3); i++)
    {
      printf ("%12d %s",tot, unit[i]);
      tot=int((tot+500)/1000);
    }
    printf ("\nAverage Mig size = %14d\n",migratedsize/migratedfiles);
    printf ("Average All size = %14d\n",(totsize+migratedsize)/files);
  }

  printf ("\nDirectories         = %11d\n",directories);
  printf ("Avg Entries per dir = %11.1f\n",(entries-1)/directories);

  printf ("\nFiles with indirect blocks        = %11d\n", ind64);
  printf ("Files with indirect blocks (wide) = %11d\n", ind32);

  printf ("\nFile%%ile represents the cummulative percentage of files.\n");
  printf ("Space%%ile represents the cummulative percentage of total space used.\n");
  printf ("AvlSpc%%ile represents the cummulative percentage used of total available space.\n");
  printf ("\nHistogram of files <= one %s block in size\n", SIZ[blocksize])
  printf ("Subblocks    Count  File%%ile  Space%%ile");
  if (availablespace > 0)
    printf (" AvlSpc%%ile");
  printf ("\n--------- -------- ---------- ----------");
  if (availablespace > 0) printf ("  ----------");
  printf ("\n");
  subblarray[0] = zerolen;
  for (j=0; j<=32; j++)
  {
    Faccum += subblarray[j];
    Baccum += j*subblarray[j]*SUB[blocksize];
    printf ("%9d%9d", j, subblarray[j]);
    printf ("%10.2f%%%10.2f%%", 100*Faccum/(files-migratedfiles), 100*Baccum/totsize);
    if (availablespace > 0)
      printf (" %10.2f%%", 100*Baccum/availablespace);
    printf ("\n");
  }

  printf ("\nHistogram of files with N %s blocks (plus end fragment)\n", SIZ[blocksize]);
  printf ("   Blocks    Count  File%%ile  Space%%ile");
  if (availablespace > 0)
    printf (" AvlSpc%%ile");
  printf ("\n--------- -------- ---------- ----------");
  if (availablespace > 0) printf ("  ----------");
  printf ("\n");
  for (j=1; j<=nbuckets; j++)
  {
    if (diskarray[j] > 0)
    {
      printf ("%9d",j);
      printf("%9d", diskarray[j]);
      Faccum += diskarray[j];
      Baccum += j*diskarray[j]*BKS[blocksize];
      printf ("%10.2f%%%10.2f%%", 100*Faccum/(files-migratedfiles), 100*Baccum/totsize);
      if (availablespace > 0)
        printf (" %10.2f%%", 100*Baccum/availablespace);
      printf ("\n");
    }
  }

  printf ("Number of files with more than %d %s blocks\n", nbuckets, SIZ[blocksize]);
  j=nbuckets+1
  printf ("%8d+",j);
  printf("%9d", diskarray[j]);
  Faccum += diskarray[j];
  Baccum = totsize;
  if (availablespace > 0)
    printf ("%10.2f%%%10.2f%% %10.2f%%\n", 100*Faccum/(files-migratedfiles), 100*Baccum/totsize, 100*Baccum/availablespace);
  else
    printf ("%10.2f%%%10.2f%%\n", 100*Faccum/(files-migratedfiles), 100*Baccum/totsize);

  if (migratedsize > 0)
  {
    printf ("\nHistogram of migrated files <= one %s block in size\n",SIZ[blocksize])
    printf ("Subblocks    Count  File%%ile  Space%%ile");
    if (availablespace > 0)
      printf (" AvlSpc%%ile");
    printf ("\n--------- -------- ---------- ----------");
    if (availablespace > 0) printf ("  ----------");
    printf ("\n");
    msubblarray[0] = 0;
    for (j=0; j<=32; j++)
    {
      if (allhist || msubblarray[j] > 0)
      {
        printf ("%9d",j);
        printf("%9d", msubblarray[j]);
        mFaccum += msubblarray[j];
        mBaccum += j*msubblarray[j]*SUB[blocksize];
        if (availablespace > 0)
          printf ("%10.2f%%%10.2f%% %10.2f%%\n", 100*mFaccum/migratedfiles, 100*mBaccum/totsize, 100*mBaccum/availablespace);
        else
          printf ("%10.2f%%%10.2f%%\n", 100*mFaccum/migratedfiles, 100*mBaccum/totsize);
      }
    }

    printf ("\nHistogram of migrated files with N %s blocks (plus end fragment)\n", SIZ[blocksize]);
    printf ("   Blocks    Count  File%%ile  Space%%ile");
    if (availablespace > 0)
      printf (" AvlSpc%%ile");
    printf ("\n--------- -------- ---------- ----------");
    if (availablespace > 0) printf ("  ----------");
    printf ("\n");
    for (j=1; j<=nbuckets; j++)
    {
      if (mdiskarray[j] > 0)
      {
        printf ("%9d",j);
        printf("%9d", mdiskarray[j]);
        mFaccum += mdiskarray[j];
        mBaccum += j*mdiskarray[j]*BKS[blocksize];
        if (availablespace > 0)
          printf ("%10.2f%%%10.2f%% %10.2f%%\n", 100*mFaccum/migratedfiles, 100*mBaccum/totsize, 100*mBaccum/availablespace);
        else
          printf ("%10.2f%%%10.2f%%\n", 100*mFaccum/migratedfiles, 100*mBaccum/totsize);
      }
    }

    printf ("Number of migrated files with more than %d %s blocks\n", nbuckets, SIZ[blocksize]);
    j=nbuckets+1
    printf ("%8d+",j);
    printf("%9d", mdiskarray[j]);
    mFaccum += mdiskarray[j];
    mBaccum = migratedsize;
    if (availablespace > 0)
      printf ("%10.2f%%%10.2f%% %10.2f%%\n", 100*mFaccum/migratedfiles, 100*mBaccum/totsize, 100*mBaccum/availablespace);
    else
      printf ("%10.2f%%%10.2f%%\n", 100*mFaccum/migratedfiles, 100*mBaccum/totsize);
  }

  #printf ("\nBlockSize        Waste %% Waste Avg Waste per File\n");
  #printf (  "--------- ------------ ------- ------------------\n");
  #printf ("     %s %12d %6.2f%% %17d\n",SIZ[blocksize],totwaste,100*totwaste/totsize,totwaste/(files-zerolen-migratedfiles));
  #printf ("     %s %12d %6.2f%% %17d\n","Real",totalactualwaste,100*totalactualwaste/totsize,totalactualwaste/(files-zerolen-migratedfiles));

}