/* IBM_PROLOG_BEGIN_TAG */ /* This is an automatically generated prolog. */ /* */ /* */ /* */ /* Licensed Materials - Property of IBM */ /* */ /* (C) COPYRIGHT International Business Machines Corp. 2004,2006 */ /* All Rights Reserved */ /* */ /* US Government Users Restricted Rights - Use, duplication or */ /* disclosure restricted by GSA ADP Schedule Contract with IBM Corp. */ /* */ /* IBM_PROLOG_END_TAG */ /* @(#)47 1.5 src/avs/fs/mmfs/samples/util/tsbackup.h, mmfs, avs_rgpfs24, rgpfs240610b 2/27/06 14:36:22 */ /* NOTES: * 1. The absolute maximum number of files in a GPFS filesystem is * 256,000,000. * 2. Size of large customer filesystems ranges from 1 to 3 TB (terrabytes). * 3. The average size of a file for a technical customer is 100MB, * while for a commercial customer it is 100KB. * 4. Therefore, the average number of files in a technical customer's file- * system is 30,000 files, and for a commecial customer it is 10 million. */ #include // #define MAXPATHLEN /* Types to use when the number of bits in a word is important. Use these instead of long, short, int, etc. for portability */ typedef short Int16; typedef int Int32; typedef long long Int64; typedef unsigned short UInt16; typedef unsigned int UInt32; typedef unsigned long long UInt64; typedef unsigned int Boolean; /* maximum number of inodes in a GPFS filesystem */ #define MAX_INODES 256000000 /* maximum number of backup clients */ #define MAX_BACKUP_CLIENTS 32 /* maximum number of processes per backup client */ #define MAX_PROCESSES_PER_CLIENT 16 /* maximum number of backup clients processes */ #define MAX_BACKUP_CLIENT_PROCESSES 512 /* maximum number of characters used for a name */ #ifdef NAME_MAX #define MAX_NAME_CHARS NAME_MAX+1 #else #define MAX_NAME_CHARS 256 #endif /* maximum number of characters in a fullpath file name */ #ifdef PATH_MAX #define MAX_FILE_NAME PATH_MAX+1 #else #define MAX_FILE_NAME 1024 #endif /* maximum number of buffers used in processing a directory of file names */ #define MAX_DIR_BUFFERS 256 /* maximum number of characters in a command to be issued */ #define MAX_COMMAND_STRING 1024 /* maximum number of characters needed to print a 32-bit integer */ #define MAX_INT32_CHARS 11 /* maximum number of characters needed to print a 64-bit integer */ #define MAX_INT64_CHARS 21 /* The current choice for the discretion value is arbitrary. 8192 was chosen because it was big enough to acommodate a directory or a number of small files. No testing has been done to ascertain the benefit of this value. */ #define PROCESS_OVERFLOW_DISCRETION 8192 #define NRECS_PER_BUFFER 500 #define LINE_LENGTH 2*MAX_FILE_NAME #define INSTALL_DIRECTORY "/usr/lpp/mmfs" #define inputCtrlSeperatorChar '=' #define RC_SUCCESS 0 #define RC_PSUCCESS 1 #define RC_FAIL 2 /* #define DEBUG_BACKUP */ #ifndef BPRINTF #ifdef DEBUG_BACKUP #define BPRINTF(x) printf x #else #define BPRINTF(x) #endif #endif #define SHELL "ksh" #ifdef GPFS_AIX #define CAT "/usr/bin/cat" #define CP "/usr/bin/cp -f" #define RM "/usr/bin/rm -f" #define SHELL_PATH "/usr/bin/ksh" #elif GPFS_LINUX #define CAT "/bin/cat" #define CP "/bin/cp -f" #define RM "/bin/rm -f" #define SHELL_PATH "/bin/ksh" #endif /************************************************************************ * structure for capturing the results of an inodescan * ************************************************************************/ typedef UInt32 *inodeBitsArray; typedef enum { zeroBit, oneBit } Bit; typedef struct inode_affinity_status { char filename[MAX_FILE_NAME]; /* the name of the anchor file (i.e., the name of the snapshot directory */ Int64 serverFilesSize; /* aggregate file size for the files stored on the server */ } inode_affinity_status_t; typedef enum { BACKUP_COMPLETION_NONE, BACKUP_COMPLETION_PARTIAL, BACKUP_COMPLETION_FULL } backup_status; /************************************************************************ * structure for information about a backup client * ************************************************************************/ typedef struct backup_client_info { char clientName[MAX_NAME_CHARS]; /* official name of client */ // BCH - clients[].clientIndex could be eliminated with careful coding Int32 clientIndex; /* unique client index number assigned to client */ } backup_client_info_t; /************************************************************************ * structure for information obtained from an input control file * ************************************************************************/ typedef struct inputControlFile { char serverName[MAX_NAME_CHARS]; /* name of server */ Int32 numberOfClients; /* number of backup clients */ Int32 processesPerClient; /* number of processes per client */ backup_client_info_t clients[MAX_BACKUP_CLIENTS]; /* list of backup clients*/ } inputCtrlFile_t; /************************************************************************** * header information for a backup control file * **************************************************************************/ typedef struct backup_control_hdr { inputCtrlFile_t icf; /* data obtained from the input control file */ Int32 numberOfProcesses; /* total number of client processes */ gpfs_fssnap_id_t snapshotId; /* snapshot level associated with the latest backup */ char snapshotName[MAX_FILE_NAME]; /* name of the latest snapshot */ Int64 inodesTotal; /* the total number of inodes */ gpfs_ino_t inodeMax; /* max inode in filesystem */ gpfs_off64_t totalSizeOfFiles; /* the aggregate size for all files of the filesystem */ backup_status completionLevel; /* completion level of the most recent backup for this snapshot */ char backupType[12]; /* backup type (full or incremental) */ } backup_control_hdr_t; /************************************************************************ * structure controlling the backup of a filesystem * ************************************************************************/ typedef struct gpfs_backup_control { backup_control_hdr_t backupHdr; /* header info for this backup control */ inode_affinity_status_t inode; /* inode information */ } gpfs_backup_control_t; /************************************************************************ * structure for the format of each record in a transactions file * ************************************************************************/ typedef struct transactionsListRecord { char filename[MAX_FILE_NAME+2]; /* full path name of the file; the +2 is for enclosing the name in quotes */ char fnLength[MAX_INT32_CHARS]; /* length of filename (including quotes) */ char filesize[MAX_INT64_CHARS]; /* For a directory, the value indicates the size of all the files in the directory. For a regular file it indicates the size of the file. */ char delimeter2; /* separator character */ } transactionRecord_t; /************************************************************************ * structure for the format of each record in a .backup_shadow file * ************************************************************************/ typedef struct backup_shadow_record { char inodenum[MAX_INT32_CHARS]; /* the file inode as a string */ transactionRecord_t tr; /* transaction record */ char inodeDir; /* Is the inode for a directory? Y|N */ char newline_char; /* a new line character */ } backup_shadow_record_t; /************************************************************************ * structure for the format of each record in the filesizes file * ************************************************************************/ typedef struct backup_filesizes { char inodenum[MAX_INT32_CHARS]; /* the file inode as a string */ char filler; /* a blank space character */ char filesize[MAX_INT64_CHARS]; /* the file size as a string */ char newline_char; /* a new line character */ } backup_filesizes_t; /* global definitions */ extern Int32 Full; /* backup type full (true or false) */ extern Int32 Incremental; /* backup type incremental (true or false) */ extern Int32 Resume; /* resume backup (true or false) */ extern Int32 ioRate; /* I/O rate */ extern Int32 Tracing; /* tracing control variable */ extern char masterNode[MAX_NAME_CHARS]; /* node on which tsbackup was invoked */ extern Int32 masterPID; /* pid for master tsbackup process */ /* Name of the filesizes file. This temporary file is created at the root of the GPFS filesystem. It will be used to hold the sizes of the inodes of the filesystem. */ extern char filesizesFile[MAX_FILE_NAME]; /* Handle for the open filesizes file */ extern int filesizesHandle; /* Pointer to the handle of the open snapshot */ extern gpfs_fssnap_handle_t *fsSnapHandleP; /* Name of the backup control file. This permanent file is created at the root of the GPFS filesystem. It is used to hold the information contained in the backup control structure at the time of the last backup. */ extern char fsBackupCtrl[MAX_FILE_NAME]; /* Name of the backup shadow file. This permanent file is created at the root of the GPFS filesystem. It is used to hold the information regarding the file names and directories of the filesystem at the time it gets backed up. */ extern char backupShadowFile[MAX_FILE_NAME]; /* The full path name of the created snapshot for a filesystem */ extern char fsSnapshotPathname[MAX_FILE_NAME]; /* Array of names of .shadow_N files which are used in creating the .backup_shadow file. */ extern char shadowFile[MAX_BACKUP_CLIENTS][MAX_FILE_NAME]; /* Array of handles for opening shadowFile[] files */ extern int shadowFileHandle[MAX_BACKUP_CLIENTS]; /* Array of number of records in shadowFile[] files */ extern Int64 shadowFileNumberOfRecords[MAX_BACKUP_CLIENTS]; /* Name of transaction list file */ extern char transactionsList[MAX_FILE_NAME]; /* Handle for opening the transactions list file */ extern int transactionsListHandle; /* Array of names of .TransactionsList_N_L files */ extern char clientTransactionsList[MAX_BACKUP_CLIENT_PROCESSES][MAX_FILE_NAME]; /* Array of handles for opening clientTransactionsList[] files */ extern int clientTransactionsListHandle[MAX_BACKUP_CLIENT_PROCESSES]; /* Name of the pending transactions file */ extern char pendingTransactionsList[MAX_FILE_NAME]; /* Array of names of .PendingTransactionsList_N_L files */ extern char clientPendingTransactionsList[MAX_BACKUP_CLIENT_PROCESSES][MAX_FILE_NAME]; /* The pointer for accessing the header information in a .backup_control_t */ extern gpfs_backup_control_t *backupControlP; /* definition of queue element used in the thread based inode walker */ typedef struct queue_element { struct queue_element *nextP; ino_t ino; char path[MAX_FILE_NAME]; } QueueElement; /* Function prototypes */ extern void traceEntry(char* filen, char* funcn, Int32 linen); extern void traceExit(char* filen, char* funcn, Int32 linen); extern void traceLine(char* filen, char* funcn, Int32 linen); extern int Bitmap(Int32 size, Bit initialValue, UInt32 **inodeBitsP); int EnqueueWork(const char *pathP, const char *dirP, ino_t inode); QueueElement *DequeueWork(void); void* ThreadBody(void *parmP); int read_dir(const char *pathP, ino_t inode); int writeToShadowFile(const char *pathP, ino_t inode, const char inodeFlag); int dirInodeWalk(const char *pathname, inodeBitsArray bit_map); extern int doInodeScan(char *fsName, inodeBitsArray *inodeBitsArrayP, gpfs_backup_control_t *backupControlHeaderP); extern int doIncrInodeScan(char *fsName, gpfs_fssnap_id_t *prevSnapId, inodeBitsArray *inodeBitsArray2P, gpfs_backup_control_t *backupControlHeaderP); extern int doFSFullBackup(char *fsName); extern int doFSIncrementalBackup(char *fsName); extern int doFSResumeBackup(char *fsName, char **backupType); extern int forkMsgChild(); extern pid_t forkit(const char *); extern int doFSBackup(char *fsName, char *fsSnapshotPathname); extern int doFSIncrementalChanges(char *fsName, char *fsSnapshotPathname); extern int doFSFileDeletions(char *fsName, char *fsSnapshotPathname); extern int doFSFileChanges(char *fsName, char *fsSnapshotPathname); extern int getFilelistDeletions(const char *oldShadow, const char *newShadow, Boolean *fileDeletions, char *deletionsFile); extern int extractChangedFiles(char *filein, inodeBitsArray bitMap, Boolean *fileChanges, char *fileout); extern int createBackupSnapshot(char *fsName, char *snapshotName); extern int createClientFilelists(gpfs_backup_control_t *backupControlHdrP); extern int createClientFilelists2(gpfs_backup_control_t *backupControlHdrP); extern int getFSFileChanges(char *fsName, char *fsSnapshotPathname, char *changesFSName); extern int createFilelist(char *fsName, gpfs_backup_control_t *backupControlHeaderP); extern int createSnapshotShadows(char *fsName, int numberOfShadowFiles, inodeBitsArray bit_map); extern int createPendingFile(char *fsName, gpfs_backup_control_t *backupControlHdrP, char *pendingFile); #if 0 extern int getValue(UInt32 *inodeBitsP, Int32 index, Bit* bitP); #endif extern int createBackupCtrlFile(char *fileName, gpfs_backup_control_t *backupControlHeaderP); extern FILE* pipeOpen(const char *cmd, const char *mode, pid_t *outPidP); extern int pipeClose(FILE *ptr, pid_t pid); extern int checkBackupCtrlFile(); extern int processBackupCtrlFile(char *fsName, char *BackupCtrlFile, gpfs_backup_control_t **backupControlHeaderP); extern int processInputCtrlFile(char *fsName, char *inputCtrlFile, gpfs_backup_control_t **backupControlHeaderP, Boolean doMalloc); extern int processFilelist(char *fsName, char *transactionCmd, char *transactionCmdOptn, gpfs_backup_control_t *backupControlHeaderP, Boolean divyBySize); extern int setToZero(inodeBitsArray inodeBitsP, Int32 index, Bit* oldValueP); extern int setToOne(UInt32* inodeBitsP, Int32 index, Bit* oldValueP); extern void setAllToZero(UInt32* bitsP, Int32 size); extern void setAllToOne(UInt32* bitsP, Int32 size); extern int sortShadowfilesByInode(Int32 numShadows); extern int sortShadowfilesByFilename(); extern const UInt32 testBit(UInt32* inodeBitsP, Int32 index); extern int updateShadowfilesFilesizes(Int32 numShadows); extern int writeBuffer(int handle, transactionsListRecord *Buffer, size_t size);