source: gpfs_3.1_ker2.6.20/lpp/mmfs/samples/util/tsbackup.h @ 152

Last change on this file since 152 was 16, checked in by rock, 18 years ago
File size: 16.6 KB
RevLine 
[16]1/* IBM_PROLOG_BEGIN_TAG                                                   */
2/* This is an automatically generated prolog.                             */
3/*                                                                        */
4/*                                                                        */
5/*                                                                        */
6/* Licensed Materials - Property of IBM                                   */
7/*                                                                        */
8/* (C) COPYRIGHT International Business Machines Corp. 2004,2006          */
9/* All Rights Reserved                                                    */
10/*                                                                        */
11/* US Government Users Restricted Rights - Use, duplication or            */
12/* disclosure restricted by GSA ADP Schedule Contract with IBM Corp.      */
13/*                                                                        */
14/* IBM_PROLOG_END_TAG                                                     */
15/* @(#)47       1.5  src/avs/fs/mmfs/samples/util/tsbackup.h, mmfs, avs_rgpfs24, rgpfs240610b 2/27/06 14:36:22 */
16
17/* NOTES:
18 * 1. The absolute maximum number of files in a GPFS filesystem is
19 *    256,000,000.
20 * 2. Size of large customer filesystems ranges from 1 to 3 TB (terrabytes).
21 * 3. The average size of a file for a technical customer is 100MB,
22 *    while for a commercial customer it is 100KB.
23 * 4. Therefore, the average number of files in a technical customer's file-
24 *    system is 30,000 files, and for a commecial customer it is 10 million.
25 */
26
27#include <sys/param.h>          // #define MAXPATHLEN
28
29/* Types to use when the number of bits in a word is important.
30   Use these instead of long, short, int, etc. for portability */
31typedef short              Int16;
32typedef int                Int32;
33typedef long long          Int64;
34typedef unsigned short     UInt16;
35typedef unsigned int       UInt32;
36typedef unsigned long long UInt64;
37typedef unsigned int       Boolean;
38
39/* maximum number of inodes in a GPFS filesystem */
40#define MAX_INODES          256000000
41
42/* maximum number of backup clients */
43#define MAX_BACKUP_CLIENTS  32
44
45/* maximum number of processes per backup client */
46#define MAX_PROCESSES_PER_CLIENT  16
47
48/* maximum number of backup clients processes */
49#define MAX_BACKUP_CLIENT_PROCESSES  512
50
51/* maximum number of characters used for a name */
52#ifdef NAME_MAX
53#define MAX_NAME_CHARS  NAME_MAX+1
54#else
55#define MAX_NAME_CHARS  256
56#endif
57
58/* maximum number of characters in a fullpath file name */
59#ifdef PATH_MAX
60#define MAX_FILE_NAME  PATH_MAX+1
61#else
62#define MAX_FILE_NAME  1024
63#endif
64
65/* maximum number of buffers used in processing a directory of file names */
66#define MAX_DIR_BUFFERS 256
67
68/* maximum number of characters in a command to be issued */
69#define MAX_COMMAND_STRING 1024
70
71/* maximum number of characters needed to print a 32-bit integer */
72#define MAX_INT32_CHARS 11
73
74/* maximum number of characters needed to print a 64-bit integer */
75#define MAX_INT64_CHARS 21
76
77/* The current choice for the discretion value is arbitrary.
78   8192 was chosen because it was big enough to acommodate
79   a directory or a number of small files.  No testing
80   has been done to ascertain the benefit of this value. */
81#define PROCESS_OVERFLOW_DISCRETION 8192
82
83#define NRECS_PER_BUFFER       500
84
85#define LINE_LENGTH            2*MAX_FILE_NAME
86
87#define INSTALL_DIRECTORY      "/usr/lpp/mmfs"
88
89#define inputCtrlSeperatorChar '='
90
91#define RC_SUCCESS  0
92#define RC_PSUCCESS 1
93#define RC_FAIL     2
94
95/* #define DEBUG_BACKUP  */
96#ifndef BPRINTF
97#ifdef  DEBUG_BACKUP
98#define BPRINTF(x)  printf x
99#else
100#define BPRINTF(x)
101#endif
102#endif
103
104#define SHELL       "ksh"
105
106#ifdef GPFS_AIX
107
108#define CAT         "/usr/bin/cat"
109#define CP          "/usr/bin/cp -f"
110#define RM          "/usr/bin/rm -f"
111#define SHELL_PATH  "/usr/bin/ksh"
112
113#elif GPFS_LINUX
114
115#define CAT         "/bin/cat"
116#define CP          "/bin/cp -f"
117#define RM          "/bin/rm -f"
118#define SHELL_PATH  "/bin/ksh"
119
120#endif
121
122
123/************************************************************************
124 * structure for capturing the results of an inodescan                  *
125 ************************************************************************/
126typedef UInt32  *inodeBitsArray;
127
128typedef enum
129{
130  zeroBit,
131  oneBit
132} Bit;
133
134typedef struct inode_affinity_status
135{
136  char  filename[MAX_FILE_NAME];   /* the name of the anchor file (i.e.,
137                                      the name of the snapshot directory */
138  Int64 serverFilesSize;           /* aggregate file size for the files
139                                                    stored on the server */
140} inode_affinity_status_t;
141
142typedef enum
143{
144  BACKUP_COMPLETION_NONE,
145  BACKUP_COMPLETION_PARTIAL,
146  BACKUP_COMPLETION_FULL
147} backup_status;
148
149
150/************************************************************************
151 * structure for information about a backup client                      *
152 ************************************************************************/
153typedef struct backup_client_info
154{
155  char  clientName[MAX_NAME_CHARS];       /* official name of client    */
156  // BCH - clients[].clientIndex could be eliminated with careful coding
157  Int32 clientIndex;                      /* unique client index number
158                                               assigned to client       */
159} backup_client_info_t;
160
161
162/************************************************************************
163 * structure for information obtained from an input control file        *
164 ************************************************************************/
165typedef struct inputControlFile
166{
167  char          serverName[MAX_NAME_CHARS];   /* name of server              */
168  Int32         numberOfClients;       /* number of backup clients           */
169  Int32         processesPerClient;    /* number of processes per client     */
170  backup_client_info_t clients[MAX_BACKUP_CLIENTS]; /* list of backup clients*/
171} inputCtrlFile_t;
172
173
174/**************************************************************************
175 * header information for a backup control file                           *
176 **************************************************************************/
177typedef struct backup_control_hdr
178{
179  inputCtrlFile_t icf;          /* data obtained from the input control file */
180  Int32         numberOfProcesses;     /* total number of client processes   */
181  gpfs_fssnap_id_t snapshotId;         /* snapshot level associated with
182                                                          the latest backup  */
183  char          snapshotName[MAX_FILE_NAME]; /* name of the latest snapshot  */
184  Int64         inodesTotal;           /* the total number of inodes         */
185  gpfs_ino_t    inodeMax;              /* max inode in filesystem            */
186  gpfs_off64_t  totalSizeOfFiles;      /* the aggregate size for all files
187                                                          of the filesystem  */
188  backup_status completionLevel;       /* completion level of the most recent
189                                                   backup for this snapshot  */
190  char          backupType[12];        /* backup type (full or incremental)  */
191} backup_control_hdr_t;
192
193
194/************************************************************************
195 * structure controlling the backup of a filesystem                     *
196 ************************************************************************/
197typedef struct gpfs_backup_control
198{
199  backup_control_hdr_t backupHdr;     /* header info for this backup control */
200  inode_affinity_status_t inode;      /* inode information                   */
201} gpfs_backup_control_t;
202
203
204/************************************************************************
205 * structure for the format of each record in a transactions file       *
206 ************************************************************************/
207typedef struct transactionsListRecord
208{
209  char filename[MAX_FILE_NAME+2];   /* full path name of the file; the +2
210                                       is for enclosing the name in quotes */
211  char fnLength[MAX_INT32_CHARS];   /* length of filename (including quotes) */
212  char filesize[MAX_INT64_CHARS];   /* For a directory, the value indicates
213                                       the size of all the files in the
214                                       directory.  For a regular file it
215                                       indicates the size of the file. */
216  char delimeter2;                  /* separator character */
217} transactionRecord_t;
218
219
220/************************************************************************
221 * structure for the format of each record in a .backup_shadow file     *
222 ************************************************************************/
223typedef struct backup_shadow_record
224{
225  char inodenum[MAX_INT32_CHARS];   /* the file inode as a string */
226  transactionRecord_t tr;           /* transaction record */
227  char inodeDir;                    /* Is the inode for a directory? Y|N */
228  char newline_char;                /* a new line character */
229} backup_shadow_record_t;
230
231
232/************************************************************************
233 * structure for the format of each record in the filesizes file        *
234 ************************************************************************/
235typedef struct backup_filesizes
236{
237  char inodenum[MAX_INT32_CHARS];   /* the file inode as a string */
238  char filler;                      /* a blank space character */
239  char filesize[MAX_INT64_CHARS];   /* the file size as a string */
240  char newline_char;                /* a new line character */
241} backup_filesizes_t;
242
243
244                   /* global definitions  */
245
246extern Int32 Full;              /* backup type full (true or false) */
247extern Int32 Incremental;       /* backup type incremental (true or false) */
248extern Int32 Resume;            /* resume backup (true or false) */
249
250extern Int32 ioRate;            /* I/O rate */
251extern Int32 Tracing;           /* tracing control variable */
252
253extern char masterNode[MAX_NAME_CHARS]; /* node on which tsbackup was invoked */
254
255extern Int32 masterPID;         /* pid for master tsbackup process */
256
257/* Name of the filesizes file.
258   This temporary file is created at the root of the GPFS filesystem.
259   It will be used to hold the sizes of the inodes of the filesystem. */
260extern char filesizesFile[MAX_FILE_NAME];
261
262/* Handle for the open filesizes file */
263extern int filesizesHandle;
264
265/* Pointer to the handle of the open snapshot */
266extern gpfs_fssnap_handle_t *fsSnapHandleP;
267
268/* Name of the backup control file.  This permanent file is created at
269   the root of the GPFS filesystem.  It is used to hold the information
270   contained in the backup control structure at the time of the last backup. */
271extern char fsBackupCtrl[MAX_FILE_NAME];
272
273/* Name of the backup shadow file.  This permanent file is created at
274   the root of the GPFS filesystem.  It is used to hold the information
275   regarding the file names and directories of the filesystem at the
276   time it gets backed up.  */
277extern char backupShadowFile[MAX_FILE_NAME];
278
279/* The full path name of the created snapshot for a filesystem */
280extern char fsSnapshotPathname[MAX_FILE_NAME];
281
282/* Array of names of .shadow_N files which are used in creating the
283   .backup_shadow file. */
284extern char shadowFile[MAX_BACKUP_CLIENTS][MAX_FILE_NAME];
285
286/* Array of handles for opening shadowFile[] files */
287extern int shadowFileHandle[MAX_BACKUP_CLIENTS];
288
289/* Array of number of records in shadowFile[] files */
290extern Int64 shadowFileNumberOfRecords[MAX_BACKUP_CLIENTS];
291
292/* Name of transaction list file */
293extern char transactionsList[MAX_FILE_NAME];
294
295/* Handle for opening the transactions list file */
296extern int transactionsListHandle;
297
298/* Array of names of .TransactionsList_N_L files */
299extern char clientTransactionsList[MAX_BACKUP_CLIENT_PROCESSES][MAX_FILE_NAME];
300
301/* Array of handles for opening clientTransactionsList[] files */
302extern int clientTransactionsListHandle[MAX_BACKUP_CLIENT_PROCESSES];
303
304/* Name of the pending transactions file */
305extern char pendingTransactionsList[MAX_FILE_NAME];
306
307/* Array of names of .PendingTransactionsList_N_L files */
308extern char clientPendingTransactionsList[MAX_BACKUP_CLIENT_PROCESSES][MAX_FILE_NAME];
309
310/* The pointer for accessing the header information in a .backup_control_t */
311extern gpfs_backup_control_t *backupControlP;
312
313/* definition of queue element used in the thread based inode walker */
314typedef struct queue_element
315{
316  struct queue_element *nextP;
317  ino_t ino;
318  char path[MAX_FILE_NAME];
319} QueueElement;
320
321
322                         /* Function prototypes  */
323
324extern void traceEntry(char* filen, char* funcn, Int32 linen);
325
326extern void traceExit(char* filen, char* funcn, Int32 linen);
327
328extern void traceLine(char* filen, char* funcn, Int32 linen);
329
330extern int Bitmap(Int32 size, Bit initialValue, UInt32 **inodeBitsP);
331
332int EnqueueWork(const char *pathP, const char *dirP, ino_t inode);
333
334QueueElement *DequeueWork(void);
335
336void* ThreadBody(void *parmP);
337
338int read_dir(const char *pathP, ino_t inode);
339
340int writeToShadowFile(const char *pathP, ino_t inode, const char inodeFlag);
341
342int dirInodeWalk(const char *pathname, inodeBitsArray bit_map);
343
344extern int doInodeScan(char *fsName,
345                       inodeBitsArray *inodeBitsArrayP,
346                       gpfs_backup_control_t *backupControlHeaderP);
347
348extern int doIncrInodeScan(char *fsName,
349                           gpfs_fssnap_id_t *prevSnapId,
350                           inodeBitsArray *inodeBitsArray2P,
351                           gpfs_backup_control_t *backupControlHeaderP);
352
353extern int doFSFullBackup(char *fsName);
354
355extern int doFSIncrementalBackup(char *fsName);
356
357extern int doFSResumeBackup(char *fsName, char **backupType);
358
359extern int forkMsgChild();
360
361extern pid_t forkit(const char *);
362
363extern int doFSBackup(char *fsName, char *fsSnapshotPathname);
364
365extern int doFSIncrementalChanges(char *fsName, char *fsSnapshotPathname);
366
367extern int doFSFileDeletions(char *fsName, char *fsSnapshotPathname);
368
369extern int doFSFileChanges(char *fsName, char *fsSnapshotPathname);
370
371extern int getFilelistDeletions(const char *oldShadow,
372                                const char *newShadow,
373                                Boolean *fileDeletions,
374                                char *deletionsFile);
375
376extern int extractChangedFiles(char *filein,
377                               inodeBitsArray bitMap,
378                               Boolean *fileChanges,
379                               char *fileout);
380
381extern int createBackupSnapshot(char *fsName, char *snapshotName);
382
383extern int createClientFilelists(gpfs_backup_control_t *backupControlHdrP);
384
385extern int createClientFilelists2(gpfs_backup_control_t *backupControlHdrP);
386
387extern int getFSFileChanges(char *fsName, char *fsSnapshotPathname,
388                            char *changesFSName);
389
390extern int createFilelist(char *fsName,
391                          gpfs_backup_control_t *backupControlHeaderP);
392
393extern int createSnapshotShadows(char *fsName,
394                                 int numberOfShadowFiles,
395                                 inodeBitsArray bit_map);
396
397extern int createPendingFile(char *fsName,
398                             gpfs_backup_control_t *backupControlHdrP,
399                             char *pendingFile);
400#if 0
401extern int getValue(UInt32 *inodeBitsP, Int32 index, Bit* bitP);
402#endif
403
404extern int createBackupCtrlFile(char *fileName,
405                                gpfs_backup_control_t *backupControlHeaderP);
406
407extern FILE* pipeOpen(const char *cmd, const char *mode, pid_t *outPidP);
408
409extern int pipeClose(FILE *ptr, pid_t pid);
410
411extern int checkBackupCtrlFile();
412
413extern int processBackupCtrlFile(char *fsName,
414                                 char *BackupCtrlFile,
415                                 gpfs_backup_control_t **backupControlHeaderP);
416
417extern int processInputCtrlFile(char *fsName,
418                                char *inputCtrlFile,
419                                gpfs_backup_control_t **backupControlHeaderP,
420                                Boolean doMalloc);
421
422extern int processFilelist(char *fsName,
423                           char *transactionCmd,
424                           char *transactionCmdOptn,
425                           gpfs_backup_control_t *backupControlHeaderP,
426                           Boolean divyBySize);
427
428extern int setToZero(inodeBitsArray inodeBitsP, Int32 index, Bit* oldValueP);
429
430extern int setToOne(UInt32* inodeBitsP, Int32 index, Bit* oldValueP);
431
432extern void setAllToZero(UInt32* bitsP, Int32 size);
433
434extern void setAllToOne(UInt32* bitsP, Int32 size);
435
436extern int sortShadowfilesByInode(Int32 numShadows);
437
438extern int sortShadowfilesByFilename();
439
440extern const UInt32 testBit(UInt32* inodeBitsP, Int32 index);
441
442extern int updateShadowfilesFilesizes(Int32 numShadows);
443
444extern int writeBuffer(int handle, transactionsListRecord *Buffer, size_t size);
445
Note: See TracBrowser for help on using the repository browser.