XRootD
Loading...
Searching...
No Matches
XrdFrmFiles.cc
Go to the documentation of this file.
1/******************************************************************************/
2/* */
3/* X r d F r m F i l e s . c c */
4/* */
5/* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */
6/* All Rights Reserved */
7/* Produced by Andrew Hanushevsky for Stanford University under contract */
8/* DE-AC02-76-SFO0515 with the Department of Energy */
9/* */
10/* This file is part of the XRootD software suite. */
11/* */
12/* XRootD is free software: you can redistribute it and/or modify it under */
13/* the terms of the GNU Lesser General Public License as published by the */
14/* Free Software Foundation, either version 3 of the License, or (at your */
15/* option) any later version. */
16/* */
17/* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18/* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19/* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20/* License for more details. */
21/* */
22/* You should have received a copy of the GNU Lesser General Public License */
23/* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24/* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25/* */
26/* The copyright holder's institutional names and contributor's names may not */
27/* be used to endorse or promote products derived from this software without */
28/* specific prior written permission of the institution or contributor. */
29/******************************************************************************/
30
31#include <cerrno>
32#include <cstring>
33#include <strings.h>
34#include <unistd.h>
35#include <fcntl.h>
36#include <sys/param.h>
37#include <sys/stat.h>
38#include <sys/types.h>
39
40#include "XrdFrc/XrdFrcTrace.hh"
42#include "XrdFrm/XrdFrmFiles.hh"
43#include "XrdOuc/XrdOucTList.hh"
45
46using namespace XrdFrc;
47using namespace XrdFrm;
48
49/******************************************************************************/
50/* C l a s s X r d F r m F i l e s e t */
51/******************************************************************************/
52/******************************************************************************/
53/* S t a t i c O b j e c t s */
54/******************************************************************************/
55
56XrdOucHash<char> XrdFrmFileset::BadFiles;
57
58/******************************************************************************/
59/* C o n s t r u c t o r */
60/******************************************************************************/
61
63 : Next(sP), dInfo(diP)
64{ memset(File, 0, sizeof(File));
65 if (diP) diP->ival[dRef]++;
66}
67
68/******************************************************************************/
69/* D e s t r u c t o r */
70/******************************************************************************/
71
73{
74 int i;
75
76// Delete all the table entries
77//
78 for (i = 0; i < XrdOssPath::sfxNum; i++) if(File[i]) delete File[i];
79
80// If there is a shared directory buffer, decrease reference count, delete if 0
81//
82 if (dInfo && ((dInfo->ival[dRef] -= 1) <= 0)) delete dInfo;
83}
84
85/******************************************************************************/
86/* d i r P a t h */
87/******************************************************************************/
88
89int XrdFrmFileset::dirPath(char *dBuff, int dBlen)
90{
91 char *dP = 0;
92 int dN = 0, i;
93
94// If we have a shared directory pointer, use that as directory information
95// Otherwise, get it from one of the files in the fileset.
96//
97 if (dInfo) {dP = dInfo->text; dN = dInfo->ival[dLen];}
98 else {for (i = 0; i < XrdOssPath::sfxNum; i++)
99 if (File[i])
100 {dP = File[i]->Path;
101 dN = File[i]->File - File[i]->Path;
102 break;
103 }
104 }
105
106// Copy out the directory path
107//
108 if (dBlen > dN && dP) strncpy(dBuff, dP, dN);
109 else dN = 0;
110 *(dBuff+dN) = '\0';
111 return dN;
112}
113
114/******************************************************************************/
115/* R e f r e s h */
116/******************************************************************************/
117
118int XrdFrmFileset::Refresh(int isMig, int doLock)
119{
120 class fdClose
121 {public:
122 int Num;
123 fdClose() : Num(-1) {}
124 ~fdClose() {if (Num >= 0) close(Num);}
125 } fnFD;
126 XrdOucNSWalk::NSEnt *lP, *bP = baseFile();
127 char pBuff[MAXPATHLEN+1], *fnP, *pnP = pBuff;
128 int n, lkFD = -1;
129
130// Get the directory path for this entry
131//
132 if (!(n = dirPath(pBuff, sizeof(pBuff)-1))) return 0;
133 fnP = pBuff+n;
134
135// If we need to lock the entry, do so. We also check if file is in use
136//
137 if (doLock && bP)
138 {strcpy(fnP, baseFile()->File);
139 if (!(lkFD = chkLock(pBuff))) return 0;
140 fnFD.Num = lkFD;
141 }
142
143// Do a new stat call on each relevant file (pin file excluded for isMig)
144//
145 if (bP)
146 {if (bP->Link) pnP = bP->Link;
147 else strcpy(fnP, bP->File);
148 if (stat(pnP, &(bP->Stat)))
149 {Say.Emsg("Refresh", errno, "stat", pnP); return 0;}
150 }
151
152 if (!isMig) pinInfo.Get(pnP, lkFD);
153
154 if ((lP = lockFile()))
155 {strcpy(fnP, lP->File);
156 if (stat(pBuff, &(lP->Stat)))
157 {Say.Emsg("Refresh", errno, "stat", pBuff); return 0;}
158 cpyInfo.Attr.cpyTime = static_cast<long long>(lP->Stat.st_mtime);
159 } else if (cpyInfo.Get(pnP, lkFD) <= 0) cpyInfo.Attr.cpyTime = 0;
160
161// All done
162//
163 return 1;
164}
165
166/******************************************************************************/
167/* S c r e e n */
168/******************************************************************************/
169
171{
172 const char *What = 0, *badFN = 0;
173
174// Verify that we have all the relevant files (old mode only)
175//
176 if (!Config.runNew && !baseFile())
177 {if (Config.Fix)
178 {if (lockFile()) Remfix("Lock", lockPath());
179 if ( pinFile()) Remfix("Pin", pinPath());
180 return 0;
181 }
182 if (lockFile()) badFN = lockPath();
183 else if ( pinFile()) badFN = pinPath();
184 else return 0;
185 What = "No base file for";
186 }
187
188// If no errors from above, try to get the copy time for this file
189//
190 if (!What)
191 {if (!needLF || setCpyTime()) return 1;
192 What = Config.runNew ? "no copy time xattr for" : "no lock file for";
193 badFN = basePath();
194 }
195
196// Issue message if we haven't issued one before
197//
198 if (!BadFiles.Add(badFN, 0, 0, Hash_data_is_key))
199 Say.Emsg("Screen", What, badFN);
200 return 0;
201}
202
203/******************************************************************************/
204/* s e t C p y T i m e */
205/******************************************************************************/
206
208{
210
211// In new run mode the copy time comes from the extended attributes
212//
213 if (Config.runNew) return cpyInfo.Get(basePath()) > 0;
214
215// If there is no lock file, indicate so
216//
217 if (!(lP = lockFile())) return 0;
218
219// Use the lock file as the source of information
220//
221 if (Refresh && stat(lockPath(), &(lP->Stat)))
222 {Say.Emsg("setCpyTime", errno, "stat", lockPath()); return 0;}
223 cpyInfo.Attr.cpyTime = static_cast<long long>(lP->Stat.st_mtime);
224 return 1;
225}
226
227/******************************************************************************/
228/* P r i v a t e M e t h o d s */
229/******************************************************************************/
230/******************************************************************************/
231/* c h k L o c k */
232/******************************************************************************/
233
234// Returns 0 if lock exists or an error occurred, o/w returns fd for the file.
235
236int XrdFrmFileset::chkLock(const char *Path)
237{
238 FLOCK_t lock_args;
239 int rc, lokFD;
240
241// Open the file appropriately
242//
243 if ((lokFD = open(Path, O_RDONLY)) < 0)
244 {Say.Emsg("chkLock", errno, "open", Path); return 0;}
245
246// Initialize the lock arguments
247//
248 bzero(&lock_args, sizeof(lock_args));
249 lock_args.l_type = F_WRLCK;
250
251// Now check if the lock can be obtained
252//
253 do {rc = fcntl(lokFD, F_GETLK, &lock_args);} while(rc < 0 && errno == EINTR);
254
255// Determine the result
256//
257 if (!rc) return lokFD;
258 Say.Emsg("chkLock", errno, "lock", Path);
259 close(lokFD);
260 return 0;
261}
262
263/******************************************************************************/
264/* M k f n */
265/******************************************************************************/
266
267const char *XrdFrmFileset::Mkfn(XrdOucNSWalk::NSEnt *fP)
268{
269
270// If we have no file for this, return the null string
271//
272 if (!fP) return "";
273
274// If we have no shared directory pointer, return the full path
275//
276 if (!dInfo) return fP->Path;
277
278// Construct the name in a non-renterant way (this is documented)
279//
280 strcpy(dInfo->text+dInfo->ival[dLen], fP->File);
281 return dInfo->text;
282}
283
284/******************************************************************************/
285/* R e m f i x */
286/******************************************************************************/
287
288void XrdFrmFileset::Remfix(const char *fType, const char *fPath)
289{
290
291// Remove the offending file
292//
293 if (unlink(fPath)) Say.Emsg("Remfix", errno, "remove orphan", fPath);
294 Say.Emsg("Remfix", fType, "file orphan fixed; removed", fPath);
295}
296
297/******************************************************************************/
298/* C l a s s X r d F r m F i l e s */
299/******************************************************************************/
300/******************************************************************************/
301/* C o n s t r u c t o r */
302/******************************************************************************/
303
304XrdFrmFiles::XrdFrmFiles(const char *dname, int opts,
306 : nsObj(&Say, dname, 0,
307 XrdOucNSWalk::retFile | XrdOucNSWalk::retLink
308 |XrdOucNSWalk::retStat | XrdOucNSWalk::skpErrs
309 |XrdOucNSWalk::retIILO
310 | (opts & CompressD ? XrdOucNSWalk::noPath : 0)
311 | (opts & Recursive ? XrdOucNSWalk::Recurse : 0), XList),
312 fsList(0), manMem(opts & NoAutoDel ? Hash_keep : Hash_default),
313 shareD(opts & CompressD), getCPT(opts & GetCpyTim)
314{
315
316// Set Call Back method
317//
318 nsObj.setCallBack(cbP);
319}
320
321/******************************************************************************/
322/* D e s t r u c t o r */
323/******************************************************************************/
324
326{
327 XrdFrmFileset *fsetP;
328
329// If manual memory is wante then we must delete any unreturned objects
330//
331 if (manMem)
332 while((fsetP = fsList))
333 {fsList = fsetP->Next; fsetP->Next = 0; delete fsetP;}
334}
335
336/******************************************************************************/
337/* G e t */
338/******************************************************************************/
339
340XrdFrmFileset *XrdFrmFiles::Get(int &rc, int noBase)
341{
343 XrdFrmFileset *fsetP;
344 const char *dPath;
345
346// Check if we have something to return
347//
348do{while ((fsetP = fsList))
349 {fsList = fsetP->Next; fsetP->Next = 0;
350 if (fsetP->File[XrdOssPath::isBase])
351 {if (getCPT) fsetP->setCpyTime();
352 rc = 0; return fsetP;
353 }
354 else if (noBase) {rc = 0; return fsetP;}
355 else if (manMem) delete fsetP;
356 }
357
358// Start with next directory (we return when no directories left).
359//
360 do {if (!(nP = nsObj.Index(rc, &dPath))) return 0;
361 fsTab.Purge(); fsList = 0;
362 } while(!Process(nP, dPath));
363
364 } while(1);
365
366// To keep the compiler happy
367//
368 return 0;
369}
370
371/******************************************************************************/
372/* P r i v a t e M e t h o d s */
373/******************************************************************************/
374/******************************************************************************/
375/* C o m p l a i n */
376/******************************************************************************/
377
378void XrdFrmFiles::Complain(const char *dPath)
379{
380 static const int OneDay = 24*60*60;
381 static XrdOucHash<char> dTab;
382
383// We want to complain about old=style directories only once every 24 hours
384//
385 if (dTab.Add(dPath, 0, OneDay, Hash_data_is_key)) return;
386
387// Complain about this directory
388//
389 Say.Emsg("Complain","Found old-style files in directory", dPath);
390 Say.Emsg("Complain","In new run mode, migrate & purge will skip them.");
391}
392
393/******************************************************************************/
394/* o l d F i l e */
395/******************************************************************************/
396
397int XrdFrmFiles::oldFile(XrdOucNSWalk::NSEnt *fP, XrdOucTList *dP, int fType)
398{
399 char pBuff[MAXPATHLEN+8], *pnP = pBuff, *fnP;
400
401// Ignore (for now): '.anew', '.fail', or '.pfn'
402//
403 if (fType == XrdOssPath::isAnew
404 || fType == XrdOssPath::isFail
405 || fType == XrdOssPath::isPfn) return 0;
406
407// If this is not a directory lock file, indicate we should complain
408//
409 if (fType >= 0) return 1;
410
411// This is a directory lock file, quietly remove it (we no longer use them)
412//
413 if (!dP) pnP = fP->Path;
414 else {strcpy(pBuff, dP->text);
415 fnP = pBuff + dP->ival[XrdFrmFileset::dLen];
416 *fnP++ = '/';
417 strcpy(fnP, fP->File);
418 }
419 unlink(pnP);
420 return 0;
421}
422
423/******************************************************************************/
424/* P r o c e s s */
425/******************************************************************************/
426
427int XrdFrmFiles::Process(XrdOucNSWalk::NSEnt *nP, const char *dPath)
428{
429 XrdOucNSWalk::NSEnt *fP;
430 XrdFrmFileset *sP;
431 XrdOucTList *dP = 0;
432 char *dotP;
433 int fType, noDLKF = 1, runOldFault = 0;
434
435// If compressed directories wanted, then setup a shared directory buffer
436// Warning! We use a hard-coded value for maximum filename length instead of
437// constantly calling pathconf().
438//
439 if (shareD)
440 {int n = strlen(dPath);
441 char *dBuff = (char *)malloc(n+264);
442 strcpy(dBuff, dPath);
443 dP = new XrdOucTList;
444 dP->text = dBuff;
445 dP->ival[XrdFrmFileset::dLen] = n;
446 dP->ival[XrdFrmFileset::dRef] = 0;
447 }
448
449// Process the file list
450//
451 while((fP = nP))
452 {nP = fP->Next; fP->Next = 0;
453 if (noDLKF && !strcmp(fP->File, Config.lockFN))
454 {oldFile(fP, dP, -1); delete fP; noDLKF = 0; continue;}
455 if (!(fType = (int)XrdOssPath::pathType(fP->File))
456 || !(dotP = rindex(fP->File, '.'))) dotP = 0;
457 else {if (Config.runNew)
458 {runOldFault |= oldFile(fP, dP, fType);
459 delete fP; continue;
460 }
461 *dotP = '\0';
462 }
463 if (!(sP = fsTab.Find(fP->File)))
464 {sP = fsList = new XrdFrmFileset(fsList, dP);
465 fsTab.Add(fP->File, sP, 0, manMem);
466 }
467 if (dotP) *dotP = '.';
468 sP->File[fType] = fP;
469 }
470
471// If we found on old-style file while in runNew, complain
472//
473 if (runOldFault) Complain(dPath);
474
475// Indicate whether we have anything here
476//
477 if (fsList) return 1;
478 if (dP) delete dP;
479 return 0;
480}
XrdOucPup XrdCmsParser::Pup & Say
@ Hash_data_is_key
Definition XrdOucHash.hh:52
@ Hash_default
Definition XrdOucHash.hh:51
@ Hash_keep
Definition XrdOucHash.hh:55
int fcntl(int fd, int cmd,...)
#define close(a)
Definition XrdPosix.hh:43
#define open
Definition XrdPosix.hh:71
#define unlink(a)
Definition XrdPosix.hh:108
#define stat(a, b)
Definition XrdPosix.hh:96
XrdOucString Path
struct myOpts opts
#define FLOCK_t
const char * lockFN
static const int CompressD
static const int NoAutoDel
XrdFrmFiles(const char *dname, int opts=Recursive, XrdOucTList *XList=0, XrdOucNSWalk::CallBack *cbP=0)
static const int GetCpyTim
XrdFrmFileset * Get(int &rc, int noBase=0)
static const int Recursive
const char * pinPath()
int setCpyTime(int Refresh=0)
int Refresh(int isMig=0, int doLock=1)
const char * basePath()
XrdOucNSWalk::NSEnt * baseFile()
const char * lockPath()
XrdOucXAttr< XrdFrcXAttrCpy > cpyInfo
XrdOucNSWalk::NSEnt * lockFile()
int Screen(int needLF=1)
int dirPath(char *dBuff, int dBlen)
XrdOucXAttr< XrdFrcXAttrPin > pinInfo
XrdOucNSWalk::NSEnt * pinFile()
XrdFrmFileset * Next
XrdFrmFileset(XrdFrmFileset *sP=0, XrdOucTList *diP=0)
static theSfx pathType(const char *Path, int chkWhat=chkAll)
T * Add(const char *KeyVal, T *KeyData, const int LifeTime=0, XrdOucHash_Options opt=Hash_default)
XrdFrmConfig Config
struct NSEnt * Next