GDAL
cpl_vsil_curl_class.h
1 /******************************************************************************
2  *
3  * Project: CPL - Common Portability Library
4  * Purpose: Declarations for /vsicurl/ and related file systems
5  * Author: Even Rouault, even.rouault at spatialys.com
6  *
7  ******************************************************************************
8  * Copyright (c) 2010-2018, Even Rouault <even.rouault at spatialys.com>
9  *
10  * Permission is hereby granted, free of charge, to any person obtaining a
11  * copy of this software and associated documentation files (the "Software"),
12  * to deal in the Software without restriction, including without limitation
13  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
14  * and/or sell copies of the Software, and to permit persons to whom the
15  * Software is furnished to do so, subject to the following conditions:
16  *
17  * The above copyright notice and this permission notice shall be included
18  * in all copies or substantial portions of the Software.
19  *
20  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
21  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
23  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
24  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
25  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
26  * DEALINGS IN THE SOFTWARE.
27  ****************************************************************************/
28 
29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
31 
32 #ifdef HAVE_CURL
33 
34 #include "cpl_aws.h"
35 #include "cpl_azure.h"
36 #include "cpl_port.h"
37 #include "cpl_json.h"
38 #include "cpl_string.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
41 
42 #include "cpl_curl_priv.h"
43 
44 #include <set>
45 #include <map>
46 #include <memory>
47 #include <mutex>
48 
50 
51 // Leave it for backward compatibility, but deprecate.
52 #define HAVE_CURLINFO_REDIRECT_URL
53 
54 void VSICurlStreamingClearCache( void ); // from cpl_vsil_curl_streaming.cpp
55 
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle, const char* pszURL,
57  const char * const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders( struct curl_slist* poDest,
59  struct curl_slist* poSrcToDestroy );
60 
61 struct curl_slist* VSICurlSetContentTypeFromExt(struct curl_slist* polist,
62  const char *pszPath);
63 
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(struct curl_slist* headers,
65  CSLConstList papszOptions,
66  const char *pszPath);
67 
68 namespace cpl {
69 
70 typedef enum
71 {
72  EXIST_UNKNOWN = -1,
73  EXIST_NO,
74  EXIST_YES,
75 } ExistStatus;
76 
77 class FileProp
78 {
79  public:
80  unsigned int nGenerationAuthParameters = 0;
81  ExistStatus eExists = EXIST_UNKNOWN;
82  vsi_l_offset fileSize = 0;
83  time_t mTime = 0;
84  time_t nExpireTimestampLocal = 0;
85  CPLString osRedirectURL{};
86  bool bHasComputedFileSize = false;
87  bool bIsDirectory = false;
88  int nMode = 0; // st_mode member of struct stat
89  bool bS3LikeRedirect = false;
90  CPLString ETag{};
91 };
92 
93 struct CachedDirList
94 {
95  bool bGotFileList = false;
96  unsigned int nGenerationAuthParameters = 0;
97  CPLStringList oFileList{}; /* only file name without path */
98 };
99 
100 struct WriteFuncStruct
101 {
102  char* pBuffer = nullptr;
103  size_t nSize = 0;
104  bool bIsHTTP = false;
105  bool bIsInHeader = false;
106  bool bMultiRange = false;
107  vsi_l_offset nStartOffset = 0;
108  vsi_l_offset nEndOffset = 0;
109  int nHTTPCode = 0;
110  vsi_l_offset nContentLength = 0;
111  bool bFoundContentRange = false;
112  bool bError = false;
113  bool bDownloadHeaderOnly = false;
114  bool bDetectRangeDownloadingError = false;
115  GIntBig nTimestampDate = 0; // Corresponds to Date: header field
116 
117  VSILFILE *fp = nullptr;
118  VSICurlReadCbkFunc pfnReadCbk = nullptr;
119  void *pReadCbkUserData = nullptr;
120  bool bInterrupted = false;
121 
122 #if !CURL_AT_LEAST_VERSION(7,54,0)
123  // Workaround to ignore extra HTTP response headers from
124  // proxies in older versions of curl.
125  // CURLOPT_SUPPRESS_CONNECT_HEADERS fixes this
126  bool bIsProxyConnectHeader = false;
127 #endif
128 };
129 
130 struct PutData
131 {
132  const GByte* pabyData = nullptr;
133  size_t nOff = 0;
134  size_t nTotalSize = 0;
135 
136  static size_t ReadCallBackBuffer( char *buffer, size_t size,
137  size_t nitems, void *instream )
138  {
139  PutData* poThis = static_cast<PutData *>(instream);
140  const size_t nSizeMax = size * nitems;
141  const size_t nSizeToWrite =
142  std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
143  memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
144  poThis->nOff += nSizeToWrite;
145  return nSizeToWrite;
146  }
147 };
148 
149 /************************************************************************/
150 /* VSICurlFilesystemHandler */
151 /************************************************************************/
152 
153 class VSICurlHandle;
154 
155 class VSICurlFilesystemHandler : public VSIFilesystemHandler
156 {
157  CPL_DISALLOW_COPY_ASSIGN(VSICurlFilesystemHandler)
158 
159  struct FilenameOffsetPair
160  {
161  std::string filename_;
162  vsi_l_offset offset_;
163 
164  FilenameOffsetPair(const std::string& filename,
165  vsi_l_offset offset) :
166  filename_(filename), offset_(offset) {}
167 
168  bool operator==(const FilenameOffsetPair& other) const
169  {
170  return filename_ == other.filename_ &&
171  offset_ == other.offset_;
172  }
173  };
174  struct FilenameOffsetPairHasher
175  {
176  std::size_t operator()(const FilenameOffsetPair& k) const
177  {
178  return std::hash<std::string>()(k.filename_) ^
179  std::hash<vsi_l_offset>()(k.offset_);
180  }
181  };
182 
183  using RegionCacheType =
184  lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
185  lru11::NullLock,
186  std::unordered_map<
187  FilenameOffsetPair,
188  typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
189  std::shared_ptr<std::string>>>::iterator,
190  FilenameOffsetPairHasher>>;
191 
192  std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{}; // do not access directly. Use GetRegionCache();
193  RegionCacheType* GetRegionCache();
194 
195  lru11::Cache<std::string, FileProp> oCacheFileProp;
196 
197  int nCachedFilesInDirList = 0;
198  lru11::Cache<std::string, CachedDirList> oCacheDirList;
199 
200  char** ParseHTMLFileList(const char* pszFilename,
201  int nMaxFiles,
202  char* pszData,
203  bool* pbGotFileList);
204 
205 protected:
206  CPLMutex *hMutex = nullptr;
207 
208  virtual VSICurlHandle* CreateFileHandle(const char* pszFilename);
209  virtual char** GetFileList(const char *pszFilename,
210  int nMaxFiles,
211  bool* pbGotFileList);
212 
213  void RegisterEmptyDir( const CPLString& osDirname );
214 
215  bool AnalyseS3FileList( const CPLString& osBaseURL,
216  const char* pszXML,
217  CPLStringList& osFileList,
218  int nMaxFiles,
219  bool bIgnoreGlacierStorageClass,
220  bool& bIsTruncated );
221 
222  void AnalyseSwiftFileList( const CPLString& osBaseURL,
223  const CPLString& osPrefix,
224  const char* pszJson,
225  CPLStringList& osFileList,
226  int nMaxFilesThisQuery,
227  int nMaxFiles,
228  bool& bIsTruncated,
229  CPLString& osNextMarker );
230 
231  static const char* GetOptionsStatic();
232 
233  static bool IsAllowedFilename( const char* pszFilename );
234 
235 public:
236  VSICurlFilesystemHandler();
237  ~VSICurlFilesystemHandler() override;
238 
239  VSIVirtualHandle *Open( const char *pszFilename,
240  const char *pszAccess,
241  bool bSetError,
242  CSLConstList /* papszOptions */ ) override;
243 
244  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
245  int nFlags ) override;
246  int Unlink( const char *pszFilename ) override;
247  int Rename( const char *oldpath, const char *newpath ) override;
248  int Mkdir( const char *pszDirname, long nMode ) override;
249  int Rmdir( const char *pszDirname ) override;
250  char **ReadDir( const char *pszDirname ) override
251  { return ReadDirEx(pszDirname, 0); }
252  char **ReadDirEx( const char *pszDirname, int nMaxFiles ) override;
253  char **SiblingFiles( const char *pszFilename ) override;
254 
255  int HasOptimizedReadMultiRange( const char* /* pszPath */ )
256  override { return true; }
257 
258  const char* GetActualURL(const char* pszFilename) override;
259 
260  const char* GetOptions() override;
261 
262  char** GetFileMetadata( const char * pszFilename, const char* pszDomain,
263  CSLConstList papszOptions ) override;
264 
265  char **ReadDirInternal( const char *pszDirname, int nMaxFiles,
266  bool* pbGotFileList );
267  void InvalidateDirContent( const char *pszDirname );
268 
269  virtual const char* GetDebugKey() const { return "VSICURL"; }
270 
271  virtual CPLString GetFSPrefix() const { return "/vsicurl/"; }
272  virtual bool AllowCachedDataFor(const char* pszFilename);
273 
274  std::shared_ptr<std::string> GetRegion( const char* pszURL,
275  vsi_l_offset nFileOffsetStart );
276 
277  void AddRegion( const char* pszURL,
278  vsi_l_offset nFileOffsetStart,
279  size_t nSize,
280  const char *pData );
281 
282  bool GetCachedFileProp( const char* pszURL,
283  FileProp& oFileProp );
284  void SetCachedFileProp( const char* pszURL,
285  FileProp& oFileProp );
286  void InvalidateCachedData( const char* pszURL );
287 
288  CURLM *GetCurlMultiHandleFor( const CPLString& osURL );
289 
290  virtual void ClearCache();
291  virtual void PartialClearCache(const char* pszFilename);
292 
293 
294  bool GetCachedDirList( const char* pszURL,
295  CachedDirList& oCachedDirList );
296  void SetCachedDirList( const char* pszURL,
297  CachedDirList& oCachedDirList );
298  bool ExistsInCacheDirList( const CPLString& osDirname, bool *pbIsDir );
299 
300  virtual CPLString GetURLFromFilename( const CPLString& osFilename );
301 };
302 
303 /************************************************************************/
304 /* VSICurlHandle */
305 /************************************************************************/
306 
307 class VSICurlHandle : public VSIVirtualHandle
308 {
309  CPL_DISALLOW_COPY_ASSIGN(VSICurlHandle)
310 
311  protected:
312  VSICurlFilesystemHandler* poFS = nullptr;
313 
314  bool m_bCached = true;
315 
316  FileProp oFileProp{};
317 
318  CPLString m_osFilename{}; // e.g "/vsicurl/http://example.com/foo"
319  char* m_pszURL = nullptr; // e.g "http://example.com/foo"
320  std::string m_osQueryString{}; // e.g. an Azure SAS
321 
322  char **m_papszHTTPOptions = nullptr;
323 
324  vsi_l_offset lastDownloadedOffset = VSI_L_OFFSET_MAX;
325  int nBlocksToDownload = 1;
326 
327  bool bStopOnInterruptUntilUninstall = false;
328  bool bInterrupted = false;
329  VSICurlReadCbkFunc pfnReadCbk = nullptr;
330  void *pReadCbkUserData = nullptr;
331 
332  int m_nMaxRetry = 0;
333  double m_dfRetryDelay = 0.0;
334 
335  CPLStringList m_aosHeaders{};
336 
337  void DownloadRegionPostProcess( const vsi_l_offset startOffset,
338  const int nBlocks,
339  const char* pBuffer,
340  size_t nSize );
341 
342  private:
343 
344  vsi_l_offset curOffset = 0;
345 
346  bool bEOF = false;
347 
348  virtual std::string DownloadRegion(vsi_l_offset startOffset, int nBlocks);
349 
350  bool m_bUseHead = false;
351  bool m_bUseRedirectURLIfNoQueryStringParams = false;
352 
353  int ReadMultiRangeSingleGet( int nRanges, void ** ppData,
354  const vsi_l_offset* panOffsets,
355  const size_t* panSizes );
356  CPLString GetRedirectURLIfValid(bool& bHasExpired);
357 
358  protected:
359  virtual struct curl_slist* GetCurlHeaders( const CPLString& /*osVerb*/,
360  const struct curl_slist* /* psExistingHeaders */)
361  { return nullptr; }
362  virtual bool AllowAutomaticRedirection() { return true; }
363  virtual bool CanRestartOnError( const char*, const char*, bool ) { return false; }
364  virtual bool UseLimitRangeGetInsteadOfHead() { return false; }
365  virtual bool IsDirectoryFromExists( const char* /*pszVerb*/, int /*response_code*/ ) { return false; }
366  virtual void ProcessGetFileSizeResult(const char* /* pszContent */ ) {}
367  void SetURL(const char* pszURL);
368  virtual bool Authenticate() { return false; }
369 
370  public:
371 
372  VSICurlHandle( VSICurlFilesystemHandler* poFS,
373  const char* pszFilename,
374  const char* pszURLIn = nullptr );
375  ~VSICurlHandle() override;
376 
377  int Seek( vsi_l_offset nOffset, int nWhence ) override;
378  vsi_l_offset Tell() override;
379  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
380  int ReadMultiRange( int nRanges, void ** ppData,
381  const vsi_l_offset* panOffsets,
382  const size_t* panSizes ) override;
383  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
384  int Eof() override;
385  int Flush() override;
386  int Close() override;
387 
388  bool IsKnownFileSize() const { return oFileProp.bHasComputedFileSize; }
389  vsi_l_offset GetFileSizeOrHeaders(bool bSetError, bool bGetHeaders);
390  virtual vsi_l_offset GetFileSize( bool bSetError ) { return GetFileSizeOrHeaders(bSetError, false); }
391  bool Exists( bool bSetError );
392  bool IsDirectory() const { return oFileProp.bIsDirectory; }
393  int GetMode() const { return oFileProp.nMode; }
394  time_t GetMTime() const { return oFileProp.mTime; }
395  const CPLStringList& GetHeaders() { return m_aosHeaders; }
396 
397  int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
398  void* pfnUserData,
399  int bStopOnInterruptUntilUninstall );
400  int UninstallReadCbk();
401 
402  const char *GetURL() const { return m_pszURL; }
403 };
404 
405 /************************************************************************/
406 /* IVSIS3LikeFSHandler */
407 /************************************************************************/
408 
409 class IVSIS3LikeFSHandler: public VSICurlFilesystemHandler
410 {
411  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeFSHandler)
412 
413  bool CopyFile(VSILFILE* fpIn,
414  vsi_l_offset nSourceSize,
415  const char* pszSource,
416  const char* pszTarget,
417  GDALProgressFunc pProgressFunc,
418  void *pProgressData);
419  virtual int MkdirInternal( const char *pszDirname, long nMode, bool bDoStatCheck );
420 
421  protected:
422  char** GetFileList( const char *pszFilename,
423  int nMaxFiles,
424  bool* pbGotFileList ) override;
425 
426  virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
427  const char* pszURI, bool bAllowNoObject) = 0;
428 
429  virtual int CopyObject( const char *oldpath, const char *newpath,
430  CSLConstList papszMetadata );
431 
432  IVSIS3LikeFSHandler() = default;
433 
434  public:
435  int Unlink( const char *pszFilename ) override;
436  int Mkdir( const char *pszDirname, long nMode ) override;
437  int Rmdir( const char *pszDirname ) override;
438  int Stat( const char *pszFilename, VSIStatBufL *pStatBuf,
439  int nFlags ) override;
440  int Rename( const char *oldpath, const char *newpath ) override;
441 
442  virtual int DeleteObject( const char *pszFilename );
443 
444  virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
445  virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
446 
447  virtual CPLString GetStreamingPath( const char* pszFilename ) const;
448 
449  bool Sync( const char* pszSource, const char* pszTarget,
450  const char* const * papszOptions,
451  GDALProgressFunc pProgressFunc,
452  void *pProgressData,
453  char*** ppapszOutputs ) override;
454 
455  VSIDIR* OpenDir( const char *pszPath, int nRecurseDepth,
456  const char* const *papszOptions) override;
457 
458  // Multipart upload
459  virtual bool SupportsParallelMultipartUpload() const { return false; }
460 
461  virtual CPLString InitiateMultipartUpload(
462  const std::string& osFilename,
463  IVSIS3LikeHandleHelper *poS3HandleHelper,
464  int nMaxRetry,
465  double dfRetryDelay,
466  CSLConstList papszOptions);
467  virtual CPLString UploadPart(const CPLString& osFilename,
468  int nPartNumber,
469  const std::string& osUploadID,
470  vsi_l_offset nPosition,
471  const void* pabyBuffer,
472  size_t nBufferSize,
473  IVSIS3LikeHandleHelper *poS3HandleHelper,
474  int nMaxRetry,
475  double dfRetryDelay);
476  virtual bool CompleteMultipart(const CPLString& osFilename,
477  const CPLString& osUploadID,
478  const std::vector<CPLString>& aosEtags,
479  vsi_l_offset nTotalSize,
480  IVSIS3LikeHandleHelper *poS3HandleHelper,
481  int nMaxRetry,
482  double dfRetryDelay);
483  virtual bool AbortMultipart(const CPLString& osFilename,
484  const CPLString& osUploadID,
485  IVSIS3LikeHandleHelper *poS3HandleHelper,
486  int nMaxRetry,
487  double dfRetryDelay);
488 };
489 
490 /************************************************************************/
491 /* IVSIS3LikeHandle */
492 /************************************************************************/
493 
494 class IVSIS3LikeHandle: public VSICurlHandle
495 {
496  CPL_DISALLOW_COPY_ASSIGN(IVSIS3LikeHandle)
497 
498  protected:
499  bool UseLimitRangeGetInsteadOfHead() override { return true; }
500  bool IsDirectoryFromExists( const char* pszVerb,
501  int response_code ) override
502  {
503  // A bit dirty, but on S3, a GET on a existing directory returns a 416
504  return response_code == 416 && EQUAL(pszVerb, "GET") &&
505  CPLString(m_pszURL).back() == '/';
506  }
507  void ProcessGetFileSizeResult( const char* pszContent ) override
508  {
509  oFileProp.bIsDirectory = strstr(pszContent, "ListBucketResult") != nullptr;
510  }
511 
512  public:
513  IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
514  const char* pszFilename,
515  const char* pszURLIn ) :
516  VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
517  ~IVSIS3LikeHandle() override {}
518 };
519 
520 /************************************************************************/
521 /* VSIS3WriteHandle */
522 /************************************************************************/
523 
524 class VSIS3WriteHandle final : public VSIVirtualHandle
525 {
526  CPL_DISALLOW_COPY_ASSIGN(VSIS3WriteHandle)
527 
528  IVSIS3LikeFSHandler *m_poFS = nullptr;
529  CPLString m_osFilename{};
530  IVSIS3LikeHandleHelper *m_poS3HandleHelper = nullptr;
531  bool m_bUseChunked = false;
532  CPLStringList m_aosOptions{};
533 
534  vsi_l_offset m_nCurOffset = 0;
535  int m_nBufferOff = 0;
536  int m_nBufferSize = 0;
537  bool m_bClosed = false;
538  GByte *m_pabyBuffer = nullptr;
539  CPLString m_osUploadID{};
540  int m_nPartNumber = 0;
541  std::vector<CPLString> m_aosEtags{};
542  bool m_bError = false;
543 
544  CURLM *m_hCurlMulti = nullptr;
545  CURL *m_hCurl = nullptr;
546  const void *m_pBuffer = nullptr;
547  CPLString m_osCurlErrBuf{};
548  size_t m_nChunkedBufferOff = 0;
549  size_t m_nChunkedBufferSize = 0;
550  size_t m_nWrittenInPUT = 0;
551 
552  int m_nMaxRetry = 0;
553  double m_dfRetryDelay = 0.0;
554  WriteFuncStruct m_sWriteFuncHeaderData{};
555 
556  bool UploadPart();
557  bool DoSinglePartPUT();
558 
559  static size_t ReadCallBackBufferChunked( char *buffer, size_t size,
560  size_t nitems, void *instream );
561  size_t WriteChunked( const void *pBuffer,
562  size_t nSize, size_t nMemb );
563  int FinishChunkedTransfer();
564 
565  void InvalidateParentDirectory();
566 
567  public:
568  VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
569  const char* pszFilename,
570  IVSIS3LikeHandleHelper* poS3HandleHelper,
571  bool bUseChunked,
572  CSLConstList papszOptions );
573  ~VSIS3WriteHandle() override;
574 
575  int Seek( vsi_l_offset nOffset, int nWhence ) override;
576  vsi_l_offset Tell() override;
577  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
578  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
579  int Eof() override;
580  int Close() override;
581 
582  bool IsOK() { return m_bUseChunked || m_pabyBuffer != nullptr; }
583 };
584 
585 /************************************************************************/
586 /* VSIAppendWriteHandle */
587 /************************************************************************/
588 
589 class VSIAppendWriteHandle : public VSIVirtualHandle
590 {
591  CPL_DISALLOW_COPY_ASSIGN(VSIAppendWriteHandle)
592 
593  protected:
594 
595  VSICurlFilesystemHandler* m_poFS = nullptr;
596  CPLString m_osFSPrefix{};
597  CPLString m_osFilename{};
598 
599  vsi_l_offset m_nCurOffset = 0;
600  int m_nBufferOff = 0;
601  int m_nBufferSize = 0;
602  int m_nBufferOffReadCallback = 0;
603  bool m_bClosed = false;
604  GByte *m_pabyBuffer = nullptr;
605  bool m_bError = false;
606 
607  static size_t ReadCallBackBuffer( char *buffer, size_t size,
608  size_t nitems, void *instream );
609  virtual bool Send(bool bIsLastBlock) = 0;
610 
611  public:
612  VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
613  const char* pszFSPrefix,
614  const char* pszFilename,
615  int nChunkSize );
616  virtual ~VSIAppendWriteHandle();
617 
618  int Seek( vsi_l_offset nOffset, int nWhence ) override;
619  vsi_l_offset Tell() override;
620  size_t Read( void *pBuffer, size_t nSize, size_t nMemb ) override;
621  size_t Write( const void *pBuffer, size_t nSize, size_t nMemb ) override;
622  int Eof() override;
623  int Close() override;
624 
625  bool IsOK() { return m_pabyBuffer != nullptr; }
626 };
627 
628 /************************************************************************/
629 /* CurlRequestHelper */
630 /************************************************************************/
631 
632 struct CurlRequestHelper
633 {
634  WriteFuncStruct sWriteFuncData{};
635  WriteFuncStruct sWriteFuncHeaderData{};
636  char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
637 
638  CurlRequestHelper();
639  ~CurlRequestHelper();
640  long perform(CURL* hCurlHandle,
641  struct curl_slist* headers, // ownership transferred
642  VSICurlFilesystemHandler *poFS,
643  IVSIS3LikeHandleHelper *poS3HandleHelper);
644 };
645 
646 /************************************************************************/
647 /* NetworkStatisticsLogger */
648 /************************************************************************/
649 
650 class NetworkStatisticsLogger
651 {
652  static int gnEnabled;
653  static NetworkStatisticsLogger gInstance;
654 
655  NetworkStatisticsLogger() = default;
656 
657  std::mutex m_mutex{};
658 
659  struct Counters
660  {
661  GIntBig nHEAD = 0;
662  GIntBig nGET = 0;
663  GIntBig nPUT = 0;
664  GIntBig nPOST = 0;
665  GIntBig nDELETE = 0;
666  GIntBig nGETDownloadedBytes = 0;
667  GIntBig nPUTUploadedBytes = 0;
668  GIntBig nPOSTDownloadedBytes = 0;
669  GIntBig nPOSTUploadedBytes = 0;
670  };
671 
672  enum class ContextPathType
673  {
674  FILESYSTEM,
675  FILE,
676  ACTION,
677  };
678 
679  struct ContextPathItem
680  {
681  ContextPathType eType;
682  CPLString osName;
683 
684  ContextPathItem(ContextPathType eTypeIn, const CPLString& osNameIn):
685  eType(eTypeIn), osName(osNameIn) {}
686 
687  bool operator< (const ContextPathItem& other ) const
688  {
689  if( static_cast<int>(eType) < static_cast<int>(other.eType) )
690  return true;
691  if( static_cast<int>(eType) > static_cast<int>(other.eType) )
692  return false;
693  return osName < other.osName;
694  }
695  };
696 
697  struct Stats
698  {
699  Counters counters{};
700  std::map<ContextPathItem, Stats> children{};
701 
702  void AsJSON(CPLJSONObject& oJSON) const;
703  };
704 
705  Stats m_stats{};
706  std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
707 
708  static void ReadEnabled();
709 
710  std::vector<Counters*> GetCountersForContext();
711 
712 public:
713 
714  static inline bool IsEnabled()
715  {
716  if( gnEnabled < 0)
717  {
718  ReadEnabled();
719  }
720  return gnEnabled == TRUE;
721  }
722 
723  static void EnterFileSystem(const char* pszName);
724 
725  static void LeaveFileSystem();
726 
727  static void EnterFile(const char* pszName);
728 
729  static void LeaveFile();
730 
731  static void EnterAction(const char* pszName);
732 
733  static void LeaveAction();
734 
735  static void LogHEAD();
736 
737  static void LogGET(size_t nDownloadedBytes);
738 
739  static void LogPUT(size_t nUploadedBytes);
740 
741  static void LogPOST(size_t nUploadedBytes,
742  size_t nDownloadedBytes);
743 
744  static void LogDELETE();
745 
746  static void Reset();
747 
748  static CPLString GetReportAsSerializedJSON();
749 };
750 
751 struct NetworkStatisticsFileSystem
752 {
753  inline explicit NetworkStatisticsFileSystem(const char* pszName) {
754  NetworkStatisticsLogger::EnterFileSystem(pszName);
755  }
756 
757  inline ~NetworkStatisticsFileSystem()
758  {
759  NetworkStatisticsLogger::LeaveFileSystem();
760  }
761 };
762 
763 struct NetworkStatisticsFile
764 {
765  inline explicit NetworkStatisticsFile(const char* pszName) {
766  NetworkStatisticsLogger::EnterFile(pszName);
767  }
768 
769  inline ~NetworkStatisticsFile()
770  {
771  NetworkStatisticsLogger::LeaveFile();
772  }
773 };
774 
775 struct NetworkStatisticsAction
776 {
777  inline explicit NetworkStatisticsAction(const char* pszName) {
778  NetworkStatisticsLogger::EnterAction(pszName);
779  }
780 
781  inline ~NetworkStatisticsAction()
782  {
783  NetworkStatisticsLogger::LeaveAction();
784  }
785 };
786 
787 
788 int VSICURLGetDownloadChunkSize();
789 
790 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
791  VSILFILE *fp,
792  VSICurlReadCbkFunc pfnReadCbk,
793  void *pReadCbkUserData );
794 size_t VSICurlHandleWriteFunc( void *buffer, size_t count,
795  size_t nmemb, void *req );
796 void MultiPerform(CURLM* hCurlMultiHandle,
797  CURL* hEasyHandle = nullptr);
798 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
799 
800 int VSICurlParseUnixPermissions(const char* pszPermissions);
801 
802 } // namespace cpl
803 
805 
806 #endif // HAVE_CURL
807 
808 #endif // CPL_VSIL_CURL_CLASS_H_INCLUDED
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:319
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140