29 #ifndef CPL_VSIL_CURL_CLASS_H_INCLUDED
30 #define CPL_VSIL_CURL_CLASS_H_INCLUDED
35 #include "cpl_azure.h"
39 #include "cpl_vsil_curl_priv.h"
40 #include "cpl_mem_cache.h"
42 #include "cpl_curl_priv.h"
52 #define HAVE_CURLINFO_REDIRECT_URL
54 void VSICurlStreamingClearCache(
void );
56 struct curl_slist* VSICurlSetOptions(CURL* hCurlHandle,
const char* pszURL,
57 const char *
const* papszOptions);
58 struct curl_slist* VSICurlMergeHeaders(
struct curl_slist* poDest,
59 struct curl_slist* poSrcToDestroy );
61 struct curl_slist* VSICurlSetContentTypeFromExt(
struct curl_slist* polist,
64 struct curl_slist* VSICurlSetCreationHeadersFromOptions(
struct curl_slist* headers,
80 unsigned int nGenerationAuthParameters = 0;
81 ExistStatus eExists = EXIST_UNKNOWN;
84 time_t nExpireTimestampLocal = 0;
86 bool bHasComputedFileSize =
false;
87 bool bIsDirectory =
false;
89 bool bS3LikeRedirect =
false;
95 bool bGotFileList =
false;
96 unsigned int nGenerationAuthParameters = 0;
100 struct WriteFuncStruct
102 char* pBuffer =
nullptr;
104 bool bIsHTTP =
false;
105 bool bIsInHeader =
false;
106 bool bMultiRange =
false;
111 bool bFoundContentRange =
false;
113 bool bDownloadHeaderOnly =
false;
114 bool bDetectRangeDownloadingError =
false;
118 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
119 void *pReadCbkUserData =
nullptr;
120 bool bInterrupted =
false;
122 #if !CURL_AT_LEAST_VERSION(7,54,0)
126 bool bIsProxyConnectHeader =
false;
132 const GByte* pabyData =
nullptr;
134 size_t nTotalSize = 0;
136 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
137 size_t nitems,
void *instream )
139 PutData* poThis =
static_cast<PutData *
>(instream);
140 const size_t nSizeMax = size * nitems;
141 const size_t nSizeToWrite =
142 std::min(nSizeMax, poThis->nTotalSize - poThis->nOff);
143 memcpy(buffer, poThis->pabyData + poThis->nOff, nSizeToWrite);
144 poThis->nOff += nSizeToWrite;
155 class VSICurlFilesystemHandler :
public VSIFilesystemHandler
159 struct FilenameOffsetPair
161 std::string filename_;
164 FilenameOffsetPair(
const std::string& filename,
166 filename_(filename), offset_(offset) {}
168 bool operator==(
const FilenameOffsetPair& other)
const
170 return filename_ == other.filename_ &&
171 offset_ == other.offset_;
174 struct FilenameOffsetPairHasher
176 std::size_t operator()(
const FilenameOffsetPair& k)
const
178 return std::hash<std::string>()(k.filename_) ^
179 std::hash<vsi_l_offset>()(k.offset_);
183 using RegionCacheType =
184 lru11::Cache<FilenameOffsetPair, std::shared_ptr<std::string>,
188 typename std::list<lru11::KeyValuePair<FilenameOffsetPair,
189 std::shared_ptr<std::string>>>::iterator,
190 FilenameOffsetPairHasher>>;
192 std::unique_ptr<RegionCacheType> m_poRegionCacheDoNotUseDirectly{};
193 RegionCacheType* GetRegionCache();
195 lru11::Cache<std::string, FileProp> oCacheFileProp;
197 int nCachedFilesInDirList = 0;
198 lru11::Cache<std::string, CachedDirList> oCacheDirList;
200 char** ParseHTMLFileList(
const char* pszFilename,
203 bool* pbGotFileList);
206 CPLMutex *hMutex =
nullptr;
208 virtual VSICurlHandle* CreateFileHandle(
const char* pszFilename);
209 virtual char** GetFileList(
const char *pszFilename,
211 bool* pbGotFileList);
213 void RegisterEmptyDir(
const CPLString& osDirname );
215 bool AnalyseS3FileList(
const CPLString& osBaseURL,
219 bool bIgnoreGlacierStorageClass,
220 bool& bIsTruncated );
222 void AnalyseSwiftFileList(
const CPLString& osBaseURL,
226 int nMaxFilesThisQuery,
231 static const char* GetOptionsStatic();
233 static bool IsAllowedFilename(
const char* pszFilename );
236 VSICurlFilesystemHandler();
237 ~VSICurlFilesystemHandler()
override;
240 const char *pszAccess,
244 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
245 int nFlags )
override;
246 int Unlink(
const char *pszFilename )
override;
247 int Rename(
const char *oldpath,
const char *newpath )
override;
248 int Mkdir(
const char *pszDirname,
long nMode )
override;
249 int Rmdir(
const char *pszDirname )
override;
250 char **ReadDir(
const char *pszDirname )
override
251 {
return ReadDirEx(pszDirname, 0); }
252 char **ReadDirEx(
const char *pszDirname,
int nMaxFiles )
override;
253 char **SiblingFiles(
const char *pszFilename )
override;
255 int HasOptimizedReadMultiRange(
const char* )
256 override {
return true; }
258 const char* GetActualURL(
const char* pszFilename)
override;
260 const char* GetOptions()
override;
262 char** GetFileMetadata(
const char * pszFilename,
const char* pszDomain,
265 char **ReadDirInternal(
const char *pszDirname,
int nMaxFiles,
266 bool* pbGotFileList );
267 void InvalidateDirContent(
const char *pszDirname );
269 virtual const char* GetDebugKey()
const {
return "VSICURL"; }
271 virtual CPLString GetFSPrefix()
const {
return "/vsicurl/"; }
272 virtual bool AllowCachedDataFor(
const char* pszFilename);
274 std::shared_ptr<std::string> GetRegion(
const char* pszURL,
277 void AddRegion(
const char* pszURL,
282 bool GetCachedFileProp(
const char* pszURL,
283 FileProp& oFileProp );
284 void SetCachedFileProp(
const char* pszURL,
285 FileProp& oFileProp );
286 void InvalidateCachedData(
const char* pszURL );
288 CURLM *GetCurlMultiHandleFor(
const CPLString& osURL );
290 virtual void ClearCache();
291 virtual void PartialClearCache(
const char* pszFilename);
294 bool GetCachedDirList(
const char* pszURL,
295 CachedDirList& oCachedDirList );
296 void SetCachedDirList(
const char* pszURL,
297 CachedDirList& oCachedDirList );
298 bool ExistsInCacheDirList(
const CPLString& osDirname,
bool *pbIsDir );
312 VSICurlFilesystemHandler* poFS =
nullptr;
314 bool m_bCached =
true;
316 FileProp oFileProp{};
319 char* m_pszURL =
nullptr;
320 std::string m_osQueryString{};
322 char **m_papszHTTPOptions =
nullptr;
325 int nBlocksToDownload = 1;
327 bool bStopOnInterruptUntilUninstall =
false;
328 bool bInterrupted =
false;
329 VSICurlReadCbkFunc pfnReadCbk =
nullptr;
330 void *pReadCbkUserData =
nullptr;
333 double m_dfRetryDelay = 0.0;
337 void DownloadRegionPostProcess(
const vsi_l_offset startOffset,
348 virtual std::string DownloadRegion(
vsi_l_offset startOffset,
int nBlocks);
350 bool m_bUseHead =
false;
351 bool m_bUseRedirectURLIfNoQueryStringParams =
false;
353 int ReadMultiRangeSingleGet(
int nRanges,
void ** ppData,
355 const size_t* panSizes );
356 CPLString GetRedirectURLIfValid(
bool& bHasExpired);
359 virtual struct curl_slist* GetCurlHeaders(
const CPLString& ,
360 const struct curl_slist* )
362 virtual bool AllowAutomaticRedirection() {
return true; }
363 virtual bool CanRestartOnError(
const char*,
const char*,
bool ) {
return false; }
364 virtual bool UseLimitRangeGetInsteadOfHead() {
return false; }
365 virtual bool IsDirectoryFromExists(
const char* ,
int ) {
return false; }
366 virtual void ProcessGetFileSizeResult(
const char* ) {}
367 void SetURL(
const char* pszURL);
368 virtual bool Authenticate() {
return false; }
372 VSICurlHandle( VSICurlFilesystemHandler* poFS,
373 const char* pszFilename,
374 const char* pszURLIn =
nullptr );
375 ~VSICurlHandle()
override;
379 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
380 int ReadMultiRange(
int nRanges,
void ** ppData,
382 const size_t* panSizes )
override;
383 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
385 int Flush()
override;
386 int Close()
override;
388 bool IsKnownFileSize()
const {
return oFileProp.bHasComputedFileSize; }
389 vsi_l_offset GetFileSizeOrHeaders(
bool bSetError,
bool bGetHeaders);
390 virtual vsi_l_offset GetFileSize(
bool bSetError ) {
return GetFileSizeOrHeaders(bSetError,
false); }
391 bool Exists(
bool bSetError );
392 bool IsDirectory()
const {
return oFileProp.bIsDirectory; }
393 int GetMode()
const {
return oFileProp.nMode; }
394 time_t GetMTime()
const {
return oFileProp.mTime; }
397 int InstallReadCbk( VSICurlReadCbkFunc pfnReadCbk,
399 int bStopOnInterruptUntilUninstall );
400 int UninstallReadCbk();
402 const char *GetURL()
const {
return m_pszURL; }
409 class IVSIS3LikeFSHandler:
public VSICurlFilesystemHandler
415 const char* pszSource,
416 const char* pszTarget,
417 GDALProgressFunc pProgressFunc,
418 void *pProgressData);
419 virtual int MkdirInternal(
const char *pszDirname,
long nMode,
bool bDoStatCheck );
422 char** GetFileList(
const char *pszFilename,
424 bool* pbGotFileList )
override;
426 virtual IVSIS3LikeHandleHelper* CreateHandleHelper(
427 const char* pszURI,
bool bAllowNoObject) = 0;
429 virtual int CopyObject(
const char *oldpath,
const char *newpath,
432 IVSIS3LikeFSHandler() =
default;
435 int Unlink(
const char *pszFilename )
override;
436 int Mkdir(
const char *pszDirname,
long nMode )
override;
437 int Rmdir(
const char *pszDirname )
override;
438 int Stat(
const char *pszFilename,
VSIStatBufL *pStatBuf,
439 int nFlags )
override;
440 int Rename(
const char *oldpath,
const char *newpath )
override;
442 virtual int DeleteObject(
const char *pszFilename );
444 virtual void UpdateMapFromHandle(IVSIS3LikeHandleHelper*) {}
445 virtual void UpdateHandleFromMap( IVSIS3LikeHandleHelper * ) {}
447 virtual CPLString GetStreamingPath(
const char* pszFilename )
const;
449 bool Sync(
const char* pszSource,
const char* pszTarget,
450 const char*
const * papszOptions,
451 GDALProgressFunc pProgressFunc,
453 char*** ppapszOutputs )
override;
455 VSIDIR* OpenDir(
const char *pszPath,
int nRecurseDepth,
456 const char*
const *papszOptions)
override;
459 virtual bool SupportsParallelMultipartUpload()
const {
return false; }
461 virtual CPLString InitiateMultipartUpload(
462 const std::string& osFilename,
463 IVSIS3LikeHandleHelper *poS3HandleHelper,
469 const std::string& osUploadID,
471 const void* pabyBuffer,
473 IVSIS3LikeHandleHelper *poS3HandleHelper,
475 double dfRetryDelay);
476 virtual bool CompleteMultipart(
const CPLString& osFilename,
478 const std::vector<CPLString>& aosEtags,
480 IVSIS3LikeHandleHelper *poS3HandleHelper,
482 double dfRetryDelay);
483 virtual bool AbortMultipart(
const CPLString& osFilename,
485 IVSIS3LikeHandleHelper *poS3HandleHelper,
487 double dfRetryDelay);
494 class IVSIS3LikeHandle:
public VSICurlHandle
499 bool UseLimitRangeGetInsteadOfHead()
override {
return true; }
500 bool IsDirectoryFromExists(
const char* pszVerb,
501 int response_code )
override
504 return response_code == 416 &&
EQUAL(pszVerb,
"GET") &&
507 void ProcessGetFileSizeResult(
const char* pszContent )
override
509 oFileProp.bIsDirectory = strstr(pszContent,
"ListBucketResult") !=
nullptr;
513 IVSIS3LikeHandle( VSICurlFilesystemHandler* poFSIn,
514 const char* pszFilename,
515 const char* pszURLIn ) :
516 VSICurlHandle(poFSIn, pszFilename, pszURLIn) {}
517 ~IVSIS3LikeHandle()
override {}
528 IVSIS3LikeFSHandler *m_poFS =
nullptr;
530 IVSIS3LikeHandleHelper *m_poS3HandleHelper =
nullptr;
531 bool m_bUseChunked =
false;
535 int m_nBufferOff = 0;
536 int m_nBufferSize = 0;
537 bool m_bClosed =
false;
538 GByte *m_pabyBuffer =
nullptr;
540 int m_nPartNumber = 0;
541 std::vector<CPLString> m_aosEtags{};
542 bool m_bError =
false;
544 CURLM *m_hCurlMulti =
nullptr;
545 CURL *m_hCurl =
nullptr;
546 const void *m_pBuffer =
nullptr;
548 size_t m_nChunkedBufferOff = 0;
549 size_t m_nChunkedBufferSize = 0;
550 size_t m_nWrittenInPUT = 0;
553 double m_dfRetryDelay = 0.0;
554 WriteFuncStruct m_sWriteFuncHeaderData{};
557 bool DoSinglePartPUT();
559 static size_t ReadCallBackBufferChunked(
char *buffer,
size_t size,
560 size_t nitems,
void *instream );
561 size_t WriteChunked(
const void *pBuffer,
562 size_t nSize,
size_t nMemb );
563 int FinishChunkedTransfer();
565 void InvalidateParentDirectory();
568 VSIS3WriteHandle( IVSIS3LikeFSHandler* poFS,
569 const char* pszFilename,
570 IVSIS3LikeHandleHelper* poS3HandleHelper,
573 ~VSIS3WriteHandle()
override;
577 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
578 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
580 int Close()
override;
582 bool IsOK() {
return m_bUseChunked || m_pabyBuffer !=
nullptr; }
595 VSICurlFilesystemHandler* m_poFS =
nullptr;
600 int m_nBufferOff = 0;
601 int m_nBufferSize = 0;
602 int m_nBufferOffReadCallback = 0;
603 bool m_bClosed =
false;
604 GByte *m_pabyBuffer =
nullptr;
605 bool m_bError =
false;
607 static size_t ReadCallBackBuffer(
char *buffer,
size_t size,
608 size_t nitems,
void *instream );
609 virtual bool Send(
bool bIsLastBlock) = 0;
612 VSIAppendWriteHandle( VSICurlFilesystemHandler* poFS,
613 const char* pszFSPrefix,
614 const char* pszFilename,
616 virtual ~VSIAppendWriteHandle();
620 size_t Read(
void *pBuffer,
size_t nSize,
size_t nMemb )
override;
621 size_t Write(
const void *pBuffer,
size_t nSize,
size_t nMemb )
override;
623 int Close()
override;
625 bool IsOK() {
return m_pabyBuffer !=
nullptr; }
632 struct CurlRequestHelper
634 WriteFuncStruct sWriteFuncData{};
635 WriteFuncStruct sWriteFuncHeaderData{};
636 char szCurlErrBuf[CURL_ERROR_SIZE+1] = {};
639 ~CurlRequestHelper();
640 long perform(CURL* hCurlHandle,
641 struct curl_slist* headers,
642 VSICurlFilesystemHandler *poFS,
643 IVSIS3LikeHandleHelper *poS3HandleHelper);
650 class NetworkStatisticsLogger
652 static int gnEnabled;
653 static NetworkStatisticsLogger gInstance;
655 NetworkStatisticsLogger() =
default;
657 std::mutex m_mutex{};
666 GIntBig nGETDownloadedBytes = 0;
668 GIntBig nPOSTDownloadedBytes = 0;
669 GIntBig nPOSTUploadedBytes = 0;
672 enum class ContextPathType
679 struct ContextPathItem
681 ContextPathType eType;
684 ContextPathItem(ContextPathType eTypeIn,
const CPLString& osNameIn):
685 eType(eTypeIn), osName(osNameIn) {}
687 bool operator< (
const ContextPathItem& other )
const
689 if(
static_cast<int>(eType) <
static_cast<int>(other.eType) )
691 if(
static_cast<int>(eType) >
static_cast<int>(other.eType) )
693 return osName < other.osName;
700 std::map<ContextPathItem, Stats> children{};
706 std::map<GIntBig, std::vector<ContextPathItem>> m_mapThreadIdToContextPath{};
708 static void ReadEnabled();
710 std::vector<Counters*> GetCountersForContext();
714 static inline bool IsEnabled()
720 return gnEnabled == TRUE;
723 static void EnterFileSystem(
const char* pszName);
725 static void LeaveFileSystem();
727 static void EnterFile(
const char* pszName);
729 static void LeaveFile();
731 static void EnterAction(
const char* pszName);
733 static void LeaveAction();
735 static void LogHEAD();
737 static void LogGET(
size_t nDownloadedBytes);
739 static void LogPUT(
size_t nUploadedBytes);
741 static void LogPOST(
size_t nUploadedBytes,
742 size_t nDownloadedBytes);
744 static void LogDELETE();
748 static CPLString GetReportAsSerializedJSON();
751 struct NetworkStatisticsFileSystem
753 inline explicit NetworkStatisticsFileSystem(
const char* pszName) {
754 NetworkStatisticsLogger::EnterFileSystem(pszName);
757 inline ~NetworkStatisticsFileSystem()
759 NetworkStatisticsLogger::LeaveFileSystem();
763 struct NetworkStatisticsFile
765 inline explicit NetworkStatisticsFile(
const char* pszName) {
766 NetworkStatisticsLogger::EnterFile(pszName);
769 inline ~NetworkStatisticsFile()
771 NetworkStatisticsLogger::LeaveFile();
775 struct NetworkStatisticsAction
777 inline explicit NetworkStatisticsAction(
const char* pszName) {
778 NetworkStatisticsLogger::EnterAction(pszName);
781 inline ~NetworkStatisticsAction()
783 NetworkStatisticsLogger::LeaveAction();
788 int VSICURLGetDownloadChunkSize();
790 void VSICURLInitWriteFuncStruct( WriteFuncStruct *psStruct,
792 VSICurlReadCbkFunc pfnReadCbk,
793 void *pReadCbkUserData );
794 size_t VSICurlHandleWriteFunc(
void *buffer,
size_t count,
795 size_t nmemb,
void *req );
796 void MultiPerform(CURLM* hCurlMultiHandle,
797 CURL* hEasyHandle =
nullptr);
798 void VSICURLResetHeaderAndWriterFunctions(CURL* hCurlHandle);
800 int VSICurlParseUnixPermissions(
const char* pszPermissions);
The CPLJSONArray class holds JSON object from CPLJSONDocument.
Definition: cpl_json.h:54
String list class designed around our use of C "char**" string lists.
Definition: cpl_string.h:442
Convenient string class based on std::string.
Definition: cpl_string.h:333
Virtual file handle.
Definition: cpl_vsi_virtual.h:56
Interface for read and write JSON documents.
Core portability definitions for CPL.
#define EQUAL(a, b)
Alias for strcasecmp() == 0.
Definition: cpl_port.h:576
#define CPL_DISALLOW_COPY_ASSIGN(ClassName)
Helper to remove the copy and assignment constructors so that the compiler will not generate the defa...
Definition: cpl_port.h:1007
char ** CSLConstList
Type of a constant null-terminated list of nul terminated strings.
Definition: cpl_port.h:1216
unsigned char GByte
Unsigned byte type.
Definition: cpl_port.h:215
long long GIntBig
Large signed integer type (generally 64-bit integer type).
Definition: cpl_port.h:248
Various convenience functions for working with strings and string lists.
#define VSI_L_OFFSET_MAX
Maximum value for a file offset.
Definition: cpl_vsi.h:142
struct VSIDIR VSIDIR
Opaque type for a directory iterator.
Definition: cpl_vsi.h:319
struct VSI_STAT64_T VSIStatBufL
Type for VSIStatL()
Definition: cpl_vsi.h:195
FILE VSILFILE
Opaque type for a FILE that implements the VSIVirtualHandle API.
Definition: cpl_vsi.h:156
GUIntBig vsi_l_offset
Type for a file offset.
Definition: cpl_vsi.h:140