bes  Updated for version 3.20.8
BESStoredDapResultCache.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 //#define DODS_DEBUG
28 
29 #include <sys/stat.h>
30 
31 #include <iostream>
32 #ifdef HAVE_TR1_FUNCTIONAL
33 #include <tr1/functional>
34 #endif
35 #include <string>
36 #include <fstream>
37 #include <sstream>
38 
39 #include <DDS.h>
40 #include <DMR.h>
41 #include <DapXmlNamespaces.h>
42 #include <ConstraintEvaluator.h>
43 #include <DDXParserSAX2.h>
44 
45 // These are needed because D4ParserSax2.h does not properly declare
46 // the classes. I think. Check on that... jhrg 3/28/14
47 #include <D4EnumDefs.h>
48 #include <D4Dimensions.h>
49 #include <D4Group.h>
50 
51 #include <D4ParserSax2.h>
52 
53 // DAP2 Stored results are not supported by default. If we do start using this.
54 // It would be better to use the CacheMarshaller and CacheUnMarshaller code
55 // since that does not translate data into network byte order. Also, there
56 // may be a bug in the XDRStreamUnMarshaller code - in/with get_opaque() - that
57 // breaks Sequence::deserialize(). jhrg 5/25/16
58 #ifdef DAP2_STORED_RESULTS
59 #include <XDRStreamMarshaller.h>
60 #include <XDRStreamUnMarshaller.h>
61 #endif
62 
63 #include <chunked_istream.h>
64 #include <D4StreamUnMarshaller.h>
65 
66 #include <debug.h>
67 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
68 #include <util.h>
69 
70 #include "BESStoredDapResultCache.h"
71 #include "BESDapResponseBuilder.h"
72 #include "BESInternalError.h"
73 
74 #include "BESUtil.h"
75 #include "TheBESKeys.h"
76 #include "BESDebug.h"
77 
78 #ifdef HAVE_TR1_FUNCTIONAL
79 #define HASH_OBJ std::tr1::hash
80 #else
81 #define HASH_OBJ std::hash
82 #endif
83 
84 #define CRLF "\r\n"
85 #define BES_DATA_ROOT "BES.Data.RootDirectory"
86 #define BES_CATALOG_ROOT "BES.Catalog.catalog.RootDirectory"
87 
88 
89 using namespace std;
90 using namespace libdap;
91 
92 BESStoredDapResultCache *BESStoredDapResultCache::d_instance = 0;
93 bool BESStoredDapResultCache::d_enabled = true;
94 
95 #if 0
96 const string BESStoredDapResultCache::SUBDIR_KEY = "DAP.StoredResultsCache.subdir";
97 const string BESStoredDapResultCache::PREFIX_KEY = "DAP.StoredResultsCache.prefix";
98 const string BESStoredDapResultCache::SIZE_KEY = "DAP.StoredResultsCache.size";
99 #endif
100 
101 unsigned long BESStoredDapResultCache::getCacheSizeFromConfig()
102 {
103  bool found;
104  string size;
105  unsigned long size_in_megabytes = 0;
106  TheBESKeys::TheKeys()->get_value(DAP_STORED_RESULTS_CACHE_SIZE_KEY, size, found);
107  if (found) {
108  istringstream iss(size);
109  iss >> size_in_megabytes;
110  }
111  else {
112  stringstream msg;
113  msg << "[ERROR] BESStoredDapResultCache::getCacheSize() - The BES Key " << DAP_STORED_RESULTS_CACHE_SIZE_KEY;
114  msg << " is not set! It MUST be set to utilize the Stored Result Caching system. ";
115  BESDEBUG("cache", msg.str() << endl);
116  throw BESInternalError(msg.str(), __FILE__, __LINE__);
117  }
118  return size_in_megabytes;
119 }
120 
121 string BESStoredDapResultCache::getSubDirFromConfig()
122 {
123  bool found;
124  string subdir = "";
125  TheBESKeys::TheKeys()->get_value(DAP_STORED_RESULTS_CACHE_SUBDIR_KEY, subdir, found);
126 
127  if (!found) {
128  stringstream msg;
129  msg << "[ERROR] BESStoredDapResultCache::getSubDirFromConfig() - The BES Key " << DAP_STORED_RESULTS_CACHE_SUBDIR_KEY;
130  msg << " is not set! It MUST be set to utilize the Stored Result Caching system. ";
131  BESDEBUG("cache", msg.str() << endl);
132  throw BESInternalError(msg.str(), __FILE__, __LINE__);
133  }
134  else {
135  while (*subdir.begin() == '/' && subdir.length() > 0) {
136  subdir = subdir.substr(1);
137  }
138  // So if it's value is "/" or the empty string then the subdir will default to the root
139  // directory of the BES data system.
140  }
141 
142  return subdir;
143 }
144 
145 string BESStoredDapResultCache::getResultPrefixFromConfig()
146 {
147  bool found;
148  string prefix = "";
149  TheBESKeys::TheKeys()->get_value(DAP_STORED_RESULTS_CACHE_PREFIX_KEY, prefix, found);
150  if (found) {
151  prefix = BESUtil::lowercase(prefix);
152  }
153  else {
154  stringstream msg;
155  msg << "[ERROR] BESStoredDapResultCache::getResultPrefix() - The BES Key " << DAP_STORED_RESULTS_CACHE_PREFIX_KEY;
156  msg << " is not set! It MUST be set to utilize the Stored Result Caching system. ";
157  BESDEBUG("cache", msg.str() << endl);
158  throw BESInternalError(msg.str(), __FILE__, __LINE__);
159  }
160 
161  return prefix;
162 }
163 
164 string BESStoredDapResultCache::getBesDataRootDirFromConfig()
165 {
166  bool found;
167  string cacheDir = "";
168  TheBESKeys::TheKeys()->get_value( BES_CATALOG_ROOT, cacheDir, found);
169  if (!found) {
170  TheBESKeys::TheKeys()->get_value( BES_DATA_ROOT, cacheDir, found);
171  if (!found) {
172  string msg = ((string) "[ERROR] BESStoredDapResultCache::getStoredResultsDir() - Neither the BES Key ")
173  + BES_CATALOG_ROOT + "or the BES key " + BES_DATA_ROOT
174  + " have been set! One MUST be set to utilize the Stored Result Caching system. ";
175  BESDEBUG("cache", msg << endl);
176  throw BESInternalError(msg, __FILE__, __LINE__);
177  }
178  }
179  return cacheDir;
180 
181 }
182 
183 BESStoredDapResultCache::BESStoredDapResultCache()
184 {
185  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - BEGIN" << endl);
186 
187  d_storedResultsSubdir = getSubDirFromConfig();
188  d_dataRootDir = getBesDataRootDirFromConfig();
189  string resultsDir = BESUtil::assemblePath(d_dataRootDir, d_storedResultsSubdir);
190 
191  d_resultFilePrefix = getResultPrefixFromConfig();
192  d_maxCacheSize = getCacheSizeFromConfig();
193 
194  BESDEBUG("cache",
195  "BESStoredDapResultCache() - Stored results cache configuration params: " << resultsDir << ", " << d_resultFilePrefix << ", " << d_maxCacheSize << endl);
196 
197  initialize(resultsDir, d_resultFilePrefix, d_maxCacheSize);
198 
199  BESDEBUG("cache", "BESStoredDapResultCache::BESStoredDapResultCache() - END" << endl);
200 }
201 
205 BESStoredDapResultCache::BESStoredDapResultCache(const string &data_root_dir, const string &stored_results_subdir,
206  const string &result_file_prefix, unsigned long long max_cache_size)
207 {
208 
209  d_storedResultsSubdir = stored_results_subdir;
210  d_dataRootDir = data_root_dir;
211  d_resultFilePrefix = result_file_prefix;
212  d_maxCacheSize = max_cache_size;
213  initialize(BESUtil::assemblePath(d_dataRootDir, stored_results_subdir), d_resultFilePrefix, d_maxCacheSize);
214 }
215 
217 BESStoredDapResultCache::get_instance(const string &data_root_dir, const string &stored_results_subdir,
218  const string &result_file_prefix, unsigned long long max_cache_size)
219 {
220  if (d_enabled && d_instance == 0) {
221  if (dir_exists(data_root_dir)) {
222  d_instance = new BESStoredDapResultCache(data_root_dir, stored_results_subdir, result_file_prefix,
223  max_cache_size);
224  d_enabled = d_instance->cache_enabled();
225  if(!d_enabled){
226  delete d_instance;
227  d_instance = NULL;
228  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
229  "Cache is DISABLED"<< endl);
230  }
231  else {
232 #ifdef HAVE_ATEXIT
233  atexit(delete_instance);
234 #endif
235  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
236  "Cache is ENABLED"<< endl);
237  }
238  }
239  }
240  return d_instance;
241 }
242 
248 {
249  if (d_enabled && d_instance == 0) {
250  d_instance = new BESStoredDapResultCache();
251  d_enabled = d_instance->cache_enabled();
252  if(!d_enabled){
253  delete d_instance;
254  d_instance = NULL;
255  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
256  "Cache is DISABLED"<< endl);
257  }
258  else {
259 #ifdef HAVE_ATEXIT
260  atexit(delete_instance);
261 #endif
262  BESDEBUG("cache", "BESStoredDapResultCache::"<<__func__ << "() - " <<
263  "Cache is ENABLED"<< endl);
264  }
265  }
266 
267  return d_instance;
268 }
269 
279 bool BESStoredDapResultCache::is_valid(const string &cache_file_name, const string &dataset)
280 {
281  // If the cached response is zero bytes in size, it's not valid.
282  // (hmmm...)
283 
284  off_t entry_size = 0;
285  time_t entry_time = 0;
286  struct stat buf;
287  if (stat(cache_file_name.c_str(), &buf) == 0) {
288  entry_size = buf.st_size;
289  entry_time = buf.st_mtime;
290  }
291  else {
292  return false;
293  }
294 
295  if (entry_size == 0) return false;
296 
297  time_t dataset_time = entry_time;
298  if (stat(dataset.c_str(), &buf) == 0) {
299  dataset_time = buf.st_mtime;
300  }
301 
302  // Trick: if the d_dataset is not a file, stat() returns error and
303  // the times stay equal and the code uses the cache entry.
304 
305  // TODO Fix this so that the code can get a LMT from the correct
306  // handler.
307  if (dataset_time > entry_time) return false;
308 
309  return true;
310 }
311 
312 #ifdef DAP2_STORED_RESULTS
324 bool BESStoredDapResultCache::read_dap2_data_from_cache(const string &cache_file_name, DDS *fdds)
325 {
326  BESDEBUG("cache",
327  "BESStoredDapResultCache::read_dap2_data_from_cache() - Opening cache file: " << cache_file_name << endl);
328 
329  int fd = 1;
330 
331  try {
332  if (get_read_lock(cache_file_name, fd)) {
333 
334  ifstream data(cache_file_name.c_str());
335 
336  // Rip off the MIME headers from the response if they are present
337  string mime = get_next_mime_header(data);
338  while (!mime.empty()) {
339  mime = get_next_mime_header(data);
340  }
341 
342  // Parse the DDX; throw an exception on error.
343  DDXParser ddx_parser(fdds->get_factory());
344 
345  // Read the MPM boundary and then read the subsequent headers
346  string boundary = read_multipart_boundary(data);
347  BESDEBUG("cache",
348  "BESStoredDapResultCache::read_dap2_data_from_cache() - MPM Boundary: " << boundary << endl);
349 
350  read_multipart_headers(data, "text/xml", dods_ddx);
351 
352  BESDEBUG("cache",
353  "BESStoredDapResultCache::read_dap2_data_from_cache() - Read the multipart haeaders" << endl);
354 
355  // Parse the DDX, reading up to and including the next boundary.
356  // Return the CID for the matching data part
357  string data_cid;
358  try {
359  ddx_parser.intern_stream(data, fdds, data_cid, boundary);
360  BESDEBUG("cache",
361  "BESStoredDapResultCache::read_dap2_data_from_cache() - Dataset name: " << fdds->get_dataset_name() << endl);
362  }
363  catch (Error &e) {
364  BESDEBUG("cache",
365  "BESStoredDapResultCache::read_dap2_data_from_cache() - DDX Parser Error: " << e.get_error_message() << endl);
366  throw;
367  }
368 
369  // Munge the CID into something we can work with
370  BESDEBUG("cache",
371  "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (before): " << data_cid << endl);
372  data_cid = cid_to_header_value(data_cid);
373  BESDEBUG("cache",
374  "BESStoredDapResultCache::read_dap2_data_from_cache() - Data CID (after): " << data_cid << endl);
375 
376  // Read the data part's MPM part headers (boundary was read by
377  // DDXParse::intern)
378  read_multipart_headers(data, "application/octet-stream", dods_data_ddx, data_cid);
379 
380  // Now read the data
381 
382  // XDRFileUnMarshaller um(data);
383  XDRStreamUnMarshaller um(data);
384  for (DDS::Vars_iter i = fdds->var_begin(); i != fdds->var_end(); i++) {
385  (*i)->deserialize(um, fdds);
386  }
387 
388  data.close();
389  unlock_and_close(cache_file_name /* was fd */);
390  return true;
391  }
392  else {
393  BESDEBUG("cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
394 
395  return false;
396  }
397  }
398  catch (...) {
399  BESDEBUG("cache",
400  "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
401  // I think this call is not needed. jhrg 10/23/12
402  if (fd != -1) unlock_and_close(cache_file_name /* was fd */);
403  throw;
404  }
405 }
406 #endif
407 
419 bool BESStoredDapResultCache::read_dap4_data_from_cache(const string &cache_file_name, libdap::DMR *dmr)
420 {
421  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - BEGIN" << endl);
422 
423  int fd = 1;
424 
425  try {
426  if (get_read_lock(cache_file_name, fd)) {
427  BESDEBUG("cache",
428  "BESStoredDapResultCache::read_dap4_data_from_cache() - Opening cache file: " << cache_file_name << endl);
429  fstream in(cache_file_name.c_str(), ios::in | ios::binary);
430 
431  // Gobble up the response's initial set of MIME headers. Normally
432  // a client would extract information from these headers.
433  // NOTE - I am dumping this call because it basically just
434  // slurps up lines until it finds a blank line, regardless of what the
435  // lines actually have in the. So basically if the stream DOESN't have
436  // a mime header then this call will read (and ignore) the entire
437  // XML encoding of the DMR. doh.
438  // remove_mime_header(in);
439 
440  chunked_istream cis(in, CHUNK_SIZE);
441 
442  bool debug = BESDebug::IsSet("parser");
443 
444  // parse the DMR, stopping when the boundary is found.
445  // force chunk read
446  // get chunk size
447  int chunk_size = cis.read_next_chunk();
448 
449  BESDEBUG("cache",
450  "BESStoredDapResultCache::read_dap4_data_from_cache() - First chunk_size: " << chunk_size << endl);
451 
452  if (chunk_size == EOF) {
453  throw InternalErr(__FILE__, __LINE__,
454  "BESStoredDapResultCache::read_dap4_data_from_cache() - Failed to read first chunk from file. Chunk size = EOF (aka "
455  + libdap::long_to_string(EOF) + ")");
456  }
457 
458  // get chunk
459  char chunk[chunk_size];
460  cis.read(chunk, chunk_size);
461  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Read first chunk." << endl);
462 
463  // parse char * with given size
464  D4ParserSax2 parser;
465  // '-2' to discard the CRLF pair
466  parser.intern(chunk, chunk_size - 2, dmr, debug);
467  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Parsed first chunk." << endl);
468 
469  D4StreamUnMarshaller um(cis, cis.twiddle_bytes());
470 
471  dmr->root()->deserialize(um, *dmr);
472  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - Deserialized data." << endl);
473 
474  BESDEBUG("cache", "BESStoredDapResultCache::read_dap4_data_from_cache() - END" << endl);
475 
476  in.close();
477  unlock_and_close(cache_file_name /* was fd */);
478 
479  return true;
480 
481  }
482  else {
483  BESDEBUG("cache", "BESStoredDapResultCache - The requested file does not exist. File: " + cache_file_name);
484 
485  return false;
486 
487  }
488  }
489  catch (...) {
490  BESDEBUG("cache",
491  "BESStoredDapResultCache::read_dap4_data_from_cache() - caught exception, unlocking cache and re-throw." << endl);
492  // I think this call is not needed. jhrg 10/23/12
493  if (fd != -1) unlock_and_close(cache_file_name /* was fd */);
494  throw;
495  }
496 }
497 
498 #ifdef DAP2_STORED_RESULTS
503 DDS *
504 BESStoredDapResultCache::get_cached_dap2_data_ddx(const string &cache_file_name, BaseTypeFactory *factory,
505  const string &filename)
506 {
507  BESDEBUG("cache",
508  "BESStoredDapResultCache::get_cached_dap2_data_ddx() - Reading cache for " << cache_file_name << endl);
509 
510  DDS *fdds = new DDS(factory);
511 
512  if (read_dap2_data_from_cache(cache_file_name, fdds)) {
513 
514  fdds->filename(filename);
515  //fdds->set_dataset_name( "function_result_" + name_path(filename) ) ;
516 
517  BESDEBUG("cache", "DDS Filename: " << fdds->filename() << endl);
518  BESDEBUG("cache", "DDS Dataset name: " << fdds->get_dataset_name() << endl);
519 
520  fdds->set_factory(0);
521 
522  // mark everything as read. and send. That is, make sure that when a response
523  // is retrieved from the cache, all of the variables are marked as to be sent
524  DDS::Vars_iter i = fdds->var_begin();
525  while (i != fdds->var_end()) {
526  (*i)->set_read_p(true);
527  (*i++)->set_send_p(true);
528  }
529 
530  return fdds;
531  }
532  else {
533  delete fdds;
534  return 0;
535  }
536 
537 }
538 #endif
539 
544 DMR *
545 BESStoredDapResultCache::get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory,
546  const string &filename)
547 {
548  BESDEBUG("cache",
549  "BESStoredDapResultCache::get_cached_dap4_data() - Reading cache for " << cache_file_name << endl);
550 
551  DMR *fdmr = new DMR(factory);
552 
553  BESDEBUG("cache", "BESStoredDapResultCache::get_cached_dap4_data() - DMR Filename: " << fdmr->filename() << endl);
554  fdmr->set_filename(filename);
555 
556  if (read_dap4_data_from_cache(cache_file_name, fdmr)) {
557  BESDEBUG("cache",
558  "BESStoredDapResultCache::get_cached_dap4_data() - DMR Dataset name: " << fdmr->name() << endl);
559 
560  fdmr->set_factory(0);
561 
562  // mark everything as read. and send. That is, make sure that when a response
563  // is retrieved from the cache, all of the variables are marked as to be sent
564  fdmr->root()->set_send_p(true);
565  fdmr->root()->set_read_p(true);
566 
567  return fdmr;
568  }
569 
570  return 0;
571 }
572 
573 #ifdef DAP2_STORED_RESULTS
578 string BESStoredDapResultCache::store_dap2_result(DDS &dds, const string &constraint, BESDapResponseBuilder *rb,
579  ConstraintEvaluator *eval)
580 {
581  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - BEGIN" << endl);
582  // These are used for the cached or newly created DDS object
583  BaseTypeFactory factory;
584 
585  // Get the cache filename for this thing. Do not use the default
586  // name mangling; instead use what build_cache_file_name() does.
587  string local_id = get_stored_result_local_id(dds.filename(), constraint, DAP_3_2);
588  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - local_id: "<< local_id << endl);
589  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
590  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - cache_file_name: "<< cache_file_name << endl);
591  int fd;
592  try {
593  // If the object in the cache is not valid, remove it. The read_lock will
594  // then fail and the code will drop down to the create_and_lock() call.
595  // is_valid() tests for a non-zero object and for d_dateset newer than
596  // the cached object.
597  if (!is_valid(cache_file_name, dds.filename())) purge_file(cache_file_name);
598 
599  if (get_read_lock(cache_file_name, fd)) {
600  BESDEBUG("cache",
601  "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
602  }
603  else if (create_and_lock(cache_file_name, fd)) {
604  // If here, the cache_file_name could not be locked for read access;
605  // try to build it. First make an empty file and get an exclusive lock on it.
606  BESDEBUG("cache",
607  "BESStoredDapResultCache::store_dap2_result() - cache_file_name " << cache_file_name << ", constraint: " << constraint << endl);
608 
609 #if 0 // I shut this off because we know that the constraint and functions have already been evaluated - ndp
610  DDS *fdds;
611 
612  fdds = new DDS(dds);
613  eval->parse_constraint(constraint, *fdds);
614 
615  if (eval->function_clauses()) {
616  DDS *temp_fdds = eval->eval_function_clauses(*fdds);
617  delete fdds;
618  fdds = temp_fdds;
619  }
620 #endif
621 
622  ofstream data_stream(cache_file_name.c_str());
623  if (!data_stream)
624  throw InternalErr(__FILE__, __LINE__,
625  "Could not open '" + cache_file_name + "' to write cached response.");
626 
627  string start = "dataddx_cache_start", boundary = "dataddx_cache_boundary";
628 
629  // Use a ConstraintEvaluator that has not parsed a CE so the code can use
630  // the send method(s)
631  ConstraintEvaluator eval;
632 
633  // Setting the version to 3.2 causes send_data_ddx to write the MIME headers that
634  // the cache expects.
635  dds.set_dap_version("3.2");
636 
637  // This is a bit of a hack, but it effectively uses ResponseBuilder to write the
638  // cached object/response without calling the machinery in one of the send_*()
639  // methods. Those methods assume they need to evaluate the BESDapResponseBuilder's
640  // CE, which is not necessary and will alter the values of the send_p property
641  // of the DDS's variables.
642  set_mime_multipart(data_stream, boundary, start, dods_data_ddx, x_plain,
643  last_modified_time(rb->get_dataset_name()));
644  //data_stream << flush;
645  rb->serialize_dap2_data_ddx(data_stream, (DDS**) &dds, eval, boundary, start);
646  //data_stream << flush;
647 
648  data_stream << CRLF << "--" << boundary << "--" << CRLF;
649 
650  data_stream.close();
651 
652  // Change the exclusive lock on the new file to a shared lock. This keeps
653  // other processes from purging the new file and ensures that the reading
654  // process can use it.
655  exclusive_to_shared_lock(fd);
656 
657  // Now update the total cache size info and purge if needed. The new file's
658  // name is passed into the purge method because this process cannot detect its
659  // own lock on the file.
660  unsigned long long size = update_cache_info(cache_file_name);
661  if (cache_too_big(size)) update_and_purge(cache_file_name);
662 
663  }
664  // get_read_lock() returns immediately if the file does not exist,
665  // but blocks waiting to get a shared lock if the file does exist.
666  else if (get_read_lock(cache_file_name, fd)) {
667  BESDEBUG("cache",
668  "BESStoredDapResultCache::store_dap2_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
669  }
670  else {
671  throw InternalErr(__FILE__, __LINE__,
672  "BESStoredDapResultCache::store_dap2_result() - Cache error during function invocation.");
673  }
674 
675  BESDEBUG("cache",
676  "BESStoredDapResultCache::store_dap2_result() - unlocking and closing cache file "<< cache_file_name << endl);
677  unlock_and_close(cache_file_name);
678  }
679  catch (...) {
680  BESDEBUG("cache",
681  "BESStoredDapResultCache::store_dap2_result() - caught exception, unlocking cache and re-throw." << endl);
682  // I think this call is not needed. jhrg 10/23/12
683  unlock_cache();
684  throw;
685  }
686 
687  BESDEBUG("cache", "BESStoredDapResultCache::store_dap2_result() - END (local_id=`"<< local_id << "')" << endl);
688  return local_id;
689 }
690 #endif
691 
699 string BESStoredDapResultCache::get_stored_result_local_id(const string &dataset, const string &ce,
700  libdap::DAPVersion version)
701 {
702  BESDEBUG("cache", "get_stored_result_local_id() - BEGIN. dataset: " << dataset << ", ce: " << ce << endl);
703  std::ostringstream ostr;
704  HASH_OBJ<std::string> str_hash;
705  string name = dataset + "#" + ce;
706  ostr << str_hash(name);
707  string hashed_name = ostr.str();
708  BESDEBUG("cache", "get_stored_result_local_id() - hashed_name: " << hashed_name << endl);
709 
710  string suffix = "";
711  switch (version) {
712 #ifdef DAP2_STORED_RESULTS
713  case DAP_2_0:
714  suffix = ".dods";
715  break;
716 
717  case DAP_3_2:
718  suffix = ".data_ddx";
719  break;
720 #endif
721  case DAP_4_0:
722  suffix = ".dap";
723  break;
724 
725  default:
726  throw BESInternalError("BESStoredDapResultCache::get_stored_result_local_id() - Unrecognized DAP version!!",
727  __FILE__, __LINE__);
728  break;
729  }
730 
731  BESDEBUG("cache", "get_stored_result_local_id() - Data file suffix: " << suffix << endl);
732 
733  string local_id = d_resultFilePrefix + hashed_name + suffix;
734  BESDEBUG("cache", "get_stored_result_local_id() - file: " << local_id << endl);
735 
736  local_id = BESUtil::assemblePath(d_storedResultsSubdir, local_id);
737 
738  BESDEBUG("cache", "get_stored_result_local_id() - END. local_id: " << local_id << endl);
739  return local_id;
740 }
741 
746 string BESStoredDapResultCache::store_dap4_result(DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
747 {
748  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - BEGIN" << endl);
749  // These are used for the cached or newly created DDS object
750  BaseTypeFactory factory;
751 
752  // Get the cache filename for this thing. Do not use the default
753  // name mangling; instead use what build_cache_file_name() does.
754  string local_id = get_stored_result_local_id(dmr.filename(), constraint, DAP_4_0);
755  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - local_id: "<< local_id << endl);
756  string cache_file_name = get_cache_file_name(local_id, /*mangle*/false);
757  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - cache_file_name: "<< cache_file_name << endl);
758  int fd;
759  try {
760  // If the object in the cache is not valid, remove it. The read_lock will
761  // then fail and the code will drop down to the create_and_lock() call.
762  // is_valid() tests for a non-zero object and for d_dateset newer than
763  // the cached object.
764  if (!is_valid(cache_file_name, dmr.filename())) {
765  BESDEBUG("cache",
766  "BESStoredDapResultCache::store_dap4_result() - File is not valid. Purging file from cache. filename: " << cache_file_name << endl);
767  purge_file(cache_file_name);
768  }
769 
770  if (get_read_lock(cache_file_name, fd)) {
771  BESDEBUG("cache",
772  "BESStoredDapResultCache::store_dap4_result() - Stored Result already exists. Not rewriting file: " << cache_file_name << endl);
773  }
774  else if (create_and_lock(cache_file_name, fd)) {
775  // If here, the cache_file_name could not be locked for read access;
776  // try to build it. First make an empty file and get an exclusive lock on it.
777  BESDEBUG("cache",
778  "BESStoredDapResultCache::store_dap4_result() - cache_file_name: " << cache_file_name << ", constraint: " << constraint << endl);
779 
780  ofstream data_stream(cache_file_name.c_str());
781  if (!data_stream)
782  throw InternalErr(__FILE__, __LINE__,
783  "Could not open '" + cache_file_name + "' to write cached response.");
784 
785  //data_stream << flush;
786  rb->serialize_dap4_data(data_stream, dmr, false);
787  //data_stream << flush;
788 
789  data_stream.close();
790 
791  // Change the exclusive lock on the new file to a shared lock. This keeps
792  // other processes from purging the new file and ensures that the reading
793  // process can use it.
794  exclusive_to_shared_lock(fd);
795 
796  // Now update the total cache size info and purge if needed. The new file's
797  // name is passed into the purge method because this process cannot detect its
798  // own lock on the file.
799  unsigned long long size = update_cache_info(cache_file_name);
800  if (cache_too_big(size)) update_and_purge(cache_file_name);
801  }
802  // get_read_lock() returns immediately if the file does not exist,
803  // but blocks waiting to get a shared lock if the file does exist.
804  else if (get_read_lock(cache_file_name, fd)) {
805  BESDEBUG("cache",
806  "BESStoredDapResultCache::store_dap4_result() - Couldn't create and lock file, But I got a read lock. " "Result may have been created by another process. " "Not rewriting file: " << cache_file_name << endl);
807  }
808  else {
809  throw InternalErr(__FILE__, __LINE__,
810  "BESStoredDapResultCache::store_dap4_result() - Cache error during function invocation.");
811  }
812 
813  BESDEBUG("cache",
814  "BESStoredDapResultCache::store_dap4_result() - unlocking and closing cache file "<< cache_file_name << endl);
815  unlock_and_close(cache_file_name);
816  }
817  catch (...) {
818  BESDEBUG("cache",
819  "BESStoredDapResultCache::store_dap4_result() - caught exception, unlocking cache and re-throw." << endl);
820  // I think this call is not needed. jhrg 10/23/12
821  unlock_cache();
822  throw;
823  }
824 
825  BESDEBUG("cache", "BESStoredDapResultCache::store_dap4_result() - END (local_id=`"<< local_id << "')" << endl);
826  return local_id;
827 
828 }
829 
virtual std::string get_dataset_name() const
Get the dataset name.
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:160
exception thrown if internal error encountered
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
libdap::DMR * get_cached_dap4_data(const string &cache_file_name, libdap::D4BaseTypeFactory *factory, const string &filename)
static BESStoredDapResultCache * get_instance()
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71