bes  Updated for version 3.20.8
FONcTransmitter.cc
1 // FONcTransmitter.cc
2 
3 // This file is part of BES Netcdf File Out Module
4 
5 // Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact University Corporation for Atmospheric Research at
23 // 3080 Center Green Drive, Boulder, CO 80301
24 
25 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
27 //
28 // Authors:
29 // pwest Patrick West <pwest@ucar.edu>
30 // jgarcia Jose Garcia <jgarcia@ucar.edu>
31 // kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32 
33 #include "config.h"
34 
35 #include <stdio.h>
36 #include <stdlib.h>
37 #include <string.h>
38 
39 #ifdef HAVE_UNISTD_H
40 #include <unistd.h>
41 #endif
42 
43 #include <sys/types.h> // For umask
44 #include <sys/stat.h>
45 
46 #include <iostream>
47 #include <fstream>
48 #include <exception>
49 #include <sstream> // std::stringstream
50 #include <libgen.h>
51 
52 #include <DataDDS.h>
53 #include <BaseType.h>
54 #include <escaping.h>
55 #include <ConstraintEvaluator.h>
56 
57 #include <TheBESKeys.h>
58 #include <BESContextManager.h>
59 #include <BESDataDDSResponse.h>
60 #include <BESDapNames.h>
61 #include <BESDataNames.h>
62 #include <BESDebug.h>
63 #include <BESUtil.h>
64 #include <TempFile.h>
65 
66 #include <BESDapResponseBuilder.h>
67 
68 #include <BESError.h>
69 #include <BESDapError.h>
70 #include <BESForbiddenError.h>
71 #include <BESInternalFatalError.h>
72 #include <DapFunctionUtils.h>
73 
74 #include "FONcBaseType.h"
75 #include "FONcRequestHandler.h"
76 #include "FONcTransmitter.h"
77 #include "FONcTransform.h"
78 
79 using namespace libdap;
80 using namespace std;
81 
82 // size of the buffer used to read from the temporary file built on disk and
83 // send data to the client over the network connection (socket/stream)
84 #define OUTPUT_FILE_BLOCK_SIZE 4096
85 
99 {
100  add_method(DATA_SERVICE, FONcTransmitter::send_data);
101  add_method(DAP4DATA_SERVICE, FONcTransmitter::send_dap4_data);
102 }
103 
107 struct wrap_temp_descriptor {
108  int d_fd;
109  wrap_temp_descriptor(int fd) : d_fd(fd) {}
110  ~wrap_temp_descriptor() { close(d_fd); }
111 };
112 
113 #if 0
114 // Replaced by code in BESHandlerUtil. jhrg 8/25/17
115 
120 struct wrap_temp_name {
121  vector<char> d_name;
122  wrap_temp_name(vector<char> &name) : d_name(name) {}
123  ~wrap_temp_name() { unlink(&d_name[0]); }
124 };
125 #endif
126 
137 void updateHistoryAttribute(DDS *dds, const string ce)
138 {
139  bool foundIt = false;
140  string cf_history_entry = BESContextManager::TheManager()->get_context("cf_history_entry", foundIt);
141  if (!foundIt) {
142  // This code will be used only when the 'cf_histroy_context' is not set,
143  // which should be never in an operating server. However, when we are
144  // testing, often only the besstandalone code is running and the existing
145  // baselines don't set the context, so we have this. It must do something
146  // so the tests are not hopelessly obscure and filter out junk that varies
147  // by host (e.g., the names of cached files that have been decompressed).
148  // jhrg 6/3/16
149 
150  string request_url = dds->filename();
151  // remove path info
152  request_url = request_url.substr(request_url.find_last_of('/')+1);
153  // remove 'uncompress' cache mangling
154  request_url = request_url.substr(request_url.find_last_of('#')+1);
155  request_url += "?" + ce;
156 
157  std::stringstream ss;
158 
159  time_t raw_now;
160  struct tm * timeinfo;
161  time(&raw_now); /* get current time; same as: timer = time(NULL) */
162  timeinfo = localtime(&raw_now);
163 
164  char time_str[100];
165  // 2000-6-1 6:00:00
166  strftime(time_str, 100, "%Y-%m-%d %H:%M:%S", timeinfo);
167 
168  ss << time_str << " " << "Hyrax" << " " << request_url;
169  cf_history_entry = ss.str();
170  }
171 
172  BESDEBUG("fonc",
173  "FONcTransmitter::updateHistoryAttribute() - Adding cf_history_entry context. '" << cf_history_entry << "'" << endl);
174 
175  vector<string> hist_entry_vec;
176  hist_entry_vec.push_back(cf_history_entry);
177  BESDEBUG("fonc",
178  "FONcTransmitter::updateHistoryAttribute() - hist_entry_vec.size(): " << hist_entry_vec.size() << endl);
179 
180  // Add the new entry to the "history" attribute
181  // Get the top level Attribute table.
182  AttrTable &globals = dds->get_attr_table();
183 
184  // Since many files support "CF" conventions the history tag may already exist in the source data
185  // and we should add an entry to it if possible.
186  bool done = false; // Used to indicate that we located a toplevel ATtrTable whose name ends in "_GLOBAL" and that has an existing "history" attribute.
187  unsigned int num_attrs = globals.get_size();
188  if (num_attrs) {
189  // Here we look for a top level AttrTable whose name ends with "_GLOBAL" which is where, by convention,
190  // data ingest handlers place global level attributes found in the source dataset.
191  AttrTable::Attr_iter i = globals.attr_begin();
192  AttrTable::Attr_iter e = globals.attr_end();
193  for (; i != e && !done; i++) {
194  AttrType attrType = globals.get_attr_type(i);
195  string attr_name = globals.get_name(i);
196  // Test the entry...
197  if (attrType == Attr_container && BESUtil::endsWith(attr_name, "_GLOBAL")) {
198  // Look promising, but does it have an existing "history" Attribute?
199  AttrTable *source_file_globals = globals.get_attr_table(i);
200  AttrTable::Attr_iter history_attrItr = source_file_globals->simple_find("history");
201  if (history_attrItr != source_file_globals->attr_end()) {
202  // Yup! Add our entry...
203  BESDEBUG("fonc",
204  "FONcTransmitter::updateHistoryAttribute() - Adding history entry to " << attr_name << endl);
205  source_file_globals->append_attr("history", "string", &hist_entry_vec);
206  done = true;
207  }
208  }
209  }
210  }
211 
212  if (!done) {
213  // We never found an existing location to place the "history" entry, so we'll just stuff it into the top level AttrTable.
214  BESDEBUG("fonc",
215  "FONcTransmitter::updateHistoryAttribute() - Adding history entry to top level AttrTable" << endl);
216  globals.append_attr("history", "string", &hist_entry_vec);
217 
218  }
219 }
220 
221 #if 0
232 void update_Dap4_HistoryAttribute(DMR *dmr, const string ce)
233 {
234  bool foundIt = false;
235  string cf_history_entry = BESContextManager::TheManager()->get_context("cf_history_entry", foundIt);
236  if (!foundIt) {
237  // This code will be used only when the 'cf_histroy_context' is not set,
238  // which should be never in an operating server. However, when we are
239  // testing, often only the besstandalone code is running and the existing
240  // baselines don't set the context, so we have this. It must do something
241  // so the tests are not hopelessly obscure and filter out junk that varies
242  // by host (e.g., the names of cached files that have been decompressed).
243  // jhrg 6/3/16
244 
245  string request_url = dds->filename();
246  // remove path info
247  request_url = request_url.substr(request_url.find_last_of('/')+1);
248  // remove 'uncompress' cache mangling
249  request_url = request_url.substr(request_url.find_last_of('#')+1);
250  request_url += "?" + ce;
251 
252  std::stringstream ss;
253 
254  time_t raw_now;
255  struct tm * timeinfo;
256  time(&raw_now); /* get current time; same as: timer = time(NULL) */
257  timeinfo = localtime(&raw_now);
258 
259  char time_str[100];
260  // 2000-6-1 6:00:00
261  strftime(time_str, 100, "%Y-%m-%d %H:%M:%S", timeinfo);
262 
263  ss << time_str << " " << "Hyrax" << " " << request_url;
264  cf_history_entry = ss.str();
265  }
266 
267  BESDEBUG("fonc",
268  "FONcTransmitter::update_Dap4_HistoryAttribute() - Adding cf_history_entry context. '" << cf_history_entry << "'" << endl);
269 
270  vector<string> hist_entry_vec;
271  hist_entry_vec.push_back(cf_history_entry);
272  BESDEBUG("fonc",
273  "FONcTransmitter::update_Dap4_HistoryAttribute() - hist_entry_vec.size(): " << hist_entry_vec.size() << endl);
274 
275  // Add the new entry to the "history" attribute
276  // Get the top level Attribute table.
277  //AttrTable &globals = dds->get_attr_table();
278  D4Group* root_grp = dmr->root();
279  //D4Attributes*d4_attrs = root_grp->attributes();
280 
281  // Since many files support "CF" conventions the history tag may already exist in the source data
282  // and we should add an entry to it if possible.
283  bool done = false; // Used to indicate that we located a toplevel ATtrTable whose name ends in "_GLOBAL" and that has an existing "history" attribute.
284  unsigned int num_attrs = globals.get_size();
285  if (num_attrs) {
286  // Here we look for a top level AttrTable whose name ends with "_GLOBAL" which is where, by convention,
287  // data ingest handlers place global level attributes found in the source dataset.
288  AttrTable::Attr_iter i = globals.attr_begin();
289  AttrTable::Attr_iter e = globals.attr_end();
290  for (; i != e && !done; i++) {
291  AttrType attrType = globals.get_attr_type(i);
292  string attr_name = globals.get_name(i);
293  // Test the entry...
294  if (attrType == Attr_container && BESUtil::endsWith(attr_name, "_GLOBAL")) {
295  // Look promising, but does it have an existing "history" Attribute?
296  AttrTable *source_file_globals = globals.get_attr_table(i);
297  AttrTable::Attr_iter history_attrItr = source_file_globals->simple_find("history");
298  if (history_attrItr != source_file_globals->attr_end()) {
299  // Yup! Add our entry...
300  BESDEBUG("fonc",
301  "FONcTransmitter::updateHistoryAttribute() - Adding history entry to " << attr_name << endl);
302  source_file_globals->append_attr("history", "string", &hist_entry_vec);
303  done = true;
304  }
305  }
306  }
307  }
308 
309  if (!done) {
310  // We never found an existing location to place the "history" entry, so we'll just stuff it into the top level AttrTable.
311  BESDEBUG("fonc",
312  "FONcTransmitter::updateHistoryAttribute() - Adding history entry to top level AttrTable" << endl);
313  globals.append_attr("history", "string", &hist_entry_vec);
314 
315  }
316 }
317 
318 #endif
319 
337 {
338  BESDEBUG("fonc", "FONcTransmitter::send_data() - BEGIN" << endl);
339 
340  try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
341  BESDapResponseBuilder responseBuilder;
342  // Use the DDS from the ResponseObject along with the parameters
343  // from the DataHandlerInterface to load the DDS with values.
344  // Note that the BESResponseObject will manage the loaded_dds object's
345  // memory. Make this a shared_ptr<>. jhrg 9/6/16
346 
347  // Now that we are ready to start reading the response data we
348  // cancel any pending timeout alarm according to the configuration.
350 
351  BESDEBUG("fonc", "FONcTransmitter::send_data() - Reading data into DataDDS" << endl);
352  DDS *loaded_dds = responseBuilder.intern_dap2_data(obj, dhi);
353 
354  // ResponseBuilder splits the CE, so use the DHI or make two calls and
355  // glue the result together: responseBuilder.get_btp_func_ce() + " " + responseBuilder.get_ce()
356  // jhrg 9/6/16
357  updateHistoryAttribute(loaded_dds, dhi.data[POST_CONSTRAINT]);
358 
359 #if 0
360  // TODO Make this code and the two struct classes that wrap the name a fd part of
361  // a utility class or file. jhrg 9/7/16
362 
363  string temp_file_name = FONcRequestHandler::temp_dir + "/ncXXXXXX";
364  vector<char> temp_file(temp_file_name.length() + 1);
365  string::size_type len = temp_file_name.copy(&temp_file[0], temp_file_name.length());
366  temp_file[len] = '\0';
367  // cover the case where older versions of mkstemp() create the file using
368  // a mode of 666.
369  mode_t original_mode = umask(077);
370  int fd = mkstemp(&temp_file[0]);
371  umask(original_mode);
372 
373  // Hack: Wrap the name and file descriptors so that the descriptor is closed
374  // and temp file in unlinked no matter how we exit. jhrg 9/7/16
375  // Except if there is a hard crash.. jhrg 3/30/17
376  wrap_temp_name w_temp_file(temp_file);
377  wrap_temp_descriptor w_fd(fd);
378 
379  if (fd == -1) throw BESInternalError("Failed to open the temporary file.", __FILE__, __LINE__);
380 #endif
381  // This object closes the file when it goes out of scope.
382  bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
383 
384  BESDEBUG("fonc", "FONcTransmitter::send_data - Building response file " << temp_file.get_name() << endl);
385  // Note that 'RETURN_CMD' is the same as the string that determines the file type:
386  // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
387  FONcTransform ft(loaded_dds, dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
388  ft.transform();
389 
390  ostream &strm = dhi.get_output_stream();
391  if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
392 
393  BESDEBUG("fonc", "FONcTransmitter::send_data - Transmitting temp file " << temp_file.get_name() << endl);
394 
395  FONcTransmitter::write_temp_file_to_stream(temp_file.get_fd(), strm); //, loaded_dds->filename(), ncVersion);
396  }
397  catch (Error &e) {
398  throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
399  }
400  catch (BESError &e) {
401  throw;
402  }
403  catch (std::exception &e) {
404  throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
405  }
406  catch (...) {
407  throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
408  }
409 
410  BESDEBUG("fonc", "FONcTransmitter::send_data - done transmitting to netcdf" << endl);
411 }
412 
431 {
432  BESDEBUG("fonc", "FONcTransmitter::send_dap4_data() - BEGIN" << endl);
433 
434  try { // Expanded try block so all DAP errors are caught. ndp 12/23/2015
435  BESDapResponseBuilder responseBuilder;
436  // Use the DDS from the ResponseObject along with the parameters
437  // from the DataHandlerInterface to load the DDS with values.
438  // Note that the BESResponseObject will manage the loaded_dds object's
439  // memory. Make this a shared_ptr<>. jhrg 9/6/16
440 
441  // Now that we are ready to start reading the response data we
442  // cancel any pending timeout alarm according to the configuration.
444 
445  BESDEBUG("fonc", "FONcTransmitter::send_dap4_data() - Reading data into DMR" << endl);
446  //DDS *loaded_dds = responseBuilder.intern_dap2_data(obj, dhi);
447  DMR *loaded_dmr = responseBuilder.intern_dap4_data(obj, dhi);
448 
449 #if 0
450 
451  // Iterate through the variables in the DataDDS and read
452  // in the data if the variable has the send flag set.
453  D4Group* root_grp = loaded_dmr->root();
454  Constructor::Vars_iter v = root_grp->var_begin();
455  for (D4Group::Vars_iter i = root_grp->var_begin(), e = root_grp->var_end(); i != e; ++i) {
456  BESDEBUG("fonc", "BESDapResponseBuilder::send_dap4_data() - "<< (*i)->name() <<endl);
457  if ((*i)->send_p()) {
458  (*i)->intern_data();
459  }
460  }
461 #endif
462 
463  // ResponseBuilder splits the CE, so use the DHI or make two calls and
464  // glue the result together: responseBuilder.get_btp_func_ce() + " " + responseBuilder.get_ce()
465  // jhrg 9/6/16
466  //update_Dap4_HistoryAttribute(dmr, dhi.data[POST_CONSTRAINT]);
467 
468 #if 0
469  // TODO Make this code and the two struct classes that wrap the name a fd part of
470  // a utility class or file. jhrg 9/7/16
471 
472  string temp_file_name = FONcRequestHandler::temp_dir + "/ncXXXXXX";
473  vector<char> temp_file(temp_file_name.length() + 1);
474  string::size_type len = temp_file_name.copy(&temp_file[0], temp_file_name.length());
475  temp_file[len] = '\0';
476  // cover the case where older versions of mkstemp() create the file using
477  // a mode of 666.
478  mode_t original_mode = umask(077);
479  int fd = mkstemp(&temp_file[0]);
480  umask(original_mode);
481 
482  // Hack: Wrap the name and file descriptors so that the descriptor is closed
483  // and temp file in unlinked no matter how we exit. jhrg 9/7/16
484  // Except if there is a hard crash.. jhrg 3/30/17
485  wrap_temp_name w_temp_file(temp_file);
486  wrap_temp_descriptor w_fd(fd);
487 
488  if (fd == -1) throw BESInternalError("Failed to open the temporary file.", __FILE__, __LINE__);
489 #endif
490  // This object closes the file when it goes out of scope.
491  bes::TempFile temp_file(FONcRequestHandler::temp_dir + "/ncXXXXXX");
492 
493  BESDEBUG("fonc", "FONcTransmitter::send_dap4_data - Building response file " << temp_file.get_name() << endl);
494  // Note that 'RETURN_CMD' is the same as the string that determines the file type:
495  // netcdf 3 or netcdf 4. Hack. jhrg 9/7/16
496  FONcTransform ft(loaded_dmr, dhi, temp_file.get_name(), dhi.data[RETURN_CMD]);
497 
498  // Call the transform function for DAP4.
499  ft.transform_dap4();
500 
501  ostream &strm = dhi.get_output_stream();
502  if (!strm) throw BESInternalError("Output stream is not set, can not return as", __FILE__, __LINE__);
503 
504  BESDEBUG("fonc", "FONcTransmitter::send_dap4_data - Transmitting temp file " << temp_file.get_name() << endl);
505 
506  FONcTransmitter::write_temp_file_to_stream(temp_file.get_fd(), strm); //, loaded_dds->filename(), ncVersion);
507  }
508  catch (Error &e) {
509  throw BESDapError("Failed to read data: " + e.get_error_message(), false, e.get_error_code(), __FILE__, __LINE__);
510  }
511  catch (BESError &e) {
512  throw;
513  }
514  catch (std::exception &e) {
515  throw BESInternalError("Failed to read data: STL Error: " + string(e.what()), __FILE__, __LINE__);
516  }
517  catch (...) {
518  throw BESInternalError("Failed to get read data: Unknown exception caught", __FILE__, __LINE__);
519  }
520 
521  BESDEBUG("fonc", "FONcTransmitter::send_dap4_data - done transmitting to netcdf" << endl);
522 }
523 
524 
534 void FONcTransmitter::write_temp_file_to_stream(int fd, ostream &strm) //, const string &filename, const string &ncVersion)
535 {
536  char block[OUTPUT_FILE_BLOCK_SIZE];
537 
538  int nbytes = read(fd, block, sizeof block);
539  while (nbytes > 0) {
540  strm.write(block, nbytes /*os.gcount()*/);
541  nbytes = read(fd, block, sizeof block);
542  }
543 }
544 
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual libdap::DMR * intern_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
virtual libdap::DDS * intern_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
Abstract base class representing a specific set of information in response to a request to the BES.
static bool endsWith(std::string const &fullString, std::string const &ending)
Definition: BESUtil.cc:942
static void conditional_timeout_cancel()
Definition: BESUtil.cc:967
Transformation object that converts an OPeNDAP DataDDS to a netcdf file.
Definition: FONcTransform.h:60
virtual void transform()
Transforms each of the variables of the DataDDS to the NetCDF file.
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
static void send_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
static void send_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
The static method registered to transmit OPeNDAP data objects as a netcdf file.
FONcTransmitter()
Construct the FONcTransmitter, adding it with name netcdf to be able to transmit a data response.
Get a new temporary file.
Definition: TempFile.h:46