bes  Updated for version 3.20.8
NCMLUtil.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 
30 #include "config.h"
31 
32 #include <ctype.h>
33 
34 #include <Array.h>
35 #include "Constructor.h"
36 #include "DAS.h"
37 #include "DDS.h"
38 #include "Grid.h"
39 #include <DataDDS.h>
40 #include <AttrTable.h>
41 
42 #include "BESDapResponse.h"
43 #include "BESDataDDSResponse.h"
44 #include "BESDDSResponse.h"
45 #include "BESDebug.h"
46 #include "BESInternalError.h"
47 
48 #include "NCMLUtil.h"
49 #include "NCMLDebug.h"
50 
51 using namespace libdap;
52 using namespace std;
53 
54 namespace ncml_module {
55 
56 const std::string NCMLUtil::WHITESPACE = " \t\n";
57 
58 int NCMLUtil::tokenize(const string& str, vector<string>& tokens, const string& delimiters)
59 {
60  BESDEBUG("ncml", "NCMLUtil::tokenize value of str:" << str << endl);
61 
62  // start empty
63  tokens.resize(0);
64  // Skip delimiters at beginning.
65  string::size_type lastPos = str.find_first_not_of(delimiters, 0);
66  // Find first "non-delimiter".
67  string::size_type pos = str.find_first_of(delimiters, lastPos);
68 
69  int count = 0; // how many we added.
70  while (string::npos != pos || string::npos != lastPos) {
71  // Found a token, add it to the vector.
72  tokens.push_back(str.substr(lastPos, pos - lastPos));
73  count++;
74  // Skip delimiters. Note the "not_of"
75  lastPos = str.find_first_not_of(delimiters, pos);
76  // Find next "non-delimiter"
77  pos = str.find_first_of(delimiters, lastPos);
78  }
79  return count;
80 }
81 
82 int NCMLUtil::tokenizeChars(const string& str, vector<string>& tokens)
83 {
84  tokens.resize(0);
85  // push each char as a token
86  for (unsigned int i = 0; i < str.size(); ++i) {
87  string val = "";
88  val += str[i];
89  tokens.push_back(val);
90  }
91  return str.size();
92 }
93 
94 bool NCMLUtil::isAscii(const string& str)
95 {
96  string::const_iterator endIt = str.end();
97  for (string::const_iterator it = str.begin(); it != endIt; ++it) {
98  if (!isascii(*it)) {
99  return false;
100  }
101  }
102  return true;
103 }
104 
105 bool NCMLUtil::isAllWhitespace(const string& str)
106 {
107  return (str.find_first_not_of(" \t\n") == string::npos);
108 }
109 
110 void NCMLUtil::trimLeft(std::string& input, const std::string& trimChars /* = WHITESPACE */)
111 {
112  size_t firstValid = input.find_first_not_of(trimChars);
113  input.erase(0, firstValid);
114 }
115 
119 void NCMLUtil::trimRight(std::string& input, const std::string& trimChars /* = WHITESPACE */)
120 {
121  size_t lastValid = input.find_last_not_of(trimChars);
122  if (lastValid != string::npos) {
123  input.erase(lastValid + 1, string::npos);
124  }
125 }
126 
127 void NCMLUtil::trimAll(std::vector<std::string>& tokens, const std::string& trimChars /* = WHITESPACE */)
128 {
129  unsigned int num = tokens.size();
130  for (unsigned int i = 0; i < num; ++i) {
131  trim(tokens[i], trimChars);
132  }
133 }
134 
135 bool NCMLUtil::toUnsignedInt(const std::string& stringVal, unsigned int& oVal)
136 {
137  bool success = true;
138  oVal = 0;
139  istringstream iss(stringVal);
140  iss >> oVal;
141  if (iss.fail() || (stringVal[0] == '-') // parsing negatives is locale-dependent, but we DO NOT want them allowed.
142  ) {
143  success = false;
144  }
145  return success;
146 }
147 
148 #if 0
159 static bool
160 has_dap2_attributes(AttrTable &a)
161 {
162  for (AttrTable::Attr_iter i = a.attr_begin(), e = a.attr_end(); i != e; ++i) {
163  if (a.get_attr_type(i) != Attr_container) {
164  return true;
165  }
166  else if (has_dap2_attributes(*a.get_attr_table(i))) {
167  return true;
168  }
169  }
170 
171  return false;
172 }
173 
183 static bool
184 has_dap2_attributes(BaseType *btp)
185 {
186  if (btp->get_attr_table().get_size() && has_dap2_attributes(btp->get_attr_table())) {
187  return true;
188  }
189 
190  Constructor *cons = dynamic_cast<Constructor *>(btp);
191  if (cons) {
192  Grid* grid = dynamic_cast<Grid*>(btp);
193  if(grid) {
194  return has_dap2_attributes(grid->get_array());
195  }
196  else {
197  for (Constructor::Vars_iter i = cons->var_begin(), e = cons->var_end(); i != e; i++) {
198  if (has_dap2_attributes(*i)) return true;
199  }
200  }
201  }
202  return false;
203 }
204 #endif
205 
206 
211 static void populateAttrTableForContainerVariableRecursive(AttrTable* dasTable, Constructor* consVar)
212 {
213  VALID_PTR(dasTable);
214  VALID_PTR(consVar);
215 
216  if(!has_dap2_attributes(consVar))
217  return;
218 
219 
220  Grid* grid = dynamic_cast<Grid*>(consVar);
221  if(grid){
222  // Here we take the Attributes from the Grid Array variable and copy them into the DAS container for the Grid.
223  // This essentially flattens the Grid in the DAS. Note too that we do now pursue the MAP vectors so they
224  // do not appear in the DAS container for the Grid.
225  BESDEBUG("ncml", __func__ << "() The variable " << grid->name() << " is a Grid. So, we promote the Grid Array AttrTable content to the DAS container for Grid " << grid->name() << endl);
226  Array *gArray = grid->get_array();
227  AttrTable arrayAT = gArray->get_attr_table();
228  for( AttrTable::Attr_iter atIter = arrayAT.attr_begin(); atIter!=arrayAT.attr_end(); ++atIter){
229  AttrType type = arrayAT.get_attr_type(atIter);
230  string childName = arrayAT.get_name(atIter);
231  if (type == Attr_container){
232  BESDEBUG("ncml", __func__ << "() Adding child AttrTable '" << childName << "' to Grid " << grid->name() << endl);
233  dasTable->append_container( new AttrTable(*arrayAT.get_attr_table(atIter)), childName);
234  }
235  else {
236  vector<string>* pAttrTokens = arrayAT.get_attr_vector(atIter);
237  // append_attr makes a copy of the vector, so we don't have to do so here.
238  BESDEBUG("ncml", __func__ << "() Adding child Attrbute '" << childName << "' to Grid " << grid->name() << endl);
239  dasTable->append_attr(childName, AttrType_to_String(type), pAttrTokens);
240  }
241  }
242  }
243  else {
244  // It's not a Grid but it's still a Constructor.
245  BESDEBUG("ncml", __func__ << "() Adding attribute tables for children of a Constructor type variable " << consVar->name() << endl);
246  Constructor::Vars_iter endIt = consVar->var_end();
247  for (Constructor::Vars_iter it = consVar->var_begin(); it != endIt; ++it) {
248  BaseType* var = *it;
249  VALID_PTR(var);
250 
251  if(has_dap2_attributes(var)){
252  BESDEBUG("ncml", __func__ << "() Adding attribute table for var: " << var->name() << endl);
253  // Make a new table for the child variable
254  AttrTable* newTable = new AttrTable(var->get_attr_table());
255  // Add it to the DAS's attribute table for the consVar scope.
256  dasTable->append_container(newTable, var->name());
257 
258  // If it's a container type, we need to recurse.
259  if (var->is_constructor_type()) {
260  Constructor* child = dynamic_cast<Constructor*>(var);
261  if (!child) {
262  throw BESInternalError("Type cast error", __FILE__, __LINE__);
263  }
264  BESDEBUG("ncml", __func__ << "() Var " << child->name() << " is Constructor type, recursively adding attribute tables" << endl);
265  populateAttrTableForContainerVariableRecursive(newTable, child);
266  }
267  }
268  else {
269  BESDEBUG("ncml", __func__ << "() Variable '" << var->name() << "' has no dap2 attributes,. Skipping."<< endl);
270  }
271  }
272  }
273 }
274 
275 // This is basically the opposite of transfer_attributes.
276 void NCMLUtil::populateDASFromDDS(DAS* das, const DDS& dds_const)
277 {
278  BESDEBUG("ncml", "Populating a DAS from a DDS...." << endl);
279 
280  VALID_PTR(das);
281 
282  // Make sure the DAS is empty to start.
283  das->erase();
284 
285  // dds is semantically const in this function, but the calls to it aren't...
286  DDS& dds = const_cast<DDS&>(dds_const);
287 
288  // First, make sure we don't have a container at top level since we're assuming for now
289  // that we only have one dataset per call (right?)
290  if (dds.container()) {
291  BESDEBUG("ncml", __func__ << "() Got unexpected container " << dds.container_name() << " and is failing." << endl);
292  throw BESInternalError("Unexpected Container Error creating DAS from DDS in NCMLHandler", __FILE__, __LINE__);
293  }
294 
295  // Copy over the global attributes table
296  //BESDEBUG("ncml", "Coping global attribute tables from DDS to DAS..." << endl);
297  *(das->get_top_level_attributes()) = dds.get_attr_table();
298 
299  // For each variable in the DDS, make a table in the DAS.
300  // If the variable in composite, then recurse
301  // BESDEBUG("ncml", "Adding attribute tables for all DDS variables into DAS recursively..." << endl);
302  DDS::Vars_iter endIt = dds.var_end();
303  for (DDS::Vars_iter it = dds.var_begin(); it != endIt; ++it) {
304  // For each BaseType*, copy its table and add to DAS under its name.
305  BaseType* var = *it;
306  VALID_PTR(var);
307 
308  // By adding this test we stop adding empty top=level containers to the DAS.
309  if(has_dap2_attributes(var)){
310  BESDEBUG("ncml", "Adding attribute table for variable: " << var->name() << endl);
311  AttrTable* clonedVarTable = new AttrTable(var->get_attr_table());
312  VALID_PTR(clonedVarTable);
313  das->add_table(var->name(), clonedVarTable);
314 
315  // If it's a container type, we need to recurse.
316  if (var->is_constructor_type()) {
317  Constructor* consVar = dynamic_cast<Constructor*>(var);
318  if (!consVar) {
319  throw BESInternalError("Type cast error", __FILE__, __LINE__);
320  }
321  populateAttrTableForContainerVariableRecursive(clonedVarTable, consVar);
322  }
323  }
324  else {
325  BESDEBUG("ncml", __func__ << "() Variable '" << var->name() << "' has no dap2 attributes,. Skipping."<< endl);
326  }
327  }
328 }
329 
330 // This function was added since DDS::operator= had some bugs we need to fix.
331 // At that point, we can just use that function, probably.
332 void NCMLUtil::copyVariablesAndAttributesInto(DDS* dds_out, const DDS& dds_in)
333 {
334  VALID_PTR(dds_out);
335 
336  // Avoid obvious bugs
337  if (dds_out == &dds_in) {
338  return;
339  }
340 
341  // handle semantic constness
342  DDS& dds = const_cast<DDS&>(dds_in);
343 
344  // Copy the global attribute table
345  dds_out->get_attr_table() = dds.get_attr_table();
346 
347  // copy the things pointed to by the variable list, not just the pointers
348  // add_var is designed to deepcopy *i, so this should get all the children
349  // as well.
350  for (DDS::Vars_iter i = dds.var_begin(); i != dds.var_end(); ++i) {
351  dds_out->add_var(*i); // add_var() dups the BaseType.
352  }
353 }
354 
355 libdap::DDS*
356 NCMLUtil::getDDSFromEitherResponse(BESDapResponse* response)
357 {
358  DDS* pDDS = 0;
359  BESDDSResponse* pDDXResponse = dynamic_cast<BESDDSResponse*>(response);
360  BESDataDDSResponse* pDataDDSResponse = dynamic_cast<BESDataDDSResponse*>(response);
361 
362  if (pDDXResponse) {
363  pDDS = pDDXResponse->get_dds();
364  }
365  else if (pDataDDSResponse) {
366  pDDS = pDataDDSResponse->get_dds(); // return as superclass ptr
367  }
368  else {
369  pDDS = 0; // return null on error
370  }
371 
372  return pDDS;
373 }
374 
375 // This little gem takes attributes that have been added to the top level
376 // attribute table (which is allowed in DAP4) and moves them all to a single
377 // container. In DAP2, only containers are allowed at the top level of the
378 // DAS. By _convention_ the name of the global attributes is NC_GLOBAL although
379 // other names are equally valid...
380 //
381 // How this works: The top-level attribute table is filled with various global
382 // attributes. To follow the spec for DAP2 that top-level container must contain
383 // _only_ other containers, each of which must be named. There are four cases...
384 //
385 // jhrg 12/15/11
386 void NCMLUtil::hackGlobalAttributesForDAP2(libdap::AttrTable &global_attributes,
387  const std::string &global_container_name)
388 {
389  if (global_container_name.empty()) return;
390 
391  // Cases: 1. only containers at the top --> return
392  // 2. only simple attrs at the top --> move them into one container
393  // 3. mixture of simple and containers --> move the simples into a new container
394  // 4. mixture ... and global_container_name exists --> move simples into that container
395 
396  // Look at the top-level container and see if it has any simple attributes.
397  // If it is empty or has only containers, do nothing.
398  bool simple_attribute_found = false;
399  AttrTable::Attr_iter i = global_attributes.attr_begin();
400  while (!simple_attribute_found && i != global_attributes.attr_end()) {
401  if (!global_attributes.is_container(i)) simple_attribute_found = true;
402  ++i;
403  }
404 
405  // Case 1
406  if (!simple_attribute_found) return;
407 #if 0
408  // Now determine if there are _only_ simple attributes
409  bool only_simple_attributes = true;
410  i = global_attributes.attr_begin();
411  while (only_simple_attributes && i != global_attributes.attr_end()) {
412  if (global_attributes.is_container(i))
413  only_simple_attributes = false;
414  ++i;
415  }
416 
417  // Case 2
418  // Note that the assignment operator first clears the destination and
419  // then performs a deep copy, so the 'new_global_attr_container' will completely
420  // replace the existing collection of attributes at the top-level.
421  if (only_simple_attributes)
422  {
423  AttrTable *new_global_attr_container = new AttrTable();
424  AttrTable *new_attr_container = new_global_attr_container->append_container(global_container_name);
425  *new_attr_container = global_attributes;
426  global_attributes = *new_global_attr_container;
427 
428  return;
429  }
430 #endif
431  // Cases 2, 3 & 4
432  AttrTable *new_attr_container = global_attributes.find_container(global_container_name);
433  if (!new_attr_container) new_attr_container = global_attributes.append_container(global_container_name);
434 
435  // Now we have a destination for all the simple attributes
436  i = global_attributes.attr_begin();
437  while (i != global_attributes.attr_end()) {
438  if (!global_attributes.is_container(i)) {
439  new_attr_container->append_attr(global_attributes.get_name(i), global_attributes.get_type(i),
440  global_attributes.get_attr_vector(i));
441  }
442  ++i;
443  }
444 
445  // Now delete the simple attributes we just moved; they are not deleted in the
446  // above loop because deleting things in a container invalidates iterators
447  i = global_attributes.attr_begin();
448  while (i != global_attributes.attr_end()) {
449  if (!global_attributes.is_container(i)) {
450  global_attributes.del_attr(global_attributes.get_name(i));
451  // delete invalidates iterators; must restart the loop
452  i = global_attributes.attr_begin();
453  }
454  else {
455  ++i;
456  }
457  }
458 
459  return;
460 }
461 
462 void NCMLUtil::setVariableNameProperly(libdap::BaseType* pVar, const std::string& name)
463 {
464  VALID_PTR(pVar);
465  pVar->set_name(name);
466  // if template, set it too since it's used to print dds...
467  BaseType* pTemplate = pVar->var();
468  if (pTemplate) {
469  pTemplate->set_name(name);
470  }
471 }
472 } // namespace ncml_module
Holds a DDS object within the BES.
libdap::DDS * get_dds()
Represents an OPeNDAP DAP response object within the BES.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
exception thrown if internal error encountered
NcML Parser for adding/modifying/removing metadata (attributes) to existing local datasets using NcML...