bes  Updated for version 3.20.8
FONcTransform.cc
1 // FONcTransform.cc
2 
3 // This file is part of BES Netcdf File Out Module
4 
5 // Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact University Corporation for Atmospheric Research at
23 // 3080 Center Green Drive, Boulder, CO 80301
24 
25 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
27 //
28 // Authors:
29 // pwest Patrick West <pwest@ucar.edu>
30 // jgarcia Jose Garcia <jgarcia@ucar.edu>
31 // kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32 
33 #include "config.h"
34 
35 #include <sstream>
36 
37 using std::ostringstream;
38 using std::istringstream;
39 
40 #include "FONcRequestHandler.h" // for the keys
41 
42 #include "FONcTransform.h"
43 #include "FONcUtils.h"
44 #include "FONcBaseType.h"
45 #include "FONcAttributes.h"
46 
47 #include <DDS.h>
48 #include <DMR.h>
49 #include <D4Group.h>
50 #include <D4Attributes.h>
51 #include <Structure.h>
52 #include <Array.h>
53 #include <Grid.h>
54 #include <Sequence.h>
55 #include <BESDebug.h>
56 #include <BESInternalError.h>
57 
58 #include "DapFunctionUtils.h"
59 
71 FONcTransform::FONcTransform(DDS *dds, BESDataHandlerInterface &dhi, const string &localfile, const string &ncVersion) :
72  _ncid(0), _dds(0)
73 {
74  if (!dds) {
75  string s = (string) "File out netcdf, " + "null DDS passed to constructor";
76  throw BESInternalError(s, __FILE__, __LINE__);
77  }
78  if (localfile.empty()) {
79  string s = (string) "File out netcdf, " + "empty local file name passed to constructor";
80  throw BESInternalError(s, __FILE__, __LINE__);
81  }
82  _localfile = localfile;
83  _dds = dds;
84  _returnAs = ncVersion;
85 
86  // if there is a variable, attribute, dimension name that is not
87  // compliant with netcdf naming conventions then we will create
88  // a new name. If the new name does not begin with an alpha
89  // character then we will prefix it with name_prefix. We will
90  // get this prefix from the type of data that we are reading in,
91  // such as nc, h4, h5, ff, jg, etc...
92  dhi.first_container();
93  if (dhi.container) {
95  }
96  else {
97  FONcUtils::name_prefix = "nc_";
98  }
99 }
111 FONcTransform::FONcTransform(DMR *dmr, BESDataHandlerInterface &dhi, const string &localfile, const string &ncVersion) :
112  _ncid(0), _dmr(0)
113 {
114  if (!dmr) {
115  string s = (string) "File out netcdf, " + "null DDS passed to constructor";
116  throw BESInternalError(s, __FILE__, __LINE__);
117  }
118  if (localfile.empty()) {
119  string s = (string) "File out netcdf, " + "empty local file name passed to constructor";
120  throw BESInternalError(s, __FILE__, __LINE__);
121  }
122  _localfile = localfile;
123  _dmr = dmr;
124  _returnAs = ncVersion;
125 
126  // if there is a variable, attribute, dimension name that is not
127  // compliant with netcdf naming conventions then we will create
128  // a new name. If the new name does not begin with an alpha
129  // character then we will prefix it with name_prefix. We will
130  // get this prefix from the type of data that we are reading in,
131  // such as nc, h4, h5, ff, jg, etc...
132  dhi.first_container();
133  if (dhi.container) {
135  }
136  else {
137  FONcUtils::name_prefix = "nc_";
138  }
139 }
140 
141 
147 {
148  bool done = false;
149  while (!done) {
150  vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
151  vector<FONcBaseType *>::iterator e = _fonc_vars.end();
152  if (i == e) {
153  done = true;
154  }
155  else {
156  // These are the FONc types, not the actual ones
157  FONcBaseType *b = (*i);
158  delete b;
159  _fonc_vars.erase(i);
160  }
161  }
162  done = false;
163  while (!done) {
164  vector<FONcBaseType *>::iterator i = _total_fonc_vars_in_grp.begin();
165  vector<FONcBaseType *>::iterator e = _total_fonc_vars_in_grp.end();
166  if (i == e) {
167  done = true;
168  }
169  else {
170  // These are the FONc types, not the actual ones
171  FONcBaseType *b = (*i);
172  delete b;
173  _total_fonc_vars_in_grp.erase(i);
174  }
175  }
176 
177 }
178 
188 {
190 
191  // Convert the DDS into an internal format to keep track of
192  // variables, arrays, shared dimensions, grids, common maps,
193  // embedded structures. It only grabs the variables that are to be
194  // sent.
195  DDS::Vars_iter vi = _dds->var_begin();
196  DDS::Vars_iter ve = _dds->var_end();
197  for (; vi != ve; vi++) {
198  if ((*vi)->send_p()) {
199  BaseType *v = *vi;
200 
201  BESDEBUG("fonc", "FONcTransform::transform() - Converting variable '" << v->name() << "'" << endl);
202 
203  // This is a factory class call, and 'fg' is specialized for 'v'
204  FONcBaseType *fb = FONcUtils::convert(v,FONcTransform::_returnAs,FONcRequestHandler::classic_model);
205 #if 0
206  fb->setVersion( FONcTransform::_returnAs );
207  if ( FONcTransform::_returnAs == RETURNAS_NETCDF4 ) {
208  if (FONcRequestHandler::classic_model)
209  fb->setNC4DataModel("NC4_CLASSIC_MODEL");
210  else
211  fb->setNC4DataModel("NC4_ENHANCED");
212  }
213 #endif
214  _fonc_vars.push_back(fb);
215 
216  vector<string> embed;
217  fb->convert(embed);
218  }
219  }
220 
221  // Open the file for writing
222  int stax;
223  if ( FONcTransform::_returnAs == RETURNAS_NETCDF4 ) {
224  if (FONcRequestHandler::classic_model){
225  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-4 cache file in classic mode. fileName: " << _localfile << endl);
226  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &_ncid);
227  }
228  else {
229  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
230  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4, &_ncid);
231  }
232  }
233  else {
234  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-3 cache file. fileName: " << _localfile << endl);
235  stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
236  }
237 
238  if (stax != NC_NOERR) {
239  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
240  }
241 
242  try {
243  // Here we will be defining the variables of the netcdf and
244  // adding attributes. To do this we must be in define mode.
245  nc_redef(_ncid);
246 
247  // For each converted FONc object, call define on it to define
248  // that object to the netcdf file. This also adds the attributes
249  // for the variables to the netcdf file
250  vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
251  vector<FONcBaseType *>::iterator e = _fonc_vars.end();
252  for (; i != e; i++) {
253  FONcBaseType *fbt = *i;
254  BESDEBUG("fonc", "FONcTransform::transform() - Defining variable: " << fbt->name() << endl);
255  fbt->define(_ncid);
256  }
257 
258  if(FONcRequestHandler::no_global_attrs == false) {
259  // Add any global attributes to the netcdf file
260  AttrTable &globals = _dds->get_attr_table();
261  BESDEBUG("fonc", "FONcTransform::transform() - Adding Global Attributes" << endl << globals << endl);
262  bool is_netCDF_enhanced = false;
263  if(FONcTransform::_returnAs == RETURNAS_NETCDF4 && FONcRequestHandler::classic_model==false)
264  is_netCDF_enhanced = true;
265  FONcAttributes::add_attributes(_ncid, NC_GLOBAL, globals, "", "",is_netCDF_enhanced);
266  }
267 
268  // We are done defining the variables, dimensions, and
269  // attributes of the netcdf file. End the define mode.
270  int stax = nc_enddef(_ncid);
271 
272  // Check error for nc_enddef. Handling of HDF failures
273  // can be detected here rather than later. KY 2012-10-25
274  if (stax != NC_NOERR) {
275  FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__, __LINE__);
276  }
277 
278  // Write everything out
279  i = _fonc_vars.begin();
280  e = _fonc_vars.end();
281  for (; i != e; i++) {
282  FONcBaseType *fbt = *i;
283  BESDEBUG("fonc", "FONcTransform::transform() - Writing data for variable: " << fbt->name() << endl);
284  fbt->write(_ncid);
285  }
286 
287  stax = nc_close(_ncid);
288  if (stax != NC_NOERR)
289  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
290  }
291  catch (BESError &e) {
292  (void) nc_close(_ncid); // ignore the error at this point
293  throw;
294  }
295 }
296 
306 {
308 
309  // Convert the DMR into an internal format to keep track of
310  // variables, arrays, shared dimensions, grids, common maps,
311  // embedded structures. It only grabs the variables that are to be
312  // sent.
313 
314  BESDEBUG("fonc", "Coming into transform_dap4() "<< endl);
315 
316  // First check if this DMR has groups etc.
317  bool support_group = check_group_support();
318 
319  if(true == support_group) {
320 
321  int stax = -1;
322  BESDEBUG("fonc", "FONcTransform::transform_dap4() - Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
323  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4, &_ncid);
324  if (stax != NC_NOERR)
325  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
326 
327  D4Group* root_grp = _dmr->root();
328 
329  // Declare the dimname to dimid map to handle netCDF-4 dimensions
330  map<string,int>fdimname_to_id;
331 
332  // Generate a list of the groups in the final netCDF file.
333  // The attributes of these groups should be included.
334  gen_included_grp_list(root_grp);
335 #if 0
336  for (std::set<string>::iterator it=_included_grp_names.begin(); it!=_included_grp_names.end(); ++it)
337  BESDEBUG("fonc","included group list name is: "<<*it<<endl);
338 #endif
339  // Build a global dimension name table for all variables if
340  // the constraint is not empty!
341  check_and_obtain_dimensions(root_grp,true);
342 
343  // Don't remove the following code, they are for debugging.
344 #if 0
345  map<string,unsigned long>:: iterator it;
346 
347  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
348  BESDEBUG("fonc", "Final GFQN dim name is: "<<it->first<<endl);
349  BESDEBUG("fonc", "Final GFQN dim size is: "<<it->second<<endl);
350  }
351 
352  for(it=VFQN_dimname_to_dimsize.begin();it!=VFQN_dimname_to_dimsize.end();++it) {
353  BESDEBUG("fonc", "Final VFQN dim name is: "<<it->first<<endl);
354  BESDEBUG("fonc", "Final VFQN dim size is: "<<it->second<<endl);
355  }
356 #endif
357 
358  // DAP4 requires the DAP4 dimension sizes defined in the group should be changed
359  // according to the corresponding variable sizes. Check section 8.6.2 at
360  // https://docs.opendap.org/index.php/DAP4:_Specification_Volume_1
361  //
362  map<string,unsigned long>:: iterator git,vit;
363  for(git=GFQN_dimname_to_dimsize.begin();git!=GFQN_dimname_to_dimsize.end();++git) {
364  for(vit=VFQN_dimname_to_dimsize.begin();vit!=VFQN_dimname_to_dimsize.end();++vit) {
365  if(git->first == vit->first) {
366  if(git->second != vit->second)
367  git->second = vit->second;
368  break;
369  }
370  }
371  }
372 
373  // Thie part of code is to address the possible dimension name confliction
374  // when variables in the constraint don't have dimension names. Fileout netCDF
375  // adds the fake dimensions such as dim1, dim2...to these variables.
376  // If these dimension names are used by
377  // the file to be handled, the dimension confliction will corrupt the final output.
378  // The idea is to find if there are any dimension names like dim1, dim2 ...
379  // under the root group.
380  // We will remember them and not use these names as fake dimension names.
381  //
382  // Obtain the dim. names under the root group
383  vector<string> root_d4_dimname_list;
384  for(git=GFQN_dimname_to_dimsize.begin();git!=GFQN_dimname_to_dimsize.end();++git) {
385  string d4_temp_dimname = git->first.substr(1);
386  //BESDEBUG("fonc", "d4_temp_dimname: "<<d4_temp_dimname<<endl);
387  if(d4_temp_dimname.find('/')==string::npos)
388  root_d4_dimname_list.push_back(d4_temp_dimname);
389  }
390 
391 #if 0
392  for(unsigned int i = 0; i <root_d4_dimname_list.size();i++)
393  BESDEBUG("fonc", "root_d4 dim name is: "<<root_d4_dimname_list[i]<<endl);
394 #endif
395 
396  // Only remember the root dimension names that are like "dim1,dim2,..."
397  vector<int> root_dim_suffix_nums;
398  for(unsigned int i = 0; i <root_d4_dimname_list.size();i++){
399  if(root_d4_dimname_list[i].size()<4)
400  continue;
401  else if(root_d4_dimname_list[i].substr(0,3)!="dim")
402  continue;
403  else {
404  string temp_suffix = root_d4_dimname_list[i].substr(3);
405  //BESDEBUG("fonc", "temp_suffix: "<<temp_suffix<<endl);
406  bool ignored_suffix = false;
407  for (unsigned int j = 0; j<temp_suffix.size();j++) {
408  if(!isdigit(temp_suffix[j])) {
409  ignored_suffix = true;
410  break;
411  }
412  }
413  if(ignored_suffix==true)
414  continue;
415  else
416  root_dim_suffix_nums.push_back(atoi(temp_suffix.c_str()));
417  }
418  }
419 
420 #if 0
421  for(unsigned int i = 0; i <root_dim_suffix_nums.size();i++)
422  BESDEBUG("fonc", "root_dim_suffix_nums: "<<root_dim_suffix_nums[i]<<endl);
423 
424 
425  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
426  BESDEBUG("fonc", "RFinal GFQN dim name is: "<<it->first<<endl);
427  BESDEBUG("fonc", "RFinal GFQN dim size is: "<<it->second<<endl);
428  }
429 
430  for(it=VFQN_dimname_to_dimsize.begin();it!=VFQN_dimname_to_dimsize.end();++it) {
431  BESDEBUG("fonc", "RFinal VFQN dim name is: "<<it->first<<endl);
432  BESDEBUG("fonc", "RFinal VFQN dim size is: "<<it->second<<endl);
433  }
434 #endif
435 
436  // Now we transform all the objects(including groups) to netCDF-4
437  transform_dap4_group(root_grp,true,_ncid,fdimname_to_id,root_dim_suffix_nums);
438  stax = nc_close(_ncid);
439  if (stax != NC_NOERR)
440  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
441 
442  }
443  else // No group, handle as the classic way
444  transform_dap4_no_group();
445 
446  return;
447 
448 }
449 
450 // Transform the DMR to a netCDF-4 file when there are no DAP4 groups.
451 // This routine is similar to transform() that handles DAP2 objects. However, DAP4 routines are needed.
452 // So still keep a separate function. May combine this function with the tranform() in the future.
453 void FONcTransform::transform_dap4_no_group() {
454 
455  D4Group* root_grp = _dmr->root();
456 
457 #if 0
458  D4Dimensions *root_dims = root_grp->dims();
459  for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
460  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
461  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
462  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
463  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
464  //cout <<"dim size is: "<<(*di)->size()<<endl;
465  //cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
466  }
467 #endif
468  Constructor::Vars_iter vi = root_grp->var_begin();
469  Constructor::Vars_iter ve = root_grp->var_end();
470 
471  for (; vi != ve; vi++) {
472  if ((*vi)->send_p()) {
473  BaseType *v = *vi;
474 
475  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Converting variable '" << v->name() << "'" << endl);
476 
477  // This is a factory class call, and 'fg' is specialized for 'v'
478  FONcBaseType *fb = FONcUtils::convert(v,FONcTransform::_returnAs,FONcRequestHandler::classic_model);
479  _fonc_vars.push_back(fb);
480 
481  vector<string> embed;
482  fb->convert(embed);
483  }
484  }
485 
486 #if 0
487  if(root_grp->grp_begin() == root_grp->grp_end())
488  BESDEBUG("fonc", "FONcTransform::transform_dap4() - No group " << endl);
489  else
490  BESDEBUG("fonc", "FONcTransform::transform_dap4() - has group " << endl);
491  for (D4Group::groupsIter gi = root_grp->grp_begin(), ge = root_grp->grp_end(); gi != ge; ++gi)
492  BESDEBUG("fonc", "FONcTransform::transform_dap4() - group name: " << (*gi)->name() << endl);
493 #endif
494 
495  // Open the file for writing
496  int stax = -1;
497  if ( FONcTransform::_returnAs == RETURNAS_NETCDF4 ) {
498  if (FONcRequestHandler::classic_model){
499  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Opening NetCDF-4 cache file in classic mode. fileName: " << _localfile << endl);
500  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &_ncid);
501  }
502  else {
503  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
504  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4, &_ncid);
505  }
506  }
507  else {
508  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Opening NetCDF-3 cache file. fileName: " << _localfile << endl);
509  stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
510  }
511 
512  if (stax != NC_NOERR) {
513  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
514  }
515 
516  try {
517  // Here we will be defining the variables of the netcdf and
518  // adding attributes. To do this we must be in define mode.
519  nc_redef(_ncid);
520 
521  // For each converted FONc object, call define on it to define
522  // that object to the netcdf file. This also adds the attributes
523  // for the variables to the netcdf file
524  vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
525  vector<FONcBaseType *>::iterator e = _fonc_vars.end();
526  for (; i != e; i++) {
527  FONcBaseType *fbt = *i;
528  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Defining variable: " << fbt->name() << endl);
529  fbt->set_is_dap4(true);
530  fbt->define(_ncid);
531  }
532 
533  if(FONcRequestHandler::no_global_attrs == false) {
534 
535  // Add any global attributes to the netcdf file
536  D4Group* root_grp=_dmr->root();
537  D4Attributes*d4_attrs = root_grp->attributes();
538 
539  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() handle GLOBAL DAP4 attributes "<< d4_attrs <<endl);
540 #if 0
541  for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
542  string name = (*ii)->name();
543  BESDEBUG("fonc", "FONcTransform::transform_dap4() GLOBAL attribute name is "<<name <<endl);
544  }
545 #endif
546  bool is_netCDF_enhanced = false;
547  if(FONcTransform::_returnAs == RETURNAS_NETCDF4 && FONcRequestHandler::classic_model==false)
548  is_netCDF_enhanced = true;
549  FONcAttributes::add_dap4_attributes(_ncid, NC_GLOBAL, d4_attrs, "", "",is_netCDF_enhanced);
550  }
551 
552  // We are done defining the variables, dimensions, and
553  // attributes of the netcdf file. End the define mode.
554  int stax = nc_enddef(_ncid);
555 
556  // Check error for nc_enddef. Handling of HDF failures
557  // can be detected here rather than later. KY 2012-10-25
558  if (stax != NC_NOERR) {
559  FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__, __LINE__);
560  }
561 
562  // Write everything out
563  i = _fonc_vars.begin();
564  e = _fonc_vars.end();
565  for (; i != e; i++) {
566  FONcBaseType *fbt = *i;
567  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Writing data for variable: " << fbt->name() << endl);
568  fbt->write(_ncid);
569  }
570 
571  stax = nc_close(_ncid);
572  if (stax != NC_NOERR)
573  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
574  }
575  catch (BESError &e) {
576  (void) nc_close(_ncid); // ignore the error at this point
577  throw;
578  }
579 
580 }
581 
582 // Transform the DMR to a netCDF-4 file when there are DAP4 groups.
583 void FONcTransform::transform_dap4_group(D4Group* grp,
584  bool is_root_grp,
585  int par_grp_id,map<string,int>&fdimname_to_id,
586  vector<int>&root_dim_suffix_nums ) {
587 
588  bool included_grp = false;
589 
590  // Always include the root and its attributes.
591  if(is_root_grp == true)
592  included_grp = true;
593  else {
594  // Check if this group is in the group list kept in the file.
595  set<string>::iterator iset;
596  if(_included_grp_names.find(grp->FQN())!=_included_grp_names.end())
597  included_grp = true;
598  }
599 
600  // Call the internal routine to transform the DMR that has groups if this group is in the group list..
601  // If this group is not in the group list, we know all its subgroups are also not in the list, just stop and return.
602  if(included_grp == true)
603  transform_dap4_group_internal(grp,is_root_grp,par_grp_id,fdimname_to_id,root_dim_suffix_nums);
604  return;
605 }
606 
607 // The internal routine to transform DMR to netCDF-4 when there are gorups.
608 void FONcTransform::transform_dap4_group_internal(D4Group* grp,
609  bool is_root_grp,
610  int par_grp_id,map<string,int>&fdimname_to_id,
611  vector<int>& rds_nums ) {
612 
613  int grp_id = -1;
614  int stax = -1;
615  if(is_root_grp == true)
616  grp_id = _ncid;
617  else {
618  stax = nc_def_grp(par_grp_id,(*grp).name().c_str(),&grp_id);
619  if (stax != NC_NOERR)
620  FONcUtils::handle_error(stax, "File out netcdf, unable to define group: " + _localfile, __FILE__, __LINE__);
621 
622  }
623 
624  D4Dimensions *grp_dims = grp->dims();
625  for(D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
626 #if 0
627  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
628  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
629  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
630  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
631 #endif
632 
633 #if 0
634  unsigned long dimsize = (*di)->size();
635  if((*di)->constrained()) {
636  dimsize = ((*di)->c_stop() -(*di)->c_start())/(*di)->c_stride() +1;
637 
638  }
639 #endif
640  unsigned long dimsize =(*di)->size();
641 
642  // The dimension size may need to be updated because of the expression constraint.
643  map<string,unsigned long>:: iterator it;
644  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
645  if(it->first == (*di)->fully_qualified_name())
646  dimsize = it->second;
647  }
648 
649  // Define dimension.
650  int g_dimid = -1;
651  stax = nc_def_dim(grp_id,(*di)->name().c_str(),dimsize,&g_dimid);
652  if (stax != NC_NOERR)
653  FONcUtils::handle_error(stax, "File out netcdf, unable to define dimension: " + _localfile, __FILE__, __LINE__);
654  // Save this dimension ID in a map.
655  fdimname_to_id[(*di)->fully_qualified_name()] = g_dimid;
656  }
657 
658  Constructor::Vars_iter vi = grp->var_begin();
659  Constructor::Vars_iter ve = grp->var_end();
660 
661  vector<FONcBaseType *> fonc_vars_in_grp;
662  for (; vi != ve; vi++) {
663  if ((*vi)->send_p()) {
664  BaseType *v = *vi;
665 
666  BESDEBUG("fonc", "FONcTransform::transform_dap4_group() - Converting variable '" << v->name() << "'" << endl);
667 
668  // This is a factory class call, and 'fg' is specialized for 'v'
669  //FONcBaseType *fb = FONcUtils::convert(v,FONcTransform::_returnAs,FONcRequestHandler::classic_model);
670  FONcBaseType *fb = FONcUtils::convert(v,RETURNAS_NETCDF4,false,fdimname_to_id,rds_nums);
671 
672  fonc_vars_in_grp.push_back(fb);
673 
674  // This is needed to avoid the memory leak.
675  _total_fonc_vars_in_grp.push_back(fb);
676 
677  vector<string> embed;
678  fb->convert(embed,true);
679  }
680  }
681 
682 #if 0
683  if(grp->grp_begin() == grp->grp_end())
684  BESDEBUG("fonc", "FONcTransform::transform_dap4() - No group " << endl);
685  else
686  BESDEBUG("fonc", "FONcTransform::transform_dap4() - has group " << endl);
687 #endif
688 
689 
690  try {
691  // Here we will be defining the variables of the netcdf and
692  // adding attributes. To do this we must be in define mode.
693  //nc_redef(_ncid);
694 
695  vector<FONcBaseType *>::iterator i = fonc_vars_in_grp.begin();
696  vector<FONcBaseType *>::iterator e = fonc_vars_in_grp.end();
697  for (; i != e; i++) {
698  FONcBaseType *fbt = *i;
699  BESDEBUG("fonc", "FONcTransform::transform_dap4_group() - Defining variable: " << fbt->name() << endl);
700  fbt->set_is_dap4(true);
701  fbt->define(grp_id);
702  }
703 
704  bool is_netCDF_enhanced = false;
705  if(FONcTransform::_returnAs == RETURNAS_NETCDF4 && FONcRequestHandler::classic_model==false)
706  is_netCDF_enhanced = true;
707 
708 
709  bool add_attr = true;
710 
711  // Only the root attribute may be ignored.
712  if(FONcRequestHandler::no_global_attrs == true && is_root_grp == true)
713  add_attr= false;
714 
715  if(true == add_attr) {
716  D4Attributes*d4_attrs = grp->attributes();
717  BESDEBUG("fonc", "FONcTransform::transform_dap4_group() - Adding Group Attributes" << endl) ;
718  // add dap4 group attributes.
719  FONcAttributes::add_dap4_attributes(grp_id, NC_GLOBAL, d4_attrs, "", "",is_netCDF_enhanced);
720  }
721 
722  // Write every variable in this group.
723  i = fonc_vars_in_grp.begin();
724  e = fonc_vars_in_grp.end();
725  for (; i != e; i++) {
726  FONcBaseType *fbt = *i;
727  BESDEBUG("fonc", "FONcTransform::transform() - Writing data for variable in group: " << fbt->name() << endl);
728  //fbt->write(_ncid);
729  fbt->write(grp_id);
730  }
731 
732  // Now handle all the child groups.
733  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
734  BESDEBUG("fonc", "FONcTransform::transform_dap4() in group - group name: " << (*gi)->name() << endl);
735  transform_dap4_group(*gi,false,grp_id,fdimname_to_id,rds_nums);
736  }
737 
738  }
739  catch (BESError &e) {
740  (void) nc_close(_ncid); // ignore the error at this point
741  throw;
742  }
743 
744 }
745 
746 
747 
748 // Group support is only on when netCDF-4 is in enhanced model and there are groups in the DMR.
749 bool FONcTransform::check_group_support() {
750  if(RETURNAS_NETCDF4 == FONcTransform::_returnAs && false == FONcRequestHandler::classic_model &&
751  (_dmr->root()->grp_begin()!=_dmr->root()->grp_end()))
752  return true;
753  else
754  return false;
755 }
756 
757 // Generate the final group list in the netCDF-4 file. Empty groups and their attributes will be removed.
758 void FONcTransform::gen_included_grp_list(D4Group*grp)
759 {
760  bool grp_has_var = false;
761  if(grp) {
762  BESDEBUG("fnoc", "<coming to the D4 group has name " << grp->name()<<endl);
763  BESDEBUG("fnoc", "<coming to the D4 group has fullpath " << grp->FQN()<<endl);
764 
765  if(grp->var_begin()!=grp->var_end()) {
766 
767  BESDEBUG("fnoc", "<has the vars " << endl);
768  Constructor::Vars_iter vi = grp->var_begin();
769  Constructor::Vars_iter ve = grp->var_end();
770 
771  for (; vi != ve; vi++) {
772 
773  // This variable is selected(in the local constraints).
774  if ((*vi)->send_p()) {
775  grp_has_var = true;
776 
777  //If a var in this group is selected, we need to include this group in the netcdf-4 file.
778  //We always include root attributes, so no need to obtain grp_names for the root.
779  if(grp->FQN()!="/")
780  _included_grp_names.insert(grp->FQN());
781  break;
782  }
783  }
784  }
785  // Loop through the subgroups to build up the list.
786  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
787  BESDEBUG("fonc", "obtain included groups - group name: " << (*gi)->name() << endl);
788  gen_included_grp_list(*gi);
789  }
790  }
791 
792  // If this group is in the final list, all its ancestors(except root, since it is always selected),should also be included.
793  if(grp_has_var == true) {
794  D4Group *temp_grp = grp;
795  while(temp_grp) {
796  if(temp_grp->get_parent()){
797  temp_grp = static_cast<D4Group*>(temp_grp->get_parent());
798  if(temp_grp->FQN()!="/")
799  _included_grp_names.insert(temp_grp->FQN());
800  }
801  else
802  temp_grp = 0;
803  }
804  }
805 
806 }
807 
808 void FONcTransform::check_and_obtain_dimensions(D4Group*grp,bool is_root_grp) {
809 
810  // We may not need to do this way,it may overkill.
811  bool included_grp = false;
812  // Always include the root attributes.
813  if(is_root_grp == true)
814  included_grp = true;
815  else {
816  // Check if this group is in the group list kept in the file.
817  set<string>::iterator iset;
818  if(_included_grp_names.find(grp->FQN())!=_included_grp_names.end())
819  included_grp = true;
820  }
821 
822  if(included_grp == true)
823  check_and_obtain_dimensions_internal(grp);
824 }
825 
826 void FONcTransform::check_and_obtain_dimensions_internal(D4Group*grp) {
827 
828  // Remember the Group Fully Qualified dimension Name and the corresponding dimension size.
829  D4Dimensions *grp_dims = grp->dims();
830  if(grp_dims) {
831  for(D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
832 #if 0
833  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
834  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
835  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
836  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
837 #endif
838  unsigned long dimsize = (*di)->size();
839  if((*di)->constrained()) {
840  dimsize = ((*di)->c_stop() -(*di)->c_start())/(*di)->c_stride() +1;
841 
842  }
843  GFQN_dimname_to_dimsize[(*di)->fully_qualified_name()] = dimsize;
844  }
845  }
846 
847  // The size of DAP4 dimension needs to be updated if the dimension size of a variable with the same dimension is
848  // different. So we also need to remember the Variable FQN dimension name and size.
849  // Check section 8.6.2 of DAP4 specification(https://docs.opendap.org/index.php/DAP4:_Specification_Volume_1)
850  Constructor::Vars_iter vi = grp->var_begin();
851  Constructor::Vars_iter ve = grp->var_end();
852  for (; vi != ve; vi++) {
853  if ((*vi)->send_p()) {
854  if((*vi)->is_vector_type()) {
855  Array *t_a = dynamic_cast<Array*>(*vi);
856  Array::Dim_iter dim_i = t_a->dim_begin();
857  Array::Dim_iter dim_e = t_a->dim_end();
858  for(;dim_i !=dim_e;dim_i++) {
859  if((*dim_i).name!="") {
860  D4Dimension* d4dim = t_a->dimension_D4dim(dim_i);
861  if(d4dim) {
862  BESDEBUG("fonc", "transform_dap4() check dim- dim name is: "<<d4dim->name()<<endl);
863  BESDEBUG("fonc", "transform_dap4() check dim- dim size is: "<<d4dim->size()<<endl);
864  BESDEBUG("fonc", "transform_dap4() check dim- fully_qualfied_dim name is: "<<d4dim->fully_qualified_name()<<endl);
865 
866 #if 0
867  unsigned long dimsize = d4dim->size();
868  if(d4dim->constrained())
869  dimsize = (d4dim->c_stop() -d4dim->c_start())/d4dim->c_stride() +1;
870  BESDEBUG("fonc", "transform_dap4() check dim- final dim size is: "<<d4dim->size()<<endl);
871 #endif
872  unsigned long dimsize = t_a->dimension_size(dim_i,true);
873  pair<map<string,unsigned long>::iterator,bool> ret_it;
874  ret_it = VFQN_dimname_to_dimsize.insert(pair<string,unsigned long>(d4dim->fully_qualified_name(),dimsize));
875  if(ret_it.second == false && ret_it.first->second!=dimsize) {
876  string err = "fileout_netcdf-4: dimension found with the same name, but different size";
877  throw BESInternalError(err, __FILE__, __LINE__);
878  }
879  //VFQN_dimname_to_dimsize[d4dim->fully_qualified_name()] = dimsize;
880  }
881  else
882  throw BESInternalError("Has dimension name but D4 dimension is NULL",__FILE__,__LINE__);
883  }
884  // No need to handle the case when the dimension name doesn't exist. This will be handled in FONcArray.cc.
885  // else { }
886  }
887  }
888  }
889  }
890 
891 #if 0
892  map<string,unsigned long>:: iterator it;
893  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
894  BESDEBUG("fonc", "GFQN dim name is: "<<it->first<<endl);
895  BESDEBUG("fonc", "GFQN dim size is: "<<it->second<<endl);
896  }
897 
898  for(it=VFQN_dimname_to_dimsize.begin();it!=VFQN_dimname_to_dimsize.end();++it) {
899  BESDEBUG("fonc", "VFQN dim name is: "<<it->first<<endl);
900  BESDEBUG("fonc", "VFQN dim size is: "<<it->second<<endl);
901  }
902 
903 #endif
904 
905  // Go through all the descendent groups.
906  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
907  BESDEBUG("fonc", "FONcTransform::check_and_obtain_dimensions() in group - group name: " << (*gi)->name() << endl);
908  check_and_obtain_dimensions(*gi,false);
909  }
910 
911 }
912 
913 
923 void FONcTransform::dump(ostream &strm) const
924 {
925  strm << BESIndent::LMarg << "FONcTransform::dump - (" << (void *) this << ")" << endl;
926  BESIndent::Indent();
927  strm << BESIndent::LMarg << "ncid = " << _ncid << endl;
928  strm << BESIndent::LMarg << "temporary file = " << _localfile << endl;
929  BESIndent::Indent();
930  vector<FONcBaseType *>::const_iterator i = _fonc_vars.begin();
931  vector<FONcBaseType *>::const_iterator e = _fonc_vars.end();
932  for (; i != e; i++) {
933  FONcBaseType *fbt = *i;
934  fbt->dump(strm);
935  }
936  BESIndent::UnIndent();
937  BESIndent::UnIndent();
938 }
939 
940 
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Structure storing information used by the BES to handle the request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
static void add_dap4_attributes(int ncid, int varid, D4Attributes *d4_attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
add_dap4_attributes
static void add_attributes(int ncid, int varid, AttrTable &attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
helper function for add_attributes
A DAP BaseType with file out netcdf information included.
Definition: FONcBaseType.h:61
virtual void setVersion(std::string version)
Identifies variable with use of NetCDF4 features.
Definition: FONcBaseType.cc:89
virtual void setNC4DataModel(std::string nc4_datamodel)
Identifies the netCDF4 data model (CLASSIC or ENHANCED)
Definition: FONcBaseType.cc:98
virtual void define(int ncid)
Define the variable in the netcdf file.
Definition: FONcBaseType.cc:54
virtual void dump(std::ostream &strm) const =0
dump the contents of this object to the specified ostream
virtual void transform()
Transforms each of the variables of the DataDDS to the NetCDF file.
FONcTransform(DDS *dds, BESDataHandlerInterface &dhi, const string &localfile, const string &netcdfVersion="netcdf")
virtual void dump(ostream &strm) const
dumps information about this transformation object for debugging purposes
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
virtual ~FONcTransform()
Destructor.
static void handle_error(int stax, const string &err, const string &file, int line)
handle any netcdf errors
Definition: FONcUtils.cc:399
static string name_prefix
If a variable name, dimension name, or attribute name begins with a character that is not supported b...
Definition: FONcUtils.h:58
static void reset()
Resets the FONc transformation for a new input and out file.
Definition: FONcUtils.cc:68
static FONcBaseType * convert(BaseType *v, const string &version, const bool classic_model)
Creates a FONc object for the given DAP object.
Definition: FONcUtils.cc:225