bes  Updated for version 3.20.8
h5gmcfdap.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
32 
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <unistd.h>
37 #include <iostream>
38 #include <sstream>
39 
40 #include <BESDebug.h>
41 #include <InternalErr.h>
42 
43 #include "HDF5RequestHandler.h"
44 #include "h5cfdaputil.h"
45 #include "h5gmcfdap.h"
46 #include "HDF5CFByte.h"
47 #include "HDF5CFUInt16.h"
48 #include "HDF5CFInt16.h"
49 #include "HDF5CFUInt32.h"
50 #include "HDF5CFInt32.h"
51 #include "HDF5CFFloat32.h"
52 #include "HDF5CFFloat64.h"
53 #include "HDF5CFStr.h"
54 #include "HDF5CFArray.h"
55 #include "HDF5GMCFMissLLArray.h"
56 #include "HDF5GMCFFillIndexArray.h"
58 #include "HDF5GMCFSpecialCVArray.h"
59 #include "HDF5GMSPCFArray.h"
60 
61 using namespace std;
62 using namespace libdap;
63 using namespace HDF5CF;
64 
65 // Map general HDF5 products to DAP DDS
66 void map_gmh5_cfdds(DDS &dds, hid_t file_id, const string& filename){
67 
68  BESDEBUG("h5","Coming to GM products DDS mapping function map_gmh5_cfdds() "<<endl);
69 
70  H5GCFProduct product_type = check_product(file_id);
71 
72  GMPattern gproduct_pattern = OTHERGMS;
73 
74  GMFile * f = NULL;
75 
76  try {
77  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
78  }
79  catch(...) {
80  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
81  }
82  // Generally don't need to handle attributes when handling DDS.
83  bool include_attr = false;
84  try {
85  // Retrieve all HDF5 info(Not the values)
86  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
87 
88  // Update product type
89  // Newer version of a product may have different layout and the
90  // product type needs to be changed to reflect it. We also want
91  // to support the older version in case people still use them.
92  // This routine will check if newer layout can be applied. If yes,
93  // update the product type.
95 
97 
98  // Need to add dimension names.
99  f->Add_Dim_Name();
100 
101  // Handle coordinate variables
102  f->Handle_CVar();
103 #if 0
104  // We need to retrieve coordinate variable attributes for memory cache use.
105  //f->Retrieve_H5_CVar_Supported_Attr_Values();
106  //if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
107  // (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
108  // f->Retrieve_H5_Supported_Attr_Values();
109 #endif
110 
111 
112  // Handle special variables
113  f->Handle_SpVar();
114 
115  // When cv memory cache is on, the unit attributes are needed to
116  // distinguish whether this is lat/lon. Generally, memory cache
117  // is not used. This snipnet will not be accessed.
118  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
119  (HDF5RequestHandler::get_srdata_mem_cache() != NULL)){
120 
121  // Handle unsupported datatypes including the attributes
122  f->Handle_Unsupported_Dtype(true);
123 
124  // Handle unsupported dataspaces including the attributes
125  f->Handle_Unsupported_Dspace(true);
126 
127  // We need to retrieve coordinate variable attributes for memory cache use.
129 
130  }
131  else {
132 
133  // Handle unsupported datatypes
134  f->Handle_Unsupported_Dtype(include_attr);
135 
136  // Handle unsupported dataspaces
137  f->Handle_Unsupported_Dspace(include_attr);
138 
139  }
140 
141  // Need to handle the "coordinate" attributes when memory cache is turned on.
142  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
143  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
144  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
145 
146  // Adjust object names(may remove redundant paths)
147  f->Adjust_Obj_Name();
148 
149  // Flatten the object names
150  f->Flatten_Obj_Name(include_attr);
151 
152  // Handle Object name clashings
153  // Only when the check_nameclashing key is turned on or
154  // general product.
155  if(General_Product == product_type ||
156  true == HDF5RequestHandler::get_check_name_clashing())
157  f->Handle_Obj_NameClashing(include_attr);
158 
159  // Adjust Dimension name
160  f->Adjust_Dim_Name();
161  if(General_Product == product_type ||
162  true == HDF5RequestHandler::get_check_name_clashing())
164 
165  f->Handle_Hybrid_EOS5();
166  if(true == f->Have_Grid_Mapping_Attrs())
168  // Need to handle the "coordinate" attributes when memory cache is turned on.
169  if((HDF5RequestHandler::get_lrdata_mem_cache() != NULL) ||
170  (HDF5RequestHandler::get_srdata_mem_cache() != NULL))
171  f->Handle_Coor_Attr();
172 
175  }
176  catch (HDF5CF::Exception &e){
177  if (f != NULL)
178  delete f;
179  throw InternalErr(e.what());
180  }
181 
182  // generate DDS.
183  try {
184  gen_gmh5_cfdds(dds,f);
185  }
186  catch(...) {
187  if (f != NULL)
188  delete f;
189  throw;
190  }
191 
192  if (f != NULL)
193  delete f;
194 }
195 
196 // Map general HDF5 products to DAP DAS
197 void map_gmh5_cfdas(DAS &das, hid_t file_id, const string& filename){
198 
199  BESDEBUG("h5","Coming to GM products DAS mapping function map_gmh5_cfdas() "<<endl);
200 
201  H5GCFProduct product_type = check_product(file_id);
202  GMPattern gproduct_pattern = OTHERGMS;
203 
204  GMFile *f = NULL;
205 
206  try {
207  f = new GMFile(filename.c_str(),file_id,product_type,gproduct_pattern);
208  }
209  catch(...) {
210  throw InternalErr(__FILE__,__LINE__,"Cannot allocate memory for GMFile ");
211  }
212 
213  bool include_attr = true;
214  try {
215  f->Retrieve_H5_Info(filename.c_str(),file_id,include_attr);
216 
217  // Update product type(check comments of map_gmh5_cfdds)
218  f->Update_Product_Type();
219 
221 
222  f->Add_Dim_Name();
223  f->Handle_CVar();
224  f->Handle_SpVar();
225  f->Handle_Unsupported_Dtype(include_attr);
226 
227  // Remove unsupported dataspace
228  f->Handle_Unsupported_Dspace(include_attr);
229 
230  // Need to retrieve the attribute values to feed DAS
232 
233  // Handle other unsupported objects,
234  // currently it mainly generates the info. for the
235  // unsupported objects other than datatype, dataspace,links and named datatype
236  // One area is maybe very long string. So we retrieve the attribute
237  // values before calling this function.
238  f->Handle_Unsupported_Others(include_attr);
239 
240 
241  // Need to add original variable name and path
242  // and other special attributes
243  // Can be turned on/off by using the check_path_attrs keys.
244  f->Add_Supplement_Attrs(HDF5RequestHandler::get_add_path_attrs());
245  f->Adjust_Obj_Name();
246  f->Flatten_Obj_Name(include_attr);
247  if(General_Product == product_type ||
248  true == HDF5RequestHandler::get_check_name_clashing())
249  f->Handle_Obj_NameClashing(include_attr);
250  if(f->HaveUnlimitedDim() == true)
251  f->Adjust_Dim_Name();
252  // Handle the "coordinate" attributes.
253  f->Handle_Coor_Attr();
254 
255  f->Handle_Hybrid_EOS5();
256  if(true == f->Have_Grid_Mapping_Attrs())
258 
260 
262 
263  if(true == HDF5RequestHandler::get_enable_coord_attr_add_path())
264  f->Add_Path_Coord_Attr();
265  }
266  catch (HDF5CF::Exception &e){
267  if (f!= NULL)
268  delete f;
269  throw InternalErr(e.what());
270  }
271 
272  // Generate the DAS attributes.
273  try {
274  gen_gmh5_cfdas(das,f);
275  }
276  catch (...) {
277  if (f!= NULL)
278  delete f;
279  throw;
280 
281  }
282 
283  if (f != NULL)
284  delete f;
285 }
286 
287 // Generate DDS mapped from general HDF5 products
288 void gen_gmh5_cfdds( DDS & dds, HDF5CF:: GMFile *f) {
289 
290  BESDEBUG("h5","Coming to GM DDS generation function gen_gmh5_cfdds() "<<endl);
291 
292  const vector<HDF5CF::Var *>& vars = f->getVars();
293  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
294  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
295  const string filename = f->getPath();
296  const hid_t fileid = f->getFileID();
297 
298  // Read Variable info.
299 
300  vector<HDF5CF::Var *>::const_iterator it_v;
301  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
302  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
303 
304  // Since we need to use dds to add das for 64-bit dmr,we need to check if
305  // this case includes 64-bit integer variables and this is for dmr response.
306  bool dmr_64bit_support = false;
307  if(HDF5RequestHandler::get_dmr_long_int()==true &&
308  HDF5RequestHandler::get_dmr_64bit_int()!=NULL) {
309  for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
310  if (H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
311  dmr_64bit_support = true;
312  break;
313  }
314  }
315  }
316 
317  // We need to remove the unsupported attributes.
318  if(true == dmr_64bit_support) {
319  //STOP: add non-support stuff
320  f->Handle_Unsupported_Dtype(true);
321 
322  // Remove unsupported dataspace
323  f->Handle_Unsupported_Dspace(true);
324 
325  }
326 
327  for (it_v = vars.begin(); it_v !=vars.end();++it_v) {
328  BESDEBUG("h5","variable full path= "<< (*it_v)->getFullPath() <<endl);
329  // Handle 64-integer DAP4 CF mapping
330  if(need_attr_values_for_dap4(*it_v) == true)
331  f->Retrieve_H5_Var_Attr_Values(*it_v);
332  gen_dap_onevar_dds(dds,*it_v,fileid, filename);
333  }
334  for (it_cv = cvars.begin(); it_cv !=cvars.end();++it_cv) {
335  BESDEBUG("h5","variable full path= "<< (*it_cv)->getFullPath() <<endl);
336  gen_dap_onegmcvar_dds(dds,*it_cv,fileid, filename);
337  }
338 
339  for (it_spv = spvars.begin(); it_spv !=spvars.end();it_spv++) {
340  BESDEBUG("h5","variable full path= "<< (*it_spv)->getFullPath() <<endl);
341  gen_dap_onegmspvar_dds(dds,*it_spv,fileid, filename);
342  }
343 
344 }
345 
346 // Generate DAS mapped from general HDF5 products
347 void gen_gmh5_cfdas( DAS & das, HDF5CF:: GMFile *f) {
348 
349  BESDEBUG("h5","Coming to GM DAS generation function gen_gmh5_cfdas() "<<endl);
350 
351  // First check if this is for generating the ignored object info.
352  if(true == f->Get_IgnoredInfo_Flag()) {
353  gen_gmh5_cf_ignored_obj_info(das, f);
354  return;
355  }
356 
357  const vector<HDF5CF::Var *>& vars = f->getVars();
358  const vector<HDF5CF::GMCVar *>& cvars = f->getCVars();
359  const vector<HDF5CF::GMSPVar *>& spvars = f->getSPVars();
360  const vector<HDF5CF::Group *>& grps = f->getGroups();
361  const vector<HDF5CF::Attribute *>& root_attrs = f->getAttributes();
362 
363 
364  vector<HDF5CF::Var *>::const_iterator it_v;
365  vector<HDF5CF::GMCVar *>::const_iterator it_cv;
366  vector<HDF5CF::GMSPVar *>::const_iterator it_spv;
367  vector<HDF5CF::Group *>::const_iterator it_g;
368  vector<HDF5CF::Attribute *>::const_iterator it_ra;
369 
370  // Handling the file attributes(attributes under the root group)
371  // The table name is "HDF_GLOBAL".
372 
373  if (false == root_attrs.empty()) {
374 
375  AttrTable *at = das.get_table(FILE_ATTR_TABLE_NAME);
376  if (NULL == at)
377  at = das.add_table(FILE_ATTR_TABLE_NAME, new AttrTable);
378 
379  for (it_ra = root_attrs.begin(); it_ra != root_attrs.end(); ++it_ra) {
380  // Check and may update the 64-bit integer attributes in DAP4.
381  check_update_int64_attr("",*it_ra);
382  gen_dap_oneobj_das(at,*it_ra,NULL);
383  }
384  }
385 
386  if (false == grps.empty()) {
387  for (it_g = grps.begin();
388  it_g != grps.end(); ++it_g) {
389  AttrTable *at = das.get_table((*it_g)->getNewName());
390  if (NULL == at)
391  at = das.add_table((*it_g)->getNewName(), new AttrTable);
392 
393  for (it_ra = (*it_g)->getAttributes().begin();
394  it_ra != (*it_g)->getAttributes().end(); ++it_ra) {
395  check_update_int64_attr((*it_g)->getNewName(),*it_ra);
396  gen_dap_oneobj_das(at,*it_ra,NULL);
397  }
398  }
399  }
400 
401  for (it_v = vars.begin();
402  it_v != vars.end(); ++it_v) {
403  if (false == ((*it_v)->getAttributes().empty())) {
404 
405  // Skip the 64-bit integer variables. The attribute mapping of
406  // DAP4 CF 64-bit integer variable support
407  // has been taken care at the routine gen_dap_onevar_dds()
408  // defined at h5commoncfdap.cc
409  if(H5INT64 == (*it_v)->getType() || H5UINT64 == (*it_v)->getType()){
410  continue;
411  }
412 
413  AttrTable *at = das.get_table((*it_v)->getNewName());
414  if (NULL == at)
415  at = das.add_table((*it_v)->getNewName(), new AttrTable);
416 
417  for (it_ra = (*it_v)->getAttributes().begin();
418  it_ra != (*it_v)->getAttributes().end(); ++it_ra) {
419  gen_dap_oneobj_das(at,*it_ra,*it_v);
420  }
421  // TODO: If a var has integer-64 bit datatype attributes, maybe
422  // we can just keep that attributes(not consistent but
423  // easy to implement) or we have to duplicate all
424  // the var in dmr and delete this var from dds.
425 
426  }
427 
428  // GPM needs to be handled in a special way(mostly _FillValue)
429  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
430  || GPM_L1 == f->getProductType())
431  update_GPM_special_attrs(das,*it_v,false);
432 
433  }
434 
435  for (it_cv = cvars.begin();
436  it_cv != cvars.end(); ++it_cv) {
437  if (false == ((*it_cv)->getAttributes().empty())) {
438 
439  // TODO: Add 64-bit int support for coordinates, this has not been tackled.
440  if(H5INT64 == (*it_cv)->getType() || H5UINT64 == (*it_cv)->getType()){
441  continue;
442  }
443 
444  AttrTable *at = das.get_table((*it_cv)->getNewName());
445  if (NULL == at)
446  at = das.add_table((*it_cv)->getNewName(), new AttrTable);
447 
448  for (it_ra = (*it_cv)->getAttributes().begin();
449  it_ra != (*it_cv)->getAttributes().end(); ++it_ra){
450  gen_dap_oneobj_das(at,*it_ra,*it_cv);
451  }
452 
453  }
454  // Though CF doesn't allow _FillValue, still keep it to keep the original form.
455  if(GPMS_L3 == f->getProductType() || GPMM_L3 == f->getProductType()
456  || GPM_L1 == f->getProductType())
457  update_GPM_special_attrs(das,*it_cv,true);
458 
459  }
460 
461  // Currently the special variables are only limited to the ACOS/OCO2 64-bit integer variables
462  for (it_spv = spvars.begin();
463  it_spv != spvars.end(); ++it_spv) {
464  if (false == ((*it_spv)->getAttributes().empty())) {
465 
466  AttrTable *at = das.get_table((*it_spv)->getNewName());
467  if (NULL == at)
468  at = das.add_table((*it_spv)->getNewName(), new AttrTable);
469 #if 0
470  // cerr<<"spv coordinate variable name "<<(*it_spv)->getNewName() <<endl;
471 #endif
472 
473  for (it_ra = (*it_spv)->getAttributes().begin();
474  it_ra != (*it_spv)->getAttributes().end(); ++it_ra)
475  gen_dap_oneobj_das(at,*it_ra,*it_spv);
476  }
477  }
478 
479  // CHECK ALL UNLIMITED DIMENSIONS from the coordinate variables based on the names.
480  if(f->HaveUnlimitedDim() == true) {
481 
482  BESDEBUG("h5","Find unlimited dimension in the GM DAS generation function gen_gmh5_cfdas() "<<endl);
483 
484  // Currently there is no way for DAP to present the unlimited dimension info.
485  // when there are no dimension names. So don't create DODS_EXTRA even if
486  // there is an unlimited dimension in the file. KY 2016-02-18
487  if(cvars.empty()==false ){
488 
489  // First check if we do have unlimited dimension in the coordinate variables.
490  // Since unsupported fakedims are removed, we may not have unlimited dimensions.
491  bool still_has_unlimited = false;
492  for (it_cv = cvars.begin();
493  it_cv != cvars.end(); ++it_cv) {
494  // Check unlimited dimension names.
495  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
496  ird != (*it_cv)->getDimensions().end(); ++ird) {
497 
498  // Currently we only check one unlimited dimension, which is the most
499  // common case. When receiving the conventions from JG, will add
500  // the support of multi-unlimited dimension. KY 2016-02-09
501  if((*ird)->HaveUnlimitedDim() == true) {
502  still_has_unlimited = true;
503  break;
504  }// if((*ird) is HaveUnlimitedDim()
505  }// for (vector<Dimension*>::
506  if(true == still_has_unlimited)
507  break;
508  }// for (it_cv=cvars.begin();
509 
510  if(true == still_has_unlimited) {
511  AttrTable* at = das.get_table("DODS_EXTRA");
512  if (NULL == at)
513  at = das.add_table("DODS_EXTRA", new AttrTable);
514 
515  string unlimited_names;
516 
517  for (it_cv = cvars.begin();
518  it_cv != cvars.end(); ++it_cv) {
519 #if 0
520  bool has_unlimited_dim = false;
521 #endif
522  // Check unlimited dimension names.
523  for (vector<Dimension*>::const_iterator ird = (*it_cv)->getDimensions().begin();
524  ird != (*it_cv)->getDimensions().end(); ++ird) {
525 
526  // Currently we only check one unlimited dimension, which is the most
527  // common case. When receiving the conventions from JG, will add
528  // the support of multi-unlimited dimension. KY 2016-02-09
529  if((*ird)->HaveUnlimitedDim() == true) {
530  if(unlimited_names=="") {
531  unlimited_names = (*ird)->getNewName();
532  if(at !=NULL)
533  at->append_attr("Unlimited_Dimension","String",unlimited_names);
534  }
535  else {
536  if(unlimited_names.rfind((*ird)->getNewName()) == string::npos) {
537  unlimited_names = unlimited_names+" "+(*ird)->getNewName();
538  if(at !=NULL)
539  at->append_attr("Unlimited_Dimension","String",(*ird)->getNewName());
540  }
541  }
542  }// if((*ird)->HaveUnlimitedDim()
543  }// for (vector<Dimension*>::
544  }// for (it_cv=cvars.begin();
545  }// if(true == still_has_unlimited)
546 
547  }//if(cvars.size()>0)
548 #if 0
549  // The following line will generate the string like "Band1 str1 str2".
550  //if(unlimited_names!="")
551  // // at->append_attr("Unlimited_Dimension","String",unlimited_names);
552 #endif
553  }
554 }
555 
556 // Generate the ignored object info. for the CF option of the general products
557 void gen_gmh5_cf_ignored_obj_info(DAS &das, HDF5CF::GMFile *f) {
558 
559  BESDEBUG("h5","Coming to gen_gmh5_cf_ignored_obj_info() "<<endl);
560  AttrTable *at = das.get_table("Ignored_Object_Info");
561  if (NULL == at)
562  at = das.add_table("Ignored_Object_Info", new AttrTable);
563 
564  at->append_attr("Message","String",f->Get_Ignored_Msg());
565 
566 }
567 
568 // Generate the DDS for a coordinate variable of the General products
569 void gen_dap_onegmcvar_dds(DDS &dds,const HDF5CF::GMCVar* cvar, const hid_t file_id, const string & filename) {
570 
571  BESDEBUG("h5","Coming to gen_dap_onegmcvar_dds() "<<endl);
572 
573  if(cvar->getType() == H5INT64 || cvar->getType() == H5UINT64)
574  return;
575  BaseType *bt = NULL;
576 
577  switch(cvar->getType()) {
578 #define HANDLE_CASE(tid,type) \
579  case tid: \
580  bt = new (type)(cvar->getNewName(),cvar->getFullPath()); \
581  break;
582 
583  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
584  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
585  HANDLE_CASE(H5CHAR,HDF5CFInt16);
586  HANDLE_CASE(H5UCHAR, HDF5CFByte);
587  HANDLE_CASE(H5INT16, HDF5CFInt16);
588  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
589  HANDLE_CASE(H5INT32, HDF5CFInt32);
590  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
591  HANDLE_CASE(H5FSTRING, Str);
592  HANDLE_CASE(H5VSTRING, Str);
593 
594  default:
595  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
596 #undef HANDLE_CASE
597  }
598 
599  if (bt) {
600 
601  const vector<HDF5CF::Dimension *>& dims = cvar->getDimensions();
602  vector <HDF5CF::Dimension*>:: const_iterator it_d;
603  vector <size_t> dimsizes;
604  dimsizes.resize(cvar->getRank());
605  for(int i = 0; i <cvar->getRank();i++)
606  dimsizes[i] = (dims[i])->getSize();
607 
608 
609  if(dims.empty())
610  throw InternalErr(__FILE__,__LINE__,"the coordinate variable cannot be a scalar");
611 
612  switch(cvar->getCVType()) {
613 
614  case CV_EXIST:
615  {
616  HDF5CFArray *ar = NULL;
617 
618  // Need to check if this CV is lat/lon. This is necessary when data memory cache is turned on.
619  bool is_latlon = cvar->isLatLon();
620 
621  try {
622  ar = new HDF5CFArray (
623  cvar->getRank(),
624  file_id,
625  filename,
626  cvar->getType(),
627  dimsizes,
628  cvar->getFullPath(),
629  cvar->getTotalElems(),
630  CV_EXIST,
631  is_latlon,
632  cvar->getCompRatio(),
633  cvar->getNewName(),
634  bt);
635  }
636  catch(...) {
637  delete bt;
638  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5CFArray. ");
639  }
640 
641  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
642  if (""==(*it_d)->getNewName())
643  ar->append_dim((*it_d)->getSize());
644  else
645  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
646  }
647 
648  dds.add_var(ar);
649  delete bt;
650  delete ar;
651  }
652  break;
653 
654  case CV_LAT_MISS:
655  case CV_LON_MISS:
656  {
657  // Using HDF5GMCFMissLLArray
658  HDF5GMCFMissLLArray *ar = NULL;
659  try {
660  ar = new HDF5GMCFMissLLArray (
661  cvar->getRank(),
662  filename,
663  file_id,
664  cvar->getType(),
665  cvar->getFullPath(),
666  cvar->getPtType(),
667  cvar->getCVType(),
668  cvar->getNewName(),
669  bt);
670  }
671  catch(...) {
672  delete bt;
673  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissLLArray. ");
674  }
675 
676 
677  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
678  if (""==(*it_d)->getNewName())
679  ar->append_dim((*it_d)->getSize());
680  else
681  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
682  }
683 
684  dds.add_var(ar);
685  delete bt;
686  delete ar;
687  }
688  break;
689 
690  case CV_NONLATLON_MISS:
691  {
692 
693  if (cvar->getRank() !=1) {
694  delete bt;
695  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
696  }
697  int nelem = (cvar->getDimensions()[0])->getSize();
698 
699  HDF5GMCFMissNonLLCVArray *ar = NULL;
700 
701  try {
702  ar = new HDF5GMCFMissNonLLCVArray(
703  cvar->getRank(),
704  nelem,
705  cvar->getNewName(),
706  bt);
707  }
708  catch(...) {
709  delete bt;
710  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
711  }
712 
713 
714  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
715  if (""==(*it_d)->getNewName())
716  ar->append_dim((*it_d)->getSize());
717  else
718  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
719  }
720  dds.add_var(ar);
721  delete bt;
722  delete ar;
723  }
724  break;
725 
726  case CV_FILLINDEX:
727  {
728 
729  if (cvar->getRank() !=1) {
730  delete bt;
731  throw InternalErr(__FILE__, __LINE__, "The rank of missing Z dimension field must be 1");
732  }
733 
734  HDF5GMCFFillIndexArray *ar = NULL;
735 
736  try {
737  ar = new HDF5GMCFFillIndexArray(
738  cvar->getRank(),
739  cvar->getType(),
740  cvar->getNewName(),
741  bt);
742  }
743  catch(...) {
744  delete bt;
745  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
746  }
747 
748 
749  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
750  if (""==(*it_d)->getNewName())
751  ar->append_dim((*it_d)->getSize());
752  else
753  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
754  }
755  dds.add_var(ar);
756  delete bt;
757  delete ar;
758  }
759  break;
760 
761 
762  case CV_SPECIAL:
763  {
764  // Currently only handle 1-D special CV.
765  if (cvar->getRank() !=1) {
766  delete bt;
767  throw InternalErr(__FILE__, __LINE__, "The rank of special coordinate variable must be 1");
768  }
769  int nelem = (cvar->getDimensions()[0])->getSize();
770 
771  HDF5GMCFSpecialCVArray * ar = NULL;
772  ar = new HDF5GMCFSpecialCVArray(
773  cvar->getType(),
774  nelem,
775  cvar->getFullPath(),
776  cvar->getPtType(),
777  cvar->getNewName(),
778  bt);
779 
780  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
781  if (""==(*it_d)->getNewName())
782  ar->append_dim((*it_d)->getSize());
783  else
784  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
785  }
786 
787  dds.add_var(ar);
788  delete bt;
789  delete ar;
790 
791  }
792  break;
793  case CV_MODIFY:
794  default:
795  delete bt;
796  throw InternalErr(__FILE__,__LINE__,"Coordinate variable type is not supported.");
797  }
798  }
799 }
800 
801 // Generate DDS for special variable in a general product
802 void gen_dap_onegmspvar_dds(DDS &dds,const HDF5CF::GMSPVar* spvar, const hid_t fileid, const string & filename) {
803 
804  BESDEBUG("h5","Coming to gen_dap_onegmspvar_dds() "<<endl);
805  BaseType *bt = NULL;
806 
807  switch(spvar->getType()) {
808 #define HANDLE_CASE(tid,type) \
809  case tid: \
810  bt = new (type)(spvar->getNewName(),spvar->getFullPath()); \
811  break;
812 
813  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32);
814  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64);
815  HANDLE_CASE(H5CHAR,HDF5CFInt16);
816  HANDLE_CASE(H5UCHAR, HDF5CFByte);
817  HANDLE_CASE(H5INT16, HDF5CFInt16);
818  HANDLE_CASE(H5UINT16, HDF5CFUInt16);
819  HANDLE_CASE(H5INT32, HDF5CFInt32);
820  HANDLE_CASE(H5UINT32, HDF5CFUInt32);
821  HANDLE_CASE(H5FSTRING, Str);
822  HANDLE_CASE(H5VSTRING, Str);
823  default:
824  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
825 #undef HANDLE_CASE
826  }
827 
828  if (bt) {
829 
830  const vector<HDF5CF::Dimension *>& dims = spvar->getDimensions();
831  vector <HDF5CF::Dimension*>:: const_iterator it_d;
832 
833  if(dims.empty())
834  throw InternalErr(__FILE__,__LINE__,"Currently don't support scalar special variables. ");
835 
836  HDF5GMSPCFArray *ar = NULL;
837 
838  try {
839  ar = new HDF5GMSPCFArray (
840  spvar->getRank(),
841  filename,
842  fileid,
843  spvar->getType(),
844  spvar->getFullPath(),
845  spvar->getOriginalType(),
846  spvar->getStartBit(),
847  spvar->getBitNum(),
848  spvar->getNewName(),
849  bt);
850  }
851  catch(...) {
852  delete bt;
853  throw InternalErr(__FILE__,__LINE__,"Unable to allocate HDF5GMCFMissNonLLCVArray. ");
854  }
855 
856 
857  for(it_d = dims.begin(); it_d != dims.end(); ++it_d) {
858  if (""==(*it_d)->getNewName())
859  ar->append_dim((*it_d)->getSize());
860  else
861  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
862  }
863 
864  dds.add_var(ar);
865  delete bt;
866  delete ar;
867  }
868 
869 }
870 
871 // When we add floating point fill value at HDF5CF.cc, the value will be changed
872 // a little bit when it changes to string representation.
873 // For example, -9999.9 becomes -9999.9000123. To reduce the misunderstanding,we
874 // just add fillvalue in the string type here. KY 2014-04-02
875 void update_GPM_special_attrs(DAS& das, const HDF5CF::Var *var,bool is_cvar) {
876 
877  BESDEBUG("h5","Coming to update_GPM_special_attrs() "<<endl);
878  if(H5FLOAT64 == var->getType() ||
879  H5FLOAT32 == var->getType() ||
880  H5INT16 == var->getType() ||
881  H5CHAR == var->getType()) {
882 
883  AttrTable *at = das.get_table(var->getNewName());
884  if (NULL == at)
885  at = das.add_table(var->getNewName(), new AttrTable);
886  bool has_fillvalue = false;
887  AttrTable::Attr_iter it = at->attr_begin();
888  while (it!=at->attr_end() && false==has_fillvalue) {
889  if (at->get_name(it) =="_FillValue")
890  {
891  has_fillvalue = true;
892  string fillvalue ="";
893  if(H5FLOAT32 == var->getType()) {
894  const string cor_fill_value = "-9999.9";
895  fillvalue = (*at->get_attr_vector(it)->begin());
896  if((fillvalue.find(cor_fill_value) == 0) && (fillvalue!= cor_fill_value)) {
897  at->del_attr("_FillValue");
898  at->append_attr("_FillValue","Float32",cor_fill_value);
899  }
900  }
901  else if(H5FLOAT64 == var->getType()) {
902  const string cor_fill_value = "-9999.9";
903  const string exist_fill_value_substr = "-9999.8999";
904  fillvalue = (*at->get_attr_vector(it)->begin());
905  if((fillvalue.find(exist_fill_value_substr) == 0) && (fillvalue!= cor_fill_value)) {
906  at->del_attr("_FillValue");
907  at->append_attr("_FillValue","Float64",cor_fill_value);
908  }
909 
910  }
911  }
912  it++;
913  }
914 
915  // Add the fill value
916  if(false == is_cvar ) {
917 
918  // Current versions of GPM don't add fillvalues. We add the fillvalue according to the document.
919  if (has_fillvalue != true ) {
920 
921  if(H5FLOAT32 == var->getType())
922  at->append_attr("_FillValue","Float32","-9999.9");
923  else if(H5FLOAT64 == var->getType())
924  at->append_attr("_FillValue","Float64","-9999.9");
925  else if (H5INT16 == var->getType())
926  at->append_attr("_FillValue","Int16","-9999");
927  else if (H5CHAR == var->getType())// H5CHAR maps to DAP int16
928  at->append_attr("_FillValue","Int16","-99");
929 
930  }
931  }
932  }
933 }
934 
935 
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class provides a way to map HDF5 byte to DAP byte for the CF option.
This class provides a way to map HDF5 float to DAP float for the CF option.
This class provides a way to map HDF5 64-bit floating-point(double) to DAP 64-bit floating-point for ...
This class provides a way to map HDF5 int16 to DAP int16 for the CF option.
This class provides a way to map HDF5 32-bit integer to DAP Int32 for the CF option.
This class provides a way to map HDF5 Str to DAP Str for the CF option.
This class provides a way to map HDF5 unsigned 16-bit integer to DAP uint16 for the CF option.
This class provides a way to map HDF5 unsigned 32-bit integer to DAP uint32 for the CF option.
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class specifies the retrieval of the values of non-lat/lon coordinate variables for general HDF5...
This class specifies the retrieval of the missing lat/lon values for general HDF5 products.
This class specifies the retrieval of data values for special HDF5 products Currently this only appli...
include the entry functions to execute the handlers
CVType getCVType() const
Get the coordinate variable type of this variable.
Definition: HDF5CF.h:372
bool HaveUnlimitedDim() const
Has unlimited dimensions.
Definition: HDF5CF.h:688
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:745
const std::string & getPath() const
Obtain the path of the file.
Definition: HDF5CF.h:664
hid_t getFileID() const
Obtain the HDF5 file ID.
Definition: HDF5CF.h:658
const std::vector< Attribute * > & getAttributes() const
Public interface to obtain information of all attributes under the root group.
Definition: HDF5CF.h:676
const std::vector< Var * > & getVars() const
Public interface to obtain information of all variables.
Definition: HDF5CF.h:670
const std::vector< Group * > & getGroups() const
Public interface to obtain all the group info.
Definition: HDF5CF.h:682
This class is a derived class of CVar. It represents a coordinate variable for general HDF5 files.
Definition: HDF5CF.h:427
H5GCFProduct getPtType() const
Get the data type of this variable.
Definition: HDF5CF.h:439
This class is a derived class of File. It includes methods applied to general HDF5 files only.
Definition: HDF5CF.h:833
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:400
virtual void Adjust_Obj_Name()
Adjust object names based on different general NASA HDF5 products.
Definition: HDF5GMCF.cc:4882
virtual void Retrieve_H5_CVar_Supported_Attr_Values()
Retrieve coordinate variable attributes.
Definition: HDF5GMCF.cc:333
void Add_Path_Coord_Attr()
Update the coordinate attribute to include path and also flatten.
Definition: HDF5GMCF.cc:6996
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5GMCF.cc:6920
virtual void Adjust_Dim_Name()
Adjust dimension name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5195
void Handle_Obj_NameClashing(bool)
Handle object name clashing for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5025
void Remove_Unused_FakeDimVars()
Unsupported datatype array may generate FakeDim. Remove them.
Definition: HDF5GMCF.cc:6928
void Update_Product_Type()
Update "product type" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:238
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool include_attr)
Retrieve DDS information from the HDF5 file; real implementation for general HDF5 products.
Definition: HDF5GMCF.cc:219
virtual void Handle_SpVar()
Handle special variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4792
virtual bool Get_IgnoredInfo_Flag()
Obtain ignored info. flag.
Definition: HDF5CF.h:931
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for general HDF5 products.
Definition: HDF5GMCF.cc:593
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes for general HDF5 products.
Definition: HDF5GMCF.cc:690
virtual const std::string & Get_Ignored_Msg()
Get the message that contains the ignored obj. info.
Definition: HDF5CF.h:937
void Remove_Unneeded_Objects()
Remove unneeded objects.
Definition: HDF5GMCF.cc:261
void Add_Dim_Name()
Add dimension name.
Definition: HDF5GMCF.cc:829
virtual void Flatten_Obj_Name(bool include_attr)
Flatten the object name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:4967
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5GMCF.cc:6924
virtual void Handle_DimNameClashing()
Definition: HDF5GMCF.cc:5130
virtual void Handle_Coor_Attr()
Handle "coordinates" attributes for general HDF5 products.
Definition: HDF5GMCF.cc:5986
void Rename_NC4_NonCoordVars()
Remove the _nc4_non_coord from the variable new names.
Definition: HDF5GMCF.cc:6976
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes for general HDF5 products.
Definition: HDF5GMCF.cc:348
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name for general NASA HDF5 products.
Definition: HDF5GMCF.cc:5257
virtual void Handle_CVar()
Handle coordinate variables for general NASA HDF5 products.
Definition: HDF5GMCF.cc:2918
This class is a derived class of Var. It represents a special general HDF5 product(currently ACOS and...
Definition: HDF5CF.h:392
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
int getRank() const
Get the dimension rank of this variable.
Definition: HDF5CF.h:305
const std::string & getFullPath() const
Get the full path of this variable.
Definition: HDF5CF.h:283
H5DataType getType() const
Get the data type of this variable(Not HDF5 datatype id)
Definition: HDF5CF.h:311
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:322
int getCompRatio() const
Get the compression ratio of this dataset.
Definition: HDF5CF.h:328
const std::string & getNewName() const
Get the new name of this variable.
Definition: HDF5CF.h:277
Helper functions for generating DAS attributes and a function to check BES Key.
Map and generate DDS and DAS for the CF option for generic HDF5 products.