bes  Updated for version 3.20.8
h5commoncfdap.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler: an HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
32 
33 #include <sys/types.h>
34 #include <sys/stat.h>
35 #include <fcntl.h>
36 #include <unistd.h>
37 #include <iostream>
38 #include <sstream>
39 
40 #include <InternalErr.h>
41 #include <BESDebug.h>
42 
43 #include "HDF5RequestHandler.h"
44 #include "h5cfdaputil.h"
45 #include "h5gmcfdap.h"
46 #include "HDF5CFByte.h"
47 #include "HDF5CFUInt16.h"
48 #include "HDF5CFInt16.h"
49 #include "HDF5CFUInt32.h"
50 #include "HDF5CFInt32.h"
51 #include "HDF5CFFloat32.h"
52 #include "HDF5CFFloat64.h"
53 #include "HDF5CFInt64.h"
54 #include "HDF5CFUInt64.h"
55 #include "HDF5CFStr.h"
56 #include "HDF5CFArray.h"
57 #include "HDF5CFGeoCF1D.h"
58 #include "HDF5CFGeoCFProj.h"
59 
60 #include "HDF5Int64.h"
61 #include "HDF5CFUtil.h"
62 
63 using namespace std;
64 using namespace libdap;
65 using namespace HDF5CF;
66 
67 // Generate DDS from one variable
68 void gen_dap_onevar_dds(DDS &dds, const HDF5CF::Var* var, const hid_t file_id, const string & filename)
69 {
70 
71  BESDEBUG("h5", "Coming to gen_dap_onevar_dds() "<<endl);
72  const vector<HDF5CF::Dimension *>& dims = var->getDimensions();
73 
74  if (dims.empty()) {
75  // Adding 64-bit integer support for DMR
76  if (H5INT64 == var->getType() || H5UINT64 == var->getType()){
77  DMR * dmr = HDF5RequestHandler::get_dmr_64bit_int();
78  if(dmr == NULL)
79  return;
80  else {
81  D4Group* root_grp = dmr->root();
82  if(H5INT64 == var->getType()) {
83  HDF5CFInt64 *sca_int64 = NULL;
84  try {
85  sca_int64 = new HDF5CFInt64(var->getNewName(), var->getFullPath(), filename);
86  }
87  catch (...) {
88  string error_message = "Cannot allocate the HDF5CFInt64: " + error_message;
89  throw InternalErr(__FILE__, __LINE__, error_message);
90  }
91  sca_int64->set_is_dap4(true);
92  map_cfh5_attrs_to_dap4(var,sca_int64);
93  root_grp->add_var_nocopy(sca_int64);
94 
95  }
96  else if(H5UINT64 == var->getType()) {
97  HDF5CFUInt64 *sca_uint64 = NULL;
98  try {
99  sca_uint64 = new HDF5CFUInt64(var->getNewName(), var->getFullPath(), filename);
100  }
101  catch (...) {
102  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFInt64.");
103  }
104  sca_uint64->set_is_dap4(true);
105  map_cfh5_attrs_to_dap4(var,sca_uint64);
106  root_grp->add_var_nocopy(sca_uint64);
107 
108  }
109 
110  }
111  }
112  else if (H5FSTRING == var->getType() || H5VSTRING == var->getType()) {
113  HDF5CFStr *sca_str = NULL;
114  try {
115  sca_str = new HDF5CFStr(var->getNewName(), filename, var->getFullPath());
116  }
117  catch (...) {
118  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFStr.");
119  }
120  dds.add_var(sca_str);
121  delete sca_str;
122  }
123  else {
124  switch (var->getType()) {
125 
126  case H5UCHAR: {
127  HDF5CFByte * sca_uchar = NULL;
128  try {
129  sca_uchar = new HDF5CFByte(var->getNewName(), var->getFullPath(), filename);
130  }
131  catch (...) {
132  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFByte.");
133  }
134  dds.add_var(sca_uchar);
135  delete sca_uchar;
136 
137  }
138  break;
139  case H5CHAR:
140  case H5INT16: {
141  HDF5CFInt16 * sca_int16 = NULL;
142  try {
143  sca_int16 = new HDF5CFInt16(var->getNewName(), var->getFullPath(), filename);
144  }
145  catch (...) {
146  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFInt16.");
147  }
148  dds.add_var(sca_int16);
149  delete sca_int16;
150  }
151  break;
152  case H5UINT16: {
153  HDF5CFUInt16 * sca_uint16 = NULL;
154  try {
155  sca_uint16 = new HDF5CFUInt16(var->getNewName(), var->getFullPath(), filename);
156  }
157  catch (...) {
158  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFUInt16.");
159  }
160  dds.add_var(sca_uint16);
161  delete sca_uint16;
162  }
163  break;
164  case H5INT32: {
165  HDF5CFInt32 * sca_int32 = NULL;
166  try {
167  sca_int32 = new HDF5CFInt32(var->getNewName(), var->getFullPath(), filename);
168  }
169  catch (...) {
170  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFInt32.");
171  }
172  dds.add_var(sca_int32);
173  delete sca_int32;
174  }
175  break;
176  case H5UINT32: {
177  HDF5CFUInt32 * sca_uint32 = NULL;
178  try {
179  sca_uint32 = new HDF5CFUInt32(var->getNewName(), var->getFullPath(), filename);
180  }
181  catch (...) {
182  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFUInt32.");
183  }
184  dds.add_var(sca_uint32);
185  delete sca_uint32;
186  }
187  break;
188  case H5FLOAT32: {
189  HDF5CFFloat32 * sca_float32 = NULL;
190  try {
191  sca_float32 = new HDF5CFFloat32(var->getNewName(), var->getFullPath(), filename);
192  }
193  catch (...) {
194  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFFloat32.");
195  }
196  dds.add_var(sca_float32);
197  delete sca_float32;
198  }
199  break;
200  case H5FLOAT64: {
201  HDF5CFFloat64 * sca_float64 = NULL;
202  try {
203  sca_float64 = new HDF5CFFloat64(var->getNewName(), var->getFullPath(), filename);
204  }
205  catch (...) {
206  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFFloat64.");
207  }
208  dds.add_var(sca_float64);
209  delete sca_float64;
210 
211  }
212  break;
213  default:
214  throw InternalErr(__FILE__, __LINE__, "unsupported data type.");
215  }
216  }
217  }
218 
219  else {
220 
221  // 64-bit integer support
222  // DMR CHECK
223  bool dap4_int64 = false;
224  if(var->getType() == H5INT64 || var->getType()==H5UINT64) {
225  DMR * dmr = HDF5RequestHandler::get_dmr_64bit_int();
226  if(dmr == NULL)
227  return;
228  else
229  dap4_int64 = true;
230  }
231 
232 #if 0
233  else {
234  D4Group* root_grp = dmr->root();
235  BaseType *bt = NULL;
236  bt = new(HDF5Int64)(var->getNewName(),var->getFullPath(),filename);
237  bt->transform_to_dap4(root_grp,root_grp);
238  delete bt;
239  return;
240  }
241 #endif
242  BaseType *bt = NULL;
243 
244  if(true == dap4_int64) {
245  if(var->getType() == H5INT64)
246  bt = new(HDF5CFInt64)(var->getNewName(),var->getFullPath());
247  else if(var->getType() == H5UINT64)
248  bt = new(HDF5CFUInt64)(var->getNewName(),var->getFullPath());
249  }
250 
251  else {
252  switch (var->getType()) {
253 #define HANDLE_CASE(tid,type) \
254  case tid: \
255  bt = new (type)(var->getNewName(),var->getFullPath()); \
256  break;
257  HANDLE_CASE(H5FLOAT32, HDF5CFFloat32)
258  ;
259  HANDLE_CASE(H5FLOAT64, HDF5CFFloat64)
260  ;
261  HANDLE_CASE(H5CHAR, HDF5CFInt16)
262  ;
263  HANDLE_CASE(H5UCHAR, HDF5CFByte)
264  ;
265  HANDLE_CASE(H5INT16, HDF5CFInt16)
266  ;
267  HANDLE_CASE(H5UINT16, HDF5CFUInt16)
268  ;
269  HANDLE_CASE(H5INT32, HDF5CFInt32)
270  ;
271  HANDLE_CASE(H5UINT32, HDF5CFUInt32)
272  ;
273  HANDLE_CASE(H5FSTRING, Str)
274  ;
275  HANDLE_CASE(H5VSTRING, Str)
276  ;
277  default:
278  throw InternalErr(__FILE__, __LINE__, "unsupported data type.");
279 #undef HANDLE_CASE
280  }
281  }
282 
283  vector<HDF5CF::Dimension*>::const_iterator it_d;
284  vector<size_t> dimsizes;
285  dimsizes.resize(var->getRank());
286  for (int i = 0; i < var->getRank(); i++)
287  dimsizes[i] = (dims[i])->getSize();
288 
289  HDF5CFArray *ar = NULL;
290  try {
291  ar = new HDF5CFArray(var->getRank(), file_id, filename, var->getType(), dimsizes, var->getFullPath(),
292  var->getTotalElems(), CV_UNSUPPORTED, false, var->getCompRatio(), var->getNewName(), bt);
293  }
294  catch (...) {
295  delete bt;
296  throw InternalErr(__FILE__, __LINE__, "Cannot allocate the HDF5CFStr.");
297  }
298 
299  for (it_d = dims.begin(); it_d != dims.end(); ++it_d) {
300  if ("" == (*it_d)->getNewName())
301  ar->append_dim((*it_d)->getSize());
302  else
303  ar->append_dim((*it_d)->getSize(), (*it_d)->getNewName());
304  }
305 
306  // When handling DAP4 CF, we need to generate dmr for 64-bit integer separately.
307  if(dap4_int64 == true) {
308  DMR * dmr = HDF5RequestHandler::get_dmr_64bit_int();
309  D4Group* root_grp = dmr->root();
310  // Dimensions need to be translated.
311  BaseType* d4_var = ar->h5cfdims_transform_to_dap4(root_grp);
312  // Attributes.
313  map_cfh5_attrs_to_dap4(var,d4_var);
314  root_grp->add_var_nocopy(d4_var);
315  }
316  else
317  dds.add_var(ar);
318 
319  delete bt;
320  delete ar;
321  }
322 
323  return;
324 
325 }
326 
327 // Currently only when the datatype of fillvalue is not the same as the datatype of the variable,
328 // special attribute handling is needed.
329 bool need_special_attribute_handling(const HDF5CF::Attribute* attr, const HDF5CF::Var* var)
330 {
331  return ((("_FillValue" == attr->getNewName()) && (var->getType() != attr->getType())) ? true : false);
332 }
333 
334 // Currently we only handle the case when the datatype of _FillValue is not the same as the variable datatype.
335 void gen_dap_special_oneobj_das(AttrTable*at, const HDF5CF::Attribute* attr, const HDF5CF::Var* var)
336 {
337 
338  BESDEBUG("h5", "Coming to gen_dap_special_oneobj_das() "<<endl);
339  if (attr->getCount() != 1) throw InternalErr(__FILE__, __LINE__, "FillValue attribute can only have one element.");
340 
341  H5DataType var_dtype = var->getType();
342  if ((true == HDF5RequestHandler::get_fillvalue_check())
343  && (false == is_fvalue_valid(var_dtype, attr))) {
344  string msg = "The attribute value is out of the range.\n";
345  msg += "The variable name: " + var->getNewName() + "\n";
346  msg += "The attribute name: " + attr->getNewName() + "\n";
347  msg += "The error occurs inside the gen_dap_special_oneobj_das function in h5commoncfdap.cc.";
348  throw InternalErr(msg);
349  }
350  string print_rep = HDF5CFDAPUtil::print_attr(attr->getType(), 0, (void*) (&(attr->getValue()[0])));
351  at->append_attr(attr->getNewName(), HDF5CFDAPUtil::print_type(var_dtype), print_rep);
352 }
353 
354 // Check if this fillvalue is in the valid datatype range when the fillvalue datatype is changed to follow the CF
355 bool is_fvalue_valid(H5DataType var_dtype, const HDF5CF::Attribute* attr)
356 {
357 
358  BESDEBUG("h5", "Coming to is_fvalue_valid() "<<endl);
359  bool ret_value = true;
360  // We only check 8-bit and 16-bit integers.
361  switch (attr->getType()) {
362  case H5CHAR: {
363  signed char final_fill_value = *((signed char*) ((void*) (&(attr->getValue()[0]))));
364  if ((var_dtype == H5UCHAR) && (final_fill_value<0))
365  ret_value = false;
366  return ret_value;
367 
368  }
369  case H5INT16: {
370  short final_fill_value = *((short*) ((void*) (&(attr->getValue()[0]))));
371  if ((var_dtype == H5UCHAR) &&(final_fill_value > 255 || final_fill_value < 0))
372  ret_value = false;
373 
374  // No need to check the var_dtype==H5CHAR case since it is mapped to int16.
375  else if ((var_dtype == H5UINT16) && (final_fill_value < 0))
376  ret_value = false;
377  return ret_value;
378  }
379  case H5UINT16: {
380  unsigned short final_fill_value = *((unsigned short*) ((void*) (&(attr->getValue()[0]))));
381  if ((var_dtype == H5UCHAR) &&(final_fill_value > 255)) {
382  ret_value = false;
383  }
384  else if ((var_dtype == H5INT16) && (final_fill_value >32767)){
385  ret_value = false;
386  }
387  return ret_value;
388 
389  }
390  // We are supposed to check the case when the datatype of fillvalue is unsigned char.
391  // However, since the variable type signed char is always mapped to int16, so there
392  // will never be an overflow case(the signed char case is the only possible one).
393  // Still the data producer should not do this. We will not check this in the handler.KY 2016-03-04
394 #if 0
395  case H5UCHAR:
396  {
397  unsigned char final_fill_value = *((unsigned char*)((void*)(&(attr->getValue()[0]))));
398  if(var_dtype == H5CHAR) {
399  if(final_fill_value >127)
400  ret_value = false;
401  }
402  return ret_value;
403  }
404 
405  case H5UCHAR:
406  case H5INT32:
407  case H5UINT32:
408 #endif
409 
410  default:
411  return ret_value;
412  }
413 
414 }
415 // Leave the old code for the time being. KY 2015-05-07
416 #if 0
417 void gen_dap_special_oneobj_das(AttrTable*at, const HDF5CF::Attribute* attr,const HDF5CF::Var* var) {
418 
419  if (attr->getCount() != 1)
420  throw InternalErr(__FILE__,__LINE__,"FillValue attribute can only have one element.");
421 
422  H5DataType var_dtype = var->getType();
423  switch(var_dtype) {
424 
425  case H5UCHAR:
426  {
427  unsigned char final_fill_value = *((unsigned char*)((void*)(&(attr->getValue()[0]))));
428  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
429  }
430  break;
431 
432  case H5CHAR:
433  {
434  // Notice HDF5 native char maps to DAP int16.
435  short final_fill_value = *((short*)((void*)(&(attr->getValue()[0]))));
436  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
437  }
438  break;
439  case H5INT16:
440  {
441  short final_fill_value = *((short*)((void*)(&(attr->getValue()[0]))));
442  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
443  }
444  break;
445  case H5UINT16:
446  {
447  unsigned short final_fill_value = *((unsigned short*)((void*)(&(attr->getValue()[0]))));
448  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
449  }
450  break;
451 
452  case H5INT32:
453  {
454  int final_fill_value = *((int*)((void*)(&(attr->getValue()[0]))));
455  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
456  }
457  break;
458  case H5UINT32:
459  {
460  unsigned int final_fill_value = *((unsigned int*)((void*)(&(attr->getValue()[0]))));
461  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
462  }
463  break;
464  case H5FLOAT32:
465  {
466  float final_fill_value = *((float*)((void*)(&(attr->getValue()[0]))));
467 // memcpy(&(attr->getValue()[0]),(void*)(&final_fill_value),sizeof(float));
468 //cerr<<"final_fill_value is "<<final_fill_value <<endl;
469  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
470  }
471  break;
472  case H5FLOAT64:
473  {
474  double final_fill_value = *((double*)((void*)(&(attr->getValue()[0]))));
475  print_rep = HDF5CFDAPUtil::print_attr(var_dtype,0,(void*)&final_fill_value);
476  }
477  break;
478  default:
479  throw InternalErr(__FILE__,__LINE__,"unsupported data type.");
480  }
481 
482  at->append_attr(attr->getNewName(), HDF5CFDAPUtil::print_type(var_dtype), print_rep);
483 }
484 #endif
485 
486 // Generate DAS from one variable
487 void gen_dap_oneobj_das(AttrTable*at, const HDF5CF::Attribute* attr, const HDF5CF::Var *var)
488 {
489 
490  BESDEBUG("h5", "Coming to gen_dap_oneobj_das() "<<endl);
491  // DMR support for 64-bit integer
492  if (H5INT64 == attr->getType() || H5UINT64 == attr->getType()) {
493  // TODO: Add code to tackle DMR for the variable datatype that is not 64-bit integer.
494  return;
495 
496  }
497  else if ((H5FSTRING == attr->getType()) || (H5VSTRING == attr->getType())) {
498  gen_dap_str_attr(at, attr);
499  }
500  else {
501 
502  if (NULL == var) {
503 
504  // HDF5 Native Char maps to DAP INT16(DAP doesn't have the corresponding datatype), so needs to
505  // obtain the mem datatype.
506  size_t mem_dtype_size = (attr->getBufSize()) / (attr->getCount());
507  H5DataType mem_dtype = HDF5CFDAPUtil::get_mem_dtype(attr->getType(), mem_dtype_size);
508 
509  for (unsigned int loc = 0; loc < attr->getCount(); loc++) {
510  string print_rep = HDF5CFDAPUtil::print_attr(mem_dtype, loc, (void*) &(attr->getValue()[0]));
511  at->append_attr(attr->getNewName(), HDF5CFDAPUtil::print_type(attr->getType()), print_rep);
512  }
513 
514  }
515 
516  else {
517 
518  // The datatype of _FillValue attribute needs to be the same as the variable datatype for an netCDF C file.
519  // To make OPeNDAP's netCDF file out work, we need to change the attribute datatype of _FillValue to be the
520  // same as the variable datatype if they are not the same. An OMI-Aura_L2-OMUVB file has such a case.
521  // The datatype of "TerrainHeight" is int32 but the datatype of the fillvalue is int16.
522  // KY 2012-11-20
523  bool special_attr_handling = need_special_attribute_handling(attr, var);
524  if (true == special_attr_handling) {
525  gen_dap_special_oneobj_das(at, attr, var);
526  }
527 
528  else {
529 
530  // HDF5 Native Char maps to DAP INT16(DAP doesn't have the corresponding datatype), so needs to
531  // obtain the mem datatype.
532  size_t mem_dtype_size = (attr->getBufSize()) / (attr->getCount());
533  H5DataType mem_dtype = HDF5CFDAPUtil::get_mem_dtype(attr->getType(), mem_dtype_size);
534 
535  for (unsigned int loc = 0; loc < attr->getCount(); loc++) {
536  string print_rep = HDF5CFDAPUtil::print_attr(mem_dtype, loc, (void*) &(attr->getValue()[0]));
537  at->append_attr(attr->getNewName(), HDF5CFDAPUtil::print_type(attr->getType()), print_rep);
538  }
539  }
540  }
541  }
542 }
543 
544 void gen_dap_str_attr(AttrTable *at, const HDF5CF::Attribute *attr)
545 {
546 
547  BESDEBUG("h5", "Coming to gen_dap_str_attr() "<<endl);
548  const vector<size_t>& strsize = attr->getStrSize();
549  unsigned int temp_start_pos = 0;
550  bool is_cset_ascii = attr->getCsetType();
551  for (unsigned int loc = 0; loc < attr->getCount(); loc++) {
552  if (strsize[loc] != 0) {
553  string tempstring(attr->getValue().begin() + temp_start_pos,
554  attr->getValue().begin() + temp_start_pos + strsize[loc]);
555  temp_start_pos += strsize[loc];
556 
557  // If the string size is longer than the current netCDF JAVA
558  // string and the "EnableDropLongString" key is turned on,
559  // No string is generated.
560  // The above statement is no longer true. The netCDF Java can handle long string
561  // attributes. The long string can be kept and I do think the
562  // performance penalty should be small. KY 2018-02-26
563  if ((attr->getNewName() != "origname") && (attr->getNewName() != "fullnamepath") && (true == is_cset_ascii))
564  tempstring = HDF5CFDAPUtil::escattr(tempstring);
565  at->append_attr(attr->getNewName(), "String", tempstring);
566  }
567  }
568 }
569 
570 //#if 0
571 // This function adds the 1-D horizontal coordinate variables as well as the dummy projection variable to the grid.
572 //Note: Since we don't add these artifical CF variables to our main engineering at HDFEOS5CF.cc, the information
573 // to handle DAS won't pass to DDS by the file pointer, we need to re-call the routines to check projection
574 // and dimension. The time to retrieve these information is trivial compared with the whole translation.
575 void add_cf_grid_cvs(DDS & dds, EOS5GridPCType cv_proj_code, float cv_point_lower, float cv_point_upper,
576  float cv_point_left, float cv_point_right, const vector<HDF5CF::Dimension*>& dims)
577 {
578 
579  //1. Check the projection information: we first just handled the sinusoidal projection.
580  // We also add the LAMAZ and PS support. These 1-D varaibles are the same as the sinusoidal one.
581  if (HE5_GCTP_SNSOID == cv_proj_code || HE5_GCTP_LAMAZ == cv_proj_code || HE5_GCTP_PS == cv_proj_code) {
582 
583  //2. Obtain the dimension information from latitude and longitude(fieldtype =1 or fieldtype =2)
584  vector<HDF5CF::Dimension*>::const_iterator it_d;
585  string dim0name = dims[0]->getNewName();
586  int dim0size = dims[0]->getSize();
587  string dim1name = dims[1]->getNewName();
588  int dim1size = dims[1]->getSize();
589 
590  //3. Add the 1-D CV variables and the dummy projection variable
591  BaseType *bt_dim0 = NULL;
592  BaseType *bt_dim1 = NULL;
593 
594  HDF5CFGeoCF1D * ar_dim0 = NULL;
595  HDF5CFGeoCF1D * ar_dim1 = NULL;
596 
597  try {
598 
599  bt_dim0 = new (HDF5CFFloat64)(dim0name, dim0name);
600  bt_dim1 = new (HDF5CFFloat64)(dim1name, dim1name);
601 
602  // Note ar_dim0 is y, ar_dim1 is x.
603  ar_dim0 = new HDF5CFGeoCF1D(HE5_GCTP_SNSOID, cv_point_upper, cv_point_lower, dim0size, dim0name, bt_dim0);
604  ar_dim0->append_dim(dim0size, dim0name);
605 
606  ar_dim1 = new HDF5CFGeoCF1D(HE5_GCTP_SNSOID, cv_point_left, cv_point_right, dim1size, dim1name, bt_dim1);
607  ar_dim1->append_dim(dim1size, dim1name);
608  dds.add_var(ar_dim0);
609  dds.add_var(ar_dim1);
610 
611  }
612  catch (...) {
613  if (bt_dim0) delete bt_dim0;
614  if (bt_dim1) delete bt_dim1;
615  if (ar_dim0) delete ar_dim0;
616  if (ar_dim1) delete ar_dim1;
617  throw InternalErr(__FILE__, __LINE__, "Unable to allocate the HDFEOS2GeoCF1D instance.");
618  }
619 
620  if (bt_dim0) delete bt_dim0;
621  if (bt_dim1) delete bt_dim1;
622  if (ar_dim0) delete ar_dim0;
623  if (ar_dim1) delete ar_dim1;
624 
625  }
626 }
627 
628 // This function adds the grid mapping variables.
629 void add_cf_grid_mapinfo_var(DDS & dds, const EOS5GridPCType grid_proj_code, const unsigned short g_suffix)
630 {
631 
632  //Add the dummy projection variable. The attributes of this variable can be used to store the grid mapping info.
633  // To handle multi-grid cases, we need to add suffixes to distinguish them.
634  string cf_projection_base = "eos_cf_projection";
635 
636  HDF5CFGeoCFProj * dummy_proj_cf = NULL;
637  if(HE5_GCTP_SNSOID == grid_proj_code) {
638  // AFAWK, one grid_mapping variable is necessary for multi-grids. So we just leave one grid here.
639  if(g_suffix == 1) {
640  dummy_proj_cf = new HDF5CFGeoCFProj(cf_projection_base, cf_projection_base);
641  dds.add_var(dummy_proj_cf);
642  }
643  }
644  else {
645  stringstream t_suffix_ss;
646  t_suffix_ss << g_suffix;
647  string cf_projection_name = cf_projection_base + "_" + t_suffix_ss.str();
648  dummy_proj_cf = new HDF5CFGeoCFProj(cf_projection_name, cf_projection_name);
649  dds.add_var(dummy_proj_cf);
650  }
651  if (dummy_proj_cf) delete dummy_proj_cf;
652 
653 }
654 
655 // This function adds 1D grid mapping CF attributes to CV and data variables.
656 #if 0
657 void add_cf_grid_cv_attrs(DAS & das, const vector<HDF5CF::Var*>& vars, EOS5GridPCType cv_proj_code,
658  float /*cv_point_lower*/, float /*cv_point_upper*/, float /*cv_point_left*/, float /*cv_point_right*/,
659  const vector<HDF5CF::Dimension*>& dims,const vector<double> &eos5_proj_params,const unsigned short g_suffix)
660 #endif
661 void add_cf_grid_cv_attrs(DAS & das, const vector<HDF5CF::Var*>& vars, EOS5GridPCType cv_proj_code,
662  const vector<HDF5CF::Dimension*>& dims,const vector<double> &eos5_proj_params,const unsigned short g_suffix)
663 {
664 
665 
666  //1. Check the projection information, now, we handle sinusoidal,PS and LAMAZ projections.
667  if (HE5_GCTP_SNSOID == cv_proj_code || HE5_GCTP_PS == cv_proj_code || HE5_GCTP_LAMAZ== cv_proj_code) {
668 
669  string dim0name = (dims[0])->getNewName();
670  int dim0size = dims[0]->getSize();
671  string dim1name = (dims[1])->getNewName();
672  int dim1size = dims[1]->getSize();
673 
674  //2. Add 1D CF attributes to the 1-D CV variables and the dummy grid_mapping variable
675  AttrTable *at = das.get_table(dim0name);
676  if (!at)
677  at = das.add_table(dim0name, new AttrTable);
678  at->append_attr("standard_name", "String", "projection_y_coordinate");
679 
680  string long_name = "y coordinate of projection ";
681  at->append_attr("long_name", "String", long_name);
682 
683  // Change this to meter.
684  at->append_attr("units", "string", "meter");
685 
686  at->append_attr("_CoordinateAxisType", "string", "GeoY");
687 
688  at = das.get_table(dim1name);
689  if (!at) at = das.add_table(dim1name, new AttrTable);
690 
691  at->append_attr("standard_name", "String", "projection_x_coordinate");
692 
693  long_name = "x coordinate of projection ";
694  at->append_attr("long_name", "String", long_name);
695 
696  // change this to meter.
697  at->append_attr("units", "string", "meter");
698 
699  // This is for CDM conventions. Adding doesn't do harm. Same as GeoY.
700  at->append_attr("_CoordinateAxisType", "string", "GeoX");
701 
702  // Add the attributes for the dummy grid_mapping variable.
703  string cf_projection_base = "eos_cf_projection";
704  string cf_projection;
705  if(HE5_GCTP_SNSOID == cv_proj_code)
706  cf_projection = cf_projection_base;
707  else {
708  stringstream t_suffix_ss;
709  t_suffix_ss << g_suffix;
710  cf_projection = cf_projection_base + "_" + t_suffix_ss.str();
711  }
712  add_cf_projection_attrs(das,cv_proj_code,eos5_proj_params,cf_projection);
713 
714  // Fill in the data fields that contains the dim0name and dim1name dimensions with the grid_mapping
715  // We only apply to >=2D data fields.
716  add_cf_grid_mapping_attr(das, vars, cf_projection, dim0name, dim0size, dim1name, dim1size);
717  }
718 
719 }
720 
721 // Add CF projection attribute
722 
723 void add_cf_projection_attrs(DAS &das,EOS5GridPCType cv_proj_code,const vector<double> &eos5_proj_params,const string& cf_projection) {
724 
725  AttrTable* at = das.get_table(cf_projection);
726  if (!at) {// Only append attributes when the table is created.
727  at = das.add_table(cf_projection, new AttrTable);
728 
729  if (HE5_GCTP_SNSOID == cv_proj_code) {
730  at->append_attr("grid_mapping_name", "String", "sinusoidal");
731  at->append_attr("longitude_of_central_meridian", "Float64", "0.0");
732  at->append_attr("earth_radius", "Float64", "6371007.181");
733  at->append_attr("_CoordinateAxisTypes", "string", "GeoX GeoY");
734  }
735  else if (HE5_GCTP_PS == cv_proj_code) {
736 
737  // The following information is added according to the HDF-EOS5 user's guide and
738  // CF 1.7 grid_mapping requirement.
739 
740  // Longitude down below pole of map
741  double vert_lon_pole = HE5_EHconvAng(eos5_proj_params[4],HE5_HDFE_DMS_DEG);
742 
743  // Latitude of true scale
744  double lat_true_scale = HE5_EHconvAng(eos5_proj_params[5],HE5_HDFE_DMS_DEG);
745 
746  // False easting
747  double fe = eos5_proj_params[6];
748 
749  // False northing
750  double fn = eos5_proj_params[7];
751 
752  at->append_attr("grid_mapping_name", "String", "polar_stereographic");
753 
754  ostringstream s_vert_lon_pole;
755  s_vert_lon_pole << vert_lon_pole;
756 
757  // I did this map is based on my best understanding. I cannot be certain about south pole. KY
758  // CF: straight_vertical_longitude_from_pole
759  at->append_attr("straight_vertical_longitude_from_pole", "Float64", s_vert_lon_pole.str());
760  ostringstream s_lat_true_scale;
761  s_lat_true_scale << lat_true_scale;
762 
763  at->append_attr("standard_parallel", "Float64", s_lat_true_scale.str());
764 
765  if(fe == 0.0)
766  at->append_attr("false_easting","Float64","0.0");
767  else {
768  ostringstream s_fe;
769  s_fe << fe;
770  at->append_attr("false_easting","Float64",s_fe.str());
771  }
772 
773 
774  if(fn == 0.0)
775  at->append_attr("false_northing","Float64","0.0");
776  else {
777  ostringstream s_fn;
778  s_fn << fn;
779  at->append_attr("false_northing","Float64",s_fn.str());
780  }
781 
782 
783  if(lat_true_scale >0)
784  at->append_attr("latitude_of_projection_origin","Float64","+90.0");
785  else
786  at->append_attr("latitude_of_projection_origin","Float64","-90.0");
787 
788 
789  at->append_attr("_CoordinateAxisTypes", "string", "GeoX GeoY");
790 
791  // From CF, PS has another parameter,
792  // Either standard_parallel (EPSG 9829) or scale_factor_at_projection_origin (EPSG 9810)
793  // I cannot find the corresponding parameter from the EOS5.
794 
795  }
796  else if(HE5_GCTP_LAMAZ == cv_proj_code) {
797  double lon_proj_origin = HE5_EHconvAng(eos5_proj_params[4],HE5_HDFE_DMS_DEG);
798  double lat_proj_origin = HE5_EHconvAng(eos5_proj_params[5],HE5_HDFE_DMS_DEG);
799  double fe = eos5_proj_params[6];
800  double fn = eos5_proj_params[7];
801 
802  at->append_attr("grid_mapping_name", "String", "lambert_azimuthal_equal_area");
803 
804  ostringstream s_lon_proj_origin;
805  s_lon_proj_origin << lon_proj_origin;
806  at->append_attr("longitude_of_projection_origin", "Float64", s_lon_proj_origin.str());
807 
808  ostringstream s_lat_proj_origin;
809  s_lat_proj_origin << lat_proj_origin;
810 
811  at->append_attr("latitude_of_projection_origin", "Float64", s_lat_proj_origin.str());
812 
813 
814  if(fe == 0.0)
815  at->append_attr("false_easting","Float64","0.0");
816  else {
817  ostringstream s_fe;
818  s_fe << fe;
819  at->append_attr("false_easting","Float64",s_fe.str());
820  }
821 
822 
823  if(fn == 0.0)
824  at->append_attr("false_northing","Float64","0.0");
825  else {
826  ostringstream s_fn;
827  s_fn << fn;
828  at->append_attr("false_northing","Float64",s_fn.str());
829  }
830 
831  at->append_attr("_CoordinateAxisTypes", "string", "GeoX GeoY");
832 
833 
834  }
835  }
836 
837 }
838 
839 
840 // This function adds the 1-D cf grid projection mapping attribute to data variables
841 // it is called by the function add_cf_grid_attrs.
842 void add_cf_grid_mapping_attr(DAS &das, const vector<HDF5CF::Var*>& vars, const string& cf_projection,
843  const string & dim0name, hsize_t dim0size, const string &dim1name, hsize_t dim1size)
844 {
845 
846 #if 0
847  cerr<<"dim0name is "<<dim0name <<endl;
848  cerr<<"dim1name is "<<dim1name <<endl;
849  cerr<<"dim0size is "<<dim0size <<endl;
850  cerr<<"dim1size is "<<dim1size <<endl;
851 #endif
852 
853  // Check >=2-D fields, check if they hold the dim0name,dim0size etc., yes, add the attribute cf_projection.
854  vector<HDF5CF::Var *>::const_iterator it_v;
855  for (it_v = vars.begin(); it_v != vars.end(); ++it_v) {
856 
857  if ((*it_v)->getRank() > 1) {
858  bool has_dim0 = false;
859  bool has_dim1 = false;
860  const vector<HDF5CF::Dimension*>& dims = (*it_v)->getDimensions();
861  for (vector<HDF5CF::Dimension *>::const_iterator j = dims.begin(); j != dims.end(); ++j) {
862  if ((*j)->getNewName() == dim0name && (*j)->getSize() == dim0size)
863  has_dim0 = true;
864  else if ((*j)->getNewName() == dim1name && (*j)->getSize() == dim1size)
865  has_dim1 = true;
866 
867  }
868  if (true == has_dim0 && true == has_dim1) { // Need to add the grid_mapping attribute
869  AttrTable *at = das.get_table((*it_v)->getNewName());
870  if (!at) at = das.add_table((*it_v)->getNewName(), new AttrTable);
871 
872  // The dummy projection name is the value of the grid_mapping attribute
873  at->append_attr("grid_mapping", "String", cf_projection);
874  }
875  }
876  }
877 }
878 // Now this is specially for LAMAZ where the NSIDC EASE-Grid may have points off the earth. So
879 // The calculated lat/lon are set to number out of the normal range. The valid_range attributes
880 // will hopefully constrain the applications not to consider those points.
881 void add_ll_valid_range(AttrTable* at, bool is_lat) {
882  if(true == is_lat) {
883  at->append_attr("valid_min", "Float64","-90.0");
884  at->append_attr("valid_max", "Float64","90.0");
885  }
886  else {
887  at->append_attr("valid_min", "Float64","-180.0");
888  at->append_attr("valid_max", "Float64","180.0");
889  }
890 }
891 
892 // This routine is for 64-bit DAP4 CF support: when var type is 64-bit integer.
893 bool need_attr_values_for_dap4(const HDF5CF::Var *var) {
894  bool ret_value = false;
895  if((HDF5RequestHandler::get_dmr_64bit_int()!=NULL) &&
896  (H5INT64 == var->getType() || H5UINT64 == var->getType()))
897  ret_value = true;
898  return ret_value;
899 }
900 
901 // This routine is for 64-bit DAP4 CF support: map all attributes to DAP4 for 64-bit integers.
902 void map_cfh5_attrs_to_dap4(const HDF5CF::Var *var,BaseType* d4_var) {
903 
904  vector<HDF5CF::Attribute *>::const_iterator it_ra;
905  for (it_ra = var->getAttributes().begin();
906  it_ra != var->getAttributes().end(); ++it_ra) {
907  // HDF5 Native Char maps to DAP INT16(DAP doesn't have the corresponding datatype), so needs to
908  // obtain the mem datatype. Keep this in DAP4 mapping.
909  size_t mem_dtype_size = ((*it_ra)->getBufSize()) / ((*it_ra)->getCount());
910  H5DataType mem_dtype = HDF5CFDAPUtil::get_mem_dtype((*it_ra)->getType(), mem_dtype_size);
911 
912  string dap2_attrtype = HDF5CFDAPUtil::print_type(mem_dtype);
913  D4AttributeType dap4_attrtype = HDF5CFDAPUtil::daptype_strrep_to_dap4_attrtype(dap2_attrtype);
914  D4Attribute *d4_attr = new D4Attribute((*it_ra)->getNewName(),dap4_attrtype);
915  if(dap4_attrtype == attr_str_c) {
916  if("coordinates" == (*it_ra)->getNewName()) {
917  bool chg_coor_value = false;
918  if((true == HDF5RequestHandler::get_enable_coord_attr_add_path())
919  &&(true == var->getCoorAttrAddPath()))
920  chg_coor_value = true;
921  string tempstring;
922  handle_coor_attr_for_int64_var((*it_ra),var->getFullPath(),tempstring,chg_coor_value);
923  d4_attr->add_value(tempstring);
924  }
925  else {
926  const vector<size_t>& strsize = (*it_ra)->getStrSize();
927  unsigned int temp_start_pos = 0;
928  for (unsigned int loc = 0; loc < (*it_ra)->getCount(); loc++) {
929  if (strsize[loc] != 0) {
930  string tempstring((*it_ra)->getValue().begin() + temp_start_pos,
931  (*it_ra)->getValue().begin() + temp_start_pos + strsize[loc]);
932  temp_start_pos += strsize[loc];
933  //The below if is not necessary since the "origname" and "fullnamepath" are not added.KY 2020-02-24
934  //if (((*it_ra)->getNewName() != "origname") && ((*it_ra)->getNewName() != "fullnamepath"))
935  tempstring = HDF5CFDAPUtil::escattr(tempstring);
936  d4_attr->add_value(tempstring);
937  }
938  }
939  }
940 
941  }
942  else {
943  for (unsigned int loc = 0; loc < (*it_ra)->getCount(); loc++) {
944  string print_rep = HDF5CFDAPUtil::print_attr(mem_dtype, loc, (void*) &((*it_ra)->getValue()[0]));
945  d4_attr->add_value(print_rep);
946  }
947  }
948  d4_var->attributes()->add_attribute_nocopy(d4_attr);
949  }
950  // Here we add the "origname" and "fullnamepath" attributes since they are crucial to DMRPP generation.
951  D4Attribute *d4_attr = new D4Attribute("origname",attr_str_c);
952  d4_attr->add_value(var->getName());
953  d4_var->attributes()->add_attribute_nocopy(d4_attr);
954  d4_attr = new D4Attribute("fullnamepath",attr_str_c);
955  d4_attr->add_value(var->getFullPath());
956  d4_var->attributes()->add_attribute_nocopy(d4_attr);
957 }
958 
959 void check_update_int64_attr(const string & obj_name, const HDF5CF::Attribute * attr) {
960  if(attr->getType() == H5INT64 || attr->getType() == H5UINT64) {
961 
962  DMR * dmr = HDF5RequestHandler::get_dmr_64bit_int();
963  if(dmr != NULL) {
964  string dap2_attrtype = HDF5CFDAPUtil::print_type(attr->getType());
965  D4AttributeType dap4_attrtype = HDF5CFDAPUtil::daptype_strrep_to_dap4_attrtype(dap2_attrtype);
966  D4Attribute *d4_attr = new D4Attribute(attr->getNewName(),dap4_attrtype);
967  for (unsigned int loc = 0; loc < attr->getCount(); loc++) {
968  string print_rep = HDF5CFDAPUtil::print_attr(attr->getType(), loc, (void*) &(attr->getValue()[0]));
969  d4_attr->add_value(print_rep);
970  }
971  D4Group * root_grp = dmr->root();
972  D4Attribute *d4_hg_container;
973  if(root_grp->attributes()->empty() == true){
974 #if 0
975  //D4Attribute *d4_hg_container = root_grp->attributes()->find("HDF5_GLOBAL");
976  //if(d4_hg_container == NULL) {
977 #endif
978  d4_hg_container = new D4Attribute;
979  d4_hg_container->set_name("HDF5_GLOBAL_integer_64");
980  d4_hg_container->set_type(attr_container_c);
981  root_grp->attributes()->add_attribute_nocopy(d4_hg_container);
982 #if 0
983  //root_grp->attributes()->add_attribute(d4_hg_container);
984 #endif
985  }
986  //else
987  d4_hg_container = root_grp->attributes()->get("HDF5_GLOBAL_integer_64");
988  if(obj_name != "") {
989  string test_obj_name = "HDF5_GLOBAL_integer_64."+obj_name;
990 #if 0
991  //D4Attribute *d4_container = root_grp->attributes()->find(obj_name);
992  //D4Attribute *d4_container = root_grp->attributes()->get(obj_name);
993 #endif
994  D4Attribute *d4_container = root_grp->attributes()->get(test_obj_name);
995  // ISSUES need to search the attributes
996  //
997 #if 0
998  //D4Attribute *d4_container = d4_hg_container->attributes()->find(obj_name);
999 #endif
1000  if(d4_container == NULL) {
1001  d4_container = new D4Attribute;
1002  d4_container->set_name(obj_name);
1003  d4_container->set_type(attr_container_c);
1004 
1005 #if 0
1006  //if(d4_hg_container->attributes()->empty()==true)
1007  // cerr<<"global container is empty"<<endl;
1008  //d4_hg_container->attributes()->add_attribute_nocopy(d4_container);
1009  //cerr<<"end of d4_container "<<endl;
1010 #endif
1011  }
1012  d4_container->attributes()->add_attribute_nocopy(d4_attr);
1013 #if 0
1014  //root_grp->attributes()->add_attribute_nocopy(d4_container);
1015 #endif
1016 //#if 0
1017  if(d4_hg_container->attributes()->get(obj_name)==NULL)
1018  d4_hg_container->attributes()->add_attribute_nocopy(d4_container);
1019 //#endif
1020  }
1021  else
1022  d4_hg_container->attributes()->add_attribute_nocopy(d4_attr);
1023  }
1024  }
1025 }
1026 void handle_coor_attr_for_int64_var(const HDF5CF::Attribute *attr,const string &var_path,string &tempstring,bool chg_coor_value) {
1027 
1028  string tempstring2(attr->getValue().begin(),attr->getValue().end());
1029  if(true == chg_coor_value) {
1030  char sep=' ';
1031  vector<string>cvalue_vec;
1032  HDF5CFUtil::Split_helper(cvalue_vec,tempstring2,sep);
1033  for (int i = 0; i<cvalue_vec.size();i++) {
1034  HDF5CFUtil::cha_co(cvalue_vec[i],var_path);
1035  string t_str = get_cf_string(cvalue_vec[i]);
1036  if(i == 0)
1037  tempstring = t_str;
1038  else
1039  tempstring += sep+t_str;
1040  }
1041  }
1042  else
1043  tempstring = tempstring2;
1044 
1045 }
1046 
1047 // Mainly copy from HDF5CF::get_CF_string. Should be
1048 // removed if we can generate DMR independently.
1049 string get_cf_string(string & s) {
1050 
1051  if(s[0] !='/')
1052  return get_cf_string_helper(s);
1053  else {
1054  // The leading underscore should be removed
1055  s.erase(0,1);
1056  return get_cf_string_helper(s);
1057  }
1058 
1059 }
1060 string get_cf_string_helper(string & s) {
1061 
1062  if ("" == s) return s;
1063  string insertString(1, '_');
1064 
1065  // Always start with _ if the first character is not a letter
1066  if (true == isdigit(s[0])) s.insert(0, insertString);
1067 
1068  for (unsigned int i = 0; i < s.length(); i++)
1069  if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1070  return s;
1071 }
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
This class provides a way to map HDF5 byte to DAP byte for the CF option.
This class provides a way to map HDF5 float to DAP float for the CF option.
This class provides a way to map HDF5 64-bit floating-point(double) to DAP 64-bit floating-point for ...
This class provides a way to map HDF5 int16 to DAP int16 for the CF option.
This class provides a way to map HDF5 32-bit integer to DAP Int32 for the CF option.
This class provides a way to map HDF5 64-bit integer to DAP4 Int64 for the CF option.
This class provides a way to map HDF5 Str to DAP Str for the CF option.
This class provides a way to map HDF5 unsigned 16-bit integer to DAP uint16 for the CF option.
This class provides a way to map HDF5 unsigned 32-bit integer to DAP uint32 for the CF option.
This class provides a way to map HDF5 64-bit unsigned integer to DAP4 UInt64 for the CF option.
This file includes several helper functions for translating HDF5 to CF-compliant.
This class provides a way to map HDF5 Int64 to DAP Int64 for the default option.
include the entry functions to execute the handlers
static string escattr(string s)
Definition: h5cfdaputil.cc:43
static D4AttributeType daptype_strrep_to_dap4_attrtype(std::string s)
Definition: h5cfdaputil.cc:303
This class represents one attribute.
Definition: HDF5CF.h:189
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
int getRank() const
Get the dimension rank of this variable.
Definition: HDF5CF.h:305
const std::string & getFullPath() const
Get the full path of this variable.
Definition: HDF5CF.h:283
const std::string & getName() const
Get the original name of this variable.
Definition: HDF5CF.h:271
H5DataType getType() const
Get the data type of this variable(Not HDF5 datatype id)
Definition: HDF5CF.h:311
const std::vector< Dimension * > & getDimensions() const
Get the list of the dimensions.
Definition: HDF5CF.h:322
int getCompRatio() const
Get the compression ratio of this dataset.
Definition: HDF5CF.h:328
const std::string & getNewName() const
Get the new name of this variable.
Definition: HDF5CF.h:277
Helper functions for generating DAS attributes and a function to check BES Key.
Map and generate DDS and DAS for the CF option for generic HDF5 products.