bes  Updated for version 3.20.6
h5dmr.cc
Go to the documentation of this file.
1 // This file is part of hdf5_handler a HDF5 file handler for the OPeNDAP
2 // data server.
3 
4 // Copyright (c) 2007-2015 The HDF Group, Inc. and OPeNDAP, Inc.
5 //
6 // This is free software; you can redistribute it and/or modify it under the
7 // terms of the GNU Lesser General Public License as published by the Free
8 // Software Foundation; either version 2.1 of the License, or (at your
9 // option) any later version.
10 //
11 // This software is distributed in the hope that it will be useful, but
12 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
13 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
14 // License for more details.
15 //
16 // You should have received a copy of the GNU Lesser General Public
17 // License along with this library; if not, write to the Free Software
18 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19 //
20 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
21 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
22 // Suite 203, Champaign, IL 61820
23 
36 // the correct DAP4 DMR layout(group's variables first and then the group).
39 
40 #include "config_hdf5.h"
41 
42 #include <InternalErr.h>
43 #include <BESDebug.h>
44 
45 #include <mime_util.h>
46 
47 #include "hdf5_handler.h"
48 #include "HDF5Int32.h"
49 #include "HDF5UInt32.h"
50 #include "HDF5UInt16.h"
51 #include "HDF5Int16.h"
52 #include "HDF5Byte.h"
53 #include "HDF5Array.h"
54 #include "HDF5Str.h"
55 #include "HDF5Float32.h"
56 #include "HDF5Float64.h"
57 #include "HDF5Url.h"
58 #include "HDF5Structure.h"
59 
60 // The HDF5CFUtil.h includes the utility function obtain_string_after_lastslash.
61 #include "HDF5CFUtil.h"
62 #include "h5dmr.h"
63 
64 using namespace std;
65 using namespace libdap;
68 
69 
71 static DS_t dt_inst;
72 
74 void map_h5_attrs_to_dap4(hid_t oid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag);
75 
76 #if 0
82 // \param par_grp DAP4 parent group
92 
93 //bool depth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname)
94 bool depth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname)
95 {
96  BESDEBUG("h5",
97  ">depth_first() for dmr "
98  << " pid: " << pid
99  << " gname: " << gname
100  << " fname: " << fname
101  << endl);
102 
104  int slinkindex = 0;
105 
106  H5G_info_t g_info;
107  hsize_t nelems = 0;
108 
110  if(H5Gget_info(pid,&g_info) <0) {
111  string msg =
112  "h5_dmr handler: counting hdf5 group elements error for ";
113  msg += gname;
114  throw InternalErr(__FILE__, __LINE__, msg);
115  }
116 
117  nelems = g_info.nlinks;
118 
119  ssize_t oname_size = 0;
120 
121  // Iterate through the file to see the members of the group from the root.
122  for (hsize_t i = 0; i < nelems; i++) {
123 
124  vector <char>oname;
125 
126  // Query the length of object name.
127  oname_size =
128  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
129  (size_t)DODS_NAMELEN, H5P_DEFAULT);
130  if (oname_size <= 0) {
131  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
132  msg += gname;
133  throw InternalErr(__FILE__, __LINE__, msg);
134  }
135 
136  // Obtain the name of the object
137  oname.resize((size_t) oname_size + 1);
138 
139  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
140  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
141  string msg =
142  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
143  msg += gname;
144  throw InternalErr(__FILE__, __LINE__, msg);
145  }
146 
147  // Check if it is the hard link or the soft link
148  H5L_info_t linfo;
149  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
150  string msg = "hdf5 link name error from: ";
151  msg += gname;
152  throw InternalErr(__FILE__, __LINE__, msg);
153  }
154 
155  // Information of soft links are stored as attributes
156  if(linfo.type == H5L_TYPE_SOFT) {
157  slinkindex++;
158  size_t val_size = linfo.u.val_size;
159  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
160  //get_softlink(par_grp,pid,gname,&oname[0],slinkindex,val_size);
161  continue;
162  }
163 
164  // Ignore external links
165  if(linfo.type == H5L_TYPE_EXTERNAL)
166  continue;
167 
168  // Obtain the object type, such as group or dataset.
169  H5O_info_t oinfo;
170 
171  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
172  i, &oinfo, H5P_DEFAULT)<0) {
173  string msg = "h5_dmr handler: Error obtaining the info for the object";
174  msg += string(oname.begin(),oname.end());
175  throw InternalErr(__FILE__, __LINE__, msg);
176  }
177 
178  H5O_type_t obj_type = oinfo.type;
179 
180  switch (obj_type) {
181 
182  case H5O_TYPE_GROUP:
183  {
184 
185  // Obtain the full path name
186  string full_path_name =
187  string(gname) + string(oname.begin(),oname.end()-1) + "/";
188 
189  BESDEBUG("h5", "=depth_first dmr ():H5G_GROUP " << full_path_name
190  << endl);
191 
192  vector <char>t_fpn;
193  t_fpn.resize(full_path_name.length()+1);
194  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
195  t_fpn[full_path_name.length()] = '\0';
196 
197  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
198  if (cgroup < 0){
199  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
200  }
201 
202  string grp_name = string(oname.begin(),oname.end()-1);
203 
204  // Check the hard link loop and break the loop if it exists.
205  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
206  if (oid == "") {
207  try {
208  D4Group* tem_d4_cgroup = new D4Group(grp_name);
209  // Map the HDF5 cgroup attributes to DAP4 group attributes.
210  // Note the last flag of map_h5_attrs_to_dap4 must be 0 for the group attribute mapping.
211  map_h5_attrs_to_dap4(cgroup,tem_d4_cgroup,NULL,NULL,0);
212 
213  // Add this new DAP4 group
214  par_grp->add_group_nocopy(tem_d4_cgroup);
215 
216  // Continue searching the objects under this group
217  //depth_first(cgroup, &t_fpn[0], dmr, tem_d4_cgroup,fname);
218  depth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname);
219  }
220  catch(...) {
221  H5Gclose(cgroup);
222  throw;
223  }
224  }
225  else {
226  // This group has been visited.
227  // Add the attribute table with the attribute name as HDF5_HARDLINK.
228  // The attribute value is the name of the group when it is first visited.
229  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
230 
231  // Note attr_str_c is the DAP4 attribute string datatype
232  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
233 
234  d4_hlinfo->add_value(obj_paths.get_name(oid));
235  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
236  par_grp->add_group_nocopy(tem_d4_cgroup);
237 
238  }
239 
240  if (H5Gclose(cgroup) < 0){
241  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
242  }
243  break;
244  }
245 
246  case H5O_TYPE_DATASET:
247  {
248 
249  // Obtain the absolute path of the HDF5 dataset
250  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
251 
252  // TOOOODOOOO
253  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
254  // All the metadata information in the handler is stored in dt_inst.
255  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
256  // Note: depth_first is for building DMR of an HDF5 file that doesn't use dim. scale.
257  // so passing the last parameter as false.
258  get_dataset(pid, full_path_name, &dt_inst,false);
259 
260  // Here we open the HDF5 dataset again to use the dataset id for dataset attributes.
261  // This is not necessary for DAP2 since DAS and DDS are separated.
262  hid_t dset_id = -1;
263  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
264  string msg = "cannot open the HDF5 dataset ";
265  msg += full_path_name;
266  throw InternalErr(__FILE__, __LINE__, msg);
267  }
268 
269  try {
270  read_objects(par_grp, full_path_name, fname,dset_id);
271  }
272  catch(...) {
273  H5Dclose(dset_id);
274  throw;
275  }
276  if(H5Dclose(dset_id)<0) {
277  string msg = "cannot close the HDF5 dataset ";
278  msg += full_path_name;
279  throw InternalErr(__FILE__, __LINE__, msg);
280  }
281  }
282  break;
283 
284  case H5O_TYPE_NAMED_DATATYPE:
285  // ignore the named datatype
286  break;
287  default:
288  break;
289  }// switch(obj_type)
290  } // for i is 0 ... nelems
291 
292  BESDEBUG("h5", "<depth_first() for dmr" << endl);
293  return true;
294 }
295 #endif
301 // \param par_grp DAP4 parent group
313 
314 
315 // The reason to use breadth_first is that the DMR representation needs to show the dimension names and the variables under the group first and then the group names.
316 // So we use this search. In the future, we may just use the breadth_first search for all cases.??
317 //bool breadth_first(hid_t pid, char *gname, DMR & dmr, D4Group* par_grp, const char *fname,bool use_dimscale)
318 bool breadth_first(hid_t pid, char *gname, D4Group* par_grp, const char *fname,bool use_dimscale)
319 {
320  BESDEBUG("h5",
321  ">breadth_first() for dmr "
322  << " pid: " << pid
323  << " gname: " << gname
324  << " fname: " << fname
325  << endl);
326 
328  int slinkindex = 0;
329 
330  // Obtain the number of objects in this group
331  H5G_info_t g_info;
332  hsize_t nelems = 0;
333  if(H5Gget_info(pid,&g_info) <0) {
334  string msg =
335  "h5_dmr handler: counting hdf5 group elements error for ";
336  msg += gname;
337  throw InternalErr(__FILE__, __LINE__, msg);
338  }
339 
340  nelems = g_info.nlinks;
341 
342  ssize_t oname_size;
343 
344  // First iterate through the HDF5 datasets under the group.
345  for (hsize_t i = 0; i < nelems; i++) {
346 
347  vector <char>oname;
348 
349  // Query the length of object name.
350  oname_size =
351  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
352  (size_t)DODS_NAMELEN, H5P_DEFAULT);
353  if (oname_size <= 0) {
354  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
355  msg += gname;
356  throw InternalErr(__FILE__, __LINE__, msg);
357  }
358 
359  // Obtain the name of the object
360  oname.resize((size_t) oname_size + 1);
361 
362  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
363  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
364  string msg =
365  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
366  msg += gname;
367  throw InternalErr(__FILE__, __LINE__, msg);
368  }
369 
370  // Check if it is the hard link or the soft link
371  H5L_info_t linfo;
372  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
373  string msg = "hdf5 link name error from: ";
374  msg += gname;
375  throw InternalErr(__FILE__, __LINE__, msg);
376  }
377 
378  // Information of soft links are stored as attributes
379  if(linfo.type == H5L_TYPE_SOFT) {
380  slinkindex++;
381 
382  // Size of a soft link value
383  size_t val_size = linfo.u.val_size;
384  get_softlink(par_grp,pid,&oname[0],slinkindex,val_size);
385  continue;
386  }
387 
388  // Ignore external links
389  if(linfo.type == H5L_TYPE_EXTERNAL)
390  continue;
391 
392  // Obtain the object type, such as group or dataset.
393  H5O_info_t oinfo;
394 
395  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
396  i, &oinfo, H5P_DEFAULT)<0) {
397  string msg = "h5_dmr handler: Error obtaining the info for the object";
398  msg += string(oname.begin(),oname.end());
399  throw InternalErr(__FILE__, __LINE__, msg);
400  }
401 
402  H5O_type_t obj_type = oinfo.type;
403 
404  if(H5O_TYPE_DATASET == obj_type) {
405 
406  // Obtain the absolute path of the HDF5 dataset
407  string full_path_name = string(gname) + string(oname.begin(),oname.end()-1);
408 
409  // TOOOODOOOO
410  // Obtain the hdf5 dataset handle stored in the structure dt_inst.
411  // All the metadata information in the handler is stored in dt_inst.
412  // Work on this later, redundant for dmr since dataset is opened twice. KY 2015-07-01
413  get_dataset(pid, full_path_name, &dt_inst,use_dimscale);
414 
415  hid_t dset_id = -1;
416  if((dset_id = H5Dopen(pid,full_path_name.c_str(),H5P_DEFAULT)) <0) {
417  string msg = "cannot open the HDF5 dataset ";
418  msg += full_path_name;
419  throw InternalErr(__FILE__, __LINE__, msg);
420  }
421 
422  try {
423  read_objects(par_grp, full_path_name, fname,dset_id);
424  }
425  catch(...) {
426  H5Dclose(dset_id);
427  throw;
428  }
429  if(H5Dclose(dset_id)<0) {
430  string msg = "cannot close the HDF5 dataset ";
431  msg += full_path_name;
432  throw InternalErr(__FILE__, __LINE__, msg);
433  }
434 
435  }
436  }
437 
438  // The attributes of this group. Doing this order to follow ncdump's way (variable,attribute then groups)
439  map_h5_attrs_to_dap4(pid,par_grp,NULL,NULL,0);
440 
441  // Then HDF5 child groups
442  for (hsize_t i = 0; i < nelems; i++) {
443 
444  vector <char>oname;
445 
446  // Query the length of object name.
447  oname_size =
448  H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,NULL,
449  (size_t)DODS_NAMELEN, H5P_DEFAULT);
450  if (oname_size <= 0) {
451  string msg = "h5_dmr handler: Error getting the size of the hdf5 object from the group: ";
452  msg += gname;
453  throw InternalErr(__FILE__, __LINE__, msg);
454  }
455 
456  // Obtain the name of the object
457  oname.resize((size_t) oname_size + 1);
458 
459  if (H5Lget_name_by_idx(pid,".",H5_INDEX_NAME,H5_ITER_NATIVE,i,&oname[0],
460  (size_t)(oname_size+1), H5P_DEFAULT) < 0){
461  string msg =
462  "h5_dmr handler: Error getting the hdf5 object name from the group: ";
463  msg += gname;
464  throw InternalErr(__FILE__, __LINE__, msg);
465  }
466 
467  // Check if it is the hard link or the soft link
468  H5L_info_t linfo;
469  if (H5Lget_info(pid,&oname[0],&linfo,H5P_DEFAULT)<0) {
470  string msg = "hdf5 link name error from: ";
471  msg += gname;
472  throw InternalErr(__FILE__, __LINE__, msg);
473  }
474 
475  // Information of soft links are handled already, the softlinks need to be ignored, otherwise
476  // the group it links will be mapped again in the block of if obj_type is H5O_TYPE_GROUP
477  if(linfo.type == H5L_TYPE_SOFT) {
478  continue;
479  }
480 
481  // Ignore external links
482  if(linfo.type == H5L_TYPE_EXTERNAL)
483  continue;
484 
485  // Obtain the object type, such as group or dataset.
486  H5O_info_t oinfo;
487 
488  if (H5Oget_info_by_idx(pid, ".", H5_INDEX_NAME, H5_ITER_NATIVE,
489  i, &oinfo, H5P_DEFAULT)<0) {
490  string msg = "h5_dmr handler: Error obtaining the info for the object in the breadth_first.";
491  throw InternalErr(__FILE__, __LINE__, msg);
492  }
493 
494  H5O_type_t obj_type = oinfo.type;
495 
496 
497  if(obj_type == H5O_TYPE_GROUP) {
498 
499  // Obtain the full path name
500  string full_path_name =
501  string(gname) + string(oname.begin(),oname.end()-1) + "/";
502 
503  BESDEBUG("h5", "=breadth_first dmr ():H5G_GROUP " << full_path_name
504  << endl);
505 
506  vector <char>t_fpn;
507  t_fpn.resize(full_path_name.length()+1);
508  copy(full_path_name.begin(),full_path_name.end(),t_fpn.begin());
509  t_fpn[full_path_name.length()] = '\0';
510 
511  hid_t cgroup = H5Gopen(pid, &t_fpn[0],H5P_DEFAULT);
512  if (cgroup < 0){
513  throw InternalErr(__FILE__, __LINE__, "h5_dmr handler: H5Gopen() failed.");
514  }
515 
516  string grp_name = string(oname.begin(),oname.end()-1);
517 
518  // Check the hard link loop and break the loop if it exists.
519  string oid = get_hardlink_dmr(cgroup, full_path_name.c_str());
520  if (oid == "") {
521  try {
522  D4Group* tem_d4_cgroup = new D4Group(grp_name);
523 
524  // Add this new DAP4 group
525  par_grp->add_group_nocopy(tem_d4_cgroup);
526 
527  // Continue searching the objects under this group
528  breadth_first(cgroup, &t_fpn[0], tem_d4_cgroup,fname,use_dimscale);
529  }
530  catch(...) {
531  H5Gclose(cgroup);
532  throw;
533  }
534  }
535  else {
536  // This group has been visited.
537  // Add the attribute table with the attribute name as HDF5_HARDLINK.
538  // The attribute value is the name of the group when it is first visited.
539  D4Group* tem_d4_cgroup = new D4Group(string(grp_name));
540 
541  // Note attr_str_c is the DAP4 attribute string datatype
542  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
543 
544  d4_hlinfo->add_value(obj_paths.get_name(oid));
545  tem_d4_cgroup->attributes()->add_attribute_nocopy(d4_hlinfo);
546  par_grp->add_group_nocopy(tem_d4_cgroup);
547  }
548 
549  if (H5Gclose(cgroup) < 0){
550  throw InternalErr(__FILE__, __LINE__, "Could not close the group.");
551  }
552  }// end if
553  } // for i is 0 ... nelems
554 
555  BESDEBUG("h5", "<breadth_first() " << endl);
556  return true;
557 }
558 
574 //
575 void
576 read_objects( D4Group * d4_grp, const string &varname, const string &filename, const hid_t dset_id)
577 {
578 
579  switch (H5Tget_class(dt_inst.type)) {
580 
581  // HDF5 compound maps to DAP structure.
582  case H5T_COMPOUND:
583  read_objects_structure(d4_grp, varname, filename,dset_id);
584  break;
585 
586  case H5T_ARRAY:
587  H5Tclose(dt_inst.type);
588  throw InternalErr(__FILE__, __LINE__, "Currently don't support accessing data of Array datatype when array datatype is not inside the compound.");
589 
590  default:
591  read_objects_base_type(d4_grp,varname, filename,dset_id);
592  break;
593  }
594  // We must close the datatype obtained in the get_dataset routine since this is the end of reading DDS.
595  if(H5Tclose(dt_inst.type)<0) {
596  throw InternalErr(__FILE__, __LINE__, "Cannot close the HDF5 datatype.");
597  }
598 }
599 
614 //
615 
616 //void
617 //read_objects_base_type(DMR & dmr, D4Group * d4_grp,const string & varname,
618 void
619 read_objects_base_type(D4Group * d4_grp,const string & varname,
620  const string & filename,hid_t dset_id)
621 {
622 
623  // Obtain the relative path of the variable name under the leaf group
624  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
625 
626  // Get a base type. It should be an HDF5 atomic datatype
627  // datatype.
628  BaseType *bt = Get_bt(newvarname, varname,filename, dt_inst.type,true);
629  if (!bt) {
630  throw
631  InternalErr(__FILE__, __LINE__,
632  "Unable to convert hdf5 datatype to dods basetype");
633  }
634 
635  // First deal with scalar data.
636  if (dt_inst.ndims == 0) {
637  // transform the DAP2 to DAP4 for this DAP base type and add it to d4_grp
638  bt->transform_to_dap4(d4_grp,d4_grp);
639  // Get it back - this may return null because the underlying type
640  // may have no DAP2 manifestation.
641  BaseType* new_var = d4_grp->var(bt->name());
642  if(new_var){
643  // Map the HDF5 dataset attributes to DAP4
644  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
645  // If this variable is a hardlink, stores the HARDLINK info. as an attribute.
646  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
647  }
648  delete bt;
649  bt = 0;
650  }
651  else {
652  // Next, deal with Array data. This 'else clause' runs to
653  // the end of the method.
654  HDF5Array *ar = new HDF5Array(newvarname, filename, bt);
655  delete bt; bt = 0;
656 
657  // set number of elements and variable name values.
658  // This essentially stores in the struct.
659  ar->set_memneed(dt_inst.need);
660  ar->set_numdim(dt_inst.ndims);
661  ar->set_numelm((int) (dt_inst.nelmts));
662  ar->set_varpath(varname);
663 
664 
665  // If we have dimension names(dimension scale is used.),we will see if we can add the names.
666  int dimnames_size = 0;
667  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
668  {
669  delete ar;
670  throw
671  InternalErr(__FILE__, __LINE__,
672  "number of dimensions: overflow");
673  }
674  dimnames_size = (int)(dt_inst.dimnames.size());
675 
676  if(dimnames_size ==dt_inst.ndims) {
677  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
678  if(dt_inst.dimnames[dim_index] !="")
679  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
680  else
681  ar->append_dim(dt_inst.size[dim_index]);
682  }
683  dt_inst.dimnames.clear();
684  }
685  else {
686  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
687  ar->append_dim(dt_inst.size[dim_index]);
688  }
689 
690  // We need to transform dimension info. to DAP4 group
691  BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp);
692 
693  // Map HDF5 dataset attributes to DAP4
694  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
695 
696  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
697  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
698 #if 0
699  // Test the attribute
700  D4Attribute *test_attr = new D4Attribute("DAP4_test",attr_str_c);
701  test_attr->add_value("test_grp_attr");
702  new_var->attributes()->add_attribute_nocopy(test_attr);
703 #endif
704  // Add this var to DAP4 group.
705  d4_grp->add_var_nocopy(new_var);
706  delete ar; ar = 0;
707  }
708 
709  BESDEBUG("h5", "<read_objects_base_type(dmr)" << endl);
710 }
711 
725 void
726 read_objects_structure(D4Group *d4_grp, const string & varname,
727  const string & filename,hid_t dset_id)
728 {
729  // Obtain the relative path of the variable name under the leaf group
730  string newvarname = HDF5CFUtil::obtain_string_after_lastslash(varname);
731 
732  // Map HDF5 compound datatype to Structure
733  Structure *structure = Get_structure(newvarname, varname,filename, dt_inst.type,true);
734 
735  try {
736  BESDEBUG("h5", "=read_objects_structure(): Dimension is "
737  << dt_inst.ndims << endl);
738 
739  if (dt_inst.ndims != 0) { // Array of Structure
740  BESDEBUG("h5", "=read_objects_structure(): array of size " <<
741  dt_inst.nelmts << endl);
742  BESDEBUG("h5", "=read_objects_structure(): memory needed = " <<
743  dt_inst.need << endl);
744 
745  // Create the Array of structure.
746  HDF5Array *ar = new HDF5Array(newvarname, filename, structure);
747  delete structure; structure = 0;
748 
749 
750  // These parameters are used in the data read function.
751  ar->set_memneed(dt_inst.need);
752  ar->set_numdim(dt_inst.ndims);
753  ar->set_numelm((int) (dt_inst.nelmts));
754  ar->set_length((int) (dt_inst.nelmts));
755  ar->set_varpath(varname);
756 
757  // If having dimension names, add the dimension names to DAP.
758  int dimnames_size = 0;
759  if((unsigned int)((int)(dt_inst.dimnames.size())) != dt_inst.dimnames.size())
760  {
761  delete ar;
762  throw
763  InternalErr(__FILE__, __LINE__,
764  "number of dimensions: overflow");
765  }
766  dimnames_size = (int)(dt_inst.dimnames.size());
767 
768 
769  if(dimnames_size ==dt_inst.ndims) {
770  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++) {
771  if(dt_inst.dimnames[dim_index] !="")
772  ar->append_dim(dt_inst.size[dim_index],dt_inst.dimnames[dim_index]);
773  else
774  ar->append_dim(dt_inst.size[dim_index]);
775  }
776  dt_inst.dimnames.clear();
777  }
778  else {
779  for (int dim_index = 0; dim_index < dt_inst.ndims; dim_index++)
780  ar->append_dim(dt_inst.size[dim_index]);
781 
782  }
783 
784  // We need to transform dimension info. to DAP4 group
785  BaseType* new_var = ar->h5dims_transform_to_dap4(d4_grp);
786 
787  // Map HDF5 dataset attributes to DAP4
788  map_h5_attrs_to_dap4(dset_id,NULL,new_var,NULL,1);
789 
790  // If this is a hardlink, map the Hardlink info. as an DAP4 attribute.
791  map_h5_dset_hardlink_to_d4(dset_id,varname,new_var,NULL,1);
792 
793  // Add this var to DAP4 group
794  if(new_var)
795  d4_grp->add_var_nocopy(new_var);
796  delete ar; ar = 0;
797  }// end if
798  else {// A scalar structure
799 
800  structure->set_is_dap4(true);
801  map_h5_attrs_to_dap4(dset_id,NULL,NULL,structure,2);
802  map_h5_dset_hardlink_to_d4(dset_id,varname,NULL,structure,2);
803  if(structure)
804  d4_grp->add_var_nocopy(structure);
805  }
806  } // try Structure
807  catch (...) {
808  delete structure;
809  throw;
810  }
811 }
812 
813 
827 //
828 
829 void map_h5_attrs_to_dap4(hid_t h5_objid,D4Group* d4g,BaseType* d4b,Structure * d4s,int flag) {
830 
831  // Get the object info
832  H5O_info_t obj_info;
833  if (H5Oget_info(h5_objid, &obj_info) <0) {
834  string msg = "Fail to obtain the HDF5 object info. .";
835  throw InternalErr(__FILE__, __LINE__, msg);
836  }
837 
838  // Obtain the number of attributes
839  int num_attr = obj_info.num_attrs;
840  if (num_attr < 0 ) {
841  string msg = "Fail to get the number of attributes for the HDF5 object. ";
842  throw InternalErr(__FILE__, __LINE__,msg);
843  }
844 
845  string print_rep;
846  vector<char>temp_buf;
847 
848  bool ignore_attr = false;
849  hid_t attr_id = -1;
850  for (int j = 0; j < num_attr; j++) {
851 
852  // Obtain attribute information.
853  DSattr_t attr_inst;
854 
855  // Ignore the attributes of which the HDF5 datatype
856  // cannot be mapped to DAP4. The ignored attribute datatypes can be found
857  // at function get_attr_info in h5get.cc.
858  attr_id = get_attr_info(h5_objid, j, true,&attr_inst, &ignore_attr);
859  if (true == ignore_attr) {
860  H5Aclose(attr_id);
861  continue;
862  }
863 
864  // Get the corresponding DAP data type of the HDF5 datatype.
865  // The following line doesn't work in HDF5 1.10.
866 #if 0
867  //hid_t ty_id = attr_inst.type;
868 #endif
869  hid_t ty_id = H5Aget_type(attr_id);
870  string dap_type = get_dap_type(ty_id,true);
871 
872  // Need to have DAP4 representation of the attribute type
873  D4AttributeType dap4_attr_type = daptype_strrep_to_dap4_attrtype(dap_type);
874 
875  // We encounter an unsupported DAP4 attribute type.
876  if(attr_null_c == dap4_attr_type) {
877  H5Tclose(ty_id);
878  H5Aclose(attr_id);
879  throw InternalErr(__FILE__, __LINE__, "unsupported DAP4 attribute type");
880  }
881 
882  string attr_name = attr_inst.name;
883 
884  // Create the DAP4 attribute mapped from HDF5
885  D4Attribute *d4_attr = new D4Attribute(attr_name,dap4_attr_type);
886 
887  // We have to handle variable length string differently.
888  if (H5Tis_variable_str(ty_id)) {
889  write_vlen_str_attrs(attr_id,ty_id,&attr_inst,d4_attr,NULL,true);
890 #if 0
891  BESDEBUG("h5","attribute name " << attr_name <<endl);
892  BESDEBUG("h5","attribute size " <<attr_inst.need <<endl);
893  //BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(attr_inst.type))<<endl);
894  BESDEBUG("h5","attribute type size " <<(int)(H5Tget_size(ty_id))<<endl);
895 
896  hid_t temp_space_id = H5Aget_space(attr_id);
897  //BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(attr_inst.type)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
898  BESDEBUG("h5","attribute calculated size "<<(int)(H5Tget_size(ty_id)) *(int)(H5Sget_simple_extent_npoints(temp_space_id)) <<endl);
899  if(temp_space_id <0) {
900  H5Tclose(ty_id);
901  H5Aclose(attr_id);
902  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
903  }
904 
905  // Variable length string attribute values only store pointers of the actual string value.
906  temp_buf.resize((size_t)attr_inst.need);
907 
908  if (H5Aread(attr_id, ty_id, &temp_buf[0]) < 0) {
909  H5Sclose(temp_space_id);
910  H5Tclose(ty_id);
911  H5Aclose(attr_id);
912  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
913  }
914 
915  char *temp_bp;
916  temp_bp = &temp_buf[0];
917  char* onestring;
918  for (unsigned int temp_i = 0; temp_i <attr_inst.nelmts; temp_i++) {
919 
920  // This line will assure that we get the real variable length string value.
921  onestring =*(char **)temp_bp;
922 
923  // Change the C-style string to C++ STD string just for easy appending the attributes in DAP.
924  if (onestring !=NULL) {
925  string tempstring(onestring);
926  d4_attr->add_value(tempstring);
927  }
928 
929  // going to the next value.
930  //temp_bp +=H5Tget_size(attr_inst.type);
931  temp_bp +=H5Tget_size(ty_id);
932  }
933  if (temp_buf.empty() != true) {
934 
935  // Reclaim any VL memory if necessary.
936  herr_t ret_vlen_claim;
937  //ret_vlen_claim = H5Dvlen_reclaim(attr_inst.type,temp_space_id,H5P_DEFAULT,&temp_buf[0]);
938  ret_vlen_claim = H5Dvlen_reclaim(ty_id,temp_space_id,H5P_DEFAULT,&temp_buf[0]);
939  if(ret_vlen_claim < 0){
940  H5Sclose(temp_space_id);
941  throw InternalErr(__FILE__, __LINE__, "Cannot reclaim the memory buffer of the HDF5 variable length string.");
942  }
943 
944  temp_buf.clear();
945  }
946  H5Sclose(temp_space_id);
947 #endif
948  }// if (H5Tis_variable_str(ty_id)
949  else {
950 
951  vector<char> value;
952  value.resize(attr_inst.need + sizeof(char));
953  BESDEBUG("h5", "arttr_inst.need=" << attr_inst.need << endl);
954 
955  // Read HDF5 attribute data.
956  if (H5Aread(attr_id, ty_id, (void *) (&value[0])) < 0) {
957  throw InternalErr(__FILE__, __LINE__, "unable to read HDF5 attribute data");
958  }
959 
960  // For scalar data, just read data once.
961  if (attr_inst.ndims == 0) {
962  for (int loc = 0; loc < (int) attr_inst.nelmts; loc++) {
963  print_rep = print_attr(ty_id, loc, &value[0]);
964  if (print_rep.c_str() != NULL) {
965  d4_attr->add_value(print_rep);
966  }
967  }
968 
969  }
970  else {// The number of dimensions is > 0
971 
972  // Get the attribute datatype size
973  int elesize = (int) H5Tget_size(ty_id);
974  if (elesize == 0) {
975  H5Tclose(ty_id);
976  H5Aclose(attr_id);
977  throw InternalErr(__FILE__, __LINE__, "unable to get attibute size");
978  }
979 
980  // Due to the implementation of print_attr, the attribute value will be
981  // written one by one.
982  char *tempvalue = &value[0];
983 
984  // Write this value. the "loc" can always be set to 0 since
985  // tempvalue will be moved to the next value.
986  for( hsize_t temp_index = 0; temp_index < attr_inst.nelmts; temp_index ++) {
987  print_rep = print_attr(ty_id, 0, tempvalue);
988  if (print_rep.c_str() != NULL) {
989 
990  d4_attr->add_value(print_rep);
991  tempvalue = tempvalue + elesize;
992  BESDEBUG("h5",
993  "tempvalue=" << tempvalue
994  << "elesize=" << elesize
995  << endl);
996 
997  }
998  else {
999  H5Tclose(ty_id);
1000  H5Aclose(attr_id);
1001  throw InternalErr(__FILE__, __LINE__, "unable to convert attibute value to DAP");
1002  }
1003  }//for(hsize_t temp_index=0; .....
1004  } // if attr_inst.ndims != 0
1005  }
1006  if(H5Tclose(ty_id) < 0) {
1007  H5Aclose(attr_id);
1008  throw InternalErr(__FILE__, __LINE__, "unable to close HDF5 type id");
1009  }
1010  if (H5Aclose(attr_id) < 0) {
1011  throw InternalErr(__FILE__, __LINE__, "unable to close attibute id");
1012  }
1013 
1014  if(0 == flag) // D4group
1015  d4g->attributes()->add_attribute_nocopy(d4_attr);
1016  else if (1 == flag) // HDF5 dataset with atomic datatypes
1017  d4b->attributes()->add_attribute_nocopy(d4_attr);
1018  else if ( 2 == flag) // HDF5 dataset with compound datatype
1019  d4s->attributes()->add_attribute_nocopy(d4_attr);
1020  } // for (int j = 0; j < num_attr; j++)
1021 
1022  return;
1023 }
1024 
1038 
1039 
1040 void map_h5_dset_hardlink_to_d4(hid_t h5_dsetid,const string & full_path, BaseType* d4b,Structure * d4s,int flag) {
1041 
1042  // Obtain the unique object number info. If no hardlinks, empty string will return.
1043  string oid = get_hardlink_dmr(h5_dsetid, full_path);
1044 
1045  // Find that this is a hardlink,add the hardlink info to a DAP4 attribute.
1046  if(false == oid.empty()) {
1047 
1048  D4Attribute *d4_hlinfo = new D4Attribute("HDF5_HARDLINK",attr_str_c);
1049  d4_hlinfo->add_value(obj_paths.get_name(oid));
1050 
1051  if (1 == flag)
1052  d4b->attributes()->add_attribute_nocopy(d4_hlinfo);
1053  else if ( 2 == flag)
1054  d4s->attributes()->add_attribute_nocopy(d4_hlinfo);
1055  else
1056  delete d4_hlinfo;
1057  }
1058 
1059 }
1060 
1073 void get_softlink(D4Group* par_grp, hid_t h5obj_id, const string & oname, int index, size_t val_size)
1074 {
1075  BESDEBUG("h5", "dap4 >get_softlink():" << oname << endl);
1076 
1077  ostringstream oss;
1078  oss << string("HDF5_SOFTLINK");
1079  oss << "_";
1080  oss << index;
1081  string temp_varname = oss.str();
1082 
1083 
1084  BESDEBUG("h5", "dap4->get_softlink():" << temp_varname << endl);
1085  D4Attribute *d4_slinfo = new D4Attribute;
1086  d4_slinfo->set_name(temp_varname);
1087 
1088  // Make the type as a container
1089  d4_slinfo->set_type(attr_container_c);
1090 
1091  string softlink_name = "linkname";
1092 
1093  D4Attribute *softlink_src = new D4Attribute(softlink_name,attr_str_c);
1094  softlink_src->add_value(oname);
1095 
1096  d4_slinfo->attributes()->add_attribute_nocopy(softlink_src);
1097  string softlink_value_name ="LINKTARGET";
1098 
1099  // Get the link target information. We always return the link value in a string format.
1100  D4Attribute *softlink_tgt = 0;
1101 
1102  try {
1103  vector<char> buf;
1104  buf.resize(val_size + 1);
1105 
1106  // get link target name
1107  if (H5Lget_val(h5obj_id, oname.c_str(), (void*) &buf[0], val_size + 1, H5P_DEFAULT) < 0) {
1108  throw InternalErr(__FILE__, __LINE__, "unable to get link value");
1109  }
1110  softlink_tgt = new D4Attribute(softlink_value_name, attr_str_c);
1111  string link_target_name = string(buf.begin(), buf.end());
1112  softlink_tgt->add_value(link_target_name);
1113 
1114  d4_slinfo->attributes()->add_attribute_nocopy(softlink_tgt);
1115  }
1116  catch (...) {
1117  delete softlink_tgt;
1118  throw;
1119  }
1120 
1121  par_grp->attributes()->add_attribute_nocopy(d4_slinfo);
1122 }
1123 
1124 
1137 string get_hardlink_dmr( hid_t h5obj_id, const string & oname) {
1138 
1139  BESDEBUG("h5", "dap4->get_hardlink_dmr():" << oname << endl);
1140 
1141  // Get the object info
1142  H5O_info_t obj_info;
1143  if (H5Oget_info(h5obj_id, &obj_info) <0) {
1144  throw InternalErr(__FILE__, __LINE__, "H5Oget_info() failed.");
1145  }
1146 
1147  // If the reference count is greater than 1,that means
1148  // hard links are found. return the original object name this
1149  // hard link points to.
1150 
1151  if (obj_info.rc >1) {
1152 
1153  ostringstream oss;
1154  oss << hex << obj_info.addr;
1155  string objno = oss.str();
1156 
1157  BESDEBUG("h5", "dap4->get_hardlink_dmr() objno=" << objno << endl);
1158 
1159  // Add this hard link to the map.
1160  // obj_paths is a global variable defined at the beginning of this file.
1161  // it is essentially a id to obj name map. See HDF5PathFinder.h.
1162  if (!obj_paths.add(objno, oname)) {
1163  return objno;
1164  }
1165  else {
1166  return "";
1167  }
1168  }
1169  else {
1170  return "";
1171  }
1172 
1173 }
A class for handling all types of array in HDF5 for the default option.
This class provides a way to map HDF5 byte to DAP Byte for the default option.
This file includes several helper functions for translating HDF5 to CF-compliant.
A class for mapping HDF5 32-bit float to DAP for the default option.
A class for mapping HDF5 64-bit float to DAP for the default option.
A class for HDF5 signed 16 bit integer type.
This class provides a way to map HDF5 32 bit integer to DAP Int32 for the default option.
This class that translates HDF5 string into DAP string for the default option.
This class converts HDF5 compound type into DAP structure for the default option.
This class provides a way to map unsigned HDF5 16 bit integer to DAP UInt16 for the default option.
This class provides a way to map unsigned HDF5 32 bit integer to DAP UInt32.
This class generates DAP URL type for the default option.
void set_numdim(int ndims)
remembers number of dimensions of this array.
Definition: HDF5Array.cc:1450
void set_numelm(int nelms)
remembers number of elements in this array.
Definition: HDF5Array.cc:1454
void set_memneed(size_t need)
remembers memory size needed.
Definition: HDF5Array.cc:1446
std::string get_name(std::string id)
bool add(std::string id, const std::string name)
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void read_objects(DAS &das, const string &varname, hid_t oid, int num_attr)
Definition: h5das.cc:295
void read_objects_base_type(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:256
void read_objects_structure(DDS &dds_table, const string &varname, const string &filename)
Definition: h5dds.cc:310
void map_h5_attrs_to_dap4(hid_t oid, D4Group *d4g, BaseType *d4b, Structure *d4s, int flag)
A function that map HDF5 attributes to DAP4.
Definition: h5dmr.cc:829
HDF5PathFinder obj_paths
A variable for remembering visited paths to break cyclic HDF5 groups.
Definition: h5dmr.cc:67
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
Definition: h5dmr.cc:318
void get_softlink(D4Group *par_grp, hid_t h5obj_id, const string &oname, int index, size_t val_size)
Definition: h5dmr.cc:1073
Data structure and retrieval processing header for the default option.
string print_attr(hid_t type, int loc, void *sm_buf)
Definition: h5get.cc:642
string get_dap_type(hid_t type, bool is_dap4)
Definition: h5get.cc:272
hid_t get_attr_info(hid_t dset, int index, bool is_dap4, DSattr_t *attr_inst_ptr, bool *ignore_attr_ptr)
Definition: h5get.cc:83
The main header of the HDF5 OPeNDAP handler.
const int DODS_NAMELEN
Maximum length of variable or attribute name(default option only).
Definition: hdf5_handler.h:64
A structure for DDS generation.
Definition: hdf5_handler.h:70
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:87
hsize_t need
Space needed.
Definition: hdf5_handler.h:89
hid_t type
HDF5 data set id.
Definition: hdf5_handler.h:78
int size[DODS_MAX_RANK]
Size of each dimension.
Definition: hdf5_handler.h:84
int ndims
HDF5 data space id.
Definition: hdf5_handler.h:82
A structure for DAS generation.
Definition: hdf5_handler.h:92
char name[DODS_NAMELEN]
Name of HDF5 group or dataset.
Definition: hdf5_handler.h:94
int ndims
Number of dimensions.
Definition: hdf5_handler.h:98
hsize_t nelmts
Number of elements.
Definition: hdf5_handler.h:102
hsize_t need
Memory space needed to hold nelmts type.
Definition: hdf5_handler.h:104