bes  Updated for version 3.20.10
HDF5CF.cc
Go to the documentation of this file.
1 // This file is part of the hdf5_handler implementing for the CF-compliant
2 // Copyright (c) 2011-2016 The HDF Group, Inc. and OPeNDAP, Inc.
3 //
4 // This is free software; you can redistribute it and/or modify it under the
5 // terms of the GNU Lesser General Public License as published by the Free
6 // Software Foundation; either version 2.1 of the License, or (at your
7 // option) any later version.
8 //
9 // This software is distributed in the hope that it will be useful, but
10 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
11 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
12 // License for more details.
13 //
14 // You should have received a copy of the GNU Lesser General Public
15 // License along with this library; if not, write to the Free Software
16 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 //
18 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
19 // You can contact The HDF Group, Inc. at 1800 South Oak Street,
20 // Suite 203, Champaign, IL 61820
21 
36 
37 #include <sstream>
38 #include <algorithm>
39 #include <functional>
40 #include <climits>
41 #include "HDF5CF.h"
42 #include "h5cfdaputil.h"
43 #include "HDF5RequestHandler.h"
44 #include "h5apicompatible.h"
45 #include "BESDebug.h"
46 
47 using namespace HDF5CF;
48 
49 Var::Var(Var *var)
50 {
51 
52  newname = var->newname;
53  name = var->name;
54  fullpath = var->fullpath;
55  rank = var->rank;
56  total_elems = var->total_elems;
57  zero_storage_size = var->zero_storage_size;
58  dtype = var->dtype;
59  comp_ratio = var->comp_ratio;
60  unsupported_attr_dtype = var->unsupported_attr_dtype;
61  unsupported_attr_dspace = var->unsupported_attr_dspace;
62  unsupported_dspace = var->unsupported_dspace;
63  unsupported_attr_dspace = var->unsupported_attr_dspace;
64  dimnameflag = var->dimnameflag;
65  coord_attr_add_path = var->coord_attr_add_path;
66 
67  for (vector<Attribute*>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
68  Attribute* attr = new Attribute();
69  attr->name = (*ira)->name;
70  attr->newname = (*ira)->newname;
71  attr->dtype = (*ira)->dtype;
72  attr->count = (*ira)->count;
73  attr->strsize = (*ira)->strsize;
74  attr->fstrsize = (*ira)->fstrsize;
75  attr->value = (*ira)->value;
76  attrs.push_back(attr);
77  }
78 
79  for (vector<Dimension*>::iterator ird = var->dims.begin(); ird != var->dims.end(); ++ird) {
80  Dimension *dim = new Dimension((*ird)->size);
81  dim->name = (*ird)->name;
82  dim->newname = (*ird)->newname;
83  dim->unlimited_dim = (*ird)->unlimited_dim;
84  dims.push_back(dim);
85  }
86 
87 }
88 
89 bool CVar::isLatLon() const
90 {
91 
92  bool ret_value = false;
93  if (CV_EXIST == this->cvartype || CV_MODIFY == this->cvartype || CV_SPECIAL == this->cvartype) {
94  string attr_name = "units";
95  string lat_unit_value = "degrees_north";
96  string lon_unit_value = "degrees_east";
97 
98  for (vector<Attribute *>::const_iterator ira = this->attrs.begin(); ira != this->attrs.end(); ira++) {
99 
100  if ((H5FSTRING == (*ira)->getType()) || (H5VSTRING == (*ira)->getType())) {
101  if (attr_name == (*ira)->newname) {
102  string attr_value1((*ira)->getValue().begin(), (*ira)->getValue().end());
103 
104  if ((*ira)->getCount() == 1) {
105  string attr_value((*ira)->getValue().begin(), (*ira)->getValue().end());
106  if (attr_value.compare(0, lat_unit_value.size(), lat_unit_value) == 0) {
107  if (attr_value.size() == lat_unit_value.size()) {
108  ret_value = true;
109  break;
110  }
111  else if (attr_value.size() == (lat_unit_value.size() + 1)) {
112  if (attr_value[attr_value.size() - 1] == '\0'
113  || attr_value[attr_value.size() - 1] == ' ') {
114  ret_value = true;
115  break;
116  }
117  }
118  }
119  else if (attr_value.compare(0, lon_unit_value.size(), lon_unit_value) == 0) {
120  if (attr_value.size() == lon_unit_value.size()) {
121  ret_value = true;
122  break;
123  }
124  else if (attr_value.size() == (lon_unit_value.size() + 1)) {
125  if (attr_value[attr_value.size() - 1] == '\0'
126  || attr_value[attr_value.size() - 1] == ' ') {
127  ret_value = true;
128  break;
129  }
130  }
131 
132  }
133 
134  }
135  }
136  }
137  }
138  }
139  else if (this->cvartype == CV_LAT_MISS || this->cvartype == CV_LON_MISS) ret_value = true;
140  return ret_value;
141 
142 }
143 File::~File()
144 {
145 
146  if (this->fileid >= 0) {
147  if (this->rootid >= 0) {
148  for_each(this->groups.begin(), this->groups.end(), delete_elem());
149  for_each(this->vars.begin(), this->vars.end(), delete_elem());
150  for_each(this->root_attrs.begin(), this->root_attrs.end(), delete_elem());
151  H5Gclose(rootid);
152  }
153  }
154 }
155 
156 Group::~Group()
157 {
158  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
159 }
160 
161 Var::~Var()
162 {
163  for_each(this->dims.begin(), this->dims.end(), delete_elem());
164  for_each(this->attrs.begin(), this->attrs.end(), delete_elem());
165 }
166 
167 Attribute::~Attribute()
168 {
169 }
170 
171 void File::Retrieve_H5_Info(const char * /*path*/, hid_t file_id, bool include_attr)
172 {
173 
174  BESDEBUG("h5", "coming to Retrieve_H5_Info" <<endl);
175 
176  if (true == include_attr) {
177 
178  // Obtain the BES key to check the ignored objects
179  // We will only use DAS to output these information.
180  this->check_ignored = HDF5RequestHandler::get_check_ignore_obj();
181  if (true == this->check_ignored) this->add_ignored_info_page_header();
182 
183  }
184 
185  hid_t root_id;
186  if ((root_id = H5Gopen(file_id, "/", H5P_DEFAULT)) < 0) {
187  throw1("Cannot open the HDF5 root group ");
188  }
189  this->rootid = root_id;
190  try {
191  this->Retrieve_H5_Obj(root_id, "/", include_attr);
192  }
193  catch (...) {
194  throw;
195  }
196 
197  // Obtain attributes only necessary
198  if (true == include_attr) {
199 
200  // Find the file(root group) attribute
201 
202  // Obtain the object type, such as group or dataset.
203  H5O_info_t oinfo;
204  int num_attrs = 0;
205 
206  if (H5OGET_INFO(root_id, &oinfo) < 0)
207  throw1("Error obtaining the info for the root group");
208 
209  num_attrs = oinfo.num_attrs;
210  bool temp_unsup_attr_atype = false;
211  bool temp_unsup_attr_dspace = false;
212 
213  for (int j = 0; j < num_attrs; j++) {
214  Attribute * attr = new Attribute();
215  try {
216  this->Retrieve_H5_Attr_Info(attr, root_id, j, temp_unsup_attr_atype, temp_unsup_attr_dspace);
217  }
218  catch (...) {
219  delete attr;
220  throw;
221 
222  }
223  this->root_attrs.push_back(attr);
224  }
225 
226  this->unsupported_attr_dtype = temp_unsup_attr_atype;
227  this->unsupported_attr_dspace = temp_unsup_attr_dspace;
228  }
229 }
230 
231 void File::Retrieve_H5_Obj(hid_t grp_id, const char*gname, bool include_attr)
232 {
233 
234  // Iterate through the file to see the members of the group from the root.
235  H5G_info_t g_info;
236  hsize_t nelems = 0;
237 
238  if (H5Gget_info(grp_id, &g_info) < 0)
239  throw2("Counting hdf5 group elements error for ", gname);
240  nelems = g_info.nlinks;
241 
242  ssize_t oname_size = 0;
243  for (hsize_t i = 0; i < nelems; i++) {
244 
245  hid_t cgroup = -1;
246  hid_t cdset = -1;
247  Group *group = NULL;
248  Var *var = NULL;
249  Attribute *attr = NULL;
250 
251  try {
252 
253  size_t dummy_name_len = 1;
254 
255  // Query the length of object name.
256  oname_size = H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, NULL, dummy_name_len,
257  H5P_DEFAULT);
258  if (oname_size <= 0)
259  throw2("Error getting the size of the hdf5 object from the group: ", gname);
260 
261  // Obtain the name of the object
262  vector<char> oname;
263  oname.resize((size_t) oname_size + 1);
264 
265  if (H5Lget_name_by_idx(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oname[0], (size_t) (oname_size + 1),
266  H5P_DEFAULT) < 0)
267  throw2("Error getting the hdf5 object name from the group: ", gname);
268 
269  // Check if it is a hard link or a soft link
270  H5L_info_t linfo;
271  if (H5Lget_info(grp_id, &oname[0], &linfo, H5P_DEFAULT) < 0)
272  throw2("HDF5 link name error from ", gname);
273 
274  // We ignore soft links and external links for the CF options
275  if (H5L_TYPE_SOFT == linfo.type || H5L_TYPE_EXTERNAL == linfo.type) {
276  if (true == include_attr && true == check_ignored) {
277  this->add_ignored_info_links_header();
278  string full_path_name;
279  string temp_oname(oname.begin(), oname.end());
280  full_path_name = (
281  (string(gname) != "/") ?
282  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
283  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
284  this->add_ignored_info_links(full_path_name);
285 
286  }
287  continue;
288  }
289 
290  // Obtain the object type, such as group or dataset.
291  H5O_info_t oinfo;
292 
293  if (H5OGET_INFO_BY_IDX(grp_id, ".", H5_INDEX_NAME, H5_ITER_NATIVE, i, &oinfo, H5P_DEFAULT) < 0)
294  throw2("Error obtaining the info for the object ", string(oname.begin(), oname.end()));
295 
296  H5O_type_t obj_type = oinfo.type;
297 
298  switch (obj_type) {
299 
300  case H5O_TYPE_GROUP: {
301 
302  // Obtain the full path name
303  string full_path_name;
304  string temp_oname(oname.begin(), oname.end());
305 
306  full_path_name = (
307  (string(gname) != "/") ?
308  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
309  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
310 
311  cgroup = H5Gopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
312  if (cgroup < 0)
313  throw2("Error opening the group ", full_path_name);
314 
315  group = new Group();
316  group->path = full_path_name;
317  group->newname = full_path_name;
318 
319  // Retrieve group attribute if the attribute flag is true
320  if (true == include_attr) {
321 
322  int num_attrs = oinfo.num_attrs;
323  bool temp_unsup_attr_dtype = false;
324  bool temp_unsup_attr_dspace = false;
325 
326  for (int j = 0; j < num_attrs; j++) {
327 
328  attr = new Attribute();
329  Retrieve_H5_Attr_Info(attr, cgroup, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
330  group->attrs.push_back(attr);
331  attr = NULL;
332  }
333 
334  group->unsupported_attr_dtype = temp_unsup_attr_dtype;
335  group->unsupported_attr_dspace = temp_unsup_attr_dspace;
336  }
337  this->groups.push_back(group);
338  Retrieve_H5_Obj(cgroup, full_path_name.c_str(), include_attr);
339  if (H5Gclose(cgroup) < 0)
340  throw2("Error closing the group ", full_path_name);
341  }
342  break;
343  case H5O_TYPE_DATASET: {
344 
345  // Obtain the absolute path of the HDF5 dataset
346  string temp_oname(oname.begin(), oname.end());
347  string full_path_name = (
348  (string(gname) != "/") ?
349  (string(gname) + "/" + temp_oname.substr(0, temp_oname.size() - 1)) :
350  ("/" + temp_oname.substr(0, temp_oname.size() - 1)));
351 
352  var = new Var();
353  var->name = temp_oname.substr(0, temp_oname.size() - 1);
354  var->fullpath = full_path_name;
355 
356  // newname is for the final CF name
357  var->newname = full_path_name;
358 
359  cdset = H5Dopen(grp_id, full_path_name.c_str(), H5P_DEFAULT);
360  if (cdset < 0)
361  throw2("Error opening the HDF5 dataset ", full_path_name);
362 
363  // Retrieve the HDF5 dataset datatype, return the flag for unsupported types.
364  bool temp_unsup_var_dtype = false;
365  Retrieve_H5_VarType(var, cdset, full_path_name, temp_unsup_var_dtype);
366 
367  // Update the unsupported datatype flag
368  if (!this->unsupported_var_dtype && temp_unsup_var_dtype) this->unsupported_var_dtype = true;
369 
370  // Retrieve the HDF5 dataset data space, return the flag for unsupported dataspaces.
371  bool temp_unsup_var_dspace = false;
372  Retrieve_H5_VarDim(var, cdset, full_path_name, temp_unsup_var_dspace);
373 
374  // Update the unsupported data space flag
375  if (!this->unsupported_var_dspace && temp_unsup_var_dspace) this->unsupported_var_dspace = true;
376 
377  hsize_t d_storage_size = H5Dget_storage_size(cdset);
378  var->zero_storage_size =(d_storage_size ==0);
379  var->comp_ratio = Retrieve_H5_VarCompRatio(var, cdset);
380 
381  // Retrieve the attribute info. if asked
382  if (true == include_attr) {
383 
384  int num_attrs = oinfo.num_attrs;
385  bool temp_unsup_attr_dtype = false;
386  bool temp_unsup_attr_dspace = false;
387 
388  for (int j = 0; j < num_attrs; j++) {
389 
390  attr = new Attribute();
391 
392  Retrieve_H5_Attr_Info(attr, cdset, j, temp_unsup_attr_dtype, temp_unsup_attr_dspace);
393  var->attrs.push_back(attr);
394  attr = NULL;
395  }
396 
397  var->unsupported_attr_dtype = temp_unsup_attr_dtype;
398  var->unsupported_attr_dspace = temp_unsup_attr_dspace;
399 
400  if (!this->unsupported_var_attr_dspace && temp_unsup_attr_dspace)
401  this->unsupported_var_attr_dspace = true;
402  }
403 
404  this->vars.push_back(var);
405  if (H5Dclose(cdset) < 0)
406  throw2("Error closing the HDF5 dataset ", full_path_name);
407  }
408  break;
409 
410  case H5O_TYPE_NAMED_DATATYPE: {
411  // ignore the named datatype
412  if (true == include_attr && true == check_ignored) {
413  this->add_ignored_info_namedtypes(string(gname), string(oname.begin(), oname.end()));
414  }
415  }
416  break;
417  default:
418  break;
419  } // "switch (obj_type)"
420  } // try
421  catch (...) {
422 
423  if (attr != NULL) {
424  delete attr;
425  attr = NULL;
426  }
427 
428  if (var != NULL) {
429  delete var;
430  var = NULL;
431  }
432 
433  if (group != NULL) {
434  delete group;
435  group = NULL;
436  }
437 
438  if (cgroup != -1) H5Gclose(cgroup);
439 
440  if (cdset != -1) H5Dclose(cdset);
441  throw;
442 
443  } // catch
444  } // "for (hsize_t i = 0; i < nelems; i++)"
445 
446 }
447 
448 // Retrieve HDF5 dataset datatype
449 float File::Retrieve_H5_VarCompRatio(Var *var, hid_t dset_id)
450 {
451 
452  float comp_ratio = 1.0;
453  // Obtain the data type of the variable.
454  hid_t dset_create_plist = H5Dget_create_plist(dset_id);
455  if (dset_create_plist < 0)
456  throw1("unable to obtain hdf5 dataset creation property list ");
457  H5D_layout_t dset_layout = H5Pget_layout(dset_create_plist);
458  if (dset_layout < 0) {
459  H5Pclose(dset_create_plist);
460  throw1("unable to obtain hdf5 dataset creation property list storage layout");
461  }
462 
463  if (dset_layout == H5D_CHUNKED) {
464 
465  hsize_t dstorage_size = H5Dget_storage_size(dset_id);
466  if (dstorage_size > 0 && var->total_elems > 0) {
467  hid_t ty_id = -1;
468 
469  // Obtain the data type of the variable.
470  if ((ty_id = H5Dget_type(dset_id)) < 0)
471  throw1("unable to obtain hdf5 datatype for the dataset ");
472  size_t type_size = H5Tget_size(ty_id);
473  comp_ratio = ((float) (var->total_elems) * type_size) / dstorage_size;
474  H5Tclose(ty_id);
475  }
476 
477  }
478  H5Pclose(dset_create_plist);
479  return comp_ratio;
480 
481 }
482 // Retrieve HDF5 dataset datatype
483 void File::Retrieve_H5_VarType(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dtype)
484 {
485 
486  hid_t ty_id = -1;
487 
488  // Obtain the data type of the variable.
489  if ((ty_id = H5Dget_type(dset_id)) < 0)
490  throw2("unable to obtain hdf5 datatype for the dataset ", varname);
491 
492  // The following datatype class and datatype will not be supported for the CF option.
493  // H5T_TIME, H5T_BITFIELD
494  // H5T_OPAQUE, H5T_ENUM
495  // H5T_REFERENCE, H5T_COMPOUND
496  // H5T_VLEN,H5T_ARRAY
497  // 64-bit integer
498 
499  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
500  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
501  // 1-D variable length of string can also be mapped for the CF option..
502  // The variable length string class is H5T_STRING rather than H5T_VLEN,
503  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
504  // support 64-bit integer. In theory, DAP2 doesn't support long double
505  // (128-bit or 92-bit floating point type).
506  //
507 
508  var->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
509  if (false == HDF5CFUtil::cf_strict_support_type(var->dtype,_is_dap4))
510  unsup_var_dtype = true;
511 
512  if (H5Tclose(ty_id) < 0)
513  throw1("Unable to close the HDF5 datatype ");;
514 }
515 
516 // Retrieve the HDF5 dataset dimension information
517 void File::Retrieve_H5_VarDim(Var *var, hid_t dset_id, const string & varname, bool &unsup_var_dspace)
518 {
519 
520  vector<hsize_t> dsize;
521  vector<hsize_t> maxsize;
522 
523  hid_t dspace_id = -1;
524  hid_t ty_id = -1;
525 
526  try {
527  if ((dspace_id = H5Dget_space(dset_id)) < 0)
528  throw2("Cannot get hdf5 dataspace id for the variable ", varname);
529 
530  H5S_class_t space_class = H5S_NO_CLASS;
531  if ((space_class = H5Sget_simple_extent_type(dspace_id)) < 0)
532  throw2("Cannot obtain the HDF5 dataspace class for the variable ", varname);
533 
534  if (H5S_NULL == space_class)
535  unsup_var_dspace = true;
536  else {
537  if (false == unsup_var_dspace) {
538 
539  hssize_t h5_total_elms = H5Sget_simple_extent_npoints(dspace_id);
540  if (h5_total_elms < 0)
541  throw2("Cannot get the total number of elements of HDF5 dataset ", varname);
542  else
543  var->total_elems = (size_t) h5_total_elms;
544  int ndims = H5Sget_simple_extent_ndims(dspace_id);
545  if (ndims < 0)
546  throw2("Cannot get the hdf5 dataspace number of dimension for the variable ", varname);
547 
548  var->rank = ndims;
549  if (ndims != 0) {
550  dsize.resize(ndims);
551  maxsize.resize(ndims);
552  }
553 
554  // The netcdf DAP client supports the representation of the unlimited dimension.
555  // So we need to check.
556  if (H5Sget_simple_extent_dims(dspace_id, &dsize[0], &maxsize[0]) < 0)
557  throw2("Cannot obtain the dim. info for the variable ", varname);
558 
559  for (int i = 0; i < ndims; i++) {
560  Dimension * dim = new Dimension(dsize[i]);
561  if (maxsize[i] == H5S_UNLIMITED) {
562  dim->unlimited_dim = true;
563  if (false == have_udim) have_udim = true;
564  }
565  var->dims.push_back(dim);
566  }
567  }
568  }
569 
570  var->unsupported_dspace = unsup_var_dspace;
571 
572  if (H5Sclose(dspace_id) < 0)
573  throw1("Cannot close the HDF5 dataspace .");
574 
575  }
576 
577  catch (...) {
578 
579  if (dspace_id != -1) H5Sclose(dspace_id);
580 
581  if (ty_id != -1) H5Tclose(ty_id);
582  throw;
583  }
584 
585 }
586 
587 // Retrieve the HDF5 attribute information.
588 void File::Retrieve_H5_Attr_Info(Attribute * attr, hid_t obj_id, const int j, bool &unsup_attr_dtype,
589  bool &unsup_attr_dspace)
590 
591 {
592 
593  hid_t attrid = -1;
594  hid_t ty_id = -1;
595  hid_t aspace_id = -1;
596  hid_t memtype = -1;
597 
598  try {
599 
600  // Obtain the attribute ID.
601  if ((attrid = H5Aopen_by_idx(obj_id, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) j, H5P_DEFAULT,
602  H5P_DEFAULT)) < 0)
603  throw1("Unable to open attribute by index ");
604 
605  // Obtain the size of attribute name.
606  ssize_t name_size = H5Aget_name(attrid, 0, NULL);
607  if (name_size < 0)
608  throw1("Unable to obtain the size of the hdf5 attribute name ");
609 
610  string attr_name;
611  attr_name.resize(name_size + 1);
612 
613  // Obtain the attribute name.
614  if ((H5Aget_name(attrid, name_size + 1, &attr_name[0])) < 0)
615  throw1("unable to obtain the hdf5 attribute name ");
616 
617  // Obtain the type of the attribute.
618  if ((ty_id = H5Aget_type(attrid)) < 0)
619  throw2("unable to obtain hdf5 datatype for the attribute ", attr_name);
620 
621  // The following datatype class and datatype will not be supported for the CF option.
622  // H5T_TIME, H5T_BITFIELD
623  // H5T_OPAQUE, H5T_ENUM
624  // H5T_REFERENCE, H5T_COMPOUND
625  // H5T_VLEN,H5T_ARRAY
626  // 64-bit integer
627 
628  // Note: H5T_REFERENCE H5T_COMPOUND and H5T_ARRAY can be mapped to DAP2 DDS for the default option.
629  // H5T_COMPOUND, H5T_ARRAY can be mapped to DAP2 DAS for the default option.
630  // 1-D variable length of string can also be mapped for the CF option..
631  // The variable length string class is H5T_STRING rather than H5T_VLEN,
632  // We also ignore the mapping of integer 64 bit since DAP2 doesn't
633  // support 64-bit integer. In theory, DAP2 doesn't support long double
634  // (128-bit or 92-bit floating point type).
635  //
636  attr->dtype = HDF5CFUtil::H5type_to_H5DAPtype(ty_id);
637  if (false == HDF5CFUtil::cf_strict_support_type(attr->dtype,_is_dap4))
638  unsup_attr_dtype = true;
639 
640  if(H5VSTRING == attr->dtype || H5FSTRING == attr->dtype) {
641  H5T_cset_t c_set_type = H5Tget_cset(ty_id);
642  if(c_set_type <0)
643  throw2("Cannot get hdf5 character set type for the attribute ", attr_name);
644  // This is a UTF-8 string
645  if(c_set_type == 1)
646  attr->is_cset_ascii = false;
647  }
648 
649  if ((aspace_id = H5Aget_space(attrid)) < 0)
650  throw2("Cannot get hdf5 dataspace id for the attribute ", attr_name);
651 
652  int ndims = H5Sget_simple_extent_ndims(aspace_id);
653  if (ndims < 0)
654  throw2("Cannot get the hdf5 dataspace number of dimension for attribute ", attr_name);
655 
656  hsize_t nelmts = 1;
657 
658  // if it is a scalar attribute, just define number of elements to be 1.
659  if (ndims != 0) {
660 
661  vector<hsize_t> asize;
662  vector<hsize_t> maxsize;
663  asize.resize(ndims);
664  maxsize.resize(ndims);
665 
666  // Obtain the attribute data space information.
667  if (H5Sget_simple_extent_dims(aspace_id, &asize[0], &maxsize[0]) < 0)
668  throw2("Cannot obtain the dim. info for the attribute ", attr_name);
669 
670  // Here we need to take care of 0-length attribute. This is legal in HDF5.
671  for (int dim_count = 0;dim_count < ndims; dim_count ++) {
672  // STOP adding unsupported_attr_dspace!
673  if (asize[dim_count] == 0) {
674  unsup_attr_dspace = true;
675  break;
676  }
677  }
678 
679  if (false == unsup_attr_dspace) {
680  // Return ndims and size[ndims].
681  for (int dim_count = 0; dim_count< ndims; dim_count++)
682  nelmts *= asize[dim_count];
683  }
684  else
685  nelmts = 0;
686  } // "if(ndims != 0)"
687 
688  size_t ty_size = H5Tget_size(ty_id);
689  if (0 == ty_size)
690  throw2("Cannot obtain the dtype size for the attribute ", attr_name);
691 
692  memtype = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
693  if (memtype < 0)
694  throw2("Cannot obtain the memory datatype for the attribute ", attr_name);
695 
696  // Store the name and the count
697  string temp_aname(attr_name.begin(), attr_name.end());
698  attr->name = temp_aname.substr(0, temp_aname.size() - 1);
699  attr->newname = attr->name;
700  attr->count = nelmts;
701 
702  // Release HDF5 resources.
703  if (H5Tclose(ty_id) < 0)
704  throw1("Cannot successfully close the attribute datatype.");
705  if (H5Tclose(memtype) < 0)
706  throw1("Cannot successfully close the attribute memory datatype.");
707  if (H5Sclose(aspace_id) < 0)
708  throw1("Cannot successfully close the HDF5 dataspace.");
709  if (H5Aclose(attrid) < 0)
710  throw1("Cannot successfully close the HDF5 attribute.");
711 
712  } // try
713  catch (...) {
714 
715  if (ty_id != -1) H5Tclose(ty_id);
716 
717  if (memtype != -1) H5Tclose(memtype);
718 
719  if (aspace_id != -1) H5Sclose(aspace_id);
720 
721  if (attrid != -1) H5Aclose(attrid);
722 
723  throw;
724  }
725 
726 }
727 
728 // Retrieve all HDF5 supported attribute values.
730 {
731 
732  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira)
733  Retrieve_H5_Attr_Value(*ira, "/");
734 
735  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
736  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
737  Retrieve_H5_Attr_Value(*ira, (*irg)->path);
738  }
739  }
740 
741  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
742  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
743  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
744  }
745  }
746 }
747 
749 {
750  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ++ira) {
751  Retrieve_H5_Attr_Value(*ira, var->fullpath);
752  }
753 }
754 
755 // Retrieve the values of a specific HDF5 attribute.
756 void File::Retrieve_H5_Attr_Value(Attribute *attr, string obj_name)
757 {
758 
759  // Define HDF5 object Ids.
760  hid_t obj_id = -1;
761  hid_t attr_id = -1;
762  hid_t ty_id = -1;
763  hid_t memtype_id = -1;
764  hid_t aspace_id = -1;
765 
766  try {
767 
768  // Open the object that hold this attribute
769  obj_id = H5Oopen(this->fileid, obj_name.c_str(), H5P_DEFAULT);
770  if (obj_id < 0)
771  throw2("Cannot open the object ", obj_name);
772 
773  attr_id = H5Aopen(obj_id, (attr->name).c_str(), H5P_DEFAULT);
774  if (attr_id < 0)
775  throw4("Cannot open the attribute ", attr->name, " of object ", obj_name);
776 
777  ty_id = H5Aget_type(attr_id);
778  if (ty_id < 0)
779  throw4("Cannot obtain the datatype of the attribute ", attr->name, " of object ", obj_name);
780 
781  memtype_id = H5Tget_native_type(ty_id, H5T_DIR_ASCEND);
782  if (memtype_id < 0)
783  throw2("Cannot obtain the memory datatype for the attribute ", attr->name);
784 
785  size_t ty_size = H5Tget_size(memtype_id);
786  if (0 == ty_size)
787  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
788 
789  size_t total_bytes = attr->count * ty_size;
790 
791  // We have to handle variable length string differently.
792  if (H5VSTRING == attr->dtype) {
793 
794  // Variable length string attribute values only store pointers of the actual string value.
795  vector<char> temp_buf;
796  temp_buf.resize(total_bytes);
797 
798  if (H5Aread(attr_id, memtype_id, &temp_buf[0]) < 0)
799  throw4("Cannot obtain the value of the attribute ", attr->name, " of object ", obj_name);
800 
801  char *temp_bp = NULL;
802  char *ptr_1stvlen_ptr = &temp_buf[0];
803  temp_bp = &temp_buf[0];
804  char* onestring = NULL;
805  string total_vstring = "";
806 
807  attr->strsize.resize(attr->count);
808 
809  for (unsigned int temp_i = 0; temp_i < attr->count; temp_i++) {
810 
811  // This line will assure that we get the real variable length string value.
812  onestring = *(char **) temp_bp;
813  if (onestring != NULL) {
814  total_vstring += string(onestring);
815  attr->strsize[temp_i] = (string(onestring)).size();
816  }
817  else
818  attr->strsize[temp_i] = 0;
819 
820  // going to the next value.
821  temp_bp += ty_size;
822  }
823 
824  if (ptr_1stvlen_ptr != NULL) {
825  aspace_id = H5Aget_space(attr_id);
826  if (aspace_id < 0)
827  throw4("Cannot obtain space id for ", attr->name, " of object ", obj_name);
828 
829  // Reclaim any VL memory if necessary.
830  if (H5Dvlen_reclaim(memtype_id, aspace_id, H5P_DEFAULT, &temp_buf[0]) < 0)
831  throw4("Cannot reclaim VL memory for ", attr->name, " of object ", obj_name);
832 
833  H5Sclose(aspace_id);
834  }
835 
836  if (HDF5CFUtil::H5type_to_H5DAPtype(ty_id) != H5VSTRING)
837  throw4("Error to obtain the VL string type for attribute ", attr->name, " of object ", obj_name);
838 
839  attr->value.resize(total_vstring.size());
840 
841  copy(total_vstring.begin(), total_vstring.end(), attr->value.begin());
842 
843  }
844  else {
845 
846  if (attr->dtype == H5FSTRING) {
847  attr->fstrsize = ty_size;
848  }
849 
850  attr->value.resize(total_bytes);
851 
852  // Read HDF5 attribute data.
853  if (H5Aread(attr_id, memtype_id, (void *) &attr->value[0]) < 0)
854  throw4("Cannot obtain the dtype size for the attribute ", attr->name, " of object ", obj_name);
855 
856  if (attr->dtype == H5FSTRING) {
857 
858  size_t sect_size = ty_size;
859  int num_sect = 1;
860  if (sect_size > 0)
861  num_sect =
862  (total_bytes % sect_size == 0) ? (total_bytes / sect_size) : (total_bytes / sect_size + 1);
863  else
864  throw4("The attribute datatype size is not a positive integer ", attr->name, " of object ",
865  obj_name);
866 
867  vector<size_t> sect_newsize;
868  sect_newsize.resize(num_sect);
869 
870  string total_fstring = string(attr->value.begin(), attr->value.end());
871 
872  string new_total_fstring = HDF5CFUtil::trim_string(memtype_id, total_fstring, num_sect, sect_size,
873  sect_newsize);
874  attr->value.resize(new_total_fstring.size());
875  copy(new_total_fstring.begin(), new_total_fstring.end(), attr->value.begin());
876  attr->strsize.resize(num_sect);
877  for (int temp_i = 0; temp_i < num_sect; temp_i++)
878  attr->strsize[temp_i] = sect_newsize[temp_i];
879 
880 #if 0
881  // "h5","new string value " <<string(attr->value.begin(), attr->value.end()) <<endl;
882  cerr<<"Attr name is "<<attr->name <<endl;
883  for (int temp_i = 0; temp_i <num_sect; temp_i ++)
884  cerr<<"string new section size = " << attr->strsize[temp_i] <<endl;
885 #endif
886  }
887  }
888 
889  if (H5Tclose(memtype_id) < 0)
890  throw1("Fail to close the HDF5 memory datatype ID.");
891  if (H5Tclose(ty_id) < 0)
892  throw1("Fail to close the HDF5 datatype ID.");
893  if (H5Aclose(attr_id) < 0)
894  throw1("Fail to close the HDF5 attribute ID.");
895  if (H5Oclose(obj_id) < 0)
896  throw1("Fail to close the HDF5 object ID.");
897 
898  }
899 
900  catch (...) {
901 
902  if (memtype_id != -1) H5Tclose(memtype_id);
903 
904  if (ty_id != -1) H5Tclose(ty_id);
905 
906  if (aspace_id != -1) H5Sclose(aspace_id);
907 
908  if (attr_id != -1) H5Aclose(attr_id);
909 
910  if (obj_id != -1) H5Oclose(obj_id);
911 
912  throw;
913  }
914 
915 }
916 
917 // Handle the unsupported datatype
918 void File::Handle_Unsupported_Dtype(bool include_attr)
919 {
920 
921  if (true == include_attr) {
922  Handle_Group_Unsupported_Dtype();
923  Handle_VarAttr_Unsupported_Dtype();
924  }
925 
926  Handle_Var_Unsupported_Dtype();
927 }
928 
929 //Leave this code here.
930 #if 0
931 // First the root attributes
932 if (true == include_attr) {
933  if (true == this->unsupported_attr_dtype) {
934  for (vector<Attribute *>::iterator ira = this->root_attrs.begin();
935  ira != this->root_attrs.end(); ) {
936  H5DataType temp_dtype = (*ira)->getType();
937  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
938  delete (*ira);
939  ira = this->root_attrs.erase(ira);
940 
941  }
942  else {
943  ++ira;
944  }
945  }
946  }
947 }
948 
949 // Then the group attributes
950 if (false == this->groups.empty()) {
951  for (vector<Group *>::iterator irg = this->groups.begin();
952  irg != this->groups.end(); ++irg) {
953  if (false == (*irg)->attrs.empty()) {
954  if (true == (*irg)->unsupported_attr_dtype) {
955  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin();
956  ira != (*irg)->attrs.end(); ) {
957  H5DataType temp_dtype = (*ira)->getType();
958  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
959  delete (*ira);
960  ira = (*irg)->attrs.erase(ira);
961  }
962  else {
963  ++ira;
964  }
965  }
966  }
967  }
968  }
969 }
970 }
971 
972  // Then the variable(HDF5 dataset) and the correponding attributes.
973 if (false == this->vars.empty()) {
974 if (true == include_attr) {
975  for (vector<Var *>::iterator irv = this->vars.begin();
976  irv != this->vars.end();++irv ) {
977  if (false == (*irv)->attrs.empty()) {
978  if (true == (*irv)->unsupported_attr_dtype) {
979  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
980  ira != (*irv)->attrs.end(); ) {
981  H5DataType temp_dtype = (*ira)->getType();
982  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
983  delete (*ira);
984  ira = (*irv)->attrs.erase(ira);
985  //ira--;
986  }
987  else {
988  ++ira;
989  }
990  }
991  }
992  }
993  }
994 }
995 if (true == this->unsupported_var_dtype) {
996  // "h5","having unsupported variable datatype" <<endl;
997  for (vector<Var *>::iterator irv = this->vars.begin();
998  irv != this->vars.end(); ) {
999  H5DataType temp_dtype = (*irv)->getType();
1000  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)) {
1001  delete (*irv);
1002  irv = this->vars.erase(irv);
1003  //irv--;
1004  }
1005  else
1006  ++irv;
1007  }
1008 }
1009 }
1010 #endif
1011 
1012 void File::Handle_Group_Unsupported_Dtype()
1013 {
1014 
1015  // First root
1016  if (false == this->root_attrs.empty()) {
1017  if (true == this->unsupported_attr_dtype) {
1018  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1019  H5DataType temp_dtype = (*ira)->getType();
1020  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1021  delete (*ira);
1022  ira = this->root_attrs.erase(ira);
1023  }
1024  else {
1025  ++ira;
1026  }
1027  }
1028  }
1029  }
1030 
1031  // Then the group attributes
1032  if (false == this->groups.empty()) {
1033  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1034  if (false == (*irg)->attrs.empty()) {
1035  if (true == (*irg)->unsupported_attr_dtype) {
1036  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1037  H5DataType temp_dtype = (*ira)->getType();
1038  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1039  delete (*ira);
1040  ira = (*irg)->attrs.erase(ira);
1041  }
1042  else {
1043  ++ira;
1044  }
1045  }
1046  }
1047  }
1048  }
1049  }
1050 }
1051 
1052 // Generate group unsupported datatype Information, this is for the BES ignored object key
1053 void File::Gen_Group_Unsupported_Dtype_Info()
1054 {
1055 
1056  // First root
1057  if (false == this->root_attrs.empty()) {
1058  //if (true == this->unsupported_attr_dtype) {
1059  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1060  H5DataType temp_dtype = (*ira)->getType();
1061  // TODO: Don't know why we still include 64-bit integer here.
1062  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1063  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1064  || temp_dtype == H5INT64 || temp_dtype == H5UINT64) {
1065  this->add_ignored_info_attrs(true, "/", (*ira)->name);
1066  }
1067  }
1068  //}
1069  }
1070 
1071  // Then the group attributes
1072  if (false == this->groups.empty()) {
1073  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1074  if (false == (*irg)->attrs.empty()) {
1075  //if (true == (*irg)->unsupported_attr_dtype) {
1076  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1077  H5DataType temp_dtype = (*ira)->getType();
1078  // TODO: Don't know why we still include 64-bit integer here.
1079  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1080  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1081  || temp_dtype == H5INT64 || temp_dtype==H5UINT64 ) {
1082  this->add_ignored_info_attrs(true, (*irg)->path, (*ira)->name);
1083  }
1084  }
1085  //}
1086  }
1087  }
1088  }
1089 }
1090 
1091 // Handler unsupported variable datatype
1092 void File::Handle_Var_Unsupported_Dtype()
1093 {
1094  if (false == this->vars.empty()) {
1095  if (true == this->unsupported_var_dtype) {
1096  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1097  H5DataType temp_dtype = (*irv)->getType();
1098  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1099  delete (*irv);
1100  irv = this->vars.erase(irv);
1101  }
1102  else {
1103  ++irv;
1104 
1105  }
1106  }
1107  }
1108  }
1109 }
1110 
1111 // Generate unsupported variable type info. This is for the ignored objects.
1112 void File::Gen_Var_Unsupported_Dtype_Info()
1113 {
1114 
1115  if (false == this->vars.empty()) {
1116  //if (true == this->unsupported_var_dtype) {
1117  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1118  H5DataType temp_dtype = (*irv)->getType();
1119  //TODO: don't know why 64-bit integer is still listed here.
1120  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype)||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1121  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1122  ||(H5INT64 == temp_dtype) ||(H5UINT64 == temp_dtype)) {
1123  this->add_ignored_info_objs(false, (*irv)->fullpath);
1124  }
1125  }
1126  //}
1127  }
1128 
1129 }
1130 
1131 // Handling unsupported datatypes for variable(HDF5 dataset) and the correponding attributes.
1132 void File::Handle_VarAttr_Unsupported_Dtype()
1133 {
1134  if (false == this->vars.empty()) {
1135  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1136  if (false == (*irv)->attrs.empty()) {
1137  if (true == (*irv)->unsupported_attr_dtype) {
1138  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1139  H5DataType temp_dtype = (*ira)->getType();
1140  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)) {
1141  delete (*ira);
1142  ira = (*irv)->attrs.erase(ira);
1143  }
1144  else {
1145  ++ira;
1146  }
1147  }
1148  }
1149  }
1150  }
1151  }
1152 }
1153 
1154 // Generated unsupported var/attribute unsupported datatype Info when the BES ignored object key is on.
1155 void File::Gen_VarAttr_Unsupported_Dtype_Info()
1156 {
1157 
1158  if (false == this->vars.empty()) {
1159  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1160  if (false == (*irv)->attrs.empty()) {
1161  //if (true == (*irv)->unsupported_attr_dtype) {
1162  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1163  H5DataType temp_dtype = (*ira)->getType();
1164  // TODO: check why 64-bit integer is still listed here.
1165  //if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype) || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1166  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1167  || (temp_dtype==H5INT64) || (temp_dtype == H5UINT64)) {
1168  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1169  }
1170  }
1171  //}
1172  }
1173  }
1174  }
1175 }
1176 
1177 // Generated unsupported datatype information for HDF5 dimension scales.
1178 // The datatypes of HDF5 dimension scales("DIMENSION_LIST" and "REFERENCE_LIST")
1179 // are not supported. However, the information
1180 // are retrieved by the handlers so we don't want to report them as ignored objects.
1181 void File::Gen_DimScale_VarAttr_Unsupported_Dtype_Info()
1182 {
1183 
1184  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1185 
1186  // If the attribute REFERENCE_LIST comes with the attribut CLASS, the
1187  // attribute REFERENCE_LIST is okay to ignore. No need to report.
1188  bool is_ignored = ignored_dimscale_ref_list((*irv));
1189  if (false == (*irv)->attrs.empty()) {
1190  //if (true == (*irv)->unsupported_attr_dtype) {
1191  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1192  H5DataType temp_dtype = (*ira)->getType();
1193  // TODO: check why 64-bit is still listed here.
1194  if (false == HDF5CFUtil::cf_strict_support_type(temp_dtype,_is_dap4)
1195  || (temp_dtype == H5INT64) || (temp_dtype == H5UINT64)) {
1196  // "DIMENSION_LIST" is okay to ignore and "REFERENCE_LIST"
1197  // is okay to ignore if the variable has another attribute
1198  // CLASS="DIMENSION_SCALE"
1199  if (("DIMENSION_LIST" != (*ira)->name)
1200  && ("REFERENCE_LIST" != (*ira)->name || true == is_ignored))
1201  this->add_ignored_info_attrs(false, (*irv)->fullpath, (*ira)->name);
1202  }
1203  }
1204  //}
1205  }
1206  }
1207 }
1208 
1209 // Handle unsupported dataspace for group attributes.
1210 void File::Handle_GroupAttr_Unsupported_Dspace()
1211 {
1212 
1213  // First root
1214  if (false == this->root_attrs.empty()) {
1215  if (true == this->unsupported_attr_dspace) {
1216  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end();) {
1217  // Remove 0-size attribute
1218  if ((*ira)->count == 0) {
1219  delete (*ira);
1220  ira = this->root_attrs.erase(ira);
1221  }
1222  else {
1223  ++ira;
1224  }
1225  }
1226  }
1227  }
1228 
1229  // Then the group attributes
1230  if (false == this->groups.empty()) {
1231  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1232  if (false == (*irg)->attrs.empty()) {
1233  if (true == (*irg)->unsupported_attr_dspace) {
1234  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end();) {
1235  if ((*ira)->count == 0) {
1236  delete (*ira);
1237  ira = (*irg)->attrs.erase(ira);
1238  }
1239  else {
1240  ++ira;
1241  }
1242  }
1243  }
1244  }
1245  }
1246  }
1247 }
1248 
1249 // Handle unsupported data space information for variable and attribute
1250 void File::Handle_VarAttr_Unsupported_Dspace()
1251 {
1252 
1253  if (false == this->vars.empty()) {
1254  if (true == this->unsupported_var_attr_dspace) {
1255  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1256  if (false == (*irv)->attrs.empty()) {
1257  if (true == (*irv)->unsupported_attr_dspace) {
1258  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end();) {
1259  if (0 == (*ira)->count) {
1260  delete (*ira);
1261  ira = (*irv)->attrs.erase(ira);
1262  }
1263  else {
1264  ++ira;
1265  }
1266  }
1267  }
1268  }
1269  }
1270  }
1271  }
1272 }
1273 
1274 // Handle unsupported data space.
1275 void File::Handle_Unsupported_Dspace(bool include_attr)
1276 {
1277 
1278  // The unsupported data space
1279  if (false == this->vars.empty()) {
1280  if (true == this->unsupported_var_dspace) {
1281  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end();) {
1282  if (true == (*irv)->unsupported_dspace) {
1283  delete (*irv);
1284  irv = this->vars.erase(irv);
1285  }
1286  else {
1287  ++irv;
1288 
1289  }
1290  }
1291  }
1292  }
1293 
1294  if (true == include_attr) {
1295  Handle_GroupAttr_Unsupported_Dspace();
1296  Handle_VarAttr_Unsupported_Dspace();
1297  }
1298 }
1299 
1300 // Generated unsupported dataspace Info when the BES ignored object key is on.
1301 void File::Gen_Unsupported_Dspace_Info()
1302 {
1303 
1304  // Notice in this function, we deliberately don't put the case when an attribute dimension has 0 length.
1305  // Since doing this requires non-trivial change of the source code and the 0-size attribute case is really, really rare,
1306  // so we just "ignore" this case in the "ignored" information.
1307  // In fact, the zero size variable is allowed in both HDF5 and DAP2. So we don't ignore 0-size HDF5 dataset. So
1308  // the only case this function checks is the H5S_NULL case.
1309  if (false == this->vars.empty()) {
1310  if (true == this->unsupported_var_dspace) {
1311  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1312  if (true == (*irv)->unsupported_dspace) {
1313  this->add_ignored_info_objs(true, (*irv)->fullpath);
1314  }
1315  }
1316  }
1317  }
1318 
1319 }
1320 
1321 // Handle other unsupported information.
1322 void File::Handle_Unsupported_Others(bool include_attr)
1323 {
1324 
1325  if (true == this->check_ignored && true == include_attr) {
1326 
1327  if (true == HDF5RequestHandler::get_drop_long_string()) {
1328 
1329  // netCDF java doesn't have limitation for attributes
1330 #if 0
1331  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1332  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1333  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1334  this->add_ignored_droplongstr_hdr();
1335  this->add_ignored_grp_longstr_info("/", (*ira)->name);
1336  }
1337  }
1338  }
1339 
1340  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1341  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1342  if (H5FSTRING == (*ira)->dtype || H5VSTRING == (*ira)->dtype) {
1343  if ((*ira)->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
1344  this->add_ignored_droplongstr_hdr();
1345  this->add_ignored_grp_longstr_info((*irg)->path, (*ira)->name);
1346  }
1347  }
1348  }
1349  }
1350 #endif
1351  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1352  if (true == Check_DropLongStr((*irv), NULL)) {
1353  this->add_ignored_droplongstr_hdr();
1354  this->add_ignored_var_longstr_info((*irv), NULL);
1355  }
1356  // netCDF java doesn't have limitation for attributes
1357 #if 0
1358  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1359  if (true == Check_DropLongStr((*irv), (*ira))) {
1360  this->add_ignored_droplongstr_hdr();
1361  this->add_ignored_var_longstr_info((*irv), (*ira));
1362  }
1363  }
1364 #endif
1365  }
1366  }
1367  }
1368 
1369 }
1370 
1371 // Flatten the object name, mainly call get_CF_string.
1372 void File::Flatten_Obj_Name(bool include_attr)
1373 {
1374 
1375  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1376  (*irv)->newname = get_CF_string((*irv)->newname);
1377 
1378  for (vector<Dimension *>::iterator ird = (*irv)->dims.begin(); ird != (*irv)->dims.end(); ++ird) {
1379  (*ird)->newname = get_CF_string((*ird)->newname);
1380  }
1381  }
1382 
1383  if (true == include_attr) {
1384 
1385  for (vector<Attribute *>::iterator ira = this->root_attrs.begin(); ira != this->root_attrs.end(); ++ira) {
1386  (*ira)->newname = get_CF_string((*ira)->newname);
1387  }
1388 
1389  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1390  (*irg)->newname = get_CF_string((*irg)->newname);
1391  for (vector<Attribute *>::iterator ira = (*irg)->attrs.begin(); ira != (*irg)->attrs.end(); ++ira) {
1392  (*ira)->newname = get_CF_string((*ira)->newname);
1393  }
1394  }
1395 
1396  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1397  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
1398  (*ira)->newname = get_CF_string((*ira)->newname);
1399  }
1400  }
1401  } // "if (true == include_attr)"
1402 }
1403 
1404 // Variable name clashing
1405 void File::Handle_Var_NameClashing(set<string>&objnameset)
1406 {
1407 
1408  Handle_General_NameClashing(objnameset, this->vars);
1409 }
1410 
1411 // Group name clashing
1412 void File::Handle_Group_NameClashing(set<string> &objnameset)
1413 {
1414 
1415  pair<set<string>::iterator, bool> setret;
1416 
1417  // Now for DAS, we need to handle name clashings for
1418  // DAS tables. Namely we need to make sure the global attribute
1419  // table(HDF5_GLOBAL) and the attribute tables mapped from
1420  // HDF5 groups will not have name clashings with the variable name
1421  // lists. If having the name clashings, the global attribute table and the
1422  // the attribute tables generated from the groups will be changed.
1423  // The file attribute name clashing
1424 
1425  setret = objnameset.insert(FILE_ATTR_TABLE_NAME);
1426  if (false == setret.second) {
1427 
1428  int clash_index = 1;
1429  string fa_clash_name = FILE_ATTR_TABLE_NAME;
1430  HDF5CFUtil::gen_unique_name(fa_clash_name, objnameset, clash_index);
1431  FILE_ATTR_TABLE_NAME = fa_clash_name;
1432  }
1433 
1434  // The group attribute name clashing
1435  Handle_General_NameClashing(objnameset, this->groups);
1436 
1437 }
1438 
1439 //Object attribute name clashing
1440 void File::Handle_Obj_AttrNameClashing()
1441 {
1442 
1443  // Now handling the possible name clashings for attributes
1444  // For attribute clashings, we only need to resolve the name clashings
1445  // for attributes within each variable, file attributes and attributes
1446  // within each group. The name clashings for attributes should be very rare.
1447  // Potentially the checking and the correcting may be costly.
1448  // This is another reason for special products, we may not even need to check
1449  // the name clashings. KY 2011-12-24
1450 
1451  set<string> objnameset;
1452 
1453  // For root attributes
1454  Handle_General_NameClashing(objnameset, this->root_attrs);
1455 
1456  // For group attributes
1457  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
1458  objnameset.clear();
1459  Handle_General_NameClashing(objnameset, (*irg)->attrs);
1460  }
1461 
1462  // For variable attributes
1463  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1464  objnameset.clear();
1465  Handle_General_NameClashing(objnameset, (*irv)->attrs);
1466  }
1467 }
1468 
1469 // Handle General name clashing
1470 //class T must have member string newname. In our case, T is either groups, attributes or vars.
1471 template<class T> void File::Handle_General_NameClashing(set<string>&objnameset, vector<T*>& objvec)
1472 {
1473 
1474 // set<string> objnameset;
1475  pair<set<string>::iterator, bool> setret;
1476  set<string>::iterator iss;
1477 
1478  vector<string> clashnamelist;
1479  vector<string>::iterator ivs;
1480 
1481  map<int, int> cl_to_ol;
1482  int ol_index = 0;
1483  int cl_index = 0;
1484 
1485  /*class*/
1486  typename vector<T*>::iterator irv;
1487 
1488  for (irv = objvec.begin(); irv != objvec.end(); ++irv) {
1489  setret = objnameset.insert((*irv)->newname);
1490  if (false == setret.second) {
1491  clashnamelist.insert(clashnamelist.end(), (*irv)->newname);
1492  cl_to_ol[cl_index] = ol_index;
1493  cl_index++;
1494  }
1495  ol_index++;
1496  }
1497 
1498  // Now change the clashed elements to unique elements,
1499  // Generate the set which has the same size as the original vector.
1500  for (ivs = clashnamelist.begin(); ivs != clashnamelist.end(); ivs++) {
1501  int clash_index = 1;
1502  string temp_clashname = *ivs + '_';
1503  HDF5CFUtil::gen_unique_name(temp_clashname, objnameset, clash_index);
1504  *ivs = temp_clashname;
1505  }
1506 
1507  // Now go back to the original vector, make it unique.
1508  for (unsigned int i = 0; i < clashnamelist.size(); i++)
1509  objvec[cl_to_ol[i]]->newname = clashnamelist[i];
1510 
1511 }
1512 
1513 // Handle General object name clashing
1514 void File::Handle_GeneralObj_NameClashing(bool include_attr, set<string>& objnameset)
1515 {
1516 
1517  Handle_Var_NameClashing(objnameset);
1518  if (true == include_attr) {
1519  Handle_Group_NameClashing(objnameset);
1520  Handle_Obj_AttrNameClashing();
1521  }
1522 }
1523 
1524 // Get CF name, flatten the path, change the non-alphanumeric letters to underscore.
1525 string File::get_CF_string(string s)
1526 {
1527 
1528  if ("" == s) return s;
1529  string insertString(1, '_');
1530 
1531  // Always start with _ if the first character is not a letter
1532  if (true == isdigit(s[0])) s.insert(0, insertString);
1533 
1534  for (unsigned int i = 0; i < s.length(); i++)
1535  if ((false == isalnum(s[i])) && (s[i] != '_')) s[i] = '_';
1536 
1537  return s;
1538 
1539 }
1540 
1541 // For the connection of variable dimensions. Build dimname to dimsize(unlimited) maps
1542 void File::Insert_One_NameSizeMap_Element(string name, hsize_t size, bool unlimited)
1543 {
1544  pair<map<string, hsize_t>::iterator, bool> mapret;
1545  mapret = dimname_to_dimsize.insert(pair<string, hsize_t>(name, size));
1546  if (false == mapret.second)
1547  throw4("The dimension name ", name, " should map to ", size);
1548 
1549  pair<map<string, bool>::iterator, bool> mapret2;
1550  mapret2 = dimname_to_unlimited.insert(pair<string, bool>(name, unlimited));
1551  if (false == mapret2.second)
1552  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1553 
1554 }
1555 
1556 // Similar to Inset_One_NameSizeMap_Element but the maps are provided as parameters.
1557 void File::Insert_One_NameSizeMap_Element2(map<string, hsize_t>& name_to_size, map<string, bool>& name_to_unlimited,
1558  string name, hsize_t size, bool unlimited)
1559 {
1560  pair<map<string, hsize_t>::iterator, bool> mapret;
1561  mapret = name_to_size.insert(pair<string, hsize_t>(name, size));
1562  if (false == mapret.second)
1563  throw4("The dimension name ", name, " should map to ", size);
1564 
1565  pair<map<string, bool>::iterator, bool> mapret2;
1566  mapret2 = name_to_unlimited.insert(pair<string, bool>(name, unlimited));
1567  if (false == mapret2.second)
1568  throw3("The dimension name ", name, " unlimited dimension info. should be provided.");
1569 
1570 }
1571 
1572 // For dimension names added by the handlers, by default,
1573 // Each dimension will have a unique dimension name. For example,
1574 // Int foo[100][200] will be Int foo[Fakedim1][Fakedim2]
1575 // If you have many variables, the dimension names may be too many.
1576 // To reduce numbers, we ASSUME that the dimension having the same
1577 // size shares the same dimension. In this way, the number of dimension names
1578 // will be reduced.
1579 // For example, Int foo2[100][300] will be Int foo2[Fakedim1][Fakedim3]
1580 // instead of foo2[Fakedim3][Fakedim4]. However, that may impose
1581 // another problem. Suppose Int Foosame[100][100] becomes
1582 // Int Foosame[FakeDim1][FakeDim1]. This doesn't make sense for some
1583 // applications. The fuction Adjust_Duplicate_FakeDim_Name will make sure
1584 // this case will not happen.
1585 void File::Add_One_FakeDim_Name(Dimension *dim)
1586 {
1587 
1588  stringstream sfakedimindex;
1589  string fakedimstr = "FakeDim";
1590  pair<set<string>::iterator, bool> setret;
1591  map<hsize_t, string>::iterator im;
1592  pair<map<hsize_t, string>::iterator, bool> mapret;
1593 
1594  sfakedimindex << addeddimindex;
1595  string added_dimname = fakedimstr + sfakedimindex.str();
1596 
1597  // Build up the size to fakedim map.
1598  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, added_dimname));
1599  if (false == mapret.second) { //The dim size exists, use the corresponding name.
1600  dim->name = dimsize_to_fakedimname[dim->size];
1601  dim->newname = dim->name;
1602  }
1603  else { // Insert this (dimsize,dimname) pair to dimsize_to_fakedimname map successfully.
1604  //First make sure this new dim name doesn't have name clashing
1605  // with previous dim names, after the checking, inserting to the
1606  // dimname list set.
1607  // dimnamelist is a private memeber of File.
1608  setret = dimnamelist.insert(added_dimname);
1609  if (false == setret.second) {
1610  int clash_index = 1;
1611  string temp_clashname = added_dimname + '_';
1612  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1613  dim->name = temp_clashname;
1614  dim->newname = dim->name;
1615  setret = dimnamelist.insert(dim->name);
1616  if (false == setret.second)
1617  throw2("Fail to insert the unique dimsizede name ", dim->name);
1618 
1619  // We have to adjust the dim. name of the dimsize_to_fakedimname map, since the
1620  // dimname has been updated for this size.
1621  dimsize_to_fakedimname.erase(dim->size);
1622  mapret = dimsize_to_fakedimname.insert(pair<hsize_t, string>(dim->size, dim->name));
1623  if (false == mapret.second)
1624  throw4("The dimension size ", dim->size, " should map to ", dim->name);
1625  } // "if(false == setret.second)"
1626 
1627  // New dim name is inserted successfully, update the dimname_to_dimsize map.
1628  dim->name = added_dimname;
1629  dim->newname = dim->name;
1630  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1631 
1632  // Increase the dimindex since the new dimname has been inserted.
1633  addeddimindex++;
1634  } // else
1635 }
1636 
1637 // See the function comments of Add_One_FakeDim_Name
1638 void File::Adjust_Duplicate_FakeDim_Name(Dimension * dim)
1639 {
1640 
1641  // No need to adjust the dimsize_to_fakedimname map, only create a new Fakedim
1642  // The simplest way is to increase the dim index and resolve any name clashings with other dim names.
1643  // Note: No need to update the dimsize_to_dimname map since the original "FakeDim??" of this size
1644  // can be used as a dimension name of other variables. But we need to update the dimname_to_dimsize map
1645  // since this is a new dim name.
1646  stringstream sfakedimindex;
1647  pair<set<string>::iterator, bool> setret;
1648 
1649  addeddimindex++;
1650  sfakedimindex << addeddimindex;
1651  string added_dimname = "FakeDim" + sfakedimindex.str();
1652  setret = dimnamelist.insert(added_dimname);
1653  if (false == setret.second) {
1654  int clash_index = 1;
1655  string temp_clashname = added_dimname + '_';
1656  HDF5CFUtil::gen_unique_name(temp_clashname, dimnamelist, clash_index);
1657  dim->name = temp_clashname;
1658  dim->newname = dim->name;
1659  setret = dimnamelist.insert(dim->name);
1660  if (false == setret.second)
1661  throw2("Fail to insert the unique dimsizede name ", dim->name);
1662  }
1663  dim->name = added_dimname;
1664  dim->newname = dim->name;
1665  Insert_One_NameSizeMap_Element(dim->name, dim->size, dim->unlimited_dim);
1666 
1667  // Need to prepare for the next unique FakeDim.
1668  addeddimindex++;
1669 }
1670 
1671 // Replace all dimension names, this function is currently not used. So comment out. May delete it in the future.
1672 #if 0
1673 void File::Replace_Dim_Name_All(const string orig_dim_name, const string new_dim_name) {
1674 
1675  // The newname of the original dimension should also be replaced by new_dim_name
1676  for (vector<Var *>::iterator irv = this->vars.begin();
1677  irv != this->vars.end(); ++irv) {
1678  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1679  ird != (*irv)->dims.end(); ++ird) {
1680  if((*ird)->name == orig_dim_name) {
1681  (*ird)->name = new_dim_name;
1682  (*ird)->newname = new_dim_name;
1683  }
1684 
1685  }
1686  }
1687 }
1688 #endif
1689 
1690 #if 0
1691 void File::Use_Dim_Name_With_Size_All(const string dim_name, const size_t dim_size) {
1692 
1693  // The newname of the original dimension should also be replaced by new_dim_name
1694  for (vector<Var *>::iterator irv = this->vars.begin();
1695  irv != this->vars.end(); ++irv) {
1696  for (vector<Dimension *>::iterator ird= (*irv)->dims.begin();
1697  ird != (*irv)->dims.end(); ++ird) {
1698  if((*ird)->size == orig_dim_name) {
1699  (*ird)->name = new_dim_name;
1700  (*ird)->newname = new_dim_name;
1701  }
1702 
1703  }
1704  }
1705 }
1706 #endif
1707 
1708 // Often times we need to add a CF attribute with string datatype because some products don't provide them
1709 // Examples are units, comment etc.
1710 void File::Add_Str_Attr(Attribute* attr, const string &attrname, const string& strvalue)
1711 {
1712 
1713  attr->name = attrname;
1714  attr->newname = attr->name;
1715  attr->dtype = H5FSTRING;
1716  attr->count = 1;
1717  attr->fstrsize = strvalue.size();
1718  attr->strsize.resize(1);
1719  attr->strsize[0] = attr->fstrsize;
1720  attr->value.resize(strvalue.size());
1721  copy(strvalue.begin(), strvalue.end(), attr->value.begin());
1722 }
1723 
1724 #if 0
1725 bool
1726 File:: Var_Has_Attr(Var*var,const string &attrname) {
1727 
1728  for (vector<Attribute *>:: iterator ira =var->attrs.begin(); ira !=var->attrs.end(); ++ira) {
1729 
1730  // We only check the original attribute name
1731  // Remove the original "coordinates" attribute.
1732  if((*ira)->name == attrname || (*ira)->newname == attrname) {
1733  return true;
1734  }
1735  }
1736  return false;
1737 }
1738 #endif
1739 
1740 // Rretrieve the variable attribute in string.var_path is the variable path.
1741 string File::Retrieve_Str_Attr_Value(Attribute *attr, const string var_path)
1742 {
1743 
1744  if (attr != NULL && var_path != "") {
1745  Retrieve_H5_Attr_Value(attr, var_path);
1746  string orig_attr_value(attr->value.begin(), attr->value.end());
1747  return orig_attr_value;
1748  }
1749  return "";
1750 
1751 }
1752 
1753 //Check if the attribute value of this variable is the input value.
1754 bool File::Is_Str_Attr(Attribute* attr, string varfullpath, const string &attrname, const string& strvalue)
1755 {
1756  bool ret_value = false;
1757  if (attrname == get_CF_string(attr->newname)) {
1758  Retrieve_H5_Attr_Value(attr, varfullpath);
1759  string attr_value(attr->value.begin(), attr->value.end());
1760  if (attr_value == strvalue) ret_value = true;
1761  }
1762  return ret_value;
1763 }
1764 
1765 // If this latitude or longitude units follows the CF
1766 bool File::has_latlon_cf_units(Attribute *attr, const string &varfullpath, bool is_lat)
1767 {
1768  string attr_name = "units";
1769  if (true == is_lat) {
1770  string lat_unit_value = "degrees_north";
1771  return Is_Str_Attr(attr, varfullpath, attr_name, lat_unit_value);
1772  }
1773  else {
1774  string lon_unit_value = "degrees_east";
1775  return Is_Str_Attr(attr, varfullpath, attr_name, lon_unit_value);
1776  }
1777 }
1778 
1779 // This function is mainly to add _FillValue.
1780 void File::Add_One_Float_Attr(Attribute* attr, const string &attrname, float float_value)
1781 {
1782  attr->name = attrname;
1783  attr->newname = attr->name;
1784  attr->dtype = H5FLOAT32;
1785  attr->count = 1;
1786  attr->value.resize(sizeof(float));
1787  memcpy(&(attr->value[0]), (void*) (&float_value), sizeof(float));
1788 }
1789 
1790 // Products like GPM use string type for MissingValue, we need to change them to the corresponding variable datatype and
1791 // get the value corrected.
1792 void File::Change_Attr_One_Str_to_Others(Attribute* attr, Var*var)
1793 {
1794 
1795  char *pEnd;
1796  // string to long int number.
1797  long int num_sli = 0;
1798  if (attr->dtype != H5FSTRING)
1799  throw2("Currently we only convert fixed-size string to other datatypes. ", attr->name);
1800  if (attr->count != 1)
1801  throw4("The fixed-size string count must be 1 and the current count is ", attr->count, " for the attribute ",
1802  attr->name);
1803 
1804  Retrieve_H5_Attr_Value(attr, var->fullpath);
1805  string attr_value;
1806  attr_value.resize(attr->value.size());
1807  copy(attr->value.begin(), attr->value.end(), attr_value.begin());
1808 
1809  switch (var->dtype) {
1810 
1811  case H5UCHAR: {
1812  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1813  if (num_sli < 0 || num_sli > UCHAR_MAX)
1814  throw5("Attribute type is unsigned char, the current attribute ", attr->name, " has the value ", num_sli,
1815  ". It is overflowed. ");
1816  else {
1817  unsigned char num_suc = (unsigned char) num_sli;
1818  attr->dtype = H5UCHAR;
1819  attr->value.resize(sizeof(unsigned char));
1820  memcpy(&(attr->value[0]), (void*) (&num_suc), sizeof(unsigned char));
1821  }
1822 
1823  }
1824  break;
1825  case H5CHAR: {
1826  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1827  if (num_sli < SCHAR_MIN || num_sli > SCHAR_MAX)
1828  throw5("Attribute type is signed char, the current attribute ", attr->name, " has the value ", num_sli,
1829  ". It is overflowed. ");
1830  else {
1831  char num_sc = (char) num_sli;
1832  attr->dtype = H5CHAR;
1833  attr->value.resize(sizeof(char));
1834  memcpy(&(attr->value[0]), (void*) (&num_sc), sizeof(char));
1835  }
1836 
1837  }
1838  break;
1839  case H5INT16: {
1840  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1841  if (num_sli < SHRT_MIN || num_sli > SHRT_MAX)
1842  throw5("Attribute type is 16-bit integer, the current attribute ", attr->name, " has the value ", num_sli,
1843  ". It is overflowed. ");
1844  else {
1845  short num_ss = (short) num_sli;
1846  attr->dtype = H5INT16;
1847  attr->value.resize(sizeof(short));
1848  memcpy(&(attr->value[0]), (void*) (&num_ss), sizeof(short));
1849  }
1850 
1851  }
1852  break;
1853  case H5UINT16: {
1854  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1855  if (num_sli < 0 || num_sli > USHRT_MAX)
1856  throw5("Attribute type is unsigned 16-bit integer, the current attribute ", attr->name, " has the value ",
1857  num_sli, ". It is overflowed. ");
1858  else {
1859  unsigned short num_uss = (unsigned short) num_sli;
1860  attr->dtype = H5UINT16;
1861  attr->value.resize(sizeof(unsigned short));
1862  memcpy(&(attr->value[0]), (void*) (&num_uss), sizeof(unsigned short));
1863  }
1864  }
1865  break;
1866  case H5INT32: {
1867  num_sli = strtol(&(attr->value[0]), &pEnd, 10);
1868  //No need to check overflow, the number will always be in the range.
1869 #if 0
1870  //if(num_sli <LONG_MIN || num_sli >LONG_MAX)
1871  // throw5("Attribute type is 32-bit integer, the current attribute ",attr->name, " has the value ",num_sli, ". It is overflowed. ");
1872 #endif
1873  attr->dtype = H5INT32;
1874  attr->value.resize(sizeof(long int));
1875  memcpy(&(attr->value[0]), (void*) (&num_sli), sizeof(long int));
1876 
1877  }
1878  break;
1879  case H5UINT32: {
1880  unsigned long int num_suli = strtoul(&(attr->value[0]), &pEnd, 10);
1881  // No need to check since num_suli will not be bigger than ULONG_MAX.
1882  attr->dtype = H5UINT32;
1883  attr->value.resize(sizeof(unsigned long int));
1884  memcpy(&(attr->value[0]), (void*) (&num_suli), sizeof(unsigned long int));
1885  }
1886  break;
1887  case H5FLOAT32: {
1888  float num_sf = strtof(&(attr->value[0]), NULL);
1889  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1890  attr->dtype = H5FLOAT32;
1891  attr->value.resize(sizeof(float));
1892  memcpy(&(attr->value[0]), (void*) (&num_sf), sizeof(float));
1893  }
1894  break;
1895  case H5FLOAT64: {
1896  double num_sd = strtod(&(attr->value[0]), NULL);
1897  // Don't think it is necessary to check if floating-point is oveflowed for this routine. ignore it now. KY 2014-09-22
1898  attr->dtype = H5FLOAT64;
1899  attr->value.resize(sizeof(double));
1900  memcpy(&(attr->value[0]), (void*) (&num_sd), sizeof(double));
1901  }
1902  break;
1903 
1904  default:
1905  throw4("Unsupported HDF5 datatype that the string is converted to for the attribute ", attr->name,
1906  " of the variable ", var->fullpath);
1907  } // "switch(var->dtype)"
1908 
1909 }
1910 
1911 // Change a string type attribute to the input value
1912 void File::Replace_Var_Str_Attr(Var* var, const string &attr_name, const string& strvalue)
1913 {
1914 
1915  bool rep_attr = true;
1916  bool rem_attr = false;
1917  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1918  if ((*ira)->name == attr_name) {
1919  if (true == Is_Str_Attr(*ira, var->fullpath, attr_name, strvalue))
1920  rep_attr = false;
1921  else
1922  rem_attr = true;
1923  break;
1924  }
1925  }
1926 
1927  // Remove the attribute if the attribute value is not strvalue
1928  if (true == rem_attr) {
1929  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
1930  if ((*ira)->name == attr_name) {
1931  delete (*ira);
1932  var->attrs.erase(ira);
1933  break;
1934  }
1935  }
1936  }
1937 
1938  // Add the attribute with strvalue
1939  if (true == rep_attr) {
1940  Attribute * attr = new Attribute();
1941  Add_Str_Attr(attr, attr_name, strvalue);
1942  var->attrs.push_back(attr);
1943  }
1944 }
1945 
1946 // Check if this variable if latitude,longitude.We check the three name pairs(lat,lon),(latitude,longitude),(Latitude,Longitude)
1947 bool File::Is_geolatlon(const string & var_name, bool is_lat)
1948 {
1949 
1950  bool ret_value = false;
1951  if (true == is_lat) {
1952  string lat1 = "lat";
1953  string lat2 = "latitude";
1954  string lat3 = "Latitude";
1955 
1956  if (var_name.compare(lat1) == 0 || var_name.compare(lat2) == 0 || var_name.compare(lat3) == 0) ret_value = true;
1957  }
1958 
1959  else {
1960  string lon1 = "lon";
1961  string lon2 = "longitude";
1962  string lon3 = "Longitude";
1963  if (var_name.compare(lon1) == 0 || var_name.compare(lon2) == 0 || var_name.compare(lon3) == 0) ret_value = true;
1964 
1965  }
1966  return ret_value;
1967 }
1968 
1969 // Add supplementary attributes.
1970 void File::Add_Supplement_Attrs(bool add_path)
1971 {
1972 
1973  if (false == add_path) return;
1974 
1975  // Adding variable original name(origname) and full path(fullpath)
1976  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1977  Attribute * attr = new Attribute();
1978  const string varname = (*irv)->name;
1979  const string attrname = "origname";
1980  Add_Str_Attr(attr, attrname, varname);
1981  (*irv)->attrs.push_back(attr);
1982  }
1983 
1984  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
1985  // Turn off the fullnamepath attribute when zero_storage_size is 0.
1986  // Use the BES key since quite a few testing cases will be affected.
1987  // KY 2020-03-23
1988  if((*irv)->zero_storage_size==false
1989  || HDF5RequestHandler::get_no_zero_size_fullnameattr() == false) {
1990  Attribute * attr = new Attribute();
1991  const string varname = (*irv)->fullpath;
1992  const string attrname = "fullnamepath";
1993  Add_Str_Attr(attr, attrname, varname);
1994  (*irv)->attrs.push_back(attr);
1995  }
1996  }
1997 
1998  // Adding group path
1999  for (vector<Group *>::iterator irg = this->groups.begin(); irg != this->groups.end(); ++irg) {
2000  // Only when this group has attributes, the original path of the group has some values. So add it.
2001  if (false == (*irg)->attrs.empty()) {
2002 
2003  Attribute * attr = new Attribute();
2004  const string varname = (*irg)->path;
2005  const string attrname = "fullnamepath";
2006  Add_Str_Attr(attr, attrname, varname);
2007  (*irg)->attrs.push_back(attr);
2008  }
2009  }
2010 
2011 }
2012 
2013 // Variable target will not be deleted, but rather its contents are replaced.
2014 // We may make this as an operator = in the future.
2015 // Note: the attributes can not be replaced.
2016 void File::Replace_Var_Info(Var *src, Var *target)
2017 {
2018 
2019 #if 0
2020  for_each (target->dims.begin (), target->dims.end (),
2021  delete_elem ());
2022  for_each (target->attrs.begin (), target->attrs.end (),
2023  delete_elem ());
2024 #endif
2025 
2026  target->newname = src->newname;
2027  target->name = src->name;
2028  target->fullpath = src->fullpath;
2029  target->rank = src->rank;
2030  target->dtype = src->dtype;
2031  target->unsupported_attr_dtype = src->unsupported_attr_dtype;
2032  target->unsupported_dspace = src->unsupported_dspace;
2033 #if 0
2034  for (vector<Attribute*>::iterator ira = target->attrs.begin();
2035  ira!=target->attrs.end(); ++ira) {
2036  delete (*ira);
2037  target->attrs.erase(ira);
2038  ira--;
2039  }
2040 #endif
2041  for (vector<Dimension*>::iterator ird = target->dims.begin(); ird != target->dims.end();) {
2042  delete (*ird);
2043  ird = target->dims.erase(ird);
2044  }
2045 
2046  // Somehow attributes cannot be replaced.
2047 #if 0
2048  for (vector<Attribute*>::iterator ira = src->attrs.begin();
2049  ira!=src->attrs.end(); ++ira) {
2050  Attribute* attr= new Attribute();
2051  attr->name = (*ira)->name;
2052  attr->newname = (*ira)->newname;
2053  attr->dtype =(*ira)->dtype;
2054  attr->count =(*ira)->count;
2055  attr->strsize = (*ira)->strsize;
2056  attr->fstrsize = (*ira)->fstrsize;
2057  attr->value =(*ira)->value;
2058  target->attrs.push_back(attr);
2059  }
2060 #endif
2061 
2062  for (vector<Dimension*>::iterator ird = src->dims.begin(); ird != src->dims.end(); ++ird) {
2063  Dimension *dim = new Dimension((*ird)->size);
2064  dim->name = (*ird)->name;
2065  dim->newname = (*ird)->newname;
2066  target->dims.push_back(dim);
2067  }
2068 
2069 }
2070 
2071 // Replace the attributes of target with src.
2072 void File::Replace_Var_Attrs(Var *src, Var *target)
2073 {
2074 
2075 #if 0
2076  for_each (target->dims.begin (), target->dims.end (),
2077  delete_elem ());
2078  for_each (target->attrs.begin (), target->attrs.end (),
2079  delete_elem ());
2080 #endif
2081 
2082  for (vector<Attribute*>::iterator ira = target->attrs.begin(); ira != target->attrs.end();) {
2083  delete (*ira);
2084  ira = target->attrs.erase(ira);
2085  }
2086  for (vector<Attribute*>::iterator ira = src->attrs.begin(); ira != src->attrs.end(); ++ira) {
2087  Attribute* attr = new Attribute();
2088  attr->name = (*ira)->name;
2089  attr->newname = (*ira)->newname;
2090  attr->dtype = (*ira)->dtype;
2091  attr->count = (*ira)->count;
2092  attr->strsize = (*ira)->strsize;
2093  attr->fstrsize = (*ira)->fstrsize;
2094  attr->value = (*ira)->value;
2095  target->attrs.push_back(attr);
2096  }
2097 
2098 }
2099 
2100 // Check if a variable with a var name is under a specific group with groupname
2101 // note: the variable's size at each dimension is also returned. The user must allocate the
2102 // memory for the dimension sizes(an array(vector is perferred).
2103 bool File::is_var_under_group(const string &varname, const string &grpname, const int var_rank,
2104  vector<size_t> & var_size)
2105 {
2106 
2107  bool ret_value = false;
2108  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2109 
2110  if ((*irv)->rank == var_rank) {
2111  if ((*irv)->name == varname) {
2112 
2113  // Obtain the variable path
2114  string var_path = HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath);
2115 
2116  // Check if we find the variable under this group
2117  if (grpname == var_path) {
2118  ret_value = true;
2119  for (int i = 0; i < var_rank; i++)
2120  var_size[i] = (*irv)->getDimensions()[i]->size;
2121  break;
2122  }
2123  }
2124  } // "if((*irv)->rank == var_rank)"
2125  } // "for (vector<Var *>::iterator irv = this->vars.begin()"
2126 
2127 
2128  return ret_value;
2129 
2130 }
2132 
2133  bool ret_value = false;
2134  for (vector<Var *>::iterator irv = this->vars.begin();
2135  irv != this->vars.end(); ++irv) {
2136  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2137  ira != (*irv)->attrs.end(); ++ira) {
2138  if((*ira)->name =="grid_mapping") {
2139  ret_value = true;
2140  break;
2141  }
2142  }
2143  if(true == ret_value)
2144  break;
2145  }
2146 
2147  return ret_value;
2148 
2149 }
2150 
2152 
2153  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2154  string attr_value;
2155  for (vector<Attribute *>::iterator ira = (*irv)->attrs.begin(); ira != (*irv)->attrs.end(); ++ira) {
2156  if((*ira)->name =="grid_mapping") {
2157  Retrieve_H5_Attr_Value(*ira, (*irv)->fullpath);
2158  attr_value.resize((*ira)->value.size());
2159  copy((*ira)->value.begin(), (*ira)->value.end(), attr_value.begin());
2160  break;
2161  }
2162 
2163  }
2164  if(attr_value.find('/') ==string::npos){
2165  string new_name = Check_Grid_Mapping_VarName(attr_value,(*irv)->fullpath);
2166  if(new_name != "")
2167  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2168 
2169  }
2170  else {
2171  string new_name = Check_Grid_Mapping_FullPath(attr_value);
2172  //Using new_name as the attribute value
2173  if(new_name != "")
2174  Replace_Var_Str_Attr((*irv),"grid_mapping",new_name);
2175  }
2176  }
2177 
2178 }
2179 
2180 string File::Check_Grid_Mapping_VarName(const string & a_value,const string & var_fpath) {
2181 
2182  string var_path = HDF5CFUtil::obtain_string_before_lastslash(var_fpath);
2183  string gmap_new_name;
2184  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2185  if((*irv)->name == a_value){
2186  if(var_path == HDF5CFUtil::obtain_string_before_lastslash((*irv)->fullpath)) {
2187  gmap_new_name = (*irv)->newname;
2188  break;
2189  }
2190  }
2191  }
2192  return gmap_new_name;
2193 }
2194 
2195 
2196 string File::Check_Grid_Mapping_FullPath(const string & a_value) {
2197 
2198  string gmap_new_name;
2199  for (vector<Var *>::iterator irv = this->vars.begin(); irv != this->vars.end(); ++irv) {
2200  if((*irv)->fullpath == a_value){
2201  gmap_new_name = (*irv)->newname;
2202  break;
2203  }
2204  }
2205 
2206  return gmap_new_name;
2207 }
2208 
2209 void File::remove_netCDF_internal_attributes(bool include_attr) {
2210 
2211  if(true == include_attr) {
2212  for (vector<Var *>::iterator irv = this->vars.begin();
2213  irv != this->vars.end(); ++irv) {
2214  bool var_has_dimscale = false;
2215 
2216  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2217  ira != (*irv)->attrs.end();) {
2218  if((*ira)->name == "CLASS") {
2219  string class_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2220 
2221  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2222  // "DIMENSION_SCALE", which is 15.
2223  if (0 == class_value.compare(0,15,"DIMENSION_SCALE")) {
2224  delete(*ira);
2225  ira = (*irv)->attrs.erase(ira);
2226  var_has_dimscale = true;
2227 
2228  }
2229 #if 0
2230  else if(1) {// Add a BES key,also delete
2231 
2232  }
2233 #endif
2234  else {
2235  ++ira;
2236  }
2237  }
2238 #if 0
2239  else if((*ira)->name == "NAME") {// Add a BES Key
2240  string name_value = Retrieve_Str_Attr_Value(*ira,(*irv)->fullpath);
2241  if( 0 == name_value.compare(0,(*irv)->name.size(),(*irv)->name)) {
2242  delete(*ira);
2243  ira =(*irv)->attrs.erase(ira);
2244  }
2245  else {
2246  string netcdf_dim_mark= "This is a netCDF dimension but not a netCDF variable";
2247  if( 0 == name_value.compare(0,netcdf_dim_mark.size(),netcdf_dim_mark)) {
2248  delete(*ira);
2249  ira =(*irv)->attrs.erase(ira);
2250  }
2251  else {
2252  ++ira;
2253  }
2254  }
2255 
2256  }
2257 #endif
2258  else if((*ira)->name == "_Netcdf4Dimid") {
2259  delete(*ira);
2260  ira =(*irv)->attrs.erase(ira);
2261  }
2262  else if((*ira)->name == "_Netcdf4Coordinates") {
2263  delete(*ira);
2264  ira =(*irv)->attrs.erase(ira);
2265  }
2266 #if 0
2267  else if((*ira)->name == "_nc3_strict") {
2268  delete((*ira));
2269  ira =(*irv)->attrs.erase(ira);
2270  }
2271 #endif
2272  else {
2273  ++ira;
2274  }
2275  }
2276 
2277  if(true == var_has_dimscale) {
2278  for(vector<Attribute *>::iterator ira = (*irv)->attrs.begin();
2279  ira != (*irv)->attrs.end();++ira) {
2280  if((*ira)->name == "NAME") {// Add a BES Key
2281  delete(*ira);
2282  ira =(*irv)->attrs.erase(ira);
2283  break;
2284  }
2285  }
2286  }
2287  }
2288  }
2289 
2290 }
2291 // Add ignored page header info. Mainly a helper message.
2292 void File::add_ignored_info_page_header()
2293 {
2294  ignored_msg =
2295  " \n This page is for HDF5 CF hyrax data providers or distributors to check if any HDF5 object or attribute information are ignored during the mapping. \n\n";
2296 }
2297 
2298 // Add ignored object header info. Mainly a helper message.
2299 void File::add_ignored_info_obj_header()
2300 {
2301 
2302  ignored_msg += " Some HDF5 objects or the object information are ignored when mapping to DAP2 by the HDF5 OPeNDAP";
2303  ignored_msg += " handler due to the restrictions of DAP2, CF conventions or CF tools.";
2304  ignored_msg += " Please use HDF5 tools(h5dump or HDFView) to check carefully and make sure that these objects";
2305  ignored_msg +=
2306  " are OK to ignore for your service. For questions or requests to find a way to handle the ignored objects, please";
2307  ignored_msg += " contact the HDF5 OPeNDAP handler developer or send an email to help@hdfgroup.org.\n";
2308 
2309  ignored_msg += " \n In general, ignored HDF5 objects include HDF5 soft links, external links and named datatype.\n";
2310  ignored_msg +=
2311  " \n The HDF5 datasets(variables in the CF term) and attributes that have the following datatypes are ignored: \n";
2312  ignored_msg +=
2313  " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2314  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2315 
2316  ignored_msg +=
2317  " \n The HDF5 datasets(variables in the CF term) and attributes associated with the following dimensions are ignored: \n";
2318  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2319  ignored_msg += " 2) attributes that have any zero size dimensions(not reported due to extreme rarity and non-trivial coding)\n\n";
2320 
2321 }
2322 
2323 // Add the ignored links information.Mainly a helper message.
2324 void File::add_ignored_info_links_header()
2325 {
2326 
2327  if (false == this->have_ignored) {
2328  add_ignored_info_obj_header();
2329  have_ignored = true;
2330  }
2331  // Add ignored datatype header.
2332  string lh_msg = "******WARNING******\n";
2333  lh_msg += "IGNORED soft links or external links are: ";
2334  if (ignored_msg.rfind(lh_msg) == string::npos) ignored_msg += lh_msg + "\n";
2335 
2336 }
2337 
2338 // Leave the code for the time being.
2339 #if 0
2340 void
2341 File:: add_ignored_info_obj_dtype_header() {
2342 
2343  // Add ignored datatype header.
2344  ignored_msg += " \n Variables and attributes ignored due to the unsupported datatypes. \n";
2345  ignored_msg += " In general, the unsupported datatypes include: \n";
2346  ignored_msg += " Signed and unsigned 64-bit integers, HDF5 compound, HDF5 variable length(excluding variable length string),";
2347  ignored_msg += " HDF5 reference, HDF5 enum, HDF5 opaque , HDF5 bitfield, HDF5 Array and HDF5 Time datatypes.\n";
2348 
2349 }
2350 
2351 void
2352 File:: add_ignored_info_obj_dspace_header() {
2353 
2354  // Add ignored dataspace header.
2355  ignored_msg += " \n Variables and attributes ignored due to the unsupported dimensions. \n";
2356  ignored_msg += " In general, the unsupported dimensions include: \n";
2357  ignored_msg += " 1) variables that have HDF5 NULL dataspace(H5S_NULL)(rarely occurred)\n";
2358  ignored_msg += " 2) variables that have any zero size dimensions\n";
2359 
2360 }
2361 #endif
2362 
2363 // Add the ignored link info.
2364 void File::add_ignored_info_links(const string & link_path)
2365 {
2366  if (ignored_msg.find("Link paths: ") == string::npos)
2367  ignored_msg += " Link paths: " + link_path;
2368  else
2369  ignored_msg += " " + link_path;
2370 }
2371 
2372 // Add the ignored name datatype info.
2373 void File::add_ignored_info_namedtypes(const string& grp_name, const string& named_dtype_name)
2374 {
2375 
2376  if (false == this->have_ignored) {
2377  add_ignored_info_obj_header();
2378  have_ignored = true;
2379  }
2380 
2381  string ignored_HDF5_named_dtype_hdr = "\n******WARNING******";
2382  ignored_HDF5_named_dtype_hdr += "\n IGNORED HDF5 named datatype objects:\n";
2383  string ignored_HDF5_named_dtype_msg = " Group name: " + grp_name + " HDF5 named datatype name: " + named_dtype_name.substr(0,named_dtype_name.size()-1)
2384  + "\n";
2385  if (ignored_msg.find(ignored_HDF5_named_dtype_hdr) == string::npos)
2386  ignored_msg += ignored_HDF5_named_dtype_hdr + ignored_HDF5_named_dtype_msg;
2387  else
2388  ignored_msg += ignored_HDF5_named_dtype_msg;
2389 
2390 }
2391 
2392 // Add the ignored attribute information. When is_grp is true, the ignored group attribute names are added.
2393 // Otherwise, the ignored dataset attribute names are added.
2394 void File::add_ignored_info_attrs(bool is_grp, const string & obj_path, const string & attr_name)
2395 {
2396 
2397  if (false == this->have_ignored) {
2398  add_ignored_info_obj_header();
2399  have_ignored = true;
2400  }
2401 
2402 
2403  string ignored_warning_str = "\n******WARNING******";
2404  string ignored_HDF5_grp_hdr = ignored_warning_str + "\n Ignored attributes under root and groups:\n";
2405  string ignored_HDF5_grp_msg = " Group path: " + obj_path + " Attribute names: " + attr_name + "\n";
2406  string ignored_HDF5_var_hdr = ignored_warning_str + "\n Ignored attributes for variables:\n";
2407  string ignored_HDF5_var_msg = " Variable path: " + obj_path + " Attribute names: " + attr_name + "\n";
2408 
2409 
2410  if (true == is_grp) {
2411  if (ignored_msg.find(ignored_HDF5_grp_hdr) == string::npos)
2412  ignored_msg += ignored_HDF5_grp_hdr + ignored_HDF5_grp_msg;
2413  else
2414  ignored_msg += ignored_HDF5_grp_msg;
2415  }
2416  else {
2417  if (ignored_msg.find(ignored_HDF5_var_hdr) == string::npos)
2418  ignored_msg += ignored_HDF5_var_hdr + ignored_HDF5_var_msg;
2419  else
2420  ignored_msg += ignored_HDF5_var_msg;
2421  }
2422 
2423 }
2424 
2425 //Ignored object information. When is_dim_related is true, ignored data space info. is present.
2426 //When is_dim_related is false, ignored data type info. is present.
2427 void File::add_ignored_info_objs(bool is_dim_related, const string & obj_path)
2428 {
2429 
2430  if (false == this->have_ignored) {
2431  add_ignored_info_obj_header();
2432  have_ignored = true;
2433  }
2434 
2435  string ignored_warning_str = "\n******WARNING******";
2436  string ignored_HDF5_dtype_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported datatypes:\n";
2437  string ignored_HDF5_dspace_var_hdr = ignored_warning_str + "\n IGNORED variables due to unsupported dimensions:\n";
2438  string ignored_HDF5_var_msg = " Variable path: " + obj_path + "\n";
2439 
2440  if (true == is_dim_related) {
2441  if (ignored_msg.find(ignored_HDF5_dspace_var_hdr) == string::npos)
2442  ignored_msg += ignored_HDF5_dspace_var_hdr + ignored_HDF5_var_msg;
2443  else
2444  ignored_msg += ignored_HDF5_var_msg;
2445 
2446  }
2447  else {
2448  if (ignored_msg.find(ignored_HDF5_dtype_var_hdr) == string::npos)
2449  ignored_msg += ignored_HDF5_dtype_var_hdr + ignored_HDF5_var_msg;
2450  else
2451  ignored_msg += ignored_HDF5_var_msg;
2452  }
2453 
2454 }
2455 
2456 // No ignored info.
2457 void File::add_no_ignored_info()
2458 {
2459 
2460  ignored_msg += "There are no ignored HDF5 objects or attributes.";
2461 
2462 }
2463 
2464 // This function should only be used when the HDF5 file is following the netCDF data model.
2465 // Check if we should not report the Dimension scale related attributes as ignored.
2466 bool File::ignored_dimscale_ref_list(Var *var)
2467 {
2468 
2469  bool ignored_dimscale = true;
2470  // Only when "General_Product == this->product_type && GENERAL_DIMSCALE== this->gproduct_pattern)"
2471 
2472  bool has_dimscale = false;
2473  bool has_reference_list = false;
2474  for (vector<Attribute *>::iterator ira = var->attrs.begin(); ira != var->attrs.end(); ira++) {
2475  if ((*ira)->name == "REFERENCE_LIST" && false == HDF5CFUtil::cf_strict_support_type((*ira)->getType(),_is_dap4))
2476  has_reference_list = true;
2477  if ((*ira)->name == "CLASS") {
2478  Retrieve_H5_Attr_Value(*ira, var->fullpath);
2479  string class_value;
2480  class_value.resize((*ira)->value.size());
2481  copy((*ira)->value.begin(), (*ira)->value.end(), class_value.begin());
2482 
2483  // Compare the attribute "CLASS" value with "DIMENSION_SCALE". We only compare the string with the size of
2484  // "DIMENSION_SCALE", which is 15.
2485  if (0 == class_value.compare(0, 15, "DIMENSION_SCALE")) {
2486  has_dimscale = true;
2487  }
2488  }
2489 
2490  if (true == has_dimscale && true == has_reference_list) {
2491  ignored_dimscale = false;
2492  break;
2493  }
2494 
2495  }
2496  //}
2497  return ignored_dimscale;
2498 }
2499 
2500 // Check if the long string can should be dropped from a dataset or an attribute. Users can set up a BES key to turn it off or on.
2501 bool File::Check_DropLongStr(Var *var, Attribute * attr)
2502 {
2503 
2504  bool drop_longstr = false;
2505  if (NULL == attr) {
2506  if (H5FSTRING == var->dtype || H5VSTRING == var->dtype) {
2507  try {
2508  drop_longstr = Check_VarDropLongStr(var->fullpath, var->dims, var->dtype);
2509  }
2510  catch (...) {
2511  throw1("Check_VarDropLongStr fails ");
2512  }
2513  }
2514  }
2515  // No limitation for the attributes. KY 2018-02-26
2516 #if 0
2517  else {
2518  if (H5FSTRING == attr->dtype || H5VSTRING == attr->dtype) {
2519  if (attr->getBufSize() > NC_JAVA_STR_SIZE_LIMIT) {
2520  drop_longstr = true;
2521  }
2522  }
2523 
2524  }
2525 #endif
2526  return drop_longstr;
2527 }
2528 
2529 // Check if a long string dataset should be dropped. Users can turn on a BES key not to drop the long string.
2530 // However, the Java clients may not access.
2531 //
2532 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype) const
2533 
2534 {
2535 
2536  bool drop_longstr = false;
2537 
2538  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2539  if (dset_id < 0)
2540  throw2("Cannot open the dataset ", varpath);
2541 
2542  hid_t dtype_id = -1;
2543  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2544  H5Dclose(dset_id);
2545  throw2("Cannot obtain the datatype of the dataset ", varpath);
2546  }
2547 
2548  size_t ty_size = H5Tget_size(dtype_id);
2549  if (ty_size == 0) {
2550  H5Tclose(dtype_id);
2551  H5Dclose(dset_id);
2552  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2553  }
2554 
2555  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2556  if (ty_size > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2557  }
2558  else if (H5VSTRING == dtype) {
2559 
2560  unsigned long long total_elms = 1;
2561  if (dims.size() != 0) {
2562  for (unsigned int i = 0; i < dims.size(); i++)
2563  total_elms = total_elms * ((dims[i])->size);
2564  }
2565  vector<char> strval;
2566  strval.resize(total_elms * ty_size);
2567  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2568  if (read_ret < 0) {
2569  H5Tclose(dtype_id);
2570  H5Dclose(dset_id);
2571  throw2("Cannot read the data of the dataset ", varpath);
2572  }
2573 
2574  vector<string> finstrval;
2575  finstrval.resize(total_elms);
2576  char*temp_bp = &strval[0];
2577  char*onestring = NULL;
2578  for (unsigned long long i = 0; i < total_elms; i++) {
2579  onestring = *(char**) temp_bp;
2580  if (onestring != NULL) {
2581  finstrval[i] = string(onestring);
2582  if(finstrval[i].size()>NC_JAVA_STR_SIZE_LIMIT) {
2583  drop_longstr = true;
2584  break;
2585  }
2586  }
2587  temp_bp += ty_size;
2588  }
2589 
2590  if (false == strval.empty()) {
2591  herr_t ret_vlen_claim;
2592  hid_t dspace_id = H5Dget_space(dset_id);
2593  if (dspace_id < 0) {
2594  H5Tclose(dtype_id);
2595  H5Dclose(dset_id);
2596  throw2("Cannot obtain the dataspace id.", varpath);
2597  }
2598  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2599  if (ret_vlen_claim < 0) {
2600  H5Tclose(dtype_id);
2601  H5Sclose(dspace_id);
2602  H5Dclose(dset_id);
2603  throw2("Cannot reclaim the vlen space ", varpath);
2604  }
2605  if (H5Sclose(dspace_id) < 0) {
2606  H5Tclose(dtype_id);
2607  H5Dclose(dset_id);
2608  throw2("Cannot close the HDF5 data space.", varpath);
2609  }
2610  }
2611  }
2612  if (H5Tclose(dtype_id) < 0) {
2613  H5Dclose(dset_id);
2614  throw2("Cannot close the HDF5 data type.", varpath);
2615  }
2616  if (H5Dclose(dset_id) < 0)
2617  throw2("Cannot close the HDF5 data type.", varpath);
2618 
2619  return drop_longstr;
2620 }
2621 #if 0
2622 bool File::Check_VarDropLongStr(const string & varpath, const vector<Dimension *>& dims, H5DataType dtype)
2623 
2624 {
2625 
2626  bool drop_longstr = false;
2627 
2628  unsigned long long total_elms = 1;
2629  if (dims.size() != 0) {
2630  for (unsigned int i = 0; i < dims.size(); i++)
2631  total_elms = total_elms * ((dims[i])->size);
2632  }
2633 
2634  if (total_elms > NC_JAVA_STR_SIZE_LIMIT)
2635  drop_longstr = true;
2636 
2637  else { // We need to check both fixed-size and variable-length strings.
2638 
2639  hid_t dset_id = H5Dopen2(this->fileid, varpath.c_str(), H5P_DEFAULT);
2640  if (dset_id < 0)
2641  throw2("Cannot open the dataset ", varpath);
2642 
2643  hid_t dtype_id = -1;
2644  if ((dtype_id = H5Dget_type(dset_id)) < 0) {
2645  H5Dclose(dset_id);
2646  throw2("Cannot obtain the datatype of the dataset ", varpath);
2647  }
2648 
2649  size_t ty_size = H5Tget_size(dtype_id);
2650  if (ty_size == 0) {
2651  H5Tclose(dtype_id);
2652  H5Dclose(dset_id);
2653  throw2("Cannot obtain the datatype size of the dataset ", varpath);
2654  }
2655 
2656  if (H5FSTRING == dtype) { // Fixed-size, just check the number of elements.
2657  if ((ty_size * total_elms) > NC_JAVA_STR_SIZE_LIMIT) drop_longstr = true;
2658  }
2659  else if (H5VSTRING == dtype) {
2660 
2661  vector<char> strval;
2662  strval.resize(total_elms * ty_size);
2663  hid_t read_ret = H5Dread(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void*) &strval[0]);
2664  if (read_ret < 0) {
2665  H5Tclose(dtype_id);
2666  H5Dclose(dset_id);
2667  throw2("Cannot read the data of the dataset ", varpath);
2668  }
2669 
2670  vector<string> finstrval;
2671  finstrval.resize(total_elms);
2672  char*temp_bp = &strval[0];
2673  char*onestring = NULL;
2674  for (unsigned long long i = 0; i < total_elms; i++) {
2675  onestring = *(char**) temp_bp;
2676  if (onestring != NULL)
2677  finstrval[i] = string(onestring);
2678  else
2679  // We will add a NULL if onestring is NULL.
2680  finstrval[i] = "";
2681  temp_bp += ty_size;
2682  }
2683 
2684  if (false == strval.empty()) {
2685  herr_t ret_vlen_claim;
2686  hid_t dspace_id = H5Dget_space(dset_id);
2687  if (dspace_id < 0) {
2688  H5Tclose(dtype_id);
2689  H5Dclose(dset_id);
2690  throw2("Cannot obtain the dataspace id.", varpath);
2691  }
2692  ret_vlen_claim = H5Dvlen_reclaim(dtype_id, dspace_id, H5P_DEFAULT, (void*) &strval[0]);
2693  if (ret_vlen_claim < 0) {
2694  H5Tclose(dtype_id);
2695  H5Sclose(dspace_id);
2696  H5Dclose(dset_id);
2697  throw2("Cannot reclaim the vlen space ", varpath);
2698  }
2699  if (H5Sclose(dspace_id) < 0) {
2700  H5Tclose(dtype_id);
2701  H5Dclose(dset_id);
2702  throw2("Cannot close the HDF5 data space.", varpath);
2703  }
2704  }
2705  unsigned long long total_str_size = 0;
2706  for (unsigned long long i = 0; i < total_elms; i++) {
2707  total_str_size += finstrval[i].size();
2708  if (total_str_size > NC_JAVA_STR_SIZE_LIMIT) {
2709  drop_longstr = true;
2710  break;
2711  }
2712  }
2713  }
2714  if (H5Tclose(dtype_id) < 0) {
2715  H5Dclose(dset_id);
2716  throw2("Cannot close the HDF5 data type.", varpath);
2717  }
2718  if (H5Dclose(dset_id) < 0)
2719  throw2("Cannot close the HDF5 data type.", varpath);
2720  }
2721  return drop_longstr;
2722 }
2723 #endif
2724 
2725 
2726 // Provide if the long string is dropped.
2727 void File::add_ignored_grp_longstr_info(const string& grp_path, const string & attr_name)
2728 {
2729 
2730  ignored_msg += "The HDF5 group: " + grp_path + " has an empty-set string attribute: " + attr_name + "\n";
2731 
2732  return;
2733 }
2734 
2735 // Provide if the long variable string is dropped.
2736 void File::add_ignored_var_longstr_info(Var *var, Attribute *attr)
2737 {
2738 
2739  if (NULL == attr)
2740  ignored_msg += "String variable: " + var->fullpath + " value is set to empty.\n";
2741  else {
2742  ignored_msg += "The variable: " + var->fullpath + " has an empty-set string attribute: " + attr->name + "\n";
2743 
2744  }
2745  return;
2746 }
2747 
2748 // The warnings of the drop of the long string header
2749 void File::add_ignored_droplongstr_hdr()
2750 {
2751 
2752  if (false == this->have_ignored) this->have_ignored = true;
2753  string hdr = "\n\n The values of the following string variables ";
2754  hdr += " are set to empty because at least one string size in this variable exceeds netCDF Java string limit(32767 bytes).\n";
2755  hdr += "To obtain the values, change the BES key H5.EnableDropLongString=true at the handler BES";
2756  hdr += " configuration file(h5.conf)\nto H5.EnableDropLongString=false.\n\n";
2757 
2758  if (ignored_msg.rfind(hdr) == string::npos) ignored_msg += hdr;
2759 
2760 }
2761 
2762 // Sometimes, we need to release the temporary added resources.
2763 void File::release_standalone_var_vector(vector<Var*>&temp_vars)
2764 {
2765 
2766  for (vector<Var *>::iterator i = temp_vars.begin(); i != temp_vars.end();) {
2767  delete (*i);
2768  i = temp_vars.erase(i);
2769  }
2770 
2771 }
This class specifies the core engineering of mapping HDF5 to DAP by following CF.
#define throw1(a1)
The followings are convenient functions to throw exceptions with different.
Definition: HDF5CF.h:128
include the entry functions to execute the handlers
This class represents one attribute.
Definition: HDF5CF.h:189
This class repersents one dimension of an HDF5 dataset(variable).
Definition: HDF5CF.h:145
std::vector< Group * > groups
Non-root group vectors.
Definition: HDF5CF.h:816
virtual void Retrieve_H5_Var_Attr_Values(Var *var)
Retrieve attribute values for a variable.
Definition: HDF5CF.cc:748
virtual void Handle_Unsupported_Dspace(bool)
Handle unsupported HDF5 dataspaces for datasets.
Definition: HDF5CF.cc:1275
std::map< hsize_t, std::string > dimsize_to_fakedimname
Handle added dimension names.
Definition: HDF5CF.h:833
virtual void Handle_Grid_Mapping_Vars()
Handle Grid Mapping Vars.
Definition: HDF5CF.cc:2151
virtual void Handle_Unsupported_Others(bool)
Handle other unmapped objects/attributes.
Definition: HDF5CF.cc:1322
virtual void Retrieve_H5_Supported_Attr_Values()
Retrieve attribute values for the supported HDF5 datatypes.
Definition: HDF5CF.cc:729
std::vector< Var * > vars
Var vectors.
Definition: HDF5CF.h:810
virtual void Add_Supplement_Attrs(bool)
Add supplemental attributes such as fullpath and original name.
Definition: HDF5CF.cc:1970
virtual void Retrieve_H5_Info(const char *path, hid_t file_id, bool)
Definition: HDF5CF.cc:171
std::vector< Attribute * > root_attrs
Root attribute vectors.
Definition: HDF5CF.h:813
virtual void Handle_Unsupported_Dtype(bool)
Handle unsupported HDF5 datatypes.
Definition: HDF5CF.cc:918
virtual void Flatten_Obj_Name(bool)
Flatten the object name.
Definition: HDF5CF.cc:1372
virtual bool Have_Grid_Mapping_Attrs()
Check if having Grid Mapping Attrs.
Definition: HDF5CF.cc:2131
This class represents an HDF5 group. The group will be flattened according to the CF conventions.
Definition: HDF5CF.h:552
This class represents one HDF5 dataset(CF variable)
Definition: HDF5CF.h:259
Helper functions for generating DAS attributes and a function to check BES Key.
static H5DataType H5type_to_H5DAPtype(hid_t h5_type_id)
Map HDF5 Datatype to the intermediate H5DAPtype for the future use.
Definition: HDF5CFUtil.cc:54
static std::string trim_string(hid_t dtypeid, const std::string s, int num_sect, size_t section_size, std::vector< size_t > &sect_newsize)
Definition: HDF5CFUtil.cc:235