bes  Updated for version 3.20.10
HDF4RequestHandler.cc
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf4_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Author: James Gallagher <jgallagher@opendap.org>
9 // Author: Muqun Yang <myang6@opendap.org>
10 // This is free software; you can redistribute it and/or modify it under the
11 // terms of the GNU Lesser General Public License as published by the Free
12 // Software Foundation; either version 2.1 of the License, or (at your
13 // option) any later version.
14 //
15 // This software is distributed in the hope that it will be useful, but
16 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18 // License for more details.
19 //
20 // You should have received a copy of the GNU Lesser General Public
21 // License along with this library; if not, write to the Free Software
22 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23 //
24 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25 
26 // HDF4RequestHandler.cc
27 
28 #include <cstdlib>
29 #include <sys/types.h>
30 #include <sys/stat.h>
31 #include <fcntl.h>
32 #include <cerrno>
33 #include <unistd.h>
34 #include <iostream>
35 #include <sstream>
36 
37 #include <libdap/DMR.h>
38 #include <libdap/D4BaseTypeFactory.h>
39 #include <BESDMRResponse.h>
40 #include <libdap/mime_util.h>
41 #include <libdap/InternalErr.h>
42 #include <libdap/Ancillary.h>
43 #include <libdap/debug.h>
44 
45 #include "HDF4RequestHandler.h"
46 #include <BESResponseNames.h>
47 #include <BESDapNames.h>
48 #include <BESDASResponse.h>
49 #include <BESDDSResponse.h>
50 #include <BESDataDDSResponse.h>
51 #include <BESInfo.h>
52 #include <BESResponseHandler.h>
53 #include <BESVersionInfo.h>
54 #include <BESServiceRegistry.h>
55 #include <BESUtil.h>
56 #include <TheBESKeys.h>
57 #include <libdap/InternalErr.h>
58 #include <BESInternalError.h>
59 #include <BESDapError.h>
60 #include <BESStopWatch.h>
61 #include <BESDebug.h>
62 #include "BESDataNames.h"
63 //#include <libdap/ConstraintEvaluator.h>
64 #include <libdap/Ancillary.h>
65 #include "config_hdf.h"
66 
67 #define HDF4_NAME "h4"
68 #include "HE2CF.h"
69 #include "HDF4_DDS.h"
70 
71 #include "HDF4_DMR.h"
72 
73 //#include "HDFCFUtil.h"
74 #include "HDFFloat32.h"
75 #include "HDFSPArray_RealField.h"
76 
77 #include "dodsutil.h"
78 //#if 0
79 #include <sys/time.h>
80 //#endif
81 
82 using namespace std;
83 using namespace libdap;
84 
85 bool check_beskeys(const string);
86 bool get_beskeys(const string,string &);
87 
88 extern void read_das(DAS & das, const string & filename);
89 extern void read_dds(DDS & dds, const string & filename);
90 
91 extern bool read_dds_hdfsp(DDS & dds, const string & filename,int32 sdfd, int32 fileid,HDFSP::File*h4file);
92 
93 extern bool read_das_hdfsp(DAS & das, const string & filename,int32 sdfd, int32 fileid,HDFSP::File**h4fileptr);
94 
95 extern void read_das_sds(DAS & das, const string & filename,int32 sdfd, bool ecs_metadata,HDFSP::File**h4fileptr);
96 extern void read_dds_sds(DDS &dds, const string & filename,int32 sdfd, HDFSP::File*h4file,bool dds_set_cache);
97 
98 #ifdef USE_HDFEOS2_LIB
99 
100 void read_das_use_eos2lib(DAS & das, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,bool ecs_metadata,HDFSP::File**h4file,HDFEOS2::File**eosfile);
101 void read_dds_use_eos2lib(DDS & dds, const string & filename,int32 sdfd,int32 fileid, int32 gridfd, int32 swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
102 void close_fileid(const int sdfd, const int fileid,const int gridfd, const int swathfd,HDFSP::File*h4file,HDFEOS2::File*eosfile);
103 
104 #endif
105 
106 void close_hdf4_fileid(const int sdfd,const int fileid,HDFSP::File*h4file);
107 bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool rw_flag);
108 bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename);
109 
110 // CF key
111 bool HDF4RequestHandler::_usecf = false;
112 
113 // Keys to tune the performance -general
114 bool HDF4RequestHandler::_pass_fileid = false;
115 bool HDF4RequestHandler::_disable_structmeta = false;
116 bool HDF4RequestHandler::_enable_special_eos = false;
117 bool HDF4RequestHandler::_disable_scaleoffset_comp = false;
118 bool HDF4RequestHandler::_disable_ecsmetadata_min = false;
119 bool HDF4RequestHandler::_disable_ecsmetadata_all = false;
120 
121 
122 // Keys to tune the performance - cache
123 bool HDF4RequestHandler::_enable_eosgeo_cachefile = false;
124 bool HDF4RequestHandler::_enable_data_cachefile = false;
125 bool HDF4RequestHandler::_enable_metadata_cachefile= false;
126 
127 // Keys to handle vdata and vgroups
128 bool HDF4RequestHandler::_enable_hybrid_vdata = false;
129 bool HDF4RequestHandler::_enable_ceres_vdata = false;
130 bool HDF4RequestHandler::_enable_vdata_attr = false;
131 bool HDF4RequestHandler::_enable_vdata_desc_attr = false;
132 bool HDF4RequestHandler::_disable_vdata_nameclashing_check = false;
133 bool HDF4RequestHandler::_enable_vgroup_attr = false;
134 
135 // Misc. keys
136 bool HDF4RequestHandler::_enable_check_modis_geo_file = false;
137 bool HDF4RequestHandler::_enable_swath_grid_attr = false;
138 bool HDF4RequestHandler::_enable_ceres_merra_short_name = false;
139 bool HDF4RequestHandler::_enable_check_scale_offset_type = false;
140 bool HDF4RequestHandler::_disable_swath_dim_map = false;
141 
142 // Cache path,prefix and size
143 bool HDF4RequestHandler::_cache_latlon_path_exist =false;
144 string HDF4RequestHandler::_cache_latlon_path ="";
145 bool HDF4RequestHandler::_cache_latlon_prefix_exist =false;
146 string HDF4RequestHandler::_cache_latlon_prefix ="";
147 bool HDF4RequestHandler::_cache_latlon_size_exist =false;
148 long HDF4RequestHandler::_cache_latlon_size =0;
149 bool HDF4RequestHandler::_cache_metadata_path_exist =false;
150 string HDF4RequestHandler::_cache_metadata_path ="";
151 
152 HDF4RequestHandler::HDF4RequestHandler(const string & name) :
153  BESRequestHandler(name) {
154  add_handler(DAS_RESPONSE, HDF4RequestHandler::hdf4_build_das);
155  add_handler(DDS_RESPONSE, HDF4RequestHandler::hdf4_build_dds);
156  add_handler(DATA_RESPONSE, HDF4RequestHandler::hdf4_build_data);
157  add_handler(DMR_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
158  add_handler(DAP4DATA_RESPONSE, HDF4RequestHandler::hdf4_build_dmr);
159  add_handler(HELP_RESPONSE, HDF4RequestHandler::hdf4_build_help);
160  add_handler(VERS_RESPONSE, HDF4RequestHandler::hdf4_build_version);
161 
162  _usecf = check_beskeys("H4.EnableCF");
163 
164  // The following keys are only effective when usecf is true.
165  // Keys to tune the performance -general
166  _pass_fileid = check_beskeys("H4.EnablePassFileID");
167  _disable_structmeta = check_beskeys("H4.DisableStructMetaAttr");
168  _enable_special_eos = check_beskeys("H4.EnableSpecialEOS");
169  _disable_scaleoffset_comp = check_beskeys("H4.DisableScaleOffsetComp");
170  _disable_ecsmetadata_min = check_beskeys("H4.DisableECSMetaDataMin");
171  _disable_ecsmetadata_all = check_beskeys("H4.DisableECSMetaDataAll");
172 
173 
174  // Keys to tune the performance - cache
175  _enable_eosgeo_cachefile = check_beskeys("H4.EnableEOSGeoCacheFile");
176  _enable_data_cachefile = check_beskeys("H4.EnableDataCacheFile");
177  _enable_metadata_cachefile = check_beskeys("H4.EnableMetaDataCacheFile");
178 
179  // Keys to handle vdata and vgroups
180  _enable_hybrid_vdata = check_beskeys("H4.EnableHybridVdata");
181  _enable_ceres_vdata = check_beskeys("H4.EnableCERESVdata");
182  _enable_vdata_attr = check_beskeys("H4.EnableVdata_to_Attr");
183  _enable_vdata_desc_attr = check_beskeys("H4.EnableVdataDescAttr");
184  _disable_vdata_nameclashing_check = check_beskeys("H4.DisableVdataNameclashingCheck");
185  _enable_vgroup_attr = check_beskeys("H4.EnableVgroupAttr");
186 
187  // Misc. keys
188  _enable_check_modis_geo_file = check_beskeys("H4.EnableCheckMODISGeoFile");
189  _enable_swath_grid_attr = check_beskeys("H4.EnableSwathGridAttr");
190  _enable_ceres_merra_short_name = check_beskeys("H4.EnableCERESMERRAShortName");
191  _enable_check_scale_offset_type = check_beskeys("H4.EnableCheckScaleOffsetType");
192 
193  _disable_swath_dim_map = check_beskeys("H4.DisableSwathDimMap");
194 
195  // Cache path etc.
196  _cache_latlon_path_exist =get_beskeys("HDF4.Cache.latlon.path",_cache_latlon_path);
197  _cache_latlon_prefix_exist =get_beskeys("HDF4.Cache.latlon.prefix",_cache_latlon_prefix);
198  string temp_cache_latlon_size;
199  _cache_latlon_size_exist =get_beskeys("HDF4.Cache.latlon.size",temp_cache_latlon_size);
200  if(_cache_latlon_size_exist == true) {
201  istringstream iss(temp_cache_latlon_size);
202  iss >> _cache_latlon_size;
203  }
204 
205  _cache_metadata_path_exist =get_beskeys("H4.Cache.metadata.path",_cache_metadata_path);
206 
207 }
208 
209 HDF4RequestHandler::~HDF4RequestHandler() {
210 }
211 
212 bool HDF4RequestHandler::hdf4_build_das(BESDataHandlerInterface & dhi) {
213 
214 
215  BESStopWatch sw;
216  if (BESDebug::IsSet(TIMING_LOG_KEY))
217  sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
218 
219  if(true == _usecf) {
220 
221  // Build DAP response only based on the HDF4 SD interfaces. Doing this
222  // way will save the use of other library open calls. Other library open
223  // calls may be expensive
224  // for an HDF4 file that only has variables created by SD interfaces.
225  // This optimization may be very useful for the aggreagation case that
226  // has many variables.
227  // Currently we only handle AIRS version 6 products. AIRS products
228  // are identified by their file names.
229  // We only obtain the filename. The path is stripped off.
230 
231  string base_file_name = basename(dhi.container->access());
232 
233  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
234  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
235  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
236  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
237  && (base_file_name.find(".v6.")!=string::npos)) {
238  return hdf4_build_das_cf_sds(dhi);
239 
240  }
241  }
242 
243  BESResponseObject *response = dhi.response_handler->get_response_object();
244  BESDASResponse *bdas = dynamic_cast<BESDASResponse *> (response);
245  if (!bdas)
246  throw BESInternalError("cast error", __FILE__, __LINE__);
247 
248  try {
250  DAS *das = bdas->get_das();
251 
252  string base_file_name = basename(dhi.container->access());
253 
254  string accessed = dhi.container->access();
255 
256  if (true == _usecf) {
257 
258  int32 sdfd = -1;
259  int32 fileid = -1;
260 
261  HDFSP::File *h4file = NULL;
262 
263  // Obtain HDF4 file IDs
264  //SDstart
265  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
266  if( -1 == sdfd){
267  string invalid_file_msg="HDF4 SDstart error for the file ";
268  invalid_file_msg +=accessed;
269  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
270  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
271  }
272 
273  // H open
274  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
275  if (-1 == fileid) {
276  SDend(sdfd);
277  string invalid_file_msg="HDF4 Hopen error for the file ";
278  invalid_file_msg +=accessed;
279  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
280  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
281  }
282 
283 #ifdef USE_HDFEOS2_LIB
284 
285  int32 gridfd = -1;
286  int32 swathfd = -1;
287  HDFEOS2::File *eosfile = NULL;
288  // Obtain HDF-EOS2 file IDs with the file open APIs.
289 
290  // Grid open
291  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
292  if (-1 == gridfd) {
293  SDend(sdfd);
294  Hclose(fileid);
295  string invalid_file_msg="HDF-EOS GDopen error for the file ";
296  invalid_file_msg +=accessed;
297  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
298  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
299  }
300 
301  // Swath open
302  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
303  if (-1 == swathfd) {
304  SDend(sdfd);
305  Hclose(fileid);
306  GDclose(gridfd);
307  string invalid_file_msg="HDF-EOS SWopen error for the file ";
308  invalid_file_msg +=accessed;
309  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
310  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
311  }
312 
313  try {
314  bool ecs_metadata = !(_disable_ecsmetadata_all);
315 #if 0
316 if(ecs_metadata == true)
317 cerr<<"output ecs metadata "<<endl;
318 else
319 cerr<<"Don't output ecs metadata "<<endl;
320 #endif
321  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
322  }
323  catch(...) {
324  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
325  throw;
326 #if 0
327  //throw InternalErr(__FILE__,__LINE__,"read_das_use_eos2lib error");
328 #endif
329  }
330  if(eosfile != NULL)
331  delete eosfile;
332  GDclose(gridfd);
333  SWclose(swathfd);
334 
335 #else
336  try {
337  read_das_hdfsp(*das,accessed,sdfd,fileid,&h4file);
338  }
339  catch(...) {
340  close_hdf4_fileid(sdfd,fileid,h4file);
341  throw;
342  //throw InternalErr(__FILE__,__LINE__,"read_das_hdfsp error");
343  }
344 #endif
345  close_hdf4_fileid(sdfd,fileid,h4file);
346  }
347  else
348  read_das(*das,accessed);
349 
350  Ancillary::read_ancillary_das(*das, accessed);
351  bdas->clear_container();
352  }
353 
354  catch (BESError & e) {
355  throw;
356  }
357  catch (InternalErr & e) {
358  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
359  __FILE__, __LINE__);
360  }
361  catch (Error & e) {
362  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
363  __FILE__, __LINE__);
364  }
365  catch (...) {
366  string s = "unknown exception caught building HDF4 DAS";
367  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
368  }
369 
370  return true;
371 }
372 
373 bool HDF4RequestHandler::hdf4_build_dds(BESDataHandlerInterface & dhi) {
374 
375 
376 
377  BESStopWatch sw;
378  if (BESDebug::IsSet(TIMING_LOG_KEY))
379  sw.start("HDF4RequestHandler::hdf4_build_das", dhi.data[REQUEST_ID]);
380 
381 
382  if(true == _usecf) {
383  // Build DAP response only based on the HDF4 SD interfaces. Doing this
384  // way will save the use of other library open calls. Other library open
385  // calls may be expensive
386  // for an HDF4 file that only has variables created by SD interfaces.
387  // This optimization may be very useful for the aggreagation case that
388  // has many variables.
389  // Currently we only handle AIRS version 6 products. AIRS products
390  // are identified by their file names.
391  // We only obtain the filename. The path is stripped off.
392 
393  string base_file_name = basename(dhi.container->access());
394 
395  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
396  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
397  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
398  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
399  && (base_file_name.find(".v6.")!=string::npos)) {
400  return hdf4_build_dds_cf_sds(dhi);
401 
402  }
403  }
404 
405 // This is for the performance check. Leave it now for future use. KY 2014-10-23
406 #ifdef KENT2
407 struct timeval start_time,end_time;
408 gettimeofday(&start_time,NULL);
409 #endif
410 
411  BESResponseObject *response = dhi.response_handler->get_response_object();
412  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *> (response);
413  if (!bdds)
414  throw BESInternalError("cast error", __FILE__, __LINE__);
415 
416  try {
418  DDS *dds = bdds->get_dds();
419 #if 0
420  //ConstraintEvaluator & ce = bdds->get_ce();
421 #endif
422 
423  string accessed = dhi.container->access();
424  dds->filename(accessed);
425 
426  DAS *das = new DAS;
427  BESDASResponse bdas(das);
429 
430  if (true == _usecf) {
431 
432  int32 sdfd = -1;
433  int32 fileid = -1;
434  HDFSP::File *h4file = NULL;
435 
436  // Obtain HDF4 file IDs
437  //SDstart
438  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
439  if( -1 == sdfd){
440  string invalid_file_msg="HDF4 SDstart error for the file ";
441  invalid_file_msg +=accessed;
442  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
443  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
444  }
445 
446  // H open
447  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
448  if (-1 == fileid) {
449  SDend(sdfd);
450  string invalid_file_msg="HDF4 Hopen error for the file ";
451  invalid_file_msg +=accessed;
452  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
453  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
454  }
455 
456 #ifdef USE_HDFEOS2_LIB
457 
458  int32 gridfd = -1;
459  int32 swathfd = -1;
460 
461  HDFEOS2::File *eosfile = NULL;
462 
463  // Obtain HDF-EOS2 file IDs with the file open APIs.
464  // Grid open
465  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
466  if (-1 == gridfd) {
467  SDend(sdfd);
468  Hclose(fileid);
469  string invalid_file_msg="HDF-EOS GDopen error for the file ";
470  invalid_file_msg +=accessed;
471  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
472  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
473  }
474 
475  // Swath open
476  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
477  if (-1 == swathfd) {
478  SDend(sdfd);
479  Hclose(fileid);
480  GDclose(gridfd);
481  string invalid_file_msg="HDF-EOS SWopen error for the file ";
482  invalid_file_msg +=accessed;
483  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
484  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
485  }
486 
487  try {
488  bool ecs_metadata = !(_disable_ecsmetadata_all);
489  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
490  Ancillary::read_ancillary_das(*das, accessed);
491 
492  // Pass file pointer(h4file, eosfile) from DAS to DDS.
493  read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
494  }
495  catch(...) {
496  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
497  throw;
498  }
499 
500  if(eosfile != NULL)
501  delete eosfile;
502 
503  GDclose(gridfd);
504  SWclose(swathfd);
505 
506 #else
507  try {
508  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
509  Ancillary::read_ancillary_das(*das, accessed);
510 
511  // Pass file pointer(h4file) from DAS to DDS.
512  read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
513  }
514  catch(...) {
515  close_hdf4_fileid(sdfd,fileid,h4file);
516  throw;
517  }
518 
519 #endif
520  close_hdf4_fileid(sdfd,fileid,h4file);
521  }
522  else {
523  read_das(*das, accessed);
524  Ancillary::read_ancillary_das(*das, accessed);
525  read_dds(*dds, accessed);
526  }
527 
528 // Leave it for future performance tests. KY 2014-10-23
529 #ifdef KENT2
530 gettimeofday(&end_time,NULL);
531 int total_time_spent = (end_time.tv_sec - start_time.tv_sec)*1000000 +end_time.tv_usec-start_time.tv_usec;
532 cerr<<"total time spent for DDS buld is "<<total_time_spent<< "micro seconds "<<endl;
533 #endif
534 
535  Ancillary::read_ancillary_dds(*dds, accessed);
536 
537  dds->transfer_attributes(das);
538 
539  bdds->set_constraint(dhi);
540 
541  bdds->clear_container();
542  }
543  catch (BESError & e) {
544  throw;
545  }
546  catch (InternalErr & e) {
547  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
548  __FILE__, __LINE__);
549  }
550  catch (Error & e) {
551  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
552  __FILE__, __LINE__);
553  }
554  catch (...) {
555  string s = "unknown exception caught building HDF4 DDS";
556  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
557  }
558 
559  return true;
560 }
561 
562 bool HDF4RequestHandler::hdf4_build_data(BESDataHandlerInterface & dhi) {
563 
564 
565  BESStopWatch sw;
566  if (BESDebug::IsSet(TIMING_LOG_KEY))
567  sw.start("HDF4RequestHandler::hdf4_build_data", dhi.data[REQUEST_ID]);
568 
569 
570  int32 sdfd = -1;
571  int32 fileid = -1;
572 
573  // Since passing file IDs requires to use the derived class and it
574  // causes the management of code structure messy, we first handle this with
575  // another method.
576  if(true == _usecf) {
577  // Build DAP response only based on the HDF4 SD interfaces. Doing this
578  // way will save the use of other library open calls. Other library open
579  // calls may be expensive
580  // for an HDF4 file that only has variables created by SD interfaces.
581  // This optimization may be very useful for the aggreagation case that
582  // has many variables.
583  // Currently we only handle AIRS version 6 products. AIRS products
584  // are identified by their file names.
585  // We only obtain the filename. The path is stripped off.
586 
587  string base_file_name = basename(dhi.container->access());
588  //AIRS.2015.01.24.L3.RetStd_IR008.v6.0.11.0.G15041143400.hdf
589  // Identify this file from product name: AIRS, product level: .L3. or .L2. and version .v6.
590  if((base_file_name.size() >12) && (base_file_name.compare(0,4,"AIRS") == 0)
591  && ((base_file_name.find(".L3.")!=string::npos) || (base_file_name.find(".L2.")!=string::npos))
592  && (base_file_name.find(".v6.")!=string::npos)) {
593 
594  BESDEBUG("h4", "Coming to read the data of AIRS level 3 or level 2 products." << endl);
595 
596  if(true == _pass_fileid)
597  return hdf4_build_data_cf_sds_with_IDs(dhi);
598  else
599  return hdf4_build_data_cf_sds(dhi);
600 
601  }
602 
603  if(true == _pass_fileid)
604  return hdf4_build_data_with_IDs(dhi);
605 
606  }
607 
608  BESResponseObject *response = dhi.response_handler->get_response_object();
609  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
610 
611  if (!bdds)
612  throw BESInternalError("cast error", __FILE__, __LINE__);
613 
614  try {
616 
617  DDS *dds = bdds->get_dds();
618 
619  // Not sure why keep the following line, it is not used.
620  //ConstraintEvaluator & ce = bdds->get_ce();
621 
622  string accessed = dhi.container->access();
623  dds->filename(accessed);
624 
625  DAS *das = new DAS;
626  BESDASResponse bdas(das);
628 
629  if (true == _usecf) {
630 
631  HDFSP::File *h4file = NULL;
632 
633  // Obtain HDF4 file IDs
634  //SDstart
635  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
636  if( -1 == sdfd) {
637  string invalid_file_msg="HDF4 SDstart error for the file ";
638  invalid_file_msg +=accessed;
639  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
640  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
641  }
642 
643  // H open
644  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
645  if (-1 == fileid) {
646  SDend(sdfd);
647  string invalid_file_msg="HDF4 Hopen error for the file ";
648  invalid_file_msg +=accessed;
649  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
650  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
651  }
652 
653 
654 #ifdef USE_HDFEOS2_LIB
655 
656  int32 gridfd = -1;
657  int32 swathfd = -1;
658  HDFEOS2::File *eosfile = NULL;
659  // Obtain HDF-EOS2 file IDs with the file open APIs.
660 
661  // Grid open
662  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
663  if (-1 == gridfd) {
664  SDend(sdfd);
665  Hclose(fileid);
666  string invalid_file_msg="HDF-EOS GDopen error for the file ";
667  invalid_file_msg +=accessed;
668  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
669  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
670  }
671 
672  // Swath open
673  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
674  if (-1 == swathfd) {
675  SDend(sdfd);
676  Hclose(fileid);
677  GDclose(gridfd);
678  string invalid_file_msg="HDF-EOS SWopen error for the file ";
679  invalid_file_msg +=accessed;
680  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
681  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
682 
683  }
684 
685  try {
686 
687  // Here we will check if ECS_Metadata key if set. For DataDDS,
688  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
689  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
690  bool ecs_metadata = true;
691 #if 0
692  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
693  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
694  ecs_metadata = false;
695 #endif
696  if((true == _disable_ecsmetadata_min)
697  || (true == _disable_ecsmetadata_all))
698  ecs_metadata = false;
699 
700  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
701  Ancillary::read_ancillary_das(*das, accessed);
702 
703  // Pass file pointer(h4file, eosfile) from DAS to DDS.
704  read_dds_use_eos2lib(*dds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
705  }
706  catch(...) {
707  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
708  throw;
709  }
710 
711  if(eosfile != NULL)
712  delete eosfile;
713  GDclose(gridfd);
714  SWclose(swathfd);
715 
716 #else
717  try {
718  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
719  Ancillary::read_ancillary_das(*das, accessed);
720 
721  // Pass file pointer(h4file) from DAS to DDS.
722  read_dds_hdfsp(*dds, accessed,sdfd,fileid,h4file);
723  }
724  catch(...) {
725  close_hdf4_fileid(sdfd,fileid,h4file);
726  throw;
727  }
728 #endif
729  close_hdf4_fileid(sdfd,fileid,h4file);
730  }
731  else {
732  read_das(*das, accessed);
733  Ancillary::read_ancillary_das(*das, accessed);
734  read_dds(*dds, accessed);
735  }
736 
737  Ancillary::read_ancillary_dds(*dds, accessed);
738  dds->transfer_attributes(das);
739  bdds->set_constraint(dhi);
740  bdds->clear_container();
741 
742  }
743 
744  catch (BESError & e) {
745  throw;
746  }
747  catch (InternalErr & e) {
748  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
749  __FILE__, __LINE__);
750  }
751  catch (Error & e) {
752  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
753  __FILE__, __LINE__);
754  }
755  catch (...) {
756  string s = "unknown exception caught building DAP2 Data Response from an HDF4 data resource";
757  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
758  }
759 
760  return true;
761 }
762 
763 bool HDF4RequestHandler::hdf4_build_data_with_IDs(BESDataHandlerInterface & dhi) {
764 
765  BESStopWatch sw;
766  if (BESDebug::IsSet(TIMING_LOG_KEY))
767  sw.start("HDF4RequestHandler::hdf4_build_data_with_IDs", dhi.data[REQUEST_ID]);
768 
769  int32 sdfd = -1;
770  int32 fileid = -1;
771  HDFSP::File *h4file = NULL;
772 #ifdef USE_HDFEOS2_LIB
773  int32 gridfd = -1;
774  int32 swathfd = -1;
775  HDFEOS2::File *eosfile = NULL;
776 #endif
777 
778  BESResponseObject *response = dhi.response_handler->get_response_object();
779  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
780 
781  if (!bdds)
782  throw BESInternalError("cast error", __FILE__, __LINE__);
783 
784  try {
786 
787  // Create a new HDF4DDS object.
788  HDF4DDS *hdds = new HDF4DDS(bdds->get_dds());
789 
790  // delete the old object.
791  delete bdds->get_dds();
792 
793  bdds->set_dds(hdds);
794 
795 #if 0
796  //ConstraintEvaluator & ce = bdds->get_ce();
797 #endif
798 
799  string accessed = dhi.container->access();
800  hdds->filename(accessed);
801 
802  DAS *das = new DAS;
803  BESDASResponse bdas(das);
805 
806 
807  // Obtain HDF4 file IDs
808  //SDstart
809  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
810  if( -1 == sdfd) {
811  string invalid_file_msg="HDF4 SDstart error for the file ";
812  invalid_file_msg +=accessed;
813  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
814  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
815  }
816  // H open
817  fileid = Hopen(const_cast<char *>(accessed.c_str()), DFACC_READ,0);
818  if (-1 == fileid) {
819  SDend(sdfd);
820  string invalid_file_msg="HDF4 Hopen error for the file ";
821  invalid_file_msg +=accessed;
822  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
823  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
824  }
825 
826 #ifdef USE_HDFEOS2_LIB
827 
828 
829  // Obtain HDF-EOS2 file IDs with the file open APIs.
830 
831  // Grid open
832  gridfd = GDopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
833  if (-1 == gridfd) {
834  SDend(sdfd);
835  Hclose(fileid);
836  string invalid_file_msg="HDF-EOS GDopen error for the file ";
837  invalid_file_msg +=accessed;
838  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
839  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
840  }
841 
842  // Swath open
843  swathfd = SWopen(const_cast < char *>(accessed.c_str()), DFACC_READ);
844  if (-1 == swathfd) {
845  SDend(sdfd);
846  Hclose(fileid);
847  GDclose(gridfd);
848  string invalid_file_msg="HDF-EOS SWopen error for the file ";
849  invalid_file_msg +=accessed;
850  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
851  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
852  }
853 
854  hdds->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
855 
856  // Here we will check if ECS_Metadata key if set. For DataDDS,
857  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
858  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
859  bool ecs_metadata = true;
860 #if 0
861  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
862  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
863 #endif
864  if((true == _disable_ecsmetadata_min)
865  || (true == _disable_ecsmetadata_all))
866  ecs_metadata = false;
867 
868  read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
869 
870 #if 0
871  //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
872 #endif
873 
874  Ancillary::read_ancillary_das(*das, accessed);
875 
876  // Pass file pointer(h4file, eosfile) from DAS to DDS.
877  read_dds_use_eos2lib(*hdds, accessed,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
878 
879  if(eosfile != NULL)
880  delete eosfile;
881 
882 #else
883  hdds->setHDF4Dataset(sdfd,fileid);
884  read_das_hdfsp(*das, accessed,sdfd,fileid,&h4file);
885  Ancillary::read_ancillary_das(*das, accessed);
886 
887  // Pass file pointer(h4file) from DAS to DDS.
888  read_dds_hdfsp(*hdds, accessed,sdfd,fileid,h4file);
889 #endif
890  if(h4file != NULL)
891  delete h4file;
892 
893  Ancillary::read_ancillary_dds(*hdds, accessed);
894 
895  hdds->transfer_attributes(das);
896 
897  bdds->set_constraint(dhi);
898 
899  bdds->clear_container();
900 
901 
902 // File IDs are closed by the derived class.
903 #if 0
904  if(true == usecf) {
905 #ifdef USE_HDFEOS2_LIB
906  GDclose(gridfd);
907  SWclose(swathfd);
908 
909 #endif
910  SDend(sdfd);
911  Hclose(fileid);
912  }
913 #endif
914  }
915 
916  catch (InternalErr & e) {
917  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
918  __FILE__, __LINE__);
919  }
920  catch (Error & e) {
921  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
922  __FILE__, __LINE__);
923  }
924  catch (...) {
925 #ifdef USE_HDFEOS2_LIB
926  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
927 #else
928  close_hdf4_fileid(sdfd,fileid,h4file);
929 #endif
930  string s = "unknown exception caught building DAP2 data response from an HDF4 data resource";
931  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
932  }
933 
934  return true;
935 }
936 
937 // Special function to build DDS for HDF4 SDS-only DDS. One can turn
938 // on cache for this function. Currently only AIRS version 6 is supported.
939 bool HDF4RequestHandler::hdf4_build_dds_cf_sds(BESDataHandlerInterface &dhi){
940 
941  int32 sdfd = -1;
942  HDFSP::File *h4file = NULL;
943 
944  BESResponseObject *response = dhi.response_handler->get_response_object();
945  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *> (response);
946 
947  if (!bdds)
948  throw BESInternalError("cast error", __FILE__, __LINE__);
949 
950  try {
952 
953  DDS *dds = bdds->get_dds();
954 
955 #if 0
956  //BaseTypeFactory* factory = new BaseTypeFactory ;
957  //dds->set_factory( factory ) ;
958 #endif
959 
960  string accessed = dhi.container->access();
961  dds->filename(accessed);
962 
963  DAS *das = new DAS;
964  BESDASResponse bdas(das);
965 
966  // Check and set up dds and das cache files.
967  string base_file_name = basename(dhi.container->access());
968  bool das_set_cache = false;
969  bool dds_set_cache = false;
970  bool dds_das_get_cache = false;
971  string das_filename;
972  string dds_filename;
973 
974  // Check if the EnableMetaData key is set.
975 #if 0
976  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
977  bool turn_on_enable_mcache_key= false;
978  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
979  if(true == turn_on_enable_mcache_key) {// the EnableMetaData key is set
980 #endif
981  if(true == _enable_metadata_cachefile) {// the EnableMetaData key is set
982 
983 
984 #if 0
985  string md_cache_dir;
986  bool found = false;
987  string key = "H4.Cache.metadata.path";
988  TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
989 #endif
990  if(true == _cache_metadata_path_exist) {// the metadata.path key is set
991 
992  // Create DAS and DDS cache file names
993  das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
994  dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
995 
996  // When the returned value das_set_cache is false, read DAS from the cache file.
997  // Otherwise, do nothing.
998  das_set_cache = rw_das_cache_file(das_filename,das,false);
999 
1000  // When the returned value dds_set_cache is false, read DDS from the cache file.
1001  // Otherwise, do nothing.
1002  dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1003 
1004  // Set the flag to obtain DDS and DAS from the cache files.
1005  // Here, we require that both DDS and DAS should be cached.
1006  // We don't support caching one and not caching another.
1007  if((false == das_set_cache)&&(false == dds_set_cache))
1008  dds_das_get_cache = true;
1009  }
1010  }
1011 
1012 
1013  // We need to go back to retrieve DDS, DAS from the HDF file.
1014  if(false == dds_das_get_cache) {
1015 
1016  // Obtain SD ID, this is the only ID we need to use.
1017  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1018  if( -1 == sdfd) {
1019 
1020  string invalid_file_msg="HDF4 SDstart error for the file ";
1021  invalid_file_msg +=accessed;
1022  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1023  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1024  }
1025 
1026 
1027  // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1028  // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1029  bool ecs_metadata = !(_disable_ecsmetadata_all);
1030 #if 0
1031  bool ecs_metadata = true;
1032  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1033  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1034  ecs_metadata = false;
1035 #endif
1036 
1037  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1038 
1039  Ancillary::read_ancillary_das(*das, accessed);
1040 
1041  // Pass file pointer(h4file) from DAS to DDS, also pass the flag
1042  // dds_set_cache to indicate if the handler needs to cache DDS.
1043  read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1044 
1045  // We also need to cache DAS if das_set_cache is set.
1046  if(true == das_set_cache) {
1047  if(das_filename =="")
1048  throw InternalErr(__FILE__,__LINE__,"DAS cache file name should be set ");
1049  rw_das_cache_file(das_filename,das,true);
1050  }
1051 
1052  }
1053 
1054  Ancillary::read_ancillary_dds(*dds, accessed);
1055  dds->transfer_attributes(das);
1056  bdds->set_constraint(dhi);
1057 
1058  bdds->clear_container();
1059 
1060  if(h4file != NULL)
1061  delete h4file;
1062 
1063  if(sdfd != -1)
1064  SDend(sdfd);
1065 #if 0
1066  // Cannot(should not) delete factory here. factory will be released by DAP? or errors occur.
1067  //if(factory != NULL)
1068  // delete factory;
1069 #endif
1070 
1071  }
1072 
1073  catch (BESError & e) {
1074  throw;
1075  }
1076  catch (InternalErr & e) {
1077  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1078  __FILE__, __LINE__);
1079  }
1080  catch (Error & e) {
1081  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1082  __FILE__, __LINE__);
1083  }
1084  catch (...) {
1085  if(sdfd != -1)
1086  SDend(sdfd);
1087  if(h4file != NULL)
1088  delete h4file;
1089  string s = "unknown exception caught building HDF4 DataDDS";
1090  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1091  }
1092 
1093  return true;
1094 }
1095 
1096 // Special function to build DAS for HDF4 SDS-only DDS. One can turn
1097 // on cache for this function. Currently only AIRS version 6 is supported.
1098 bool HDF4RequestHandler::hdf4_build_das_cf_sds(BESDataHandlerInterface &dhi){
1099 
1100  int32 sdfd = -1;
1101  HDFSP::File *h4file = NULL;
1102 
1103  BESResponseObject *response = dhi.response_handler->get_response_object();
1104  BESDASResponse *bdas = dynamic_cast<BESDASResponse *> (response);
1105 
1106  if (!bdas)
1107  throw BESInternalError("cast error", __FILE__, __LINE__);
1108 
1109  try {
1111 
1112  DAS *das = bdas->get_das();
1113  string base_file_name = basename(dhi.container->access());
1114 
1115  string accessed = dhi.container->access();
1116 
1117  // Check if the enable metadata cache key is set and set the key appropriate.
1118  bool das_set_cache = false;
1119  bool das_get_cache = false;
1120  string das_filename;
1121 #if 0
1122  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1123  bool turn_on_enable_mcache_key= false;
1124  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1125  if(true == turn_on_enable_mcache_key) { // find the metadata cache key.
1126 #endif
1127  if(true == _enable_metadata_cachefile) { // find the metadata cache key.
1128 #if 0
1129  string md_cache_dir;
1130  string key = "H4.Cache.metadata.path";
1131  bool found = false;
1132  TheBESKeys::TheKeys()->get_value(key,md_cache_dir,found);
1133  if(true == found) { // Also find the metadata cache.
1134 #endif
1135  if(true == _cache_metadata_path_exist) {
1136 
1137  // Create the DAS cache file name.
1138  das_filename = _cache_metadata_path + "/" + base_file_name +"_das";
1139 
1140  // Read the DAS from the cached file, if das_set_cache is false.
1141  // When the das_set_cache is true, need to create a das cache file.
1142  das_set_cache = rw_das_cache_file(das_filename,das,false);
1143 
1144  // Not set cache, must get the das from cache, so das_get_cache should be true.
1145  if(false == das_set_cache)
1146  das_get_cache = true;
1147  }
1148  }
1149 
1150  // Need to retrieve DAS from the HDF4 file.
1151  if(false == das_get_cache) {
1152  // Obtain SD ID.
1153  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1154  if( -1 == sdfd){
1155  string invalid_file_msg="HDF4 SDstart error for the file ";
1156  invalid_file_msg +=accessed;
1157  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1158  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1159  }
1160 
1161 
1162 #if 0
1163  bool ecs_metadata = true;
1164  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1165  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1166  ecs_metadata = false;
1167 #endif
1168  // Here we will check if ECS_Metadata key if set. For DDS and DAS,
1169  // only when the DisableECSMetaDataAll key is set, ecs_metadata is off.
1170  bool ecs_metadata = !(_disable_ecsmetadata_all);
1171 
1172  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1173  Ancillary::read_ancillary_das(*das, accessed);
1174 
1175  // Generate DAS cache file if the cache flag is set.
1176  if(true == das_set_cache)
1177  rw_das_cache_file(das_filename,das,true);
1178 
1179  }
1180 
1181  bdas->clear_container();
1182 
1183  if(h4file != NULL)
1184  delete h4file;
1185 
1186  if(sdfd != -1)
1187  SDend(sdfd);
1188 
1189  }
1190 
1191  catch (BESError & e) {
1192  throw;
1193  }
1194  catch (InternalErr & e) {
1195  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1196  __FILE__, __LINE__);
1197  }
1198  catch (Error & e) {
1199  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1200  __FILE__, __LINE__);
1201  }
1202  catch (...) {
1203  if(sdfd != -1)
1204  SDend(sdfd);
1205  if(h4file != NULL)
1206  delete h4file;
1207  string s = "unknown exception caught building HDF4 DataDDS";
1208  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1209  }
1210  return true;
1211 }
1212 // Special function to build data for HDF4 SDS-only DDS. One can turn
1213 // on cache for this function. Currently only AIRS version 6 is supported.
1214 bool HDF4RequestHandler::hdf4_build_data_cf_sds(BESDataHandlerInterface &dhi){
1215 
1216  int32 sdfd = -1;
1217  HDFSP::File *h4file = NULL;
1218 
1219  BESResponseObject *response = dhi.response_handler->get_response_object();
1220  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1221 
1222  if (!bdds)
1223  throw BESInternalError("cast error", __FILE__, __LINE__);
1224 
1225  try {
1227 
1228  DDS *dds = bdds->get_dds();
1229 
1230  string accessed = dhi.container->access();
1231  dds->filename(accessed);
1232 
1233  DAS *das = new DAS;
1234  BESDASResponse bdas(das);
1236 
1237 #if 0
1238  //BaseTypeFactory* factory = new BaseTypeFactory ;
1239  //dds->set_factory( factory ) ;
1240 #endif
1241 
1242  string base_file_name = basename(dhi.container->access());
1243 
1244  bool das_set_cache = false;
1245  bool dds_set_cache = false;
1246  bool dds_das_get_cache = false;
1247  string das_filename;
1248  string dds_filename;
1249 
1250 #if 0
1251  string check_enable_mcache_key="H4.EnableMetaDataCacheFile";
1252  bool _disable_ecsmetadata_allturn_on_enable_mcache_key= false;
1253  turn_on_enable_mcache_key = HDFCFUtil::check_beskeys(check_enable_mcache_key);
1254  if(true == turn_on_enable_mcache_key) {
1255 #endif
1256  if(true == _enable_metadata_cachefile) {
1257 
1258  if(true == _cache_metadata_path_exist) {
1259  BESDEBUG("h4", "H4.Cache.metadata.path key is set and metadata cache key is set." << endl);
1260 
1261  // Notice, since the DAS output may be different between DAS/DDS service and DataDDS service.
1262  // See comments about ecs_metadata below.
1263  // So we create a different DAS file. This can be optimized in the future if necessary.
1264  das_filename = _cache_metadata_path + "/" + base_file_name +"_das_dd";
1265  dds_filename = _cache_metadata_path + "/" + base_file_name +"_dds";
1266 
1267  // If das_set_cache is true, data is read from the DAS cache.
1268  das_set_cache = rw_das_cache_file(das_filename,das,false);
1269 
1270  // If dds_set_cache is true, data is read from the DDS cache.
1271  dds_set_cache = r_dds_cache_file(dds_filename,dds,accessed);
1272  // Need to set a flag to generate DAS and DDS cache files
1273  if((false == das_set_cache)&&(false == dds_set_cache))
1274  dds_das_get_cache = true;
1275  }
1276  }
1277  if(false == dds_das_get_cache) {
1278 
1279  // Obtain HDF4 SD ID
1280  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1281  if( -1 == sdfd){
1282  string invalid_file_msg="HDF4 SDstart error for the file ";
1283  invalid_file_msg +=accessed;
1284  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1285  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1286  }
1287 
1288 
1289  // Here we will check if ECS_Metadata key if set. For DataDDS,
1290  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1291  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1292  bool ecs_metadata = true;
1293 #if 0
1294  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1295  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1296 #endif
1297  if((true == _disable_ecsmetadata_min)
1298  || (true == _disable_ecsmetadata_all))
1299  ecs_metadata = false;
1300 
1301  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1302 
1303  Ancillary::read_ancillary_das(*das, accessed);
1304 
1305  // Need to write DAS to a cache file.
1306  if(true == das_set_cache) {
1307  rw_das_cache_file(das_filename,das,true);
1308  }
1309 
1310  // Pass file pointer h4file from DAS to DDS. Also need to pass the
1311  // flag that indicates if a DDS file needs to be created.
1312  read_dds_sds(*dds, accessed,sdfd,h4file,dds_set_cache);
1313 
1314  }
1315 
1316  Ancillary::read_ancillary_dds(*dds, accessed);
1317 
1318  dds->transfer_attributes(das);
1319 
1320  bdds->set_constraint(dhi);
1321 
1322  bdds->clear_container();
1323 
1324  if(h4file != NULL)
1325  delete h4file;
1326 
1327  if(sdfd != -1)
1328  SDend(sdfd);
1329  }
1330 
1331  catch (BESError & e) {
1332  throw;
1333  }
1334  catch (InternalErr & e) {
1335  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1336  __FILE__, __LINE__);
1337  }
1338  catch (Error & e) {
1339  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1340  __FILE__, __LINE__);
1341  }
1342  catch (...) {
1343  if(sdfd != -1)
1344  SDend(sdfd);
1345  if(h4file != NULL)
1346  delete h4file;
1347  string s = "unknown exception caught building HDF4 DataDDS";
1348  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1349  }
1350 
1351 
1352  return true;
1353 }
1354 
1355 // Notice the metadata cache doesn't apply when the EnableFileID key is set.
1356 // This is because the file ID generated by creating DDS/DAS WO cache is needed to access the data.
1357 // So to make this work at any time, we have to create SDS ID even when the EnableFileID key is set.
1358 // This is against the purpose of EnableFileID key. To acheieve the same purpose for AIRS,
1359 // one can set the DataCache key, the performance is similar or even better than just using the EnableFileID key.
1360 bool HDF4RequestHandler::hdf4_build_data_cf_sds_with_IDs(BESDataHandlerInterface &dhi){
1361 
1362  int32 sdfd = -1;
1363  HDFSP::File *h4file = NULL;
1364 
1365  BESResponseObject *response = dhi.response_handler->get_response_object();
1366  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1367 
1368  if (!bdds)
1369  throw BESInternalError("cast error", __FILE__, __LINE__);
1370 
1371  try {
1373 
1374  HDF4DDS *hdds = new HDF4DDS(bdds->get_dds());
1375 
1376  delete bdds->get_dds();
1377 
1378  bdds->set_dds(hdds);
1379 
1380 
1381  string accessed = dhi.container->access();
1382  hdds->filename(accessed);
1383 
1384  DAS *das = new DAS;
1385  BESDASResponse bdas(das);
1387 
1388  //Obtain SD ID.
1389  sdfd = SDstart (const_cast < char *>(accessed.c_str()), DFACC_READ);
1390  if( -1 == sdfd) {
1391  string invalid_file_msg="HDF4 SDstart error for the file ";
1392  invalid_file_msg +=accessed;
1393  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1394  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1395  }
1396 
1397  hdds->setHDF4Dataset(sdfd,-1);
1398 
1399  // Here we will check if ECS_Metadata key if set. For DataDDS,
1400  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1401  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1402  bool ecs_metadata = true;
1403 #if 0
1404  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1405  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1406 #endif
1407  if((true == _disable_ecsmetadata_min)
1408  || (true == _disable_ecsmetadata_all))
1409  ecs_metadata = false;
1410 
1411  read_das_sds(*das, accessed,sdfd,ecs_metadata,&h4file);
1412 #if 0
1413  //read_das_use_eos2lib(*das, accessed,sdfd,fileid,gridfd,swathfd,true);
1414 #endif
1415 
1416  Ancillary::read_ancillary_das(*das, accessed);
1417 
1418  // Pass file pointer(h4file, eosfile) from DAS to DDS.
1419  read_dds_sds(*hdds, accessed,sdfd,h4file,false);
1420 
1421  if(h4file != NULL)
1422  delete h4file;
1423 
1424  Ancillary::read_ancillary_dds(*hdds, accessed);
1425 
1426  hdds->transfer_attributes(das);
1427 
1428  bdds->set_constraint(dhi);
1429 
1430  bdds->clear_container();
1431 
1432  }
1433 
1434  catch (BESError & e) {
1435  throw;
1436  }
1437  catch (InternalErr & e) {
1438  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1439  __FILE__, __LINE__);
1440  }
1441  catch (Error & e) {
1442  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1443  __FILE__, __LINE__);
1444  }
1445  catch (...) {
1446  if(sdfd != -1)
1447  SDend(sdfd);
1448  if(h4file != NULL)
1449  delete h4file;
1450  string s = "unknown exception caught building HDF4 DataDDS";
1451  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1452  }
1453 
1454 
1455  return true;
1456 }
1457 bool HDF4RequestHandler::hdf4_build_dmr(BESDataHandlerInterface &dhi)
1458 {
1459 
1460  BESStopWatch sw;
1461  if (BESDebug::IsSet(TIMING_LOG_KEY))
1462  sw.start("HDF4RequestHandler::hdf4_build_dmr", dhi.data[REQUEST_ID]);
1463 
1464  // Because this code does not yet know how to build a DMR directly, use
1465  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1466  // First step, build the 'full DDS'
1467  string data_path = dhi.container->access();
1468 
1469  BaseTypeFactory factory;
1470  DDS dds(&factory, name_path(data_path), "3.2");
1471  dds.filename(data_path);
1472 
1473  DAS das;
1474 
1475  int32 sdfd = -1;
1476  int32 fileid = -1;
1477 
1478  // Since passing file IDs requires to use the derived class and it
1479  // causes the management of code structure messy, we first handle this with
1480  // another method.
1481  if(true == _usecf) {
1482 
1483  if(true == _pass_fileid)
1484  return hdf4_build_dmr_with_IDs(dhi);
1485 
1486  }
1487 
1488 
1489  try {
1490 
1491  if (true == _usecf) {
1492 
1493  HDFSP::File *h4file = NULL;
1494 
1495  // Obtain HDF4 file IDs
1496  //SDstart
1497  sdfd = SDstart (const_cast < char *>(data_path.c_str()), DFACC_READ);
1498  if( -1 == sdfd){
1499  string invalid_file_msg="HDF4 SDstart error for the file ";
1500  invalid_file_msg +=data_path;
1501  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1502  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1503  }
1504 
1505  // H open
1506  fileid = Hopen(const_cast<char *>(data_path.c_str()), DFACC_READ,0);
1507  if (-1 == fileid) {
1508  SDend(sdfd);
1509  string invalid_file_msg="HDF4 Hopen error for the file ";
1510  invalid_file_msg +=data_path;
1511  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1512  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1513  }
1514 
1515 #ifdef USE_HDFEOS2_LIB
1516 
1517  int32 gridfd = -1;
1518  int32 swathfd = -1;
1519 
1520  HDFEOS2::File *eosfile = NULL;
1521 
1522  // Obtain HDF-EOS2 file IDs with the file open APIs.
1523  // Grid open
1524  gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1525  if (-1 == gridfd) {
1526  SDend(sdfd);
1527  Hclose(fileid);
1528  string invalid_file_msg="HDF-EOS GDopen error for the file ";
1529  invalid_file_msg +=data_path;
1530  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1531  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1532  }
1533 
1534  // Swath open
1535  swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1536  if (-1 == swathfd) {
1537  SDend(sdfd);
1538  Hclose(fileid);
1539  GDclose(gridfd);
1540  string invalid_file_msg="HDF-EOS SWopen error for the file ";
1541  invalid_file_msg +=data_path;
1542  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1543  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1544  }
1545 
1546 
1547  // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1548  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1549  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1550  // This is one difference between DAP2 and DAP4 mapping. Since
1551  // people can use BES key to turn on the ECS metadata, so this is okay.
1552  // KY 2014-10-23
1553  bool ecs_metadata = true;
1554 #if 0
1555  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1556  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1557 #endif
1558  if((true == _disable_ecsmetadata_min)
1559  || (true == _disable_ecsmetadata_all))
1560  ecs_metadata = false;
1561 
1562  try {
1563 
1564  read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1565 #if 0
1566  //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1567 #endif
1568  Ancillary::read_ancillary_das(das, data_path);
1569 
1570  // Pass file pointer(h4file, eosfile) from DAS to DDS
1571  read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1572  }
1573 
1574  catch(...) {
1575  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1576  throw;
1577  }
1578  if(eosfile != NULL)
1579  delete eosfile;
1580  GDclose(gridfd);
1581  SWclose(swathfd);
1582 
1583 #else
1584  try {
1585  read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1586  Ancillary::read_ancillary_das(das, data_path);
1587 
1588  // Pass file pointer(h4file) from DAS to DDS.
1589  read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1590  }
1591  catch(...) {
1592  close_hdf4_fileid(sdfd,fileid,h4file);
1593  throw;
1594  //throw InternalErr(__FILE__,__LINE__,"read_dds_hdfsp error");
1595  }
1596 #endif
1597  close_hdf4_fileid(sdfd,fileid,h4file);
1598 
1599  }
1600  else {
1601  read_das(das, data_path);
1602  Ancillary::read_ancillary_das(das, data_path);
1603  read_dds(dds, data_path);
1604  }
1605 
1606 
1607  Ancillary::read_ancillary_dds(dds, data_path);
1608 
1609  dds.transfer_attributes(&das);
1610 
1611  }
1612 
1613  catch (InternalErr & e) {
1614  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1615  __FILE__, __LINE__);
1616  }
1617  catch (Error & e) {
1618  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1619  __FILE__, __LINE__);
1620  }
1621  catch (...) {
1622  string s = "unknown exception caught building HDF4 DataDDS";
1623  throw BESDapError(s, true, unknown_error, __FILE__, __LINE__);
1624  }
1625 
1626 #if 0
1627  //dds.print(cout);
1628  //dds.print_das(cout);
1629 #endif
1630  //
1631  // Extract the DMR Response object - this holds the DMR used by the
1632  // other parts of the framework.
1633  BESResponseObject *response = dhi.response_handler->get_response_object();
1634  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1635 
1636  DMR *dmr = bes_dmr.get_dmr();
1637 
1638  D4BaseTypeFactory MyD4TypeFactory;
1639  dmr->set_factory(&MyD4TypeFactory);
1640 #if 0
1641  //dmr->set_factory(new D4BaseTypeFactory);
1642 #endif
1643  dmr->build_using_dds(dds);
1644 
1645 #if 0
1646  //dmr->print(cout);
1647 #endif
1648 
1649  // Instead of fiddling with the internal storage of the DHI object,
1650  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1651  // methods to set the constraints. But, why? Ans: from Patrick is that
1652  // in the 'container' mode of BES each container can have a different
1653  // CE.
1654  bes_dmr.set_dap4_constraint(dhi);
1655  bes_dmr.set_dap4_function(dhi);
1656  dmr->set_factory(0);
1657 
1658  return true;
1659 }
1660 
1661 bool HDF4RequestHandler::hdf4_build_dmr_with_IDs(BESDataHandlerInterface & dhi) {
1662 
1663  BESStopWatch sw;
1664  if (BESDebug::IsSet(TIMING_LOG_KEY))
1665  sw.start("HDF4RequestHandler::hdf4_build_dmr_with_IDs", dhi.data[REQUEST_ID]);
1666 
1667  // Because this code does not yet know how to build a DMR directly, use
1668  // the DMR ctor that builds a DMR using a 'full DDS' (a DDS with attributes).
1669  // First step, build the 'full DDS'
1670  string data_path = dhi.container->access();
1671 
1672  BaseTypeFactory factory;
1673  DDS dds(&factory, name_path(data_path), "3.2");
1674  dds.filename(data_path);
1675 
1676  DAS das;
1677 
1678  int32 sdfd = -1;
1679  int32 fileid = -1;
1680 
1681  HDFSP::File *h4file = NULL;
1682 
1683  // Obtain HDF4 file IDs
1684  //SDstart
1685  sdfd = SDstart (const_cast < char *>(data_path.c_str()), DFACC_READ);
1686  if( -1 == sdfd){
1687  string invalid_file_msg="HDF4 SDstart error for the file ";
1688  invalid_file_msg +=data_path;
1689  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1690  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1691  }
1692 
1693  // H open
1694  fileid = Hopen(const_cast<char *>(data_path.c_str()), DFACC_READ,0);
1695  if (-1 == fileid) {
1696  SDend(sdfd);
1697  string invalid_file_msg="HDF4 SDstart error for the file ";
1698  invalid_file_msg +=data_path;
1699  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1700  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1701  }
1702 
1703 
1704 #ifdef USE_HDFEOS2_LIB
1705 
1706  int32 gridfd = -1;
1707  int32 swathfd = -1;
1708 
1709  HDFEOS2::File *eosfile = NULL;
1710  // Obtain HDF-EOS2 file IDs with the file open APIs.
1711  // Grid open
1712  gridfd = GDopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1713  if (-1 == gridfd) {
1714  SDend(sdfd);
1715  Hclose(fileid);
1716  string invalid_file_msg="HDF4 SDstart error for the file ";
1717  invalid_file_msg +=data_path;
1718  invalid_file_msg +=". It is very possible that this file is not an HDF4 file. ";
1719  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1720 
1721  }
1722 
1723  // Swath open
1724  swathfd = SWopen(const_cast < char *>(data_path.c_str()), DFACC_READ);
1725  if (-1 == swathfd) {
1726  SDend(sdfd);
1727  Hclose(fileid);
1728  GDclose(gridfd);
1729  string invalid_file_msg="HDF-EOS SWopen error for the file ";
1730  invalid_file_msg +=data_path;
1731  invalid_file_msg +=". It is very possible that this file is not an HDF-EOS2 file. ";
1732  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1733 
1734  }
1735 
1736  // Here we will check if ECS_Metadata key if set. For DAP4's DMR,
1737  // if either H4.DisableECSMetaDataMin or H4.DisableECSMetaDataAll is set,
1738  // the HDF-EOS2 coremetadata or archivemetadata will not be passed to DAP.
1739  // This is one difference between DAP2 and DAP4 mapping. Since
1740  // people can use BES key to turn on the ECS metadata, so this is okay.
1741  // KY 2014-10-23
1742  bool ecs_metadata = true;
1743 #if 0
1744  if((true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataMin"))
1745  || (true == HDFCFUtil::check_beskeys("H4.DisableECSMetaDataAll")))
1746 #endif
1747  if((true == _disable_ecsmetadata_min)
1748  || (true == _disable_ecsmetadata_all))
1749  ecs_metadata = false;
1750 
1751  try {
1752 
1753  read_das_use_eos2lib(das, data_path,sdfd,fileid,gridfd,swathfd,ecs_metadata,&h4file,&eosfile);
1754 #if 0
1755  //read_das_use_eos2lib(*das, data_path,sdfd,fileid,gridfd,swathfd,true);
1756 #endif
1757  Ancillary::read_ancillary_das(das, data_path);
1758 
1759  // Pass file pointer(h4file, eosfile) from DAS to DDS.
1760  read_dds_use_eos2lib(dds, data_path,sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1761  }
1762 
1763  catch(...) {
1764  close_fileid(sdfd,fileid,gridfd,swathfd,h4file,eosfile);
1765  throw;
1766  }
1767  if(eosfile != NULL)
1768  delete eosfile;
1769 
1770 #else
1771  try {
1772  read_das_hdfsp(das, data_path,sdfd,fileid,&h4file);
1773  Ancillary::read_ancillary_das(das, data_path);
1774 
1775  // Pass file pointer(h4file) from DAS to DDS.
1776  read_dds_hdfsp(dds, data_path,sdfd,fileid,h4file);
1777  }
1778  catch(...) {
1779  close_hdf4_fileid(sdfd,fileid,h4file);
1780  throw;
1781  }
1782 #endif
1783  if(h4file != NULL)
1784  delete h4file;
1785 
1786  Ancillary::read_ancillary_dds(dds, data_path);
1787 
1788  dds.transfer_attributes(&das);
1789 
1790 
1791 // File IDs are closed by the derived class.
1792 #if 0
1793  if(true == usecf) {
1794 #ifdef USE_HDFEOS2_LIB
1795  GDclose(gridfd);
1796  SWclose(swathfd);
1797 
1798 #endif
1799  SDend(sdfd);
1800  Hclose(fileid);
1801  }
1802 
1803  //dds.print(cout);
1804  //dds.print_das(cout);
1805 #endif
1806  // Extract the DMR Response object - this holds the DMR used by the
1807  // other parts of the framework.
1808  BESResponseObject *response = dhi.response_handler->get_response_object();
1809  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1810 
1811  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1812  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1813  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1814  // the BES runs the DMR's destructor.
1815  DMR *dmr = bes_dmr.get_dmr();
1816  D4BaseTypeFactory MyD4TypeFactory;
1817  dmr->set_factory(&MyD4TypeFactory);
1818  dmr->build_using_dds(dds);
1819  HDF4DMR *hdf4_dmr = new HDF4DMR(dmr);
1820 #ifdef USE_HDFEOS2_LIB
1821  hdf4_dmr->setHDF4Dataset(sdfd,fileid,gridfd,swathfd);
1822 #else
1823  hdf4_dmr->setHDF4Dataset(sdfd,fileid);
1824 #endif
1825  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1826  bes_dmr.set_dmr(hdf4_dmr); // BESDMRResponse will delete hdf4_dmr
1827 
1828  // Instead of fiddling with the internal storage of the DHI object,
1829  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1830  // methods to set the constraints. But, why? Ans: from Patrick is that
1831  // in the 'container' mode of BES each container can have a different
1832  // CE.
1833  bes_dmr.set_dap4_constraint(dhi);
1834  bes_dmr.set_dap4_function(dhi);
1835  hdf4_dmr->set_factory(0);
1836 
1837  return true;
1838 
1839 }
1840 
1841 bool HDF4RequestHandler::hdf4_build_help(BESDataHandlerInterface & dhi) {
1842  BESResponseObject *response = dhi.response_handler->get_response_object();
1843  BESInfo *info = dynamic_cast<BESInfo *> (response);
1844  if (!info)
1845  throw BESInternalError("cast error", __FILE__, __LINE__);
1846 
1847  map < string, string > attrs;
1848  attrs["name"] = MODULE_NAME ;
1849  attrs["version"] = MODULE_VERSION ;
1850  list < string > services;
1851  BESServiceRegistry::TheRegistry()->services_handled(HDF4_NAME, services);
1852  if (services.size() > 0) {
1853  string handles = BESUtil::implode(services, ',');
1854  attrs["handles"] = handles;
1855  }
1856  info->begin_tag("module", &attrs);
1857  info->end_tag("module");
1858 
1859  return true;
1860 }
1861 
1862 bool HDF4RequestHandler::hdf4_build_version(BESDataHandlerInterface & dhi) {
1863  BESResponseObject *response = dhi.response_handler->get_response_object();
1864  BESVersionInfo *info = dynamic_cast<BESVersionInfo *> (response);
1865  if (!info)
1866  throw BESInternalError("cast error", __FILE__, __LINE__);
1867 
1868  info->add_module(MODULE_NAME, MODULE_VERSION);
1869 
1870  return true;
1871 }
1872 
1873 #ifdef USE_HDFEOS2_LIB
1874 void close_fileid(int sdfd, int fileid,int gridfd, int swathfd,HDFSP:: File* h4file, HDFEOS2::File*eosfile) {
1875  if(h4file !=NULL)
1876  delete h4file;
1877  if(sdfd != -1)
1878  SDend(sdfd);
1879  if(fileid != -1)
1880  Hclose(fileid);
1881 
1882 
1883  if(eosfile !=NULL)
1884  delete eosfile;
1885  if(gridfd != -1)
1886  GDclose(gridfd);
1887  if(swathfd != -1)
1888  SWclose(swathfd);
1889 
1890 }
1891 #endif
1892 void close_hdf4_fileid(int sdfd, int fileid,HDFSP::File*h4file) {
1893 
1894  if(h4file !=NULL)
1895  delete h4file;
1896 
1897  if(sdfd != -1)
1898  SDend(sdfd);
1899  if(fileid != -1)
1900  Hclose(fileid);
1901 
1902 }
1903 
1904 // Handling DAS and DDS cache. Currently we only apply the cache to special products(AIRS version 6 level 3 or 2).
1905 // read or write DAS from/to a cache file.
1906 bool rw_das_cache_file(const string & filename, DAS *das_ptr,bool w_flag) {
1907 
1908  bool das_set_cache = false;
1909  FILE *das_file = NULL;
1910 
1911  if(false == w_flag) // open a cache file for reading.
1912  das_file = fopen(filename.c_str(),"r");
1913  else
1914  das_file = fopen(filename.c_str(),"w");
1915 
1916  if(NULL == das_file) {
1917  if(ENOENT == errno) {
1918  // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
1919  // it will generates a cache file, so here we set a flag to indicate if a cache file needs to be generated.
1920  if(false == w_flag) {
1921  BESDEBUG("h4", "DAS set cache key is true." << endl);
1922  das_set_cache = true;
1923  }
1924  }
1925  else
1926  throw BESInternalError( "An error occurred trying to open a das cache file " + get_errno(), __FILE__, __LINE__);
1927  }
1928  else {
1929 
1930  int fd_das = fileno(das_file);
1931 
1932  // Set a corresponding read(shared) or write(exclusive) lock.
1933  struct flock *l_das;
1934  if(false == w_flag)
1935  l_das = lock(F_RDLCK);
1936  else
1937  l_das = lock(F_WRLCK);
1938 
1939  // Hold a lock.
1940  if(fcntl(fd_das,F_SETLKW,l_das) == -1) {
1941  fclose(das_file);
1942  ostringstream oss;
1943  oss << "cache process: " << l_das->l_pid << " triggered a locking error: " << get_errno();
1944  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1945  }
1946 
1947  if(false == w_flag){
1948  // Read DAS from a cache file
1949  BESDEBUG("h4", "Obtaining DAS from the cache file" << endl);
1950  try {
1951  das_ptr->parse(das_file);
1952  }
1953  catch(...) {
1954  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1955  fclose(das_file);
1956  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1957  }
1958  fclose(das_file);
1959  throw InternalErr(__FILE__,__LINE__,"Fail to parse the das from a das file.");
1960  }
1961  }
1962  else {
1963  // Write DAS to a cache file
1964  BESDEBUG("h4", "write DAS to a cache file" << endl);
1965  try {
1966  das_ptr->print(das_file);
1967  }
1968  catch(...) {
1969  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1970  fclose(das_file);
1971  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1972  }
1973  fclose(das_file);
1974  throw InternalErr(__FILE__,__LINE__,"Fail to generate a das cache file.");
1975  }
1976 
1977  }
1978 
1979  // Unlock the cache file
1980  if(fcntl(fd_das,F_SETLK,lock(F_UNLCK)) == -1) {
1981  fclose(das_file);
1982  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1983  }
1984  fclose(das_file);
1985 
1986  }
1987 
1988  return das_set_cache;
1989 
1990 }
1991 
1992 // Read dds from a cache file.
1993 bool r_dds_cache_file(const string & cache_filename, DDS *dds_ptr,const string & hdf4_filename) {
1994 
1995  bool dds_set_cache = false;
1996  FILE *dds_file = NULL;
1997  dds_file = fopen(cache_filename.c_str(),"rb");
1998 
1999  if(NULL == dds_file) {
2000  if(ENOENT == errno) {
2001  // Since the das service always tries to read the data from a cache and if the cache file doesn't exist,
2002  // it is supposed that the handler should generate a cache file,
2003  // so set a flag to indicate if a cache file needs to be generated.
2004  dds_set_cache = true;
2005  }
2006  else
2007  throw BESInternalError( "An error occurred trying to open a dds cache file " + get_errno(), __FILE__, __LINE__);
2008  }
2009  else {
2010 
2011  int fd_dds = fileno(dds_file);
2012  struct flock *l_dds;
2013  l_dds = lock(F_RDLCK);
2014 
2015  // hold a read(shared) lock to read dds from a file.
2016  if(fcntl(fd_dds,F_SETLKW,l_dds) == -1) {
2017  fclose(dds_file);
2018  ostringstream oss;
2019  oss << "cache process: " << l_dds->l_pid << " triggered a locking error: " << get_errno();
2020  throw BESInternalError( oss.str(), __FILE__, __LINE__);
2021  }
2022 
2023  try {
2024  HDFCFUtil::read_sp_sds_dds_cache(dds_file,dds_ptr,cache_filename,hdf4_filename);
2025  }
2026  catch(...) {
2027  if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2028  fclose(dds_file);
2029  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2030  }
2031 
2032  fclose(dds_file);
2033  throw InternalErr(__FILE__,__LINE__,"Fail to generate a dds cache file.");
2034  }
2035 
2036  if(fcntl(fd_dds,F_SETLK,lock(F_UNLCK)) == -1) {
2037  fclose(dds_file);
2038  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2039  }
2040 
2041  fclose(dds_file);
2042 
2043  }
2044 
2045  return dds_set_cache;
2046 
2047 }
2048 
2049 bool check_beskeys(const string key) {
2050 
2051  bool found = false;
2052  string doset ="";
2053  const string dosettrue ="true";
2054  const string dosetyes = "yes";
2055 
2056  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2057  if( true == found ) {
2058  doset = BESUtil::lowercase( doset ) ;
2059  if( dosettrue == doset || dosetyes == doset )
2060  return true;
2061  }
2062  return false;
2063 
2064 }
2065 
2066 bool get_beskeys(const string key,string &key_value) {
2067 
2068  bool found = false;
2069 
2070  TheBESKeys::TheKeys()->get_value( key, key_value, found ) ;
2071  return found;
2072 
2073 }
2074 
2075 #if 0
2076 int get_cachekey_int(const string key) {
2077 
2078  bool found = false;
2079  int ret_value = 0;
2080  string doset ="";
2081 
2082  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2083  if( true == found ) {
2084  istringstream iss(doset);
2085  iss >>ret_value;
2086  }
2087  return ret_value;
2088 
2089 }
2090 #endif
2091 #if 0
2092 void test_func(HDFSP::File**h4file) {
2093 cerr<<"OK to pass pointer of a NULL pointer "<<endl;
2094 
2095 }
2096 #endif
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
informational response object
Definition: BESInfo.h:63
exception thrown if internal error encountered
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:206
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:657
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
void close_fileid(hid_t fid)
Definition: h5get.cc:433