39 #include <sys/types.h>
45 #include <D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
50 #include <mime_util.h>
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <Ancillary.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
71 #include <BESStopWatch.h>
73 #include "config_hdf5.h"
75 #define HDF5_NAME "h5"
81 #define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
84 void get_attr_contents(AttrTable* temp_table);
88 void write_das_to_file(DAS*das_ptr,FILE* das_file);
91 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
94 void write_container_name_to_file(
const string&,FILE* das_file);
97 void write_das_attr_info(AttrTable* dtp,
const string&,
const string&,FILE * das_file);
100 char* copy_str(
char *temp_ptr,
const string & str);
104 char* obtain_str(
char*temp_ptr,
string & str);
107 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at);
110 bool check_beskeys(
const string);
113 static unsigned int get_uint_key(
const string &key,
unsigned int def_val);
114 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val);
117 static float get_float_key(
const string &key,
float def_val);
120 static string get_beskeys(
const string&);
123 extern void read_cfdas(DAS &das,
const string & filename,hid_t fileid);
124 extern void read_cfdds(DDS &dds,
const string & filename,hid_t fileid);
128 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
129 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
130 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
131 float HDF5RequestHandler::_cache_purge_level = 0.2;
136 ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
139 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
140 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
143 bool HDF5RequestHandler::_usecf =
false;
144 bool HDF5RequestHandler::_pass_fileid =
false;
145 bool HDF5RequestHandler::_disable_structmeta =
false;
146 bool HDF5RequestHandler::_disable_ecsmeta =
false;
147 bool HDF5RequestHandler::_keep_var_leading_underscore =
false;
148 bool HDF5RequestHandler::_check_name_clashing =
false;
149 bool HDF5RequestHandler::_add_path_attrs =
false;
150 bool HDF5RequestHandler::_drop_long_string =
false;
151 bool HDF5RequestHandler::_fillvalue_check =
false;
152 bool HDF5RequestHandler::_check_ignore_obj =
false;
153 bool HDF5RequestHandler::_flatten_coor_attr =
false;
154 bool HDF5RequestHandler::_default_handle_dimension =
false;
155 bool HDF5RequestHandler::_eos5_rm_convention_attr_path =
true;
156 bool HDF5RequestHandler::_dmr_long_int =
false;
157 bool HDF5RequestHandler::_no_zero_size_fullnameattr =
false;
158 bool HDF5RequestHandler::_enable_coord_attr_add_path =
false;
160 bool HDF5RequestHandler::_common_cache_dirs =
false;
162 bool HDF5RequestHandler::_use_disk_cache =
false;
163 bool HDF5RequestHandler::_use_disk_dds_cache =
false;
164 string HDF5RequestHandler::_disk_cache_dir =
"";
165 string HDF5RequestHandler::_disk_cachefile_prefix =
"";
166 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
169 bool HDF5RequestHandler::_disk_cache_comp_data =
false;
170 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =
false;
171 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
172 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
174 bool HDF5RequestHandler::_use_disk_meta_cache =
false;
175 string HDF5RequestHandler::_disk_meta_cache_path =
"";
177 bool HDF5RequestHandler::_use_latlon_disk_cache =
false;
178 long HDF5RequestHandler::_latlon_disk_cache_size =0;
179 string HDF5RequestHandler::_latlon_disk_cache_dir =
"";
180 string HDF5RequestHandler::_latlon_disk_cachefile_prefix=
"";
182 DMR* HDF5RequestHandler::dmr_int64 = 0;
189 string HDF5RequestHandler::_stp_east_filename;
190 string HDF5RequestHandler::_stp_north_filename;
191 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
192 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
193 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
199 HDF5RequestHandler::HDF5RequestHandler(
const string & name)
203 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
205 add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
206 add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
207 add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
208 add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
209 add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
211 add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
212 add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
216 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
219 HDF5RequestHandler::~HDF5RequestHandler()
224 delete datadds_cache;
226 delete lrdata_mem_cache;
227 delete srdata_mem_cache;
234 void HDF5RequestHandler::load_config()
236 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
238 if (BESISDEBUG(HDF5_NAME)){
239 sw.
start(prolog,
"ClockTheBESKeys");
243 HDF5RequestHandler::_mdcache_entries = get_uint_key(
"H5.MetaDataMemCacheEntries", 0);
244 HDF5RequestHandler::_lrdcache_entries = get_uint_key(
"H5.LargeDataMemCacheEntries", 0);
245 HDF5RequestHandler::_srdcache_entries = get_uint_key(
"H5.SmallDataMemCacheEntries", 0);
246 HDF5RequestHandler::_cache_purge_level = get_float_key(
"H5.CachePurgeLevel", 0.2);
248 if (get_mdcache_entries()) {
249 das_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
250 dds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
251 datadds_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
252 dmr_cache =
new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
256 _usecf = check_beskeys(
"H5.EnableCF");
257 BESDEBUG(HDF5_NAME, prolog <<
"H5.EnableCF: " << (_usecf?
"true":
"false") << endl);
260 _pass_fileid = check_beskeys(
"H5.EnablePassFileID");
261 _disable_structmeta = check_beskeys(
"H5.DisableStructMetaAttr");
262 _disable_ecsmeta = check_beskeys(
"H5.DisableECSMetaAttr");
263 _keep_var_leading_underscore = check_beskeys(
"H5.KeepVarLeadingUnderscore");
264 _check_name_clashing = check_beskeys(
"H5.EnableCheckNameClashing");
265 _add_path_attrs = check_beskeys(
"H5.EnableAddPathAttrs");
266 _drop_long_string = check_beskeys(
"H5.EnableDropLongString");
267 _fillvalue_check = check_beskeys(
"H5.EnableFillValueCheck");
268 _check_ignore_obj = check_beskeys(
"H5.CheckIgnoreObj");
269 _flatten_coor_attr = check_beskeys(
"H5.ForceFlattenNDCoorAttr");
270 _default_handle_dimension = check_beskeys(
"H5.DefaultHandleDimension");
271 _eos5_rm_convention_attr_path= check_beskeys(
"H5.RmConventionAttrPath");
272 _dmr_long_int = check_beskeys(
"H5.EnableDMR64bitInt");
273 _no_zero_size_fullnameattr = check_beskeys(
"H5.NoZeroSizeFullnameAttr");
274 _enable_coord_attr_add_path = check_beskeys(
"H5.EnableCoorattrAddPath");
276 _use_disk_cache = check_beskeys(
"H5.EnableDiskDataCache");
277 _disk_cache_dir = get_beskeys(
"H5.DiskCacheDataPath");
278 _disk_cachefile_prefix = get_beskeys(
"H5.DiskCacheFilePrefix");
279 _disk_cache_size = get_ulong_key(
"H5.DiskCacheSize",0);
281 _disk_cache_comp_data = check_beskeys(
"H5.DiskCacheComp");
282 _disk_cache_float_only_comp_data = check_beskeys(
"H5.DiskCacheFloatOnlyComp");
283 _disk_cache_comp_threshold = get_float_key(
"H5.DiskCacheCompThreshold",1.0);
284 _disk_cache_var_size = 1024*get_uint_key(
"H5.DiskCacheCompVarSize",0);
286 _use_disk_meta_cache = check_beskeys(
"H5.EnableDiskMetaDataCache");
287 _use_disk_dds_cache = check_beskeys(
"H5.EnableDiskDDSCache");
288 _disk_meta_cache_path = get_beskeys(
"H5.DiskMetaDataCachePath");
290 _use_latlon_disk_cache = check_beskeys(
"H5.EnableEOSGeoCacheFile");
291 _latlon_disk_cache_size = get_uint_key(
"H5.Cache.latlon.size",0);
292 _latlon_disk_cache_dir = get_beskeys(
"H5.Cache.latlon.path");
293 _latlon_disk_cachefile_prefix= get_beskeys(
"H5.Cache.latlon.prefix");
296 if(get_lrdcache_entries()) {
297 lrdata_mem_cache =
new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
298 if(
true == check_beskeys(
"H5.LargeDataMemCacheConfig")) {
299 _common_cache_dirs =obtain_lrd_common_cache_dirs();
301 if(
false == _common_cache_dirs)
302 cerr<<
"No specific cache info"<<endl;
307 if(get_srdcache_entries()) {
309 BESDEBUG(HDF5_NAME, prolog <<
"Generate memory cache for smaller coordinate variables" << endl);
310 srdata_mem_cache =
new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
314 if(_disk_cache_comp_data ==
true && _use_disk_cache ==
true) {
315 if(_disk_cache_comp_threshold < 1.0) {
317 ss<< _disk_cache_comp_threshold;
318 string _comp_threshold_str(ss.str());
319 string invalid_comp_threshold =
"The Compression Threshold is the total size of the variable array";
320 invalid_comp_threshold+=
" divided by the storage size of compressed array. It should always be >1";
321 invalid_comp_threshold+=
" The current threhold set at h5.conf is ";
322 invalid_comp_threshold+=_comp_threshold_str;
323 invalid_comp_threshold+=
" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
327 _stp_east_filename = get_beskeys(
"H5.STPEastFileName");
328 _stp_north_filename = get_beskeys(
"H5.STPNorthFileName");
330 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
337 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
338 #if DYNAMIC_CONFIG_ENABLED
343 hid_t cf_fileid = -1;
358 DAS *das = bdas->get_das();
361 DAS *cached_das_ptr = 0;
362 bool use_das_cache =
false;
364 cached_das_ptr =
static_cast<DAS*
>(das_cache->
get(filename));
366 use_das_cache =
true;
368 if (
true == use_das_cache) {
371 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
372 *das = *cached_das_ptr;
376 bool das_from_dc =
false;
377 string das_cache_fname;
380 if(_use_disk_meta_cache ==
true) {
382 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
383 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
385 if(access(das_cache_fname.c_str(),F_OK) !=-1)
391 if(
true == das_from_dc) {
392 read_das_from_disk_cache(das_cache_fname,das);
397 BESDEBUG(HDF5_NAME, prolog <<
"HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
398 das_cache->
add(
new DAS(*das), filename);
403 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
404 if (
true == _usecf) {
406 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
408 string invalid_file_msg=
"Could not open this HDF5 file ";
409 invalid_file_msg +=filename;
410 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
411 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
412 invalid_file_msg +=
" distributor.";
416 if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
417 HDF5RequestHandler::set_dmr_64bit_int(NULL);
418 read_cfdas( *das,filename,cf_fileid);
424 string invalid_file_msg=
"Could not open this HDF5 file ";
425 invalid_file_msg +=filename;
426 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
427 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
428 invalid_file_msg +=
" distributor.";
438 Ancillary::read_ancillary_das( *das, filename ) ;
442 AttrTable* top_table = das->get_top_level_attributes();
443 get_attr_contents(top_table);
446 AttrTable::Attr_iter start_aiter=das->var_begin();
447 AttrTable::Attr_iter it = start_aiter;
448 AttrTable::Attr_iter end_aiter = das->var_end();
449 while(it != end_aiter) {
450 AttrTable* temp_table = das->get_table(it);
452 cerr<<
"var_begin"<<endl;
453 temp_table->print(cerr);
461 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
462 das_cache->
add(
new DAS(*das), filename);
467 if(das_cache_fname!=
"") {
468 BESDEBUG(HDF5_NAME, prolog <<
"HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
469 write_das_to_disk_cache(das_cache_fname,das);
479 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
482 catch(InternalErr & e) {
486 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
493 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
500 string s =
"unknown exception caught building HDF5 DAS";
504 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
511 void HDF5RequestHandler::get_dds_with_attributes(
BESDDSResponse*bdds,
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename,
const string &dds_cache_fname,
const string &das_cache_fname,
bool dds_from_dc,
bool das_from_dc,
bool build_data)
514 if(
true == build_data)
515 dds = data_bdds->get_dds();
520 hid_t cf_fileid = -1;
525 DDS* cached_dds_ptr = 0;
526 bool use_dds_cache =
false;
528 cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename));
530 use_dds_cache =
true;
531 if (
true == use_dds_cache) {
534 BESDEBUG(HDF5_NAME, prolog <<
"DDS Metadata Cached hit for : " << filename << endl);
535 *dds = *cached_dds_ptr;
537 else if (
true ==dds_from_dc) {
538 read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
541 BESDEBUG(HDF5_NAME, prolog <<
"Build DDS from the HDF5 file. " << filename << endl);
542 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
543 dds->filename(filename);
548 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
550 string invalid_file_msg=
"Could not open this HDF5 file ";
551 invalid_file_msg +=filename;
552 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
553 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
554 invalid_file_msg +=
" distributor.";
559 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
560 HDF5RequestHandler::set_dmr_64bit_int(NULL);
561 read_cfdds(*dds,filename,cf_fileid);
567 string invalid_file_msg=
"Could not open this HDF5 file ";
568 invalid_file_msg +=filename;
569 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
570 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
571 invalid_file_msg +=
" distributor.";
575 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
579 if (!dds->check_semantics()) {
581 throw InternalErr(__FILE__, __LINE__,
582 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
585 Ancillary::read_ancillary_dds( *dds, filename ) ;
588 if(dds_cache_fname!=
"" && dds_from_dc ==
false)
589 write_dds_to_disk_cache(dds_cache_fname,dds);
598 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
604 BESDEBUG(HDF5_NAME, prolog <<
"DDS added to the cache for : " << filename << endl);
605 dds_cache->
add(
new DDS(*dds), filename);
616 catch(InternalErr & e) {
624 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
634 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
644 string s =
"unknown exception caught building HDF5 DDS";
650 void HDF5RequestHandler::get_dds_without_attributes_datadds(
BESDataDDSResponse*data_bdds,
const string &container_name,
const string& filename)
652 DDS *dds = data_bdds->get_dds();
656 hid_t cf_fileid = -1;
661 DDS* cached_dds_ptr = 0;
662 bool use_datadds_cache =
false;
664 cached_dds_ptr =
static_cast<DDS*
>(datadds_cache->
get(filename));
666 use_datadds_cache =
true;
667 if (
true == use_datadds_cache) {
670 BESDEBUG(HDF5_NAME, prolog <<
"DataDDS Metadata Cached hit for : " << filename << endl);
671 *dds = *cached_dds_ptr;
674 BESDEBUG(HDF5_NAME, prolog <<
"Build DDS from the HDF5 file. " << filename << endl);
675 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
676 dds->filename(filename);
681 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
683 string invalid_file_msg=
"Could not open this HDF5 file ";
684 invalid_file_msg +=filename;
685 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
686 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
687 invalid_file_msg +=
" distributor.";
692 if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
693 HDF5RequestHandler::set_dmr_64bit_int(NULL);
694 read_cfdds(*dds,filename,cf_fileid);
700 string invalid_file_msg=
"Could not open this HDF5 file ";
701 invalid_file_msg +=filename;
702 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
703 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
704 invalid_file_msg +=
" distributor.";
708 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
712 if (!dds->check_semantics()) {
714 throw InternalErr(__FILE__, __LINE__,
715 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
718 Ancillary::read_ancillary_dds( *dds, filename ) ;
728 add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
735 BESDEBUG(HDF5_NAME, prolog <<
"DataDDS added to the cache for : " << filename << endl);
736 datadds_cache->
add(
new DDS(*dds), filename);
745 BESDEBUG(HDF5_NAME, prolog <<
"Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
746 data_bdds->set_ia_flag(
false);
749 catch(InternalErr & e) {
757 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
767 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
777 string s =
"unknown exception caught building HDF5 DDS";
787 void HDF5RequestHandler::get_dds_with_attributes(
const string &filename,
const string &container_name, DDS*dds) {
791 hid_t cf_fileid = -1;
796 DDS* cached_dds_ptr = 0;
797 if (dds_cache && (cached_dds_ptr =
static_cast<DDS*
>(dds_cache->
get(filename)))) {
800 BESDEBUG(HDF5_NAME, prolog <<
"DDS Cached hit for : " << filename << endl);
801 *dds = *cached_dds_ptr;
806 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
807 if (!container_name.empty())
808 dds->container_name(container_name);
809 dds->filename(filename);
815 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
816 string dds_filename =
"/tmp/"+base_filename+
"_dds";
817 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
818 cerr<<
"before parsing "<<endl;
820 DDS tdds(&tf,name_path(filename),
"3.2");
821 tdds.filename(filename);
823 tdds.parse(dds_file);
825 cache_dds =
new DDS(tdds);
831 cerr<<
"after parsing "<<endl;
840 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
842 string invalid_file_msg=
"Could not open this HDF5 file ";
843 invalid_file_msg +=filename;
844 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
845 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
846 invalid_file_msg +=
" distributor.";
850 read_cfdds(*dds,filename,cf_fileid);
853 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
854 string dds_filename =
"/tmp/"+base_filename+
"_dds";
855 FILE *dds_file = fopen(dds_filename.c_str(),
"w");
856 dds->print(dds_file);
864 string invalid_file_msg=
"Could not open this HDF5 file ";
865 invalid_file_msg +=filename;
866 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
867 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
868 invalid_file_msg +=
" distributor.";
872 depth_first(fileid, (
char*)
"/", *dds, filename.c_str());
878 if (!dds->check_semantics()) {
880 throw InternalErr(__FILE__, __LINE__,
881 "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
884 Ancillary::read_ancillary_dds( *dds, filename ) ;
890 if (das_cache && (das =
static_cast<DAS*
>(das_cache->
get(filename)))) {
891 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
892 dds->transfer_attributes(das);
899 if (!container_name.empty())
900 das->container_name(container_name);
902 if (
true == _usecf) {
905 read_cfdas( *das,filename,cf_fileid);
918 Ancillary::read_ancillary_das( *das, filename ) ;
920 dds->transfer_attributes(das);
926 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
928 das_cache->
add(das, filename);
937 BESDEBUG(HDF5_NAME, prolog <<
"DDS added to the cache for : " << filename << endl);
938 dds_cache->
add(
new DDS(*dds), filename);
946 catch(InternalErr & e) {
954 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
964 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
974 string s =
"unknown exception caught building HDF5 DDS";
984 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
985 #if DYNAMIC_CONFIG_ENABLED
1001 bool dds_from_dc =
false;
1002 bool das_from_dc =
false;
1003 bool build_data =
false;
1004 string dds_cache_fname;
1005 string das_cache_fname;
1007 if(_use_disk_meta_cache) {
1009 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1012 if(_use_disk_dds_cache) {
1013 dds_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_dds";
1014 if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1018 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1020 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1025 get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1029 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1030 string dds_filename =
"/tmp/"+base_filename+
"_dds";
1033 DDS tdds(&tf,name_path(filename),
"3.2");
1034 tdds.filename(filename);
1037 FILE *dds_file = fopen(dds_filename.c_str(),
"r");
1038 tdds.parse(dds_file);
1040 DDS* cache_dds =
new DDS(tdds);
1052 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1055 catch(InternalErr & e) {
1057 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1058 __FILE__, __LINE__);
1062 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1063 __FILE__, __LINE__);
1067 string s =
"unknown exception caught building HDF5 DDS";
1071 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1077 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1078 #if DYNAMIC_CONFIG_ENABLED
1084 if(
true == _pass_fileid)
1085 return hdf5_build_data_with_IDs(dhi);
1099 bool dds_from_dc =
false;
1100 bool das_from_dc =
false;
1101 bool build_data =
true;
1102 string dds_cache_fname;
1103 string das_cache_fname;
1107 if(_use_disk_meta_cache ==
true) {
1109 string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1110 das_cache_fname = _disk_meta_cache_path+
"/" +base_filename+
"_das";
1112 if(access(das_cache_fname.c_str(),F_OK) !=-1)
1118 get_dds_without_attributes_datadds(bdds,container_name,filename);
1125 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1128 catch(InternalErr & e) {
1130 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1131 __FILE__, __LINE__);
1135 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1136 __FILE__, __LINE__);
1140 string s =
"unknown exception caught building HDF5 DDS";
1144 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1151 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1152 #if DYNAMIC_CONFIG_ENABLED
1156 BESDEBUG(HDF5_NAME,prolog <<
"Building DataDDS by passing file IDs. "<<endl);
1157 hid_t cf_fileid = -1;
1161 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1162 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1164 string invalid_file_msg=
"Could not open this HDF5 file ";
1165 invalid_file_msg +=filename;
1166 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1167 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1168 invalid_file_msg +=
" distributor.";
1182 delete bdds->get_dds();
1186 hdds->setHDF5Dataset(cf_fileid);
1188 read_cfdds( *hdds,filename,cf_fileid);
1190 if (!hdds->check_semantics()) {
1192 throw InternalErr(__FILE__, __LINE__,
1193 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1196 Ancillary::read_ancillary_dds( *hdds, filename ) ;
1198 DAS *das =
new DAS ;
1201 read_cfdas( *das,filename,cf_fileid);
1202 Ancillary::read_ancillary_das( *das, filename ) ;
1204 hdds->transfer_attributes(das);
1212 H5Fclose(cf_fileid);
1213 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1216 catch(InternalErr & e) {
1218 H5Fclose(cf_fileid);
1219 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1220 __FILE__, __LINE__);
1224 H5Fclose(cf_fileid);
1225 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1226 __FILE__, __LINE__);
1230 H5Fclose(cf_fileid);
1231 string s =
"unknown exception caught building HDF5 DataDDS";
1235 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1241 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1242 #if DYNAMIC_CONFIG_ENABLED
1253 DMR *dmr = bes_dmr_response.get_dmr();
1257 hid_t cf_fileid = -1;
1261 DMR* cached_dmr_ptr = 0;
1263 BESDEBUG(HDF5_NAME, prolog <<
"Checking DMR cache for : " << filename << endl);
1264 cached_dmr_ptr =
static_cast<DMR*
>(dmr_cache->
get(filename));
1267 if (cached_dmr_ptr) {
1269 BESDEBUG(HDF5_NAME, prolog <<
"DMR cache hit for : " << filename << endl);
1270 *dmr = *cached_dmr_ptr;
1275 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1276 D4BaseTypeFactory MyD4TypeFactory;
1277 dmr->set_factory(&MyD4TypeFactory);
1281 if(
true == _pass_fileid)
1282 return hdf5_build_dmr_with_IDs(dhi);
1284 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1286 string invalid_file_msg=
"Could not open this HDF5 file ";
1287 invalid_file_msg +=filename;
1288 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1289 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1290 invalid_file_msg +=
" distributor.";
1294 BaseTypeFactory factory;
1295 DDS dds(&factory, name_path(filename),
"3.2");
1296 dds.filename(filename);
1302 HDF5RequestHandler::set_dmr_64bit_int(dmr);
1303 read_cfdds( dds,filename,cf_fileid);
1304 if (!dds.check_semantics()) {
1306 throw InternalErr(__FILE__, __LINE__,
1307 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1310 read_cfdas(das,filename,cf_fileid);
1311 Ancillary::read_ancillary_das( das, filename ) ;
1313 dds.transfer_attributes(&das);
1317 H5Fclose(cf_fileid);
1319 dmr->build_using_dds(dds);
1327 string invalid_file_msg=
"Could not open this HDF5 file ";
1328 invalid_file_msg +=filename;
1329 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1330 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1331 invalid_file_msg +=
" distributor.";
1335 bool use_dimscale =
false;
1336 if(
true == _default_handle_dimension)
1337 use_dimscale = check_dimscale(fileid);
1338 dmr->set_name(name_path(filename));
1339 dmr->set_filename(name_path(filename));
1344 D4Group* root_grp = dmr->root();
1345 BESDEBUG(
"h5",
"use_dimscale is "<< use_dimscale <<endl);
1346 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),use_dimscale);
1348 BESDEBUG(
"h5",
"build_dmr - before obtain dimensions"<< endl);
1349 D4Dimensions *root_dims = root_grp->dims();
1350 for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1351 BESDEBUG(
"fonc",
"transform_dap4() - check dimensions"<< endl);
1352 BESDEBUG(
"fonc",
"transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1353 BESDEBUG(
"fonc",
"transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1354 BESDEBUG(
"fonc",
"transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1358 BESDEBUG(
"h5",
"build_dmr - after obtain dimensions"<< endl);
1362 if(
true == use_dimscale)
1364 breadth_first(fileid,(
char*)
"/",root_grp,filename.c_str(),
true);
1366 depth_first(fileid,(
char*)
"/",root_grp,filename.c_str());
1377 BESDEBUG(HDF5_NAME, prolog <<
"DMR added to the cache for : " << filename << endl);
1378 dmr_cache->
add(
new DMR(*dmr), filename);
1384 H5Fclose(cf_fileid);
1387 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1390 catch(InternalErr & e) {
1393 H5Fclose(cf_fileid);
1397 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1398 __FILE__, __LINE__);
1403 H5Fclose(cf_fileid);
1406 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1407 __FILE__, __LINE__);
1412 H5Fclose(cf_fileid);
1415 string s =
"unknown exception caught building HDF5 DMR";
1428 dmr->set_factory(0);
1430 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1437 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
1438 #if DYNAMIC_CONFIG_ENABLED
1442 BESDEBUG(
"h5",
"Building DMR with passing file IDs. "<<endl);
1444 hid_t cf_fileid = -1;
1446 H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1447 cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1449 string invalid_file_msg=
"Could not open this HDF5 file ";
1450 invalid_file_msg +=filename;
1451 invalid_file_msg +=
". It is very possible that this file is not an HDF5 file ";
1452 invalid_file_msg +=
" but with the .h5/.HDF5 suffix. Please check with the data";
1453 invalid_file_msg +=
" distributor.";
1457 BaseTypeFactory factory;
1458 DDS dds(&factory, name_path(filename),
"3.2");
1459 dds.filename(filename);
1467 read_cfdds( dds,filename,cf_fileid);
1469 if (!dds.check_semantics()) {
1471 throw InternalErr(__FILE__, __LINE__,
1472 "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1475 Ancillary::read_ancillary_dds( dds, filename ) ;
1478 read_cfdas(das,filename,cf_fileid);
1480 Ancillary::read_ancillary_das( das, filename ) ;
1482 dds.transfer_attributes(&das);
1491 H5Fclose(cf_fileid);
1492 BESDEBUG(HDF5_NAME, prolog <<
"Caught BESError! Message: " << e.
get_message() << endl);
1495 catch(InternalErr & e) {
1498 H5Fclose(cf_fileid);
1500 throw BESDapError(e.get_error_message(),
true, e.get_error_code(),
1501 __FILE__, __LINE__);
1506 H5Fclose(cf_fileid);
1508 throw BESDapError(e.get_error_message(),
false, e.get_error_code(),
1509 __FILE__, __LINE__);
1514 H5Fclose(cf_fileid);
1516 string s =
"unknown exception caught building HDF5 DataDDS";
1530 DMR *dmr = bes_dmr.get_dmr();
1531 D4BaseTypeFactory MyD4TypeFactory;
1532 dmr->set_factory(&MyD4TypeFactory);
1533 dmr->build_using_dds(dds);
1536 hdf5_dmr->setHDF5Dataset(cf_fileid);
1538 bes_dmr.set_dmr(hdf5_dmr);
1547 hdf5_dmr->set_factory(0);
1549 BESDEBUG(HDF5_NAME, prolog <<
"END" << endl);
1560 string add_info=
"Just for Test";
1562 map<string,string> attrs ;
1563 attrs[
"name"] = MODULE_NAME ;
1564 attrs[
"version"] = MODULE_VERSION ;
1565 list<string> services ;
1566 BESServiceRegistry::TheRegistry()->
services_handled( HDF5_NAME, services );
1567 if( services.size() > 0 )
1570 attrs[
"handles"] = handles ;
1572 info->begin_tag(
"module", &attrs ) ;
1573 info->end_tag(
"module" ) ;
1586 info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1592 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1594 string lrd_config_fpath;
1595 string lrd_config_fname;
1598 lrd_config_fpath = get_beskeys(
"H5.DataCachePath");
1601 lrd_config_fname = get_beskeys(
"H5.LargeDataMemCacheFileName");
1604 if(lrd_config_fpath==
"" || lrd_config_fname==
"")
1611 string mcache_config_fname = lrd_config_fpath+
"/"+lrd_config_fname;
1615 ifstream mcache_config_file(mcache_config_fname.c_str());
1618 if(mcache_config_file.is_open()==
false){
1619 BESDEBUG(HDF5_NAME, prolog <<
"The large data memory cache configure file "<<mcache_config_fname );
1620 BESDEBUG(HDF5_NAME, prolog <<
" cannot be opened."<<endl);
1625 while(getline(mcache_config_file,temp_line)) {
1628 if(temp_line.size()>1 && temp_line.at(1)==
' ') {
1630 string subline = temp_line.substr(2);
1631 vector<string> temp_name_list;
1634 if(temp_line.at(0)==
'1') {
1635 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1637 lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1640 else if(temp_line.at(0)==
'0'){
1641 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1643 lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1646 else if(temp_line.at(0)==
'2') {
1652 for(
unsigned int i = 0; i<subline.size();i++){
1653 if(subline[i]==
'"') {
1654 dq_pos.push_back(i);
1656 else if(subline[i]==
'\'')
1657 sq_pos.push_back(i);
1659 if(dq_pos.size()==0 && sq_pos.size()==0)
1660 HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1661 else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1662 unsigned int dq_index= 0;
1663 while(dq_index < dq_pos.size()){
1664 if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1665 temp_name_list.push_back
1666 (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1668 dq_index = dq_index + 2;
1671 else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1672 unsigned int sq_index= 0;
1673 while(sq_index < sq_pos.size()){
1674 if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1675 temp_name_list.push_back
1676 (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1678 sq_index = sq_index+2;
1682 lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1690 for(
int i =0; i<lrd_cache_dir_list.size();i++)
1691 cerr<<
"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1692 for(
int i =0; i<lrd_non_cache_dir_list.size();i++)
1693 cerr<<
"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1694 for(
int i =0; i<lrd_var_cache_file_list.size();i++)
1695 cerr<<
"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1699 mcache_config_file.close();
1700 if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1707 bool HDF5RequestHandler::read_das_from_disk_cache(
const string & cache_filename,DAS *das_ptr) {
1709 BESDEBUG(HDF5_NAME, prolog <<
"Coming to read_das_from_disk_cache() " << cache_filename << endl);
1710 bool ret_value =
true;
1711 FILE *md_file = NULL;
1712 md_file = fopen(cache_filename.c_str(),
"rb");
1714 if(NULL == md_file) {
1715 string bes_error =
"An error occurred trying to open a metadata cache file " + cache_filename;
1720 int fd_md = fileno(md_file);
1722 l_md = lock(F_RDLCK);
1725 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1728 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1735 if(stat(cache_filename.c_str(),&sb) != 0) {
1736 string bes_error =
"An error occurred trying to stat a metadata cache file size " + cache_filename;
1742 size_t bytes_expected_read=(size_t)sb.st_size;
1743 BESDEBUG(HDF5_NAME, prolog <<
"DAS Disk cache file size is " << bytes_expected_read << endl);
1746 buf.resize(bytes_expected_read);
1747 size_t bytes_to_read =fread((
void*)&buf[0],1,bytes_expected_read,md_file);
1748 if(bytes_to_read != bytes_expected_read)
1749 throw InternalErr(__FILE__,__LINE__,
"Fail to read the data from the das cache file.");
1751 char* temp_pointer =&buf[0];
1753 AttrTable*at = NULL;
1756 temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1762 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1764 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1768 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a das cache file.");
1772 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1774 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1783 bool HDF5RequestHandler::write_dds_to_disk_cache(
const string& dds_cache_fname,DDS *dds_ptr) {
1785 BESDEBUG(HDF5_NAME, prolog <<
"Write DDS to disk cache " << dds_cache_fname << endl);
1786 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"w");
1788 if(NULL == dds_file) {
1789 string bes_error =
"An error occurred trying to open a metadata cache file " + dds_cache_fname;
1794 int fd_md = fileno(dds_file);
1796 l_md = lock(F_WRLCK);
1799 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1802 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1807 dds_ptr->print(dds_file);
1810 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1812 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1816 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1819 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1821 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1831 bool HDF5RequestHandler::write_das_to_disk_cache(
const string & das_cache_fname, DAS *das_ptr) {
1833 BESDEBUG(HDF5_NAME, prolog <<
"Write DAS to disk cache " << das_cache_fname << endl);
1834 FILE *das_file = fopen(das_cache_fname.c_str(),
"wb");
1835 if(NULL == das_file) {
1836 string bes_error =
"An error occurred trying to open a metadata cache file " + das_cache_fname;
1840 int fd_md = fileno(das_file);
1842 l_md = lock(F_WRLCK);
1845 if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1848 oss <<
"cache process: " << l_md->l_pid <<
" triggered a locking error: " << get_errno();
1853 write_das_to_file(das_ptr,das_file);
1856 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1858 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1862 throw InternalErr(__FILE__,__LINE__,
"Fail to parse a dds cache file.");
1865 if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1867 throw BESInternalError(
"An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1879 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
1882 uint8_t category_flag = 2;
1883 AttrTable* top_table = das_ptr->get_top_level_attributes();
1884 write_das_table_to_file(top_table,das_file);
1887 fwrite((
const void*)&category_flag,1,1,das_file);
1893 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
1895 if(temp_table !=NULL) {
1898 uint8_t category_flag = 2;
1901 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
1902 AttrTable::Attr_iter top_endit = temp_table->attr_end();
1903 AttrTable::Attr_iter top_it = top_startit;
1904 while(top_it !=top_endit) {
1905 AttrType atype = temp_table->get_attr_type(top_it);
1906 if(atype == Attr_unknown)
1907 throw InternalErr(__FILE__,__LINE__,
"Unsupported DAS Attribute type");
1908 else if(atype!=Attr_container) {
1909 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr name is: "
1910 << temp_table->get_name(top_it) << endl);
1911 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr type is: "
1912 << temp_table->get_type(top_it) << endl);
1921 write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
1924 BESDEBUG(HDF5_NAME, prolog <<
"DAS to the disk cache, attr container name is: "
1925 << (*top_it)->name << endl);
1927 AttrTable* sub_table = temp_table->get_attr_table(top_it);
1928 write_container_name_to_file(sub_table->get_name(),das_file);
1929 write_das_table_to_file(sub_table,das_file);
1932 fwrite((
const void*)&category_flag,1,1,das_file);
1942 void write_container_name_to_file(
const string& cont_name,FILE *das_file) {
1945 uint8_t category_flag = 1;
1947 size_t bytes_to_write = cont_name.size()+
sizeof(size_t)+1;
1948 buf.resize(bytes_to_write);
1949 char*temp_pointer =&buf[0];
1950 memcpy((
void*)temp_pointer,(
void*)&category_flag,1);
1952 temp_pointer=copy_str(temp_pointer,cont_name);
1954 size_t bytes_to_be_written = fwrite((
const void*)&buf[0],1,bytes_to_write,das_file);
1955 if(bytes_to_be_written != bytes_to_write)
1956 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS container name to a cache");
1962 void write_das_attr_info(AttrTable* dtp,
const string& attr_name,
const string & attr_type,FILE * das_file) {
1965 uint8_t category_flag = 0;
1967 unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
1968 vector<string> attr_values;
1969 size_t total_attr_values_size = 0;
1970 for (
unsigned int i = 0; i <num_attr_elems;i++){
1971 attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
1972 total_attr_values_size += attr_values[i].size();
1976 size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2*
sizeof(size_t);
1982 bytes_to_write_attr +=
sizeof(
unsigned int) + num_attr_elems*
sizeof(
size_t)+total_attr_values_size;
1984 vector<char>attr_buf;
1985 attr_buf.resize(bytes_to_write_attr);
1986 char* temp_attrp =&attr_buf[0];
1989 memcpy((
void*)temp_attrp,(
void*)&category_flag,1);
1993 temp_attrp=copy_str(temp_attrp,attr_name);
1994 temp_attrp=copy_str(temp_attrp,attr_type);
1997 memcpy((
void*)temp_attrp,(
void*)&num_attr_elems,
sizeof(
unsigned int));
1998 temp_attrp+=
sizeof(
unsigned int);
2001 for (
unsigned int i = 0; i <num_attr_elems;i++)
2002 temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2004 size_t bytes_to_be_written = fwrite((
const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
2005 if(bytes_to_be_written != bytes_to_write_attr)
2006 throw InternalErr(__FILE__, __LINE__,
"Failed to write a DAS attribute to a cache");
2014 bool build_data,
const string & container_name,
const string & h5_fname,
2015 const string & dds_cache_fname,
const string &das_cache_fname, hid_t h5_fd,
2019 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2022 if(
true == build_data)
2023 dds = data_bdds->get_dds();
2029 DDS tdds(&tf,name_path(h5_fname),
"3.2");
2030 tdds.filename(h5_fname);
2032 FILE *dds_file = fopen(dds_cache_fname.c_str(),
"r");
2033 tdds.parse(dds_file);
2034 DDS* cache_dds =
new DDS(tdds);
2036 cerr<<
"before dds "<<endl;
2038 cerr<<
"after dds "<<endl;
2039 cerr<<
"before tdds "<<endl;
2040 cache_dds->dump(cerr);
2041 cerr<<
"after tdds "<<endl;
2046 Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2048 add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2049 if(
true == build_data)
2050 data_bdds->
set_dds(cache_dds);
2057 BESDEBUG(HDF5_NAME, prolog <<
"For memory cache, DDS added to the cache for : " << h5_fname << endl);
2058 dds_cache->
add(
new DDS(*cache_dds), h5_fname);
2064 void HDF5RequestHandler::add_das_to_dds(DDS *dds,
const string &,
const string &filename,
2065 const string &das_cache_fname, hid_t h5_fd,
bool das_from_dc) {
2067 BESDEBUG(HDF5_NAME, prolog <<
"BEGIN" << endl);
2071 bool use_das_cache =
false;
2073 das =
static_cast<DAS*
>(das_cache->
get(filename));
2075 use_das_cache =
true;
2077 if (
true == use_das_cache) {
2078 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
2079 dds->transfer_attributes(das);
2086 if (!container_name.empty())
2087 das->container_name(container_name);
2089 if(das_from_dc ==
true)
2090 read_das_from_disk_cache(das_cache_fname,das);
2093 bool h5_file_open =
true;
2095 h5_file_open =
false;
2096 if (
true == _usecf) {
2098 if(h5_file_open ==
false)
2099 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2101 read_cfdas( *das,filename,h5_fd);
2102 if(h5_file_open ==
false)
2106 if(h5_file_open ==
false)
2110 if(h5_file_open ==
false)
2114 Ancillary::read_ancillary_das( *das, filename ) ;
2116 if(das_cache_fname!=
"" && das_from_dc ==
false)
2117 write_das_to_disk_cache(das_cache_fname,das);
2120 dds->transfer_attributes(das);
2124 BESDEBUG(HDF5_NAME, prolog <<
"For memory cache, DAS added to the cache for : " << filename << endl);
2125 das_cache->
add(
new DAS(*das), filename);
2133 bool check_beskeys(
const string key) {
2137 const string dosettrue =
"true";
2138 const string dosetyes =
"yes";
2141 if(
true == found ) {
2144 BESDEBUG(HDF5_NAME, prolog <<
"Key: " << key << (found?(
" was found. value: "+doset):
" was not found.") << endl);
2145 return found && (dosettrue == doset || dosetyes == doset);
2151 static unsigned int get_uint_key(
const string &key,
unsigned int def_val)
2157 if (
true == found) {
2159 return atoi(doset.c_str());
2166 static unsigned long get_ulong_key(
const string &key,
unsigned long def_val)
2172 if (
true == found) {
2174 return atol(doset.c_str());
2180 static float get_float_key(
const string &key,
float def_val)
2186 if (
true == found) {
2187 return atof(doset.c_str());
2194 static string get_beskeys(
const string &key) {
2197 string ret_value =
"";
2205 char* copy_str(
char*temp_ptr,
const string & str) {
2207 size_t str_size=str.size();
2208 memcpy((
void*)temp_ptr,(
void*)&str_size,
sizeof(
size_t));
2209 temp_ptr+=
sizeof(size_t);
2210 vector<char>temp_vc2(str.begin(),str.end());
2211 memcpy((
void*)temp_ptr,(
void*)&temp_vc2[0],str.size());
2212 temp_ptr+=str.size();
2221 char* obtain_str(
char*temp_ptr,
string & str) {
2223 size_t oname_size = *((
size_t *)temp_ptr);
2224 temp_ptr = temp_ptr +
sizeof(size_t);
2226 for(
unsigned int i =0; i<oname_size; i++){
2227 oname.push_back(*temp_ptr);
2238 char* get_attr_info_from_dc(
char*temp_pointer,DAS *das,AttrTable *at_par) {
2243 flag = *((uint8_t*)(temp_pointer));
2244 BESDEBUG(HDF5_NAME, prolog <<
"Build DAS from the disk cache file flag: "
2245 <<
" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2246 <<
" flag = 3; the initial value to get the attribute retrieval process started."
2247 <<
" The flag value is "
2248 << (
int)flag <<endl);
2252 string container_name;
2253 temp_pointer = obtain_str(temp_pointer,container_name);
2254 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, container name is " << container_name << endl);
2257 AttrTable*temp_at_par = at_par;
2259 at_par = das->add_table(container_name,
new AttrTable);
2261 at_par = at_par->append_container(container_name);
2263 temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2265 at_par = temp_at_par;
2268 else if(flag == 0) {
2271 throw BESInternalError(
"The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2275 temp_pointer = obtain_str(temp_pointer,attr_name);
2276 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attr name is: " << attr_name << endl);
2280 temp_pointer = obtain_str(temp_pointer,attr_type);
2281 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attr type is: " << attr_type << endl);
2284 unsigned int num_values = *((
unsigned int*)(temp_pointer));
2285 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, number of attribute values is: " << num_values << endl);
2286 temp_pointer+=
sizeof(
unsigned int);
2288 vector <string> attr_values;
2290 for(
unsigned int i = 0; i<num_values; i++) {
2292 temp_pointer = obtain_str(temp_pointer,attr_value);
2293 attr_values.push_back(attr_value);
2294 BESDEBUG(HDF5_NAME, prolog <<
"DAS from the disk cache, attribute value is: " << attr_value << endl);
2297 at_par->append_attr(attr_name,attr_type,&attr_values);
2301 return temp_pointer;
2306 void get_attr_contents(AttrTable*temp_table) {
2307 if(temp_table !=NULL) {
2308 AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2309 AttrTable::Attr_iter top_endit = temp_table->attr_end();
2310 AttrTable::Attr_iter top_it = top_startit;
2311 while(top_it !=top_endit) {
2312 AttrType atype = temp_table->get_attr_type(top_it);
2313 if(atype == Attr_unknown)
2314 cerr<<
"unsupported DAS attributes" <<endl;
2315 else if(atype!=Attr_container) {
2317 cerr<<
"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2318 cerr<<
"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2319 unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2320 cerr<<
"Attribute values are "<<endl;
2321 for (
unsigned int i = 0; i <num_attrs;i++)
2322 cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<
" ";
2326 cerr<<
"Coming to the attribute container. "<<endl;
2327 cerr<<
"container name is "<<(*top_it)->name <<endl;
2328 AttrTable* sub_table = temp_table->get_attr_table(top_it);
2329 cerr<<
"container table name is "<<sub_table->get_name() <<endl;
2330 get_attr_contents(sub_table);
2346 DDS *dds = bdds->get_dds();
2350 bool das_from_mcache =
false;
2352 das =
static_cast<DAS*
>(das_cache->
get(filename));
2354 BESDEBUG(HDF5_NAME, prolog <<
"DAS Cached hit for : " << filename << endl);
2355 dds->transfer_attributes(das);
2356 das_from_mcache =
true;
2360 if(
false == das_from_mcache) {
2364 if (!container_name.empty()) das->container_name(container_name);
2367 if (
true == _usecf) {
2369 h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2371 read_cfdas( *das,filename,h5_fd);
2383 Ancillary::read_ancillary_das(*das, filename);
2385 dds->transfer_attributes(das);
2390 BESDEBUG(HDF5_NAME, prolog <<
"DAS added to the cache for : " << filename << endl);
2391 das_cache->
add(das, filename);
2397 BESDEBUG(HDF5_NAME, prolog <<
"Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2398 bdds->set_ia_flag(
true);
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
virtual std::string get_message()
get the error message for this exception
informational response object
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
static std::string lowercase(const std::string &s)
static std::string implode(const std::list< std::string > &values, char delim)
An in-memory cache for DapObj (DAS, DDS, ...) objects.
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
static TheBESKeys * TheKeys()
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
void find_gloattr(hid_t file, DAS &das)
bool breadth_first(hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale)
hid_t get_fileid(const char *filename)
void close_fileid(hid_t fid)
The main header of the HDF5 OPeNDAP handler.