bes  Updated for version 3.20.10
DmrppCommon.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2016 OPeNDAP, Inc.
6 // Author: James Gallagher <jgallagher@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 
24 #include "config.h"
25 
26 #include <string>
27 #include <sstream>
28 #include <vector>
29 #include <iterator>
30 #include <cstdio>
31 #include <cstdlib>
32 #include <cstring>
33 
34 #include <curl/curl.h>
35 
36 #include <libdap/BaseType.h>
37 #include <libdap/D4Attributes.h>
38 #include <libdap/XMLWriter.h>
39 #include <libdap/util.h>
40 
41 #if 1
42 #define PUGIXML_NO_XPATH
43 #define PUGIXML_HEADER_ONLY
44 #include <pugixml.hpp>
45 #endif
46 
47 #include "url_impl.h"
48 #include "BESIndent.h"
49 #include "BESDebug.h"
50 #include "BESUtil.h"
51 #include "BESInternalError.h"
52 
53 #include "DmrppRequestHandler.h"
54 #include "DmrppCommon.h"
55 #include "Chunk.h"
56 #include "byteswap_compat.h"
57 
58 
59 using namespace std;
60 using namespace libdap;
61 
62 #define prolog std::string("DmrppCommon::").append(__func__).append("() - ")
63 
64 namespace dmrpp {
65 
66 // Used with BESDEBUG
67 static const string dmrpp_3 = "dmrpp:3";
68 static const string dmrpp_4 = "dmrpp:4";
69 
70 bool DmrppCommon::d_print_chunks = false;
71 string DmrppCommon::d_dmrpp_ns = "http://xml.opendap.org/dap/dmrpp/1.0.0#";
72 string DmrppCommon::d_ns_prefix = "dmrpp";
73 
85 void join_threads(pthread_t threads[], unsigned int num_threads)
86 {
87  int status;
88  for (unsigned int i = 0; i < num_threads; ++i) {
89  if (threads[i]) {
90  BESDEBUG(dmrpp_3, "Join thread " << i << " after an exception was caught." << endl);
91  string *error = NULL;
92  if ((status = pthread_join(threads[i], (void **) &error)) < 0) {
93  BESDEBUG(dmrpp_3, "Could not join thread " << i << ", " << strerror(status)<< endl);
94  // ERROR_LOG("Failed to join thread " << i << " during clean up from an exception: " << strerror(status) << endl);
95  }
96  else if (error != NULL) {
97  BESDEBUG(dmrpp_3, "Joined thread " << i << ", error exit: " << *error << endl);
98  // ERROR_LOG("Joined thread " << i << ", error exit" << *error << endl);
99  }
100  else {
101  BESDEBUG(dmrpp_3, "Joined thread " << i << ", successful exit." << endl);
102  }
103  }
104  }
105 }
106 
108 void DmrppCommon::set_filter(const string &value) {
109  if (DmrppRequestHandler::d_emulate_original_filter_order_behavior) {
110  d_filters = "";
111  if (value.find("shuffle") != string::npos)
112  d_filters.append(" shuffle");
113  if (value.find("deflate") != string::npos)
114  d_filters.append(" deflate");
115  if (value.find("fletcher32") != string::npos)
116  d_filters.append(" fletcher32");
117 
119  }
120  else {
121  d_filters = value;
122  }
123 }
124 
134 void DmrppCommon::parse_chunk_dimension_sizes(const string &chunk_dims_string)
135 {
136  d_chunk_dimension_sizes.clear();
137 
138  if (chunk_dims_string.empty()) return;
139 
140  string chunk_dims = chunk_dims_string;
141  // If the input is anything other than integers and spaces, throw
142  if (chunk_dims.find_first_not_of("1234567890 ") != string::npos)
143  throw BESInternalError("while processing chunk dimension information, illegal character(s)", __FILE__, __LINE__);
144 
145  string space(" ");
146  size_t strPos = 0;
147  string strVal;
148 
149  // Are there spaces or multiple values?
150  if (chunk_dims.find(space) != string::npos) {
151  // Process space delimited content
152  while ((strPos = chunk_dims.find(space)) != string::npos) {
153  strVal = chunk_dims.substr(0, strPos);
154 
155  d_chunk_dimension_sizes.push_back(strtol(strVal.c_str(), nullptr, 10));
156  chunk_dims.erase(0, strPos + space.length());
157  }
158  }
159 
160  // If it's multivalued there's still one more value left to process
161  // If it's single valued the same is true, so let's ingest that.
162  d_chunk_dimension_sizes.push_back(strtol(chunk_dims.c_str(), nullptr, 10));
163 }
164 
171 void DmrppCommon::ingest_compression_type(const string &compression_type_string)
172 {
173  if (compression_type_string.empty()) return;
174  set_filter(compression_type_string);
175 }
176 
182 void DmrppCommon::ingest_byte_order(const string &byte_order_string) {
183 
184  if (byte_order_string.empty()) return;
185 
186  // Process content
187  if (byte_order_string.compare("LE") == 0) {
188  d_byte_order = "LE";
189  d_twiddle_bytes = is_host_big_endian();
190  } else {
191  if (byte_order_string.compare("BE") == 0) {
192  d_byte_order = "BE";
193  d_twiddle_bytes = !(is_host_big_endian());
194  } else {
195  throw BESInternalError("Did not recognize byteOrder.", __FILE__, __LINE__);
196  }
197  }
198 }
199 
204 unsigned long DmrppCommon::add_chunk(
205  std::shared_ptr<http::url> data_url,
206  const string &byte_order,
207  unsigned long long size,
208  unsigned long long offset,
209  const string &position_in_array)
210 {
211  vector<unsigned long long> cpia_vector;
212  Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
213  return add_chunk(move(data_url), byte_order, size, offset, cpia_vector);
214 }
215 
216 unsigned long DmrppCommon::add_chunk(
217  std::shared_ptr<http::url> data_url,
218  const string &byte_order,
219  unsigned long long size,
220  unsigned long long offset,
221  const vector<unsigned long long> &position_in_array)
222 {
223  std::shared_ptr<Chunk> chunk(new Chunk(move(data_url), byte_order, size, offset, position_in_array));
224 #if 0
225  auto array = dynamic_cast<dmrpp::DmrppArray *>(this);
226  if(!array){
227  stringstream msg;
228  msg << prolog << "ERROR DmrrpCommon::add_chunk() may only be called on an instance of DmrppArray. ";
229  msg << "The variable";
230  auto bt = dynamic_cast<libdap::BaseType *>(this);
231  if(bt){
232  msg << " " << bt->type_name() << " " << bt->name();
233  }
234  msg << " is not an instance of DmrppArray.";
235  msg << "this: " << (void **) this << " ";
236  msg << "byte_order: " << byte_order << " ";
237  msg << "size: " << size << " ";
238  msg << "offset: " << offset << " ";
239  throw BESInternalError(msg.str(),__FILE__, __LINE__);
240  }
241 
242  if(d_super_chunks.empty())
243  d_super_chunks.push_back( shared_ptr<SuperChunk>(new SuperChunk()));
244 
245  auto currentSuperChunk = d_super_chunks.back();
246 
247  bool chunk_was_added = currentSuperChunk->add_chunk(chunk);
248  if(!chunk_was_added){
249  if(currentSuperChunk->empty()){
250  stringstream msg;
251  msg << prolog << "ERROR! Failed to add a Chunk to an empty SuperChunk. This should not happen.";
252  throw BESInternalError(msg.str(),__FILE__,__LINE__);
253  }
254  // This means that the chunk was not contiguous with the currentSuperChunk
255  currentSuperChunk = shared_ptr<SuperChunk>(new SuperChunk());
256  chunk_was_added = currentSuperChunk->add_chunk(chunk);
257  if(!chunk_was_added) {
258  stringstream msg;
259  msg << prolog << "ERROR! Failed to add a Chunk to an empty SuperChunk. This should not happen.";
260  throw BESInternalError(msg.str(),__FILE__,__LINE__);
261  }
262  d_super_chunks.push_back(currentSuperChunk);
263  }
264 #endif
265 
266  d_chunks.push_back(chunk);
267  return d_chunks.size();
268 }
269 
270 unsigned long DmrppCommon::add_chunk(
271  const string &byte_order,
272  unsigned long long size,
273  unsigned long long offset,
274  const string &position_in_array)
275 {
276  vector<unsigned long long> cpia_vector;
277  Chunk::parse_chunk_position_in_array_string(position_in_array, cpia_vector);
278  return add_chunk(byte_order, size, offset, cpia_vector);
279 }
280 
281 unsigned long DmrppCommon::add_chunk(
282  const string &byte_order,
283  unsigned long long size,
284  unsigned long long offset,
285  const vector<unsigned long long> &position_in_array)
286 {
287  std::shared_ptr<Chunk> chunk(new Chunk( byte_order, size, offset, position_in_array));
288 
289  d_chunks.push_back(chunk);
290  return d_chunks.size();
291 }
292 
310 char *
311 DmrppCommon::read_atomic(const string &name)
312 {
313  if (get_chunks_size() != 1)
314  throw BESInternalError(string("Expected only a single chunk for variable ") + name, __FILE__, __LINE__);
315 
316  auto chunk = get_immutable_chunks()[0];
317 
318  chunk->read_chunk();
319 
320  return chunk->get_rbuf();
321 }
322 
329 void
330 DmrppCommon::print_chunks_element(XMLWriter &xml, const string &name_space)
331 {
332  // Start element "chunks" with dmrpp namespace and attributes:
333  if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunks", NULL) < 0)
334  throw BESInternalError("Could not start chunks element.", __FILE__, __LINE__);
335 
336  if (!d_filters.empty())
337  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "compressionType", (const xmlChar*) d_filters.c_str()) < 0)
338  throw BESInternalError("Could not write compression attribute.", __FILE__, __LINE__);
339 
340 
341  if(get_chunks_size() != 0) { // FIXME !get_chunks().empty()){
342  auto first_chunk = get_immutable_chunks().front();
343  if (!first_chunk->get_byte_order().empty()) {
344  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar *) "byteOrder",
345  (const xmlChar *) first_chunk->get_byte_order().c_str()) < 0)
346  throw BESInternalError("Could not write attribute byteOrder", __FILE__, __LINE__);
347  }
348  }
349 
350  if (d_chunk_dimension_sizes.size() > 0) {
351  // Write element "chunkDimensionSizes" with dmrpp namespace:
352  ostringstream oss;
353  copy(d_chunk_dimension_sizes.begin(), d_chunk_dimension_sizes.end(), ostream_iterator<unsigned int>(oss, " "));
354  string sizes = oss.str();
355  sizes.erase(sizes.size() - 1, 1); // trim the trailing space
356 
357  if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar*) name_space.c_str(), (const xmlChar*) "chunkDimensionSizes", NULL,
358  (const xmlChar*) sizes.c_str()) < 0) throw BESInternalError("Could not write chunkDimensionSizes attribute.", __FILE__, __LINE__);
359  }
360 
361  // Start elements "chunk" with dmrpp namespace and attributes:
362  // for (vector<Chunk>::iterator i = get_chunks().begin(), e = get_chunks().end(); i != e; ++i) {
363 
364  for(auto chunk: get_immutable_chunks()){
365 
366  if (xmlTextWriterStartElementNS(xml.get_writer(), (const xmlChar*)name_space.c_str(), (const xmlChar*) "chunk", NULL) < 0)
367  throw BESInternalError("Could not start element chunk", __FILE__, __LINE__);
368 
369  // Get offset string:
370  ostringstream offset;
371  offset << chunk->get_offset();
372  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "offset", (const xmlChar*) offset.str().c_str()) < 0)
373  throw BESInternalError("Could not write attribute offset", __FILE__, __LINE__);
374 
375  // Get nBytes string:
376  ostringstream nBytes;
377  nBytes << chunk->get_size();
378  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "nBytes", (const xmlChar*) nBytes.str().c_str()) < 0)
379  throw BESInternalError("Could not write attribute nBytes", __FILE__, __LINE__);
380 
381  if (chunk->get_position_in_array().size() > 0) {
382  // Get position in array string:
383  vector<unsigned long long> pia = chunk->get_position_in_array();
384  ostringstream oss;
385  oss << "[";
386  copy(pia.begin(), pia.end(), ostream_iterator<unsigned int>(oss, ","));
387  string pia_str = oss.str();
388  if (pia.size() > 0) pia_str.replace(pia_str.size() - 1, 1, "]"); // replace the trailing ',' with ']'
389  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "chunkPositionInArray", (const xmlChar*) pia_str.c_str()) < 0)
390  throw BESInternalError("Could not write attribute position in array", __FILE__, __LINE__);
391  }
392 
393  // End element "chunk":
394  if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunk element", __FILE__, __LINE__);
395  }
396 
397  if (xmlTextWriterEndElement(xml.get_writer()) < 0) throw BESInternalError("Could not end chunks element", __FILE__, __LINE__);
398 }
399 
403 void
404 DmrppCommon::print_compact_element(XMLWriter &xml, const string &name_space, const std::string &encoded)
405 {
406  // Write element "compact" with dmrpp namespace:
407  ostringstream oss;
408  copy(encoded.begin(), encoded.end(), ostream_iterator<char>(oss, ""));
409  string sizes = oss.str();
410 
411  if (xmlTextWriterWriteElementNS(xml.get_writer(), (const xmlChar *) name_space.c_str(),
412  (const xmlChar *) "compact", NULL,
413  (const xmlChar *) sizes.c_str()) < 0)
414  throw BESInternalError("Could not write compact element.", __FILE__, __LINE__);
415 }
416 
427 void DmrppCommon::print_dmrpp(XMLWriter &xml, bool constrained /*false*/)
428 {
429  BaseType &bt = dynamic_cast<BaseType&>(*this);
430  if (constrained && !bt.send_p())
431  return;
432 
433  if (xmlTextWriterStartElement(xml.get_writer(), (const xmlChar*)bt.type_name().c_str()) < 0)
434  throw InternalErr(__FILE__, __LINE__, "Could not write " + bt.type_name() + " element");
435 
436  if (!bt.name().empty())
437  if (xmlTextWriterWriteAttribute(xml.get_writer(), (const xmlChar*) "name", (const xmlChar*)bt.name().c_str()) < 0)
438  throw InternalErr(__FILE__, __LINE__, "Could not write attribute for name");
439 
440  if (bt.is_dap4())
441  bt.attributes()->print_dap4(xml);
442 
443  if (!bt.is_dap4() && bt.get_attr_table().get_size() > 0)
444  bt.get_attr_table().print_xml_writer(xml);
445 
446  // This is the code added to libdap::BaseType::print_dap4(). jhrg 5/10/18
447  if (DmrppCommon::d_print_chunks && get_chunks_size() > 0)
448  print_chunks_element(xml, DmrppCommon::d_ns_prefix);
449 
450  if (xmlTextWriterEndElement(xml.get_writer()) < 0)
451  throw InternalErr(__FILE__, __LINE__, "Could not end " + bt.type_name() + " element");
452 }
453 
454 void DmrppCommon::dump(ostream & strm) const
455 {
456  strm << BESIndent::LMarg << "is_filters_empty: " << (is_filters_empty() ? "true" : "false") << endl;
457  strm << BESIndent::LMarg << "filters: " << (d_filters.c_str()) << endl;
458 
459  const vector<unsigned long long> &chunk_dim_sizes = get_chunk_dimension_sizes();
460 
461  strm << BESIndent::LMarg << "chunk dimension sizes: [";
462  for (unsigned int i = 0; i < chunk_dim_sizes.size(); i++) {
463  strm << (i ? "][" : "") << chunk_dim_sizes[i];
464  }
465  strm << "]" << endl;
466 
467  auto chunk_refs = get_immutable_chunks();
468  strm << BESIndent::LMarg << "Chunks (aka chunks):" << (chunk_refs.size() ? "" : "None Found.") << endl;
469  BESIndent::Indent();
470  for (auto & chunk_ref : chunk_refs) {
471  strm << BESIndent::LMarg;
472  chunk_ref->dump(strm);
473  strm << endl;
474  }
475 }
476 
481 void
482 DmrppCommon::load_chunks(BaseType *btp) {
483  d_dmz->load_chunks(btp);
484 }
485 
490 void
491 DmrppCommon::load_attributes(libdap::BaseType *btp)
492 {
493  d_dmz->load_attributes(btp);
494 }
495 
496 } // namespace dmrpp
497 
exception thrown if internal error encountered
static void removeLeadingAndTrailingBlanks(std::string &key)
Definition: BESUtil.cc:485
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68