bes  Updated for version 3.20.10
AggregationUtil.cc
1 // This file is part of the "NcML Module" project, a BES module designed
3 // to allow NcML files to be used to be used as a wrapper to add
4 // AIS to existing datasets of any format.
5 //
6 // Copyright (c) 2009 OPeNDAP, Inc.
7 // Author: Michael Johnson <m.johnson@opendap.org>
8 //
9 // For more information, please also see the main website: http://opendap.org/
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // Please see the files COPYING and COPYRIGHT for more information on the GLPL.
26 //
27 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
29 #include "config.h"
30 
31 #include "AggregationUtil.h"
32 
33 // agg_util includes
34 #include "AggMemberDataset.h"
35 #include "AggregationException.h"
36 #include "Dimension.h"
37 
38 // libdap includes
39 #include <libdap/Array.h> // libdap
40 #include <libdap/AttrTable.h>
41 #include <libdap/BaseType.h>
42 #include <libdap/DataDDS.h>
43 #include <libdap/DDS.h>
44 #include <libdap/Grid.h>
45 #include "BESDebug.h"
46 #include "BESStopWatch.h"
47 
48 // Outside includes (MINIMIZE THESE!)
49 #include "NCMLDebug.h" // This the ONLY dependency on NCML Module I want in this class since the macros there are general it's ok...
50 
51 using libdap::Array;
52 using libdap::AttrTable;
53 using libdap::BaseType;
54 using libdap::Constructor;
55 using libdap::DataDDS;
56 using libdap::DDS;
57 using libdap::Grid;
58 using libdap::Vector;
59 using std::string;
60 using std::vector;
61 
62 namespace agg_util {
63 // Static class member used to track the position of the last CVs insertion
64 // when building a JoinExisting aggregation.
65 int AggregationUtil::d_last_added_cv_position = 0;
66 
68 // ArrayGetterInterface impls
69 
70 /* virtual */
71 ArrayGetterInterface::~ArrayGetterInterface()
72 {
73 }
74 
76 // TopLevelArrayGetter impl
77 
78 TopLevelArrayGetter::TopLevelArrayGetter() :
79  ArrayGetterInterface()
80 {
81 }
82 
83 /* virtual */
84 TopLevelArrayGetter::~TopLevelArrayGetter()
85 {
86 }
87 
88 /* virtual */
89 TopLevelArrayGetter*
90 TopLevelArrayGetter::clone() const
91 {
92  return new TopLevelArrayGetter(*this);
93 }
94 
95 /* virtual */
96 libdap::Array*
97 TopLevelArrayGetter::readAndGetArray(const std::string& name, const libdap::DDS& dds,
98  const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
99 {
100 
101  BESStopWatch sw;
102  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("TopLevelArrayGetter::readAndGetArray", "");
103 
104  // First, look up the BaseType
105  BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
106 
107  // Next, if it's not there, throw exception.
108  if (!pBT) {
109  throw AggregationException("TopLevelArrayGetter: "
110  "Did not find a variable named \"" + name + "\" at the top-level of the DDS!");
111  }
112 
113  // Next, make sure it's an Array before we cast it
114  // Prefer using the enum type for speed rather than RTTI
115  if (pBT->type() != libdap::dods_array_c) {
116  throw AggregationException("TopLevelArrayGetter: "
117  "The top-level DDS variable named \"" + name + "\" was not of the expected type!"
118  " Expected:Array Found:" + pBT->type_name());
119  }
120 
121  libdap::Array* pDatasetArray = static_cast<libdap::Array*>(pBT);
122 
123  // If given, copy the constraints over to the found Array
124  if (pConstraintTemplate) {
125  agg_util::AggregationUtil::transferArrayConstraints(pDatasetArray, // into this dataset array to be read
126  *pConstraintTemplate, // from this template
127  false, // same rank Array's in template and loaded, don't skip first dim
128  false, // or the to dimension. copy whole thing.
129  !(debugChannel.empty()), // printDebug
130  debugChannel);
131  }
132 
133  // Force a read() perhaps with constraints
134  pDatasetArray->set_send_p(true);
135  pDatasetArray->set_in_selection(true);
136  pDatasetArray->read();
137 
138  return pDatasetArray;
139 }
140 
142 // TopLevelGridDataArrayGetter impl
143 
144 TopLevelGridDataArrayGetter::TopLevelGridDataArrayGetter() :
146 {
147 }
148 
149 /* virtual */
150 TopLevelGridDataArrayGetter::~TopLevelGridDataArrayGetter()
151 {
152 }
153 
154 /* virtual */
155 TopLevelGridDataArrayGetter*
157 {
158  return new TopLevelGridDataArrayGetter(*this);
159 }
160 
161 /* virtual */
162 libdap::Array*
163 TopLevelGridDataArrayGetter::readAndGetArray(const std::string& name, const libdap::DDS& dds,
164  const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
165 {
166  BESStopWatch sw;
167  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("TopLevelGridDataArrayGetter::readAndGetArray", "");
168 
169  // First, look up the BaseType
170  BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, name);
171 
172  // Next, if it's not there, throw exception.
173  if (!pBT) {
174  throw AggregationException("TopLevelGridArrayGetter: "
175  "Did not find a variable named \"" + name + "\" at the top-level of the DDS!");
176  }
177 
178  // Next, make sure it's a Grid before we cast it
179  // Prefer using the enum type for speed rather than RTTI
180  if (pBT->type() != libdap::dods_grid_c) {
181  throw AggregationException("TopLevelGridArrayGetter: "
182  "The top-level DDS variable named \"" + name + "\" was not of the expected type!"
183  " Expected:Grid Found:" + pBT->type_name());
184  }
185 
186  // Grab the array and return it.
187  Grid* pDataGrid = static_cast<Grid*>(pBT);
188  Array* pDataArray = static_cast<Array*>(pDataGrid->array_var());
189  if (!pDataArray) {
190  throw AggregationException("TopLevelGridArrayGetter: "
191  "The data Array var for variable name=\"" + name + "\" was unexpectedly null!");
192  }
193 
194  // If given, copy the constraints over to the found Array
195  if (pConstraintTemplate) {
196  agg_util::AggregationUtil::transferArrayConstraints(pDataArray, // into this data array to be read
197  *pConstraintTemplate, // from this template
198  false, // same rank Array's in template and loaded, don't skip first dim
199  false, // also don't skip in the to array
200  !(debugChannel.empty()), // printDebug
201  debugChannel);
202  }
203 
204  // Force the read() on the Grid level since some handlers
205  // cannot handle a read on a subobject unless read() is called
206  // on the parent object. We have given the constraints to the
207  // data Array already.
208  // TODO make an option on whether to load the Grid's map
209  // vectors or not! I think for these cases we do not want them ever!
210  pDataGrid->set_send_p(true);
211  pDataGrid->set_in_selection(true);
212  pDataGrid->read();
213 
214  // Also make sure the Array was read and if not call it as well.
215  if (!pDataArray->read_p()) {
216  pDataArray->set_send_p(true);
217  pDataArray->set_in_selection(true);
218  pDataArray->read();
219  }
220 
221  return pDataArray;
222 }
223 
225 // TopLevelGridMapArrayGetter impl
226 
227 TopLevelGridMapArrayGetter::TopLevelGridMapArrayGetter(const std::string& gridName) :
228  ArrayGetterInterface(), _gridName(gridName)
229 {
230 }
231 
232 /* virtual */
233 TopLevelGridMapArrayGetter::~TopLevelGridMapArrayGetter()
234 {
235 }
236 
237 /* virtual */
238 TopLevelGridMapArrayGetter*
240 {
241  return new TopLevelGridMapArrayGetter(*this);
242 }
243 
244 /* virtual */
245 libdap::Array*
246 TopLevelGridMapArrayGetter::readAndGetArray(const std::string& arrayName, const libdap::DDS& dds,
247  const libdap::Array* const pConstraintTemplate, const std::string& debugChannel) const
248 {
249 
250  BESStopWatch sw;
251  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("TopLevelGridMapArrayGetter::readAndGetArray", "");
252 
253  // First, look up the Grid the map is in
254  BaseType* pBT = AggregationUtil::getVariableNoRecurse(dds, _gridName);
255 
256  // Next, if it's not there, throw exception.
257  if (!pBT) {
258  throw AggregationException("Did not find a variable named \"" + _gridName + "\" at the top-level of the DDS!");
259  }
260 
261  // Next, make sure it's a Grid before we cast it
262  // Prefer using the enum type for speed rather than RTTI
263  if (pBT->type() != libdap::dods_grid_c) {
264  throw AggregationException(
265  "The top-level DDS variable named \"" + _gridName + "\" was not of the expected type!"
266  " Expected:Grid Found:" + pBT->type_name());
267  }
268 
269  // Find the correct map
270  Grid* pDataGrid = static_cast<Grid*>(pBT);
271  Array* pMap = const_cast<Array*>(AggregationUtil::findMapByName(*pDataGrid, arrayName));
272  NCML_ASSERT_MSG(pMap,
273  "Expected to find the map with name " + arrayName + " within the Grid " + _gridName + " but failed to find it!");
274 
275  // Prepare it to be read in so we can get the data
276  pMap->set_send_p(true);
277  pMap->set_in_selection(true);
278 
279  // If given, copy the constraints over to the found Array
280  if (pConstraintTemplate) {
281  agg_util::AggregationUtil::transferArrayConstraints(pMap, // into this data array to be read
282  *pConstraintTemplate, // from this template
283  false, // same rank Array's in template and loaded, don't skip first dim
284  false, // also don't skip in the to array
285  !(debugChannel.empty()), // printDebug
286  debugChannel);
287  }
288 
289  // Do the read
290  pMap->read();
291 
292  return pMap;
293 }
294 
295 /*********************************************************************************************************
296  * AggregationUtil Impl
297  */
298 void AggregationUtil::performUnionAggregation(DDS* pOutputUnion, const ConstDDSList& datasetsInOrder)
299 {
300  VALID_PTR(pOutputUnion);
301 
302  // Union any non-aggregated variables from the template dataset into the aggregated dataset
303  // Because we want the joinExistingaggregation to build up the Coordinate Variables (CVs)
304  // in the order they are declared in the NCML file, we need to track the current position
305  // where the last one was inserted. We can do that with a field in the AggregationUtil
306  // class. Here we reset that field so that it starts at position 0. 12.13.11 jhrg
308 
309  vector<const DDS*>::const_iterator endIt = datasetsInOrder.end();
310  vector<const DDS*>::const_iterator it;
311  for (it = datasetsInOrder.begin(); it != endIt; ++it) {
312  const DDS* pDDS = *it;
313  VALID_PTR(pDDS);
314 
315  // Merge in the global attribute tables
316  unionAttrsInto(&(pOutputUnion->get_attr_table()),
317  // TODO there really should be const version of this in libdap::DDS
318  const_cast<DDS*>(pDDS)->get_attr_table());
319 
320  // Merge in the variables, which carry their tables along with them since the AttrTable is
321  // within the variable's "namespace", or lexical scope.
322  unionAllVariablesInto(pOutputUnion, *pDDS);
323  }
324 }
325 
326 void AggregationUtil::unionAttrsInto(AttrTable* pOut, const AttrTable& fromTableIn)
327 {
328  // semantically const
329  AttrTable& fromTable = const_cast<AttrTable&>(fromTableIn);
330  AttrTable::Attr_iter endIt = fromTable.attr_end();
331  AttrTable::Attr_iter it;
332  for (it = fromTable.attr_begin(); it != endIt; ++it) {
333  const string& name = fromTable.get_name(it);
334  AttrTable::Attr_iter attrInOut;
335  bool foundIt = findAttribute(*pOut, name, attrInOut);
336  // If it's already in the output, then skip it
337  if (foundIt) {
338  BESDEBUG("ncml",
339  "Union of AttrTable: an attribute named " << name << " already exist in output, skipping it..." << endl);
340  continue;
341  }
342  else // put a copy of it into the output
343  {
344  // containers need deep copy
345  if (fromTable.is_container(it)) {
346  AttrTable* pOrigAttrContainer = fromTable.get_attr_table(it);
347  NCML_ASSERT_MSG(pOrigAttrContainer,
348  "AggregationUtil::mergeAttrTables(): expected non-null AttrTable for the attribute container: "
349  + name);
350  AttrTable* pClonedAttrContainer = new AttrTable(*pOrigAttrContainer);
351  VALID_PTR(pClonedAttrContainer);
352  pOut->append_container(pClonedAttrContainer, name);
353  BESDEBUG("ncml",
354  "Union of AttrTable: adding a deep copy of attribute=" << name << " to the merged output." << endl);
355  }
356  else // for a simple type
357  {
358  string type = fromTable.get_type(it);
359  vector<string>* pAttrTokens = fromTable.get_attr_vector(it);
360  VALID_PTR(pAttrTokens);
361  // append_attr makes a copy of the vector, so we don't have to do so here.
362  pOut->append_attr(name, type, pAttrTokens);
363  }
364  }
365  }
366 }
367 
368 bool AggregationUtil::findAttribute(const AttrTable& inTable, const string& name, AttrTable::Attr_iter& attr)
369 {
370  AttrTable& inTableSemanticConst = const_cast<AttrTable&>(inTable);
371  attr = inTableSemanticConst.simple_find(name);
372  return (attr != inTableSemanticConst.attr_end());
373 }
374 
375 void AggregationUtil::unionAllVariablesInto(libdap::DDS* pOutputUnion, const ConstDDSList& datasetsInOrder)
376 {
377  ConstDDSList::const_iterator endIt = datasetsInOrder.end();
378  ConstDDSList::const_iterator it;
379  for (it = datasetsInOrder.begin(); it != endIt; ++it) {
380  unionAllVariablesInto(pOutputUnion, *(*it));
381  }
382 }
383 
384 void AggregationUtil::unionAllVariablesInto(libdap::DDS* pOutputUnion, const libdap::DDS& fromDDS, bool add_at_top)
385 {
386  DDS& dds = const_cast<DDS&>(fromDDS); // semantically const
387  DDS::Vars_iter endIt = dds.var_end();
388  DDS::Vars_iter it;
389  for (it = dds.var_begin(); it != endIt; ++it) {
390  BaseType* var = *it;
391  if (var) {
392  bool addedVar = addCopyOfVariableIfNameIsAvailable(pOutputUnion, *var, add_at_top);
393  if (addedVar) {
394  BESDEBUG("ncml", "Variable name=" << var->name() << " wasn't in the union yet and was added." << endl);
395  }
396  else {
397  BESDEBUG("ncml",
398  "Variable name=" << var->name() << " was already in the union and was skipped." << endl);
399  }
400  }
401  }
402 }
403 
404 // This method is used to 'initialize' a new JoinExisting aggregation so that
405 // A set of Coordinate Variables (CVs) will be inserted _in the order they are
406 // listed_ in the .ncml file.
408 {
409  //cerr << "Called resetCVInsertionPosition" << endl;
410  d_last_added_cv_position = 0;
411 }
412 
413 bool AggregationUtil::addCopyOfVariableIfNameIsAvailable(libdap::DDS* pOutDDS, const libdap::BaseType& varProto,
414  bool add_at_top)
415 {
416  bool ret = false;
417  BaseType* existingVar = findVariableAtDDSTopLevel(*pOutDDS, varProto.name());
418  if (!existingVar) {
419  // Add the var. add_var does a clone, so we don't need to.
420  BESDEBUG("ncml2", "AggregationUtil::addCopyOfVariableIfNameIsAvailable: " << varProto.name() << endl);
421  if (add_at_top) {
422  // This provides a way to remember where the last CV was inserted and adds
423  // this one after it. That provides the behavior that all of the CVs are
424  // added at the beginning of the DDS but in the order they appear in the NCML.
425  // That will translate into a greater chance of success for users, I think ...
426  //
427  // See also similar code in AggregationElement::createAndAddCoordinateVariableForNewDimensio
428  // jhrg 10/17/11
429  //cerr << "d_last_added_cv_position: " << d_last_added_cv_position << endl;
430  DDS::Vars_iter pos = pOutDDS->var_begin() + d_last_added_cv_position;
431 
432  pOutDDS->insert_var(pos, const_cast<BaseType*>(&varProto));
433 
434  ++d_last_added_cv_position;
435  }
436  else {
437  pOutDDS->add_var(const_cast<BaseType*>(&varProto));
438  }
439 
440  ret = true;
441  }
442  return ret;
443 }
444 
445 void AggregationUtil::addOrReplaceVariableForName(libdap::DDS* pOutDDS, const libdap::BaseType& varProto)
446 {
447  BaseType* existingVar = findVariableAtDDSTopLevel(*pOutDDS, varProto.name());
448 
449  // If exists, nuke it.
450  if (existingVar) {
451  pOutDDS->del_var(varProto.name());
452  }
453 
454  // Add the var. add_var does a clone, so we don't need to clone it here.
455  pOutDDS->add_var(const_cast<BaseType*>(&varProto));
456 }
457 
458 libdap::BaseType*
459 AggregationUtil::findVariableAtDDSTopLevel(const libdap::DDS& dds_const, const string& name)
460 {
461  BaseType* ret = 0;
462  DDS& dds = const_cast<DDS&>(dds_const); // semantically const
463  DDS::Vars_iter endIt = dds.var_end();
464  DDS::Vars_iter it;
465  for (it = dds.var_begin(); it != endIt; ++it) {
466  BaseType* var = *it;
467  if (var && var->name() == name) {
468  ret = var;
469  break;
470  }
471  }
472  return ret;
473 }
474 
475 template<class LibdapType>
476 LibdapType*
477 AggregationUtil::findTypedVariableAtDDSTopLevel(const libdap::DDS& dds, const string& name)
478 {
479  BaseType* pBT = findVariableAtDDSTopLevel(dds, name);
480  if (pBT) {
481  return dynamic_cast<LibdapType*>(pBT);
482  }
483  else {
484  return 0;
485  }
486 }
487 
488 #if 0
489 void AggregationUtil::produceOuterDimensionJoinedArray(Array* pJoinedArray, const std::string& joinedArrayName,
490  const std::string& newOuterDimName, const std::vector<libdap::Array*>& fromVars, bool copyData)
491 {
492  string funcName = "AggregationUtil::createOuterDimensionJoinedArray:";
493 
494  NCML_ASSERT_MSG(fromVars.size() > 0, funcName + "Must be at least one Array in input!");
495 
496  // uses the first one as template for type and shape
497  if (!validateArrayTypesAndShapesMatch(fromVars, true)) {
498  throw AggregationException(
499  funcName + " The input arrays must all have the same data type and dimensions but do not!");
500  }
501 
502  // The first will be used to "set up" the pJoinedArray
503  Array* templateArray = fromVars[0];
504  VALID_PTR(templateArray);
505  BaseType* templateVar = templateArray->var();
506  NCML_ASSERT_MSG(templateVar, funcName + "Expected a non-NULL prototype BaseType in the first Array!");
507 
508  // Set the template var for the type.
509  pJoinedArray->add_var(templateVar);
510  // and force the name to be the desired one, not the prototype's
511  pJoinedArray->set_name(joinedArrayName);
512 
513  // Copy the attribute table from the template over... We're not merging or anything.
514  pJoinedArray->set_attr_table(templateArray->get_attr_table());
515 
516  // Create a new outer dimension based on the number of inputs we have.
517  // These append_dim calls go left to right, so we need to add the new dim FIRST.
518  pJoinedArray->append_dim(fromVars.size(), newOuterDimName);
519 
520  // Use the template to add inner dimensions to the new array
521  for (Array::Dim_iter it = templateArray->dim_begin(); it != templateArray->dim_end(); ++it) {
522  int dimSize = templateArray->dimension_size(it);
523  string dimName = templateArray->dimension_name(it);
524  pJoinedArray->append_dim(dimSize, dimName);
525  }
526 
527  if (copyData) {
528  // Make sure we have capacity for the full length of the up-ranked shape.
529  pJoinedArray->reserve_value_capacity(pJoinedArray->length());
530  // Glom the data together in
531  joinArrayData(pJoinedArray, fromVars, false, // we already reserved the space
532  true); // but please clear the Vector buffers after you use each Array in fromVars to help on memory.
533  }
534 }
535 #endif
536 
537 bool AggregationUtil::validateArrayTypesAndShapesMatch(const std::vector<libdap::Array*>& arrays,
538  bool enforceMatchingDimNames)
539 {
540  NCML_ASSERT(arrays.size() > 0);
541  bool valid = true;
542  Array* pTemplate = 0;
543  for (vector<Array*>::const_iterator it = arrays.begin(); it != arrays.end(); ++it) {
544  // Set the template from the first one.
545  if (!pTemplate) {
546  pTemplate = *it;
547  VALID_PTR(pTemplate);
548  continue;
549  }
550 
551  valid = (valid && doTypesMatch(*pTemplate, **it) && doShapesMatch(*pTemplate, **it, enforceMatchingDimNames));
552  // No use continuing
553  if (!valid) {
554  break;
555  }
556  }
557  return valid;
558 }
559 
560 bool AggregationUtil::doTypesMatch(const libdap::Array& lhsC, const libdap::Array& rhsC)
561 {
562  // semantically const
563  Array& lhs = const_cast<Array&>(lhsC);
564  Array& rhs = const_cast<Array&>(rhsC);
565  return (lhs.var() && rhs.var() && lhs.var()->type() == rhs.var()->type());
566 }
567 
568 bool AggregationUtil::doShapesMatch(const libdap::Array& lhsC, const libdap::Array& rhsC, bool checkDimNames)
569 {
570  // semantically const
571  Array& lhs = const_cast<Array&>(lhsC);
572  Array& rhs = const_cast<Array&>(rhsC);
573 
574  // Check the number of dims matches first.
575  bool valid = true;
576  if (lhs.dimensions() != rhs.dimensions()) {
577  valid = false;
578  }
579  else {
580  // Then iterate on both in sync and compare.
581  Array::Dim_iter rhsIt = rhs.dim_begin();
582  for (Array::Dim_iter lhsIt = lhs.dim_begin(); lhsIt != lhs.dim_end(); (++lhsIt, ++rhsIt)) {
583  valid = (valid && (lhs.dimension_size(lhsIt) == rhs.dimension_size(rhsIt)));
584 
585  if (checkDimNames) {
586  valid = (valid && (lhs.dimension_name(lhsIt) == rhs.dimension_name(rhsIt)));
587  }
588  }
589  }
590  return valid;
591 }
592 
593 unsigned int AggregationUtil::collectVariableArraysInOrder(std::vector<Array*>& varArrays,
594  const std::string& collectVarName, const ConstDDSList& datasetsInOrder)
595 {
596  unsigned int count = 0;
597  ConstDDSList::const_iterator endIt = datasetsInOrder.end();
598  ConstDDSList::const_iterator it;
599  for (it = datasetsInOrder.begin(); it != endIt; ++it) {
600  DDS* pDDS = const_cast<DDS*>(*it);
601  VALID_PTR(pDDS);
602  Array* pVar = dynamic_cast<Array*>(findVariableAtDDSTopLevel(*pDDS, collectVarName));
603  if (pVar) {
604  varArrays.push_back(pVar);
605  count++;
606  }
607  }
608  return count;
609 }
610 
612 {
613  Array* pArr = dynamic_cast<Array*>(pBT);
614  if (pArr && (pArr->dimensions() == 1)) {
615  // only one dimension, so grab the first and make sure we only got one.
616  Array::Dim_iter it = pArr->dim_begin();
617  bool matches = (pArr->dimension_name(it) == pArr->name());
618  NCML_ASSERT_MSG((++it == pArr->dim_end()),
619  "Logic error: NCMLUtil::isCoordinateVariable(): expected one dimension from Array, but got more!");
620  return matches;
621  }
622  else {
623  return false;
624  }
625 }
626 
627 #if 0
628 void AggregationUtil::joinArrayData(Array* pAggArray, const std::vector<Array*>& varArrays,
629  bool reserveStorage/*=true*/, bool clearDataAfterUse/*=false*/)
630 {
631  // Make sure we get a pAggArray with a type var we can deal with.
632  VALID_PTR(pAggArray->var());
633  NCML_ASSERT_MSG(pAggArray->var()->is_simple_type(),
634  "AggregationUtil::joinArrayData: the output Array is not of a simple type! Can't aggregate!");
635 
636  // If the caller wants us to do it, sum up length() and reserve that much.
637  if (reserveStorage) {
638  // Figure it how much we need...
639  unsigned int totalLength = 0;
640  {
641  vector<Array*>::const_iterator it;
642  vector<Array*>::const_iterator endIt = varArrays.end();
643  for (it = varArrays.begin(); it != endIt; ++it) {
644  Array* pArr = *it;
645  if (pArr) {
646  totalLength += pArr->length();
647  }
648  }
649  }
650  pAggArray->reserve_value_capacity(totalLength);
651  }
652 
653  // For each Array, make sure it's read in and copy its data into the output.
654  unsigned int nextElt = 0; // keeps track of where we are to write next in the output
655  vector<Array*>::const_iterator it;
656  vector<Array*>::const_iterator endIt = varArrays.end();
657  for (it = varArrays.begin(); it != endIt; ++it) {
658  Array* pArr = *it;
659  VALID_PTR(pArr);
660  NCML_ASSERT_MSG(pArr->var() && (pArr->var()->type() == pAggArray->var()->type()),
661  "AggregationUtil::joinArrayData: one of the arrays to join has different type than output! Can't aggregate!");
662 
663  // Make sure it was read in...
664  if (!pArr->read_p()) {
665  pArr->read();
666  }
667 
668  // Copy it in with the Vector call and update our location
669  nextElt += pAggArray->set_value_slice_from_row_major_vector(*pArr, nextElt);
670 
671  if (clearDataAfterUse) {
672  pArr->clear_local_data();
673  }
674  }
675 
676  // That's all folks!
677 }
678 #endif
679 
681 // struct dimension
682 // {
683 // int size; ///< The unconstrained dimension size.
684 // string name; ///< The name of this dimension.
685 // int start; ///< The constraint start index
686 // int stop; ///< The constraint end index
687 // int stride; ///< The constraint stride
688 // int c_size; ///< Size of dimension once constrained
689 // };
690 
692 void AggregationUtil::printDimensions(std::ostream& os, const libdap::Array& fromArray)
693 {
694  os << "Array dimensions: " << endl;
695  Array& theArray = const_cast<Array&>(fromArray);
696  Array::Dim_iter it;
697  Array::Dim_iter endIt = theArray.dim_end();
698  for (it = theArray.dim_begin(); it != endIt; ++it) {
699  Array::dimension d = *it;
700  os << "Dim = {" << endl;
701  os << "name=" << d.name << endl;
702  os << "size=" << d.size << endl;
703  os << " }" << endl;
704  }
705  os << "End Array dimensions." << endl;
706 }
707 
708 void AggregationUtil::printConstraints(std::ostream& os, const Array& rcArray)
709 {
710  os << "Array constraints: " << endl;
711  Array& theArray = const_cast<Array&>(rcArray);
712  Array::Dim_iter it;
713  Array::Dim_iter endIt = theArray.dim_end();
714  for (it = theArray.dim_begin(); it != endIt; ++it) {
715  Array::dimension d = *it;
716  os << "Dim = {" << endl;
717  os << "name=" << d.name << endl;
718  os << "start=" << d.start << endl;
719  os << "stride=" << d.stride << endl;
720  os << "stop=" << d.stop << endl;
721  os << " }" << endl;
722  }
723  os << "End Array constraints" << endl;
724 }
725 
726 void AggregationUtil::printConstraintsToDebugChannel(const std::string& debugChannel, const libdap::Array& fromArray)
727 {
728  ostringstream oss;
729  BESDEBUG(debugChannel, "Printing constraints for Array: " << fromArray.name() << ": " << oss.str() << endl);
730  AggregationUtil::printConstraints(oss, fromArray);
731  BESDEBUG(debugChannel, oss.str() << endl);
732 }
733 
734 void AggregationUtil::transferArrayConstraints(Array* pToArray, const Array& fromArrayConst, bool skipFirstFromDim,
735  bool skipFirstToDim, bool printDebug /* = false */, const std::string& debugChannel /* = "agg_util" */)
736 {
737  VALID_PTR(pToArray);
738  Array& fromArray = const_cast<Array&>(fromArrayConst);
739 
740  // Make sure there's no constraints on output. Shouldn't be, but...
741  pToArray->reset_constraint();
742 
743  // Ensure the dimensionalities will work out.
744  int skipDelta = ((skipFirstFromDim) ? (1) : (0));
745  // If skipping output as well, subtract out the delta.
746  // If we go negative, also an error.
747  if (skipFirstToDim) {
748  skipDelta -= 1;
749  }
750  if (skipDelta < 0 || (pToArray->dimensions() + skipDelta != const_cast<Array&>(fromArrayConst).dimensions())) {
751  throw AggregationException("AggregationUtil::transferArrayConstraints: "
752  "Mismatched dimensionalities!");
753  }
754 
755  if (printDebug) {
756  BESDEBUG(debugChannel,
757  "Printing constraints on fromArray name= " << fromArray.name() << " before transfer..." << endl);
758  printConstraintsToDebugChannel(debugChannel, fromArray);
759  }
760 
761  // Only real way to the constraints is with the iterator,
762  // so we'll iterator on the fromArray and move
763  // to toarray iterator in sync.
764  Array::Dim_iter fromArrIt = fromArray.dim_begin();
765  Array::Dim_iter fromArrEndIt = fromArray.dim_end();
766  Array::Dim_iter toArrIt = pToArray->dim_begin();
767  for (; fromArrIt != fromArrEndIt; ++fromArrIt) {
768  if (skipFirstFromDim && (fromArrIt == fromArray.dim_begin())) {
769  // If we skip first to array as well, increment
770  // before the next call.
771  if (skipFirstToDim) {
772  ++toArrIt;
773  }
774  continue;
775  }
776 // If aggregates with renaming dimensions do not match each other. SK 07/26/18
777 // NCML_ASSERT_MSG(fromArrIt->name == toArrIt->name, "GAggregationUtil::transferArrayConstraints: "
778 // "Expected the dimensions to have the same name but they did not.");
779  pToArray->add_constraint(toArrIt, fromArrIt->start, fromArrIt->stride, fromArrIt->stop);
780  ++toArrIt;
781  }
782 
783  if (printDebug) {
784  BESDEBUG(debugChannel, "Printing constrains on pToArray after transfer..." << endl);
785  printConstraintsToDebugChannel(debugChannel, *pToArray);
786  }
787 }
788 
789 BaseType*
790 AggregationUtil::getVariableNoRecurse(const libdap::DDS& ddsConst, const std::string& name)
791 {
792  BaseType* ret = 0;
793  DDS& dds = const_cast<DDS&>(ddsConst); // semantically const
794  DDS::Vars_iter endIt = dds.var_end();
795  DDS::Vars_iter it;
796  for (it = dds.var_begin(); it != endIt; ++it) {
797  BaseType* var = *it;
798  if (var && var->name() == name) {
799  ret = var;
800  break;
801  }
802  }
803  return ret;
804 }
805 
806 // Ugh cut and pasted from the other...
807 // TODO REFACTOR DDS and Constructor really need a common abstract interface,
808 // like IVariableContainer that declares the iterators and associated methods.
809 BaseType*
810 AggregationUtil::getVariableNoRecurse(const libdap::Constructor& varContainerConst, const std::string& name)
811 {
812  BaseType* ret = 0;
813  Constructor& varContainer = const_cast<Constructor&>(varContainerConst); // semantically const
814  Constructor::Vars_iter endIt = varContainer.var_end();
815  Constructor::Vars_iter it;
816  for (it = varContainer.var_begin(); it != endIt; ++it) {
817  BaseType* var = *it;
818  if (var && var->name() == name) {
819  ret = var;
820  break;
821  }
822  }
823  return ret;
824 }
825 
826 /*static*/
827 Array*
829 {
830  if (!pBT) {
831  return 0;
832  }
833 
834  // After switch():
835  // if Array, will be cast to Array.
836  // if Grid, will be cast data Array member of Grid.
837  // Other types, will be null.
838  libdap::Array* pArray(0);
839  switch (pBT->type()) {
840  case libdap::dods_array_c:
841  pArray = static_cast<libdap::Array*>(pBT);
842  break;
843 
844  case libdap::dods_grid_c:
845  pArray = static_cast<Grid*>(pBT)->get_array();
846  break;
847 
848  default:
849  pArray = 0;
850  break;
851  }
852  return pArray;
853 }
854 
855 const Array*
856 AggregationUtil::findMapByName(const libdap::Grid& inGrid, const string& findName)
857 {
858  Grid& grid = const_cast<Grid&>(inGrid);
859  Array* pRet = 0;
860  Grid::Map_iter it;
861  Grid::Map_iter endIt = grid.map_end();
862  for (it = grid.map_begin(); it != endIt; ++it) {
863  if ((*it)->name() == findName) {
864  pRet = static_cast<Array*>(*it);
865  break;
866  }
867  }
868  return pRet;
869 }
870 
871 Array* AggregationUtil::readDatasetArrayDataForAggregation(const Array& constrainedTemplateArray,
872  const std::string& varName, AggMemberDataset& dataset, const ArrayGetterInterface& arrayGetter,
873  const std::string& debugChannel)
874 {
875  BESStopWatch sw;
876  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("AggregationUtil::readDatasetArrayDataForAggregation", "");
877 
878  const libdap::DDS* pDDS = dataset.getDDS();
879  NCML_ASSERT_MSG(pDDS, "GridAggregateOnOuterDimension::read(): Got a null DataDDS "
880  "while loading dataset = " + dataset.getLocation());
881 
882  // Grab the Array from the DataDDS with the getter
883  Array* pDatasetArray = arrayGetter.readAndGetArray(varName, *pDDS, &constrainedTemplateArray, debugChannel);
884  NCML_ASSERT_MSG(pDatasetArray, "In aggregation member dataset, failed to get the array! "
885  "Dataset location = " + dataset.getLocation());
886 
887  // Make sure that the data was read in or I dunno what went on.
888  if (!pDatasetArray->read_p()) {
889  NCML_ASSERT_MSG(pDatasetArray->read_p(),
890  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: pDatasetArray was not read_p()!");
891  }
892 
893  // Make sure it matches the prototype or somthing went wrong
894  if (!AggregationUtil::doTypesMatch(constrainedTemplateArray, *pDatasetArray)) {
895  throw AggregationException(
896  "Invalid aggregation! "
897  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
898  "We found the aggregation variable name=" + varName
899  + " but it was not of the same type as the prototype variable!");
900  }
901 
902  // Make sure the subshapes match! (true means check dimension names too... debate this)
903  if (!AggregationUtil::doShapesMatch(constrainedTemplateArray, *pDatasetArray, true)) {
904  throw AggregationException(
905  "Invalid aggregation! "
906  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
907  "We found the aggregation variable name=" + varName
908  + " but it was not of the same shape as the prototype!");
909  }
910 
911  // Make sure the length of the data array also matches the proto
912  if (constrainedTemplateArray.length() != pDatasetArray->length()) {
913  NCML_ASSERT_MSG(constrainedTemplateArray.length() == pDatasetArray->length(),
914  "AggregationUtil::addDatasetArrayDataToAggregationOutputArray: "
915  "The prototype array and the loaded dataset array length()'s were not equal, even "
916  "though their shapes matched. Logic problem.");
917  }
918 
919  return pDatasetArray;
920 }
921 
922 void AggregationUtil::addDatasetArrayDataToAggregationOutputArray(libdap::Array& oOutputArray, unsigned int atIndex,
923  const Array& constrainedTemplateArray, const std::string& varName, AggMemberDataset& dataset,
924  const ArrayGetterInterface& arrayGetter, const std::string& debugChannel)
925 {
926  BESStopWatch sw;
927  if (BESDebug::IsSet(TIMING_LOG_KEY)) sw.start("AggregationUtil::addDatasetArrayDataToAggregationOutputArray", "");
928 
929  libdap::Array* pDatasetArray = readDatasetArrayDataForAggregation(constrainedTemplateArray, varName, dataset, arrayGetter,
930  debugChannel);
931 
932  // FINALLY, we get to stream the data!
933  oOutputArray.set_value_slice_from_row_major_vector(*pDatasetArray, atIndex);
934 
935  // Now that we have copied it - let the memory go! Free! Let the bytes be freed! - ndp 08/12/2015
936  pDatasetArray->clear_local_data();
937 }
938 
939 void AggregationUtil::gatherMetadataChangesFrom(BaseType* pIntoVar, const BaseType& fromVarC)
940 {
941  BaseType& fromVar = const_cast<BaseType&>(fromVarC); //semantic const
942  // The output will end up here.
943  AttrTable finalAT;
944 
945  // First, seed it with the changes in the fromVar.
946  unionAttrsInto(&finalAT, fromVar.get_attr_table());
947 
948  // Then union in the items from the to var
949  unionAttrsInto(&finalAT, pIntoVar->get_attr_table());
950 
951  // HACK BUG In the set_attr_table call through AttrTable operator=
952  // means we keep bad memory around. Workaround until fixed!
953  pIntoVar->get_attr_table().erase();
954 
955  // Finally, replace the output var's table with the constructed one!
956  pIntoVar->set_attr_table(finalAT);
957 }
958 
959 } // namespace agg_util
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:168
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
const std::string & getLocation() const
virtual const libdap::DDS * getDDS()=0
static void addDatasetArrayDataToAggregationOutputArray(libdap::Array &oOutputArray, unsigned int atIndex, const libdap::Array &constrainedTemplateArray, const string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const string &debugChannel)
static void printConstraintsToDebugChannel(const std::string &debugChannel, const libdap::Array &fromArray)
static void gatherMetadataChangesFrom(libdap::BaseType *pIntoVar, const libdap::BaseType &fromVar)
static bool doTypesMatch(const libdap::Array &lhs, const libdap::Array &rhs)
static unsigned int collectVariableArraysInOrder(std::vector< libdap::Array * > &varArrays, const std::string &collectVarName, const ConstDDSList &datasetsInOrder)
static LibdapType * findTypedVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
static void transferArrayConstraints(libdap::Array *pToArray, const libdap::Array &fromArray, bool skipFirstFromDim, bool skipFirstToDim, bool printDebug=false, const std::string &debugChannel="agg_util")
static void printDimensions(std::ostream &os, const libdap::Array &fromArray)
static libdap::Array * readDatasetArrayDataForAggregation(const libdap::Array &constrainedTemplateArray, const std::string &varName, AggMemberDataset &dataset, const ArrayGetterInterface &arrayGetter, const std::string &debugChannel)
static void performUnionAggregation(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
static libdap::BaseType * getVariableNoRecurse(const libdap::DDS &dds, const std::string &name)
static void unionAllVariablesInto(libdap::DDS *pOutputUnion, const ConstDDSList &datasetsInOrder)
static libdap::Array * getAsArrayIfPossible(libdap::BaseType *pBT)
static bool couldBeCoordinateVariable(libdap::BaseType *pBT)
static const libdap::Array * findMapByName(const libdap::Grid &inGrid, const std::string &findName)
static bool addCopyOfVariableIfNameIsAvailable(libdap::DDS *pOutDDS, const libdap::BaseType &varProto, bool add_at_top=false)
static void addOrReplaceVariableForName(libdap::DDS *pOutDDS, const libdap::BaseType &varProto)
static void unionAttrsInto(libdap::AttrTable *pOut, const libdap::AttrTable &fromTable)
static bool doShapesMatch(const libdap::Array &lhs, const libdap::Array &rhs, bool checkDimNames)
static bool findAttribute(const libdap::AttrTable &inTable, const string &name, libdap::AttrTable::Attr_iter &attr)
static void resetCVInsertionPosition()
static bool validateArrayTypesAndShapesMatch(const std::vector< libdap::Array * > &arrays, bool enforceMatchingDimNames)
static void printConstraints(std::ostream &os, const libdap::Array &fromArray)
static libdap::BaseType * findVariableAtDDSTopLevel(const libdap::DDS &dds, const string &name)
Helper class for temporarily hijacking an existing dhi to load a DDX response for one particular file...
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const =0
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const
virtual TopLevelGridDataArrayGetter * clone() const
virtual TopLevelGridMapArrayGetter * clone() const
virtual libdap::Array * readAndGetArray(const std::string &name, const libdap::DDS &dds, const libdap::Array *const pConstraintTemplate, const std::string &debugChannel) const