bes  Updated for version 3.20.8
BESDapResponseBuilder.cc
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of libdap, A C++ implementation of the OPeNDAP Data
4 // Access Protocol.
5 
6 // Copyright (c) 2011 OPeNDAP, Inc.
7 // Author: James Gallagher <jgallagher@opendap.org>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
24 
25 #include "config.h"
26 
27 #include <signal.h>
28 #include <unistd.h>
29 #include <sys/stat.h>
30 
31 #ifdef HAVE_UUID_UUID_H
32 #include <uuid/uuid.h> // used to build CID header value for data ddx
33 #elif defined(HAVE_UUID_H)
34 #include <uuid.h>
35 #else
36 #error "Could not find UUID library header"
37 #endif
38 
39 
40 #ifndef WIN32
41 #include <sys/wait.h>
42 #else
43 #include <io.h>
44 #include <fcntl.h>
45 #include <process.h>
46 #endif
47 
48 #include <iostream>
49 #include <string>
50 #include <sstream>
51 #include <fstream>
52 
53 #include <cstring>
54 #include <ctime>
55 
56 //#define DODS_DEBUG
57 #define CLEAR_LOCAL_DATA
58 #undef USE_LOCAL_TIMEOUT_SCHEME
59 
60 #include <DAS.h>
61 #include <DDS.h>
62 #include <Structure.h>
63 #include <ConstraintEvaluator.h>
64 #include <DDXParserSAX2.h>
65 #include <Ancillary.h>
66 #include <XDRStreamMarshaller.h>
67 #include <XDRFileUnMarshaller.h>
68 
69 #include <DMR.h>
70 #include <D4Group.h>
71 #include <D4Attributes.h>
72 #include <XMLWriter.h>
73 #include <D4AsyncUtil.h>
74 #include <D4StreamMarshaller.h>
75 #include <chunked_ostream.h>
76 #include <chunked_istream.h>
77 #include <D4ConstraintEvaluator.h>
78 #include <D4FunctionEvaluator.h>
79 #include <D4BaseTypeFactory.h>
80 
81 #include <ServerFunctionsList.h>
82 
83 #include <mime_util.h> // for last_modified_time() and rfc_822_date()
84 #include <escaping.h>
85 #include <util.h>
86 
87 #if USE_LOCAL_TIMEOUT_SCHEME
88 #ifndef WIN32
89 #include <SignalHandler.h>
90 #include <EventHandler.h>
91 #include <AlarmHandler.h>
92 #endif
93 #endif
94 
95 #include "TheBESKeys.h"
96 #include "BESDapResponseBuilder.h"
97 #include "BESContextManager.h"
98 #include "BESDapFunctionResponseCache.h"
99 #include "BESStoredDapResultCache.h"
100 
101 
102 #include "BESResponseObject.h"
103 #include "BESDDSResponse.h"
104 #include "BESDataDDSResponse.h"
105 #include "BESDMRResponse.h"
106 #include "BESDataHandlerInterface.h"
107 #include "BESInternalFatalError.h"
108 #include "BESSyntaxUserError.h"
109 #include "BESDataNames.h"
110 
111 #include "BESRequestHandler.h"
112 #include "BESRequestHandlerList.h"
113 #include "BESNotFoundError.h"
114 
115 #include "BESUtil.h"
116 #include "BESDebug.h"
117 #include "BESLog.h"
118 #include "BESStopWatch.h"
119 #include "DapFunctionUtils.h"
120 
121 using namespace std;
122 using namespace libdap;
123 
124 const string CRLF = "\r\n"; // Change here, expr-test.cc
125 const string BES_KEY_TIMEOUT_CANCEL = "BES.CancelTimeoutOnSend";
126 
127 #define MODULE "dap"
128 #define prolog std::string("BESDapResponseBuilder::").append(__func__).append("() - ")
129 
135 {
136  bool found = false;
137  string cancel_timeout_on_send = "";
138  TheBESKeys::TheKeys()->get_value(BES_KEY_TIMEOUT_CANCEL, cancel_timeout_on_send, found);
139  if (found && !cancel_timeout_on_send.empty()) {
140  // The default value is false.
141  downcase(cancel_timeout_on_send);
142  if (cancel_timeout_on_send == "yes" || cancel_timeout_on_send == "true")
143  d_cancel_timeout_on_send = true;
144  }
145 }
146 
147 BESDapResponseBuilder::~BESDapResponseBuilder()
148 {
149 #if USE_LOCAL_TIMEOUT_SCHEME
150  // If an alarm was registered, delete it. The register code in SignalHandler
151  // always deletes the old alarm handler object, so only the one returned by
152  // remove_handler needs to be deleted at this point.
153  delete dynamic_cast<AlarmHandler*>(SignalHandler::instance()->remove_handler(SIGALRM));
154 #endif
155 }
156 
164 {
165  return d_dap2ce;
166 }
167 
179 {
180  d_dap2ce = www2id(_ce, "%", "%20");
181 }
182 
187 {
188  return d_dap4ce;
189 }
190 
202 {
203  d_dap4ce = www2id(_ce, "%", "%20");
204 }
205 
210 {
211  return d_dap4function;
212 }
213 
226 {
227  d_dap4function = www2id(_func, "%", "%20");
228 }
229 
230 std::string BESDapResponseBuilder::get_store_result() const
231 {
232  return d_store_result;
233 }
234 
235 void BESDapResponseBuilder::set_store_result(std::string _sr)
236 {
237  d_store_result = _sr;
238  BESDEBUG(MODULE, prolog << "store_result: " << _sr << endl);
239 }
240 
241 std::string BESDapResponseBuilder::get_async_accepted() const
242 {
243  return d_async_accepted;
244 }
245 
246 void BESDapResponseBuilder::set_async_accepted(std::string _aa)
247 {
248  d_async_accepted = _aa;
249  BESDEBUG(MODULE, prolog << "set_async_accepted() - async_accepted: " << _aa << endl);
250 }
251 
261 {
262  return d_dataset;
263 }
264 
276 {
277  d_dataset = www2id(ds, "%", "%20");
278 }
279 
286 {
287  d_timeout = t;
288 }
289 
292 {
293  return d_timeout;
294 }
295 
302 void
304 {
305 #if USE_LOCAL_TIMEOUT_SCHEME
306 #ifndef WIN32
307  alarm(d_timeout);
308 #endif
309 #endif
310 }
311 
317 void
319 {
320 #if USE_LOCAL_TIMEOUT_SCHEME
321 #ifndef WIN32
322  alarm(0);
323 #endif
324 #endif
325 }
326 
342 {
343  if (d_cancel_timeout_on_send)
344  alarm(0);
345 }
346 
356 {
357 #if USE_LOCAL_TIMEOUT_SCHEME
358 #ifndef WIN32
359  SignalHandler *sh = SignalHandler::instance();
360  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
361  delete old_eh;
362 #endif
363 #endif
364 }
365 
366 
373 {
374 #if USE_LOCAL_TIMEOUT_SCHEME
375 #ifndef WIN32
376  if (d_timeout > 0) {
377  SignalHandler *sh = SignalHandler::instance();
378  EventHandler *old_eh = sh->register_handler(SIGALRM, new AlarmHandler());
379  delete old_eh;
380  alarm(d_timeout);
381  }
382 #endif
383 #endif
384 }
385 
400 static string::size_type find_closing_paren(const string &ce, string::size_type pos)
401 {
402  // Iterate over the string finding all ( or ) characters until the matching ) is found.
403  // For each ( found, increment count. When a ) is found and count is zero, it is the
404  // matching closing paren, otherwise, decrement count and keep looking.
405  int count = 1;
406  do {
407  pos = ce.find_first_of("()", pos + 1);
408  if (pos == string::npos)
409  throw Error(malformed_expr, "Expected to find a matching closing parenthesis in " + ce);
410 
411  if (ce[pos] == '(')
412  ++count;
413  else
414  --count; // must be ')'
415 
416  } while (count > 0);
417 
418  return pos;
419 }
420 
427 void BESDapResponseBuilder::split_ce(ConstraintEvaluator &eval, const string &expr)
428 {
429  BESDEBUG(MODULE, prolog << "source expression: " << expr << endl);
430 
431  string ce;
432  if (!expr.empty())
433  ce = expr;
434  else
435  ce = d_dap2ce;
436 
437  string btp_function_ce = "";
438  string::size_type pos = 0;
439 
440  // This hack assumes that the functions are listed first. Look for the first
441  // open paren and the last closing paren to accommodate nested function calls
442  string::size_type first_paren = ce.find("(", pos);
443  string::size_type closing_paren = string::npos;
444  if (first_paren != string::npos) closing_paren = find_closing_paren(ce, first_paren); //ce.find(")", pos);
445 
446  while (first_paren != string::npos && closing_paren != string::npos) {
447  // Maybe a BTP function; get the name of the potential function
448  string name = ce.substr(pos, first_paren - pos);
449 
450  // is this a BTP function
451  btp_func f;
452  if (eval.find_function(name, &f)) {
453  // Found a BTP function
454  if (!btp_function_ce.empty()) btp_function_ce += ",";
455  btp_function_ce += ce.substr(pos, closing_paren + 1 - pos);
456  ce.erase(pos, closing_paren + 1 - pos);
457  if (ce[pos] == ',') ce.erase(pos, 1);
458  }
459  else {
460  pos = closing_paren + 1;
461  // exception?
462  if (pos < ce.length() && ce.at(pos) == ',') ++pos;
463  }
464 
465  first_paren = ce.find("(", pos);
466  closing_paren = ce.find(")", pos);
467  }
468 
469  d_dap2ce = ce;
470  d_btp_func_ce = btp_function_ce;
471 
472  BESDEBUG(MODULE, prolog << "Modified constraint: " << d_dap2ce << endl);
473  BESDEBUG(MODULE, prolog << "BTP Function part: " << btp_function_ce << endl);
474  BESDEBUG(MODULE, prolog << "END" << endl);
475 }
476 
483 static void
484 throw_if_dap2_response_too_big(DDS *dds)
485 {
486  if (dds->get_response_limit() != 0 && ((dds->get_request_size(true)) > dds->get_response_limit())) {
487  string msg = "The Request for " + long_to_string(dds->get_request_size(true) / 1024)
488  + "KB is too large; requests on this server are limited to "
489  + long_to_string(dds->get_response_limit() /1024) + "KB.";
490  throw Error(msg);
491  }
492 }
493 
508 void BESDapResponseBuilder::send_das(ostream &out, DAS &das, bool with_mime_headers) const
509 {
510  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
511 
512  das.print(out);
513 
514  out << flush;
515 }
516 
534 void BESDapResponseBuilder::send_das(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
535  bool with_mime_headers)
536 {
537 #if USE_LOCAL_TIMEOUT_SCHEME
538  // Set up the alarm.
539  establish_timeout(out);
540  dds.set_timeout(d_timeout);
541 #endif
542  if (!constrained) {
543  if (with_mime_headers) set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), "2.0");
544 
545  conditional_timeout_cancel();
546 
547  (*dds)->print_das(out);
548  out << flush;
549 
550  return;
551  }
552 
553  split_ce(eval);
554 
555  // If there are functions, parse them and eval.
556  // Use that DDS and parse the non-function ce
557  // Serialize using the second ce and the second dds
558  if (!d_btp_func_ce.empty()) {
559  ConstraintEvaluator func_eval;
560  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
561 
562  DDS *fdds = 0; // nulll_ptr
563  if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
564  fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
565  }
566  else {
567  func_eval.parse_constraint(get_btp_func_ce(), **dds);
568  fdds = func_eval.eval_function_clauses(**dds);
569  }
570 
571  delete *dds; *dds = 0;
572  *dds = fdds;
573 
574  if (with_mime_headers)
575  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
576 
577  conditional_timeout_cancel();
578 
579  (*dds)->print_das(out);
580  }
581  else {
582  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
583 
584  if (with_mime_headers)
585  set_mime_text(out, dods_das, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
586 
587  conditional_timeout_cancel();
588 
589  (*dds)->print_das(out);
590  }
591 
592  out << flush;
593 }
594 
595 
614 void BESDapResponseBuilder::send_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool constrained,
615  bool with_mime_headers)
616 {
617  if (!constrained) {
618  if (with_mime_headers)
619  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
620 
621  conditional_timeout_cancel();
622 
623  (*dds)->print(out);
624  out << flush;
625  return;
626  }
627 
628 #if USE_LOCAL_TIMEOUT_SCHEME
629  // Set up the alarm.
630  establish_timeout(out);
631  dds.set_timeout(d_timeout);
632 #endif
633 
634  // Split constraint into two halves
635  split_ce(eval);
636 
637  // If there are functions, parse them and eval.
638  // Use that DDS and parse the non-function ce
639  // Serialize using the second ce and the second dds
640  if (!d_btp_func_ce.empty()) {
641  ConstraintEvaluator func_eval;
642 
643  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
644 
645  DDS *fdds = 0; // nulll_ptr
646  if (responseCache && responseCache->can_be_cached(*dds, get_btp_func_ce())) {
647  fdds = responseCache->get_or_cache_dataset(*dds, get_btp_func_ce());
648  }
649  else {
650  func_eval.parse_constraint(get_btp_func_ce(), **dds);
651  fdds = func_eval.eval_function_clauses(**dds);
652  }
653 
654  delete *dds; *dds = 0;
655  *dds = fdds;
656 
657  // Server functions might mark variables to use their read()
658  // methods. Clear that so the CE in d_dap2ce will control what is
659  // sent. If that is empty (there was only a function call) all
660  // of the variables in the intermediate DDS (i.e., the function
661  // result) will be sent.
662  (*dds)->mark_all(false);
663 
664  // This next step utilizes a well known static method (so really it's a function;),
665  // promote_function_output_structures() to look for
666  // one or more top level Structures whose name indicates (by way of ending with
667  // "_unwrap") that their contents should be promoted (aka moved) to the top level.
668  // This is in support of a hack around the current API where server side functions
669  // may only return a single DAP object and not a collection of objects. The name suffix
670  // "_unwrap" is used as a signal from the function to the the various response
671  // builders and transmitters that the representation needs to be altered before
672  // transmission, and that in fact is what happens in our friend
673  // promote_function_output_structures()
674  promote_function_output_structures(*dds);
675 
676  eval.parse_constraint(d_dap2ce, **dds);
677 
678  if (with_mime_headers)
679  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
680 
681 
682  conditional_timeout_cancel();
683 
684  (*dds)->print_constrained(out);
685  }
686  else {
687  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
688 
689  if (with_mime_headers)
690  set_mime_text(out, dods_dds, x_plain, last_modified_time(d_dataset),(*dds)->get_dap_version());
691 
692  conditional_timeout_cancel();
693 
694  (*dds)->print_constrained(out);
695  }
696 
697  out << flush;
698 }
699 
700 #ifdef DAP2_STORED_RESULTS
715 bool BESDapResponseBuilder::store_dap2_result(ostream &out, DDS &dds, ConstraintEvaluator &eval)
716 {
717  if (get_store_result().empty()) return false;
718 
719  string serviceUrl = get_store_result();
720 
721  XMLWriter xmlWrtr;
722  D4AsyncUtil d4au;
723 
724  // FIXME Keys should be read in initialize(). Also, I think the D4AsyncUtil should
725  // be removed from libdap - it is much more about how the BES processes these kinds
726  // of operations. Change this when working on the response caching for ODSIP. But...
727  // do we really need to put the style sheet in the bes.conf file? Should it be baked
728  // into the code (because we don't want people to change it)?
729  bool found;
730  string *stylesheet_ref = 0, ss_ref_value;
731  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
732  if (found && ss_ref_value.length() > 0) {
733  stylesheet_ref = &ss_ref_value;
734  }
735 
737  if (resultCache == NULL) {
738 
744  string msg = "The Stored Result request cannot be serviced. ";
745  msg += "Unable to acquire StoredResultCache instance. ";
746  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
747 
748  BESDEBUG(MODULE, prolog << "[WARNING] " << msg << endl);
749 
750  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
751  out << xmlWrtr.get_doc();
752  out << flush;
753 
754  BESDEBUG(MODULE,prolog << "Sent AsyncRequestRejected" << endl);
755  }
756  else if (get_async_accepted().length() != 0) {
757 
761  BESDEBUG(MODULE, prolog << "serviceUrl="<< serviceUrl << endl);
762 
764  string storedResultId = "";
765  storedResultId = resultCache->store_dap2_result(dds, get_ce(), this, &eval);
766 
767  BESDEBUG(MODULE, prolog << "storedResultId='"<< storedResultId << "'" << endl);
768 
769  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
770  BESDEBUG(MODULE, prolog << "targetURL='"<< targetURL << "'" << endl);
771 
772  XMLWriter xmlWrtr;
773  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
774  out << xmlWrtr.get_doc();
775  out << flush;
776 
777  BESDEBUG(MODULE, prolog << "Sent DAP4 AsyncAccepted response" << endl);
778  }
779  else {
784  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
785  out << xmlWrtr.get_doc();
786  out << flush;
787 
788  BESDEBUG(MODULE, prolog << "Sent DAP4 AsyncRequired response" << endl);
789  }
790 
791  return true;
792 
793 }
794 #endif
795 
799 void BESDapResponseBuilder::serialize_dap2_data_dds(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool ce_eval)
800 {
801  BESStopWatch sw;
802  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
803 
804  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
805 
806  (*dds)->print_constrained(out);
807  out << "Data:\n";
808  out << flush;
809 
810  XDRStreamMarshaller m(out);
811 
812  // This only has an effect when the timeout in BESInterface::execute_request()
813  // is set. Otherwise it does nothing.
814  conditional_timeout_cancel();
815 
816  // Send all variables in the current projection (send_p())
817  for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
818  if ((*i)->send_p()) {
819  (*i)->serialize(eval, **dds, m, ce_eval);
820 #ifdef CLEAR_LOCAL_DATA
821  (*i)->clear_local_data();
822 #endif
823  }
824  }
825 
826  BESDEBUG(MODULE, prolog << "END" << endl);
827 }
828 
829 #ifdef DAP2_STORED_RESULTS
838 void BESDapResponseBuilder::serialize_dap2_data_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval,
839  const string &boundary, const string &start, bool ce_eval)
840 {
841  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
842 
843  // Write the MPM headers for the DDX (text/xml) part of the response
844  libdap::set_mime_ddx_boundary(out, boundary, start, dods_ddx, x_plain);
845 
846  // Make cid
847  uuid_t uu;
848  uuid_generate(uu);
849  char uuid[37];
850  uuid_unparse(uu, &uuid[0]);
851  char domain[256];
852  if (getdomainname(domain, 255) != 0 || strlen(domain) == 0) strncpy(domain, "opendap.org", 255);
853 
854  string cid = string(&uuid[0]) + "@" + string(&domain[0]);
855 
856  // Send constrained DDX with a data blob reference.
857  // Note: CID passed but ignored jhrg 10/20/15
858  (*dds)->print_xml_writer(out, true, cid);
859 
860  // write the data part mime headers here
861  set_mime_data_boundary(out, boundary, cid, dods_data_ddx /* old value dap4_data*/, x_plain);
862 
863  XDRStreamMarshaller m(out);
864 
865  conditional_timeout_cancel();
866 
867 
868  // Send all variables in the current projection (send_p()).
869  for (DDS::Vars_iter i = (*dds)->var_begin(); i != (*dds)->var_end(); i++) {
870  if ((*i)->send_p()) {
871  (*i)->serialize(eval, **dds, m, ce_eval);
872 #ifdef CLEAR_LOCAL_DATA
873  (*i)->clear_local_data();
874 #endif
875  }
876  }
877 
878  BESDEBUG(MODULE, prolog << "END" << endl);
879 }
880 #endif
881 
899 {
900 #if USE_LOCAL_TIMEOUT_SCHEME
901  alarm(0);
902 #endif
903 }
904 
918 libdap::DDS *
920 {
921  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
922 
923  dhi.first_container();
924 
925  BESDDSResponse *bdds = dynamic_cast<BESDDSResponse *>(obj);
926  if (!bdds) throw BESInternalFatalError("Expected a BESDDSResponse instance", __FILE__, __LINE__);
927 
928  DDS *dds = bdds->get_dds();
929 
930  set_dataset_name(dds->filename());
931  set_ce(dhi.data[POST_CONSTRAINT]);
932  set_async_accepted(dhi.data[ASYNC]);
933  set_store_result(dhi.data[STORE_RESULT]);
934 
935  ConstraintEvaluator &eval = bdds->get_ce();
936 
937  // Split constraint into two halves
938  split_ce(eval);
939 
940  // If there are functions, parse them and eval.
941  // Use that DDS and parse the non-function ce
942  // Serialize using the second ce and the second dds
943  if (!d_btp_func_ce.empty()) {
944  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
945 
946  ConstraintEvaluator func_eval;
947  DDS *fdds = 0; // nulll_ptr
948  if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
949  fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
950  }
951  else {
952  func_eval.parse_constraint(get_btp_func_ce(), *dds);
953  fdds = func_eval.eval_function_clauses(*dds);
954  }
955 
956  delete dds; // Delete so that we can ...
957  bdds->set_dds(fdds); // Transfer management responsibility
958  dds = fdds;
959 
960  dds->mark_all(false);
961 
962  promote_function_output_structures(dds);
963  }
964 
965  eval.parse_constraint(d_dap2ce, *dds); // Throws Error if the ce doesn't parse.
966  BESDEBUG(MODULE, prolog << "END"<< endl);
967 
968  return dds;
969 }
970 
988 libdap::DDS *
990 {
991  BESStopWatch sw;
992  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
993 
994  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
995 
996  dhi.first_container();
997 
998  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
999  if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
1000 
1001  DDS *dds = bdds->get_dds();
1002 
1003  set_dataset_name(dds->filename());
1004  set_ce(dhi.data[POST_CONSTRAINT]);
1005  set_async_accepted(dhi.data[ASYNC]);
1006  set_store_result(dhi.data[STORE_RESULT]);
1007 
1008 
1009  // This function is used by all fileout modules and they need to include the attributes in data access.
1010  // So obtain the attributes if necessary. KY 2019-10-30
1011  if(bdds->get_ia_flag() == false) {
1012  BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(dhi.container->get_container_type());
1013  besRH->add_attributes(dhi);
1014  }
1015 
1016  ConstraintEvaluator &eval = bdds->get_ce();
1017 
1018  // Split constraint into two halves; stores the function and non-function parts in this instance.
1019  split_ce(eval);
1020 
1021  // If there are functions, parse them and eval.
1022  // Use that DDS and parse the non-function ce
1023  // Serialize using the second ce and the second dds
1024  if (!get_btp_func_ce().empty()) {
1025  BESDEBUG(MODULE,prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1026 
1027  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
1028 
1029  ConstraintEvaluator func_eval;
1030  DDS *fdds = 0; // nulll_ptr
1031  if (responseCache && responseCache->can_be_cached(dds, get_btp_func_ce())) {
1032  fdds = responseCache->get_or_cache_dataset(dds, get_btp_func_ce());
1033  }
1034  else {
1035  func_eval.parse_constraint(get_btp_func_ce(), *dds);
1036  fdds = func_eval.eval_function_clauses(*dds);
1037  }
1038 
1039  delete dds; // Delete so that we can ...
1040  bdds->set_dds(fdds); // Transfer management responsibility
1041  dds = fdds;
1042 
1043  // Server functions might mark (i.e. setting send_p) so variables will use their read()
1044  // methods. Clear that so the CE in d_dap2ce will control what is
1045  // sent. If that is empty (there was only a function call) all
1046  // of the variables in the intermediate DDS (i.e., the function
1047  // result) will be sent.
1048  dds->mark_all(false);
1049 
1050  // Look for one or more top level Structures whose name indicates (by way of ending with
1051  // "_uwrap") that their contents should be moved to the top level.
1052  //
1053  // This is in support of a hack around the current API where server side functions
1054  // may only return a single DAP object and not a collection of objects. The name suffix
1055  // "_unwrap" is used as a signal from the function to the the various response
1056  // builders and transmitters that the representation needs to be altered before
1057  // transmission, and that in fact is what happens in our friend
1058  // promote_function_output_structures()
1059  promote_function_output_structures(dds);
1060  }
1061 
1062  // evaluate the rest of the CE - the part that follows the function calls.
1063  eval.parse_constraint(get_ce(), *dds);
1064 
1065  dds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1066 
1067  throw_if_dap2_response_too_big(dds);
1068 
1069  // Iterate through the variables in the DataDDS and read
1070  // in the data if the variable has the send flag set.
1071  for (DDS::Vars_iter i = dds->var_begin(), e = dds->var_end(); i != e; ++i) {
1072  if ((*i)->send_p()) {
1073  try {
1074  (*i)->intern_data(eval, *dds);
1075  }
1076  catch(std::exception &e) {
1077  throw BESSyntaxUserError(string("Caught a C++ standard exception while working on '") + (*i)->name() + "' The error was: " + e.what(), __FILE__, __LINE__);
1078  }
1079  }
1080  }
1081 
1082  BESDEBUG(MODULE, prolog << "END"<< endl);
1083 
1084  return dds;
1085 }
1086 
1087 
1100 void BESDapResponseBuilder::send_dap2_data(ostream &data_stream, DDS **dds, ConstraintEvaluator &eval,
1101  bool with_mime_headers)
1102 {
1103  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1104 
1105 #if USE_LOCAL_TIMEOUT_SCHEME
1106  // Set up the alarm.
1107  establish_timeout(data_stream);
1108  dds.set_timeout(d_timeout);
1109 #endif
1110 
1111  // Split constraint into two halves
1112  split_ce(eval);
1113 
1114  // If there are functions, parse them and eval.
1115  // Use that DDS and parse the non-function ce
1116  // Serialize using the second ce and the second dds
1117  if (!get_btp_func_ce().empty()) {
1118  BESDEBUG(MODULE,prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1119 
1120  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1121 
1122  ConstraintEvaluator func_eval;
1123  DDS *fdds = 0; // nulll_ptr
1124  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1125  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1126  }
1127  else {
1128  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1129  fdds = func_eval.eval_function_clauses(**dds);
1130  }
1131 
1132  delete *dds; *dds = 0;
1133  *dds = fdds;
1134 
1135  (*dds)->mark_all(false);
1136 
1137  promote_function_output_structures(*dds);
1138 
1139  // evaluate the rest of the CE - the part that follows the function calls.
1140  eval.parse_constraint(get_ce(), **dds);
1141 
1142  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1143 
1144  throw_if_dap2_response_too_big(*dds);
1145 
1146  if (with_mime_headers)
1147  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1148 
1149 #if STORE_DAP2_RESULT_FEATURE
1150  // This means: if we are not supposed to store the result, then serialize it.
1151  if (!store_dap2_result(data_stream, **dds, eval)) {
1152  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1153  }
1154 #else
1155  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1156 #endif
1157 
1158  }
1159  else {
1160  BESDEBUG(MODULE, prolog << "Simple constraint" << endl);
1161 
1162  eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1163 
1164  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1165 
1166  throw_if_dap2_response_too_big(*dds);
1167 
1168  if (with_mime_headers)
1169  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1170 
1171 #if STORE_DAP2_RESULT_FEATURE
1172  // This means: if we are not supposed to store the result, then serialize it.
1173  if (!store_dap2_result(data_stream, **dds, eval)) {
1174  serialize_dap2_data_dds(data_stream, dds, eval);
1175  }
1176 #else
1177  serialize_dap2_data_dds(data_stream, dds, eval);
1178 #endif
1179  }
1180 
1181  data_stream << flush;
1182 
1183  BESDEBUG(MODULE, prolog << "END"<< endl);
1184 
1185 }
1186 
1187 void BESDapResponseBuilder::send_dap2_data(BESDataHandlerInterface &dhi, DDS **dds, ConstraintEvaluator &eval,
1188  bool with_mime_headers)
1189 {
1190  BESDEBUG(MODULE, prolog << "BEGIN"<< endl);
1191 
1192  ostream & data_stream = dhi.get_output_stream();
1193 #if USE_LOCAL_TIMEOUT_SCHEME
1194  // Set up the alarm.
1195  establish_timeout(data_stream);
1196  dds.set_timeout(d_timeout);
1197 #endif
1198 
1199  // Split constraint into two halves
1200  split_ce(eval);
1201 
1202  // If there are functions, parse them and eval.
1203  // Use that DDS and parse the non-function ce
1204  // Serialize using the second ce and the second dds
1205  if (!get_btp_func_ce().empty()) {
1206  BESDEBUG(MODULE, prolog << "Found function(s) in CE: " << get_btp_func_ce() << endl);
1207 
1208  // Server-side functions need to include the attributes in data access.
1209  // So obtain the attributes if necessary. KY 2019-10-30
1210  {
1211  BESResponseObject *response = dhi.response_handler->get_response_object();
1212  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *> (response);
1213  if (!bdds)
1214  throw BESInternalError("cast error", __FILE__, __LINE__);
1215 
1216  if(bdds->get_ia_flag() == false) {
1217  BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(dhi.container->get_container_type());
1218  besRH->add_attributes(dhi);
1219  }
1220  }
1221 
1222  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1223  ConstraintEvaluator func_eval;
1224  DDS *fdds = 0; // nulll_ptr
1225  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1226  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1227  }
1228  else {
1229  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1230  fdds = func_eval.eval_function_clauses(**dds);
1231  }
1232 
1233  delete *dds; *dds = 0;
1234  *dds = fdds;
1235 
1236  (*dds)->mark_all(false);
1237 
1238  promote_function_output_structures(*dds);
1239 
1240  // evaluate the rest of the CE - the part that follows the function calls.
1241  eval.parse_constraint(get_ce(), **dds);
1242 
1243  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1244 
1245  throw_if_dap2_response_too_big(*dds);
1246 
1247  if (with_mime_headers)
1248  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1249 
1250 #if STORE_DAP2_RESULT_FEATURE
1251  // This means: if we are not supposed to store the result, then serialize it.
1252  if (!store_dap2_result(data_stream, **dds, eval)) {
1253  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1254  }
1255 #else
1256  serialize_dap2_data_dds(data_stream, dds, eval, true /* was 'false'. jhrg 3/10/15 */);
1257 #endif
1258 
1259  }
1260  else {
1261  BESDEBUG(MODULE, prolog << "Simple constraint" << endl);
1262 
1263  eval.parse_constraint(get_ce(), **dds); // Throws Error if the ce doesn't parse.
1264 
1265  (*dds)->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
1266 
1267  throw_if_dap2_response_too_big(*dds);
1268 
1269  if (with_mime_headers)
1270  set_mime_binary(data_stream, dods_data, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1271 
1272 #if STORE_DAP2_RESULT_FEATURE
1273  // This means: if we are not supposed to store the result, then serialize it.
1274  if (!store_dap2_result(data_stream, **dds, eval)) {
1275  serialize_dap2_data_dds(data_stream, dds, eval);
1276  }
1277 #else
1278  serialize_dap2_data_dds(data_stream, dds, eval);
1279 #endif
1280  }
1281 
1282  data_stream << flush;
1283 
1284  BESDEBUG(MODULE, prolog << "END"<< endl);
1285 
1286 }
1300 void BESDapResponseBuilder::send_ddx(ostream &out, DDS **dds, ConstraintEvaluator &eval, bool with_mime_headers)
1301 {
1302  if (d_dap2ce.empty()) {
1303  if (with_mime_headers)
1304  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1305 
1306  (*dds)->print_xml_writer(out, false /*constrained */, "");
1307  //dds.print(out);
1308  out << flush;
1309  return;
1310  }
1311 
1312 #if USE_LOCAL_TIMEOUT_SCHEME
1313  // Set up the alarm.
1314  establish_timeout(out);
1315  dds.set_timeout(d_timeout);
1316 #endif
1317 
1318  // Split constraint into two halves
1319  split_ce(eval);
1320 
1321  // If there are functions, parse them and eval.
1322  // Use that DDS and parse the non-function ce
1323  // Serialize using the second ce and the second dds
1324  if (!d_btp_func_ce.empty()) {
1325  BESDapFunctionResponseCache *response_cache = BESDapFunctionResponseCache::get_instance();
1326 
1327  ConstraintEvaluator func_eval;
1328  DDS *fdds = 0; // nulll_ptr
1329  if (response_cache && response_cache->can_be_cached(*dds, get_btp_func_ce())) {
1330  fdds = response_cache->get_or_cache_dataset(*dds, get_btp_func_ce());
1331  }
1332  else {
1333  func_eval.parse_constraint(get_btp_func_ce(), **dds);
1334  fdds = func_eval.eval_function_clauses(**dds);
1335  }
1336 
1337  delete *dds; *dds = 0;
1338  *dds = fdds;
1339 
1340  (*dds)->mark_all(false);
1341 
1342  promote_function_output_structures(*dds);
1343 
1344  eval.parse_constraint(d_dap2ce, **dds);
1345 
1346  if (with_mime_headers)
1347  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1348 
1349  conditional_timeout_cancel();
1350 
1351  (*dds)->print_xml_writer(out, true, "");
1352  }
1353  else {
1354  eval.parse_constraint(d_dap2ce, **dds); // Throws Error if the ce doesn't parse.
1355 
1356  if (with_mime_headers)
1357  set_mime_text(out, dods_ddx, x_plain, last_modified_time(d_dataset), (*dds)->get_dap_version());
1358 
1359  conditional_timeout_cancel();
1360 
1361 
1362  // dds.print_constrained(out);
1363  (*dds)->print_xml_writer(out, true, "");
1364  }
1365 
1366  out << flush;
1367 }
1368 
1369 void BESDapResponseBuilder::send_dmr(ostream &out, DMR &dmr, bool with_mime_headers)
1370 {
1371  // If the CE is not empty, parse it. The projections, etc., are set as a side effect.
1372  // If the parser returns false, the expression did not parse. The parser may also
1373  // throw Error
1374  if (!d_dap4ce.empty()) {
1375 
1376  BESDEBUG(MODULE, prolog << "Parsing DAP4 constraint: '"<< d_dap4ce << "'"<< endl);
1377 
1378  D4ConstraintEvaluator parser(&dmr);
1379  bool parse_ok = parser.parse(d_dap4ce);
1380  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1381  }
1382  // with an empty CE, send everything. Even though print_dap4() and serialize()
1383  // don't need this, other code may depend on send_p being set. This may change
1384  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1385  else {
1386  dmr.root()->set_send_p(true);
1387  }
1388 
1389  if (with_mime_headers) set_mime_text(out, dap4_dmr, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1390 
1391  conditional_timeout_cancel();
1392 
1393  BESDEBUG(MODULE, prolog << "dmr.request_xml_base(): '"<< dmr.request_xml_base() << "' (dmr: " << (void *) &dmr << ")" << endl);
1394 
1395  XMLWriter xml;
1396  dmr.print_dap4(xml, /*constrained &&*/!d_dap4ce.empty() /* true == constrained */);
1397  out << xml.get_doc() << flush;
1398 }
1399 
1400 void BESDapResponseBuilder::send_dap4_data_using_ce(ostream &out, DMR &dmr, bool with_mime_headers)
1401 {
1402  if (!d_dap4ce.empty()) {
1403  BESDEBUG("dap", "BESDapResponseBuilder::send_dap4_data_using_ce() - expression constraint is not empty. " <<endl);
1404  D4ConstraintEvaluator parser(&dmr);
1405  bool parse_ok = parser.parse(d_dap4ce);
1406  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1407  }
1408  // with an empty CE, send everything. Even though print_dap4() and serialize()
1409  // don't need this, other code may depend on send_p being set. This may change
1410  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1411  else {
1412  dmr.root()->set_send_p(true);
1413  }
1414 
1415  if (dmr.response_limit() != 0 && (dmr.request_size(true) > dmr.response_limit())) {
1416  string msg = "The Request for " + long_to_string(dmr.request_size(true))
1417  + "KB is too large; requests for this server are limited to " + long_to_string(dmr.response_limit())
1418  + "KB.";
1419  throw Error(msg);
1420  }
1421 
1422  // The following block is for debugging purpose. KY 05/13/2020
1423 #if 0
1424  for (D4Group::Vars_iter i = dmr.root()->var_begin(), e = dmr.root()->var_end(); i != e; ++i) {
1425  BESDEBUG("dap", "BESDapResponseBuilder::send_dap4_data_ce() - "<< (*i)->name() <<endl);
1426  if ((*i)->send_p()) {
1427  BESDEBUG("dap", "BESDapResponseBuilder::send_dap4_data() Obtain data- "<< (*i)->name() <<endl);
1428  D4Attributes*d4_attrs = (*i)->attributes();
1429  BESDEBUG("dap", "BESDapResponseBuilder::send_dap4_data() number of attributes "<< d4_attrs <<endl);
1430  for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
1431  string name = (*ii)->name();
1432  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() attribute name is "<<name <<endl);
1433  }
1434 
1435 
1436  }
1437  }
1438 #endif
1439 
1440 
1441  if (!store_dap4_result(out, dmr)) {
1442  serialize_dap4_data(out, dmr, with_mime_headers);
1443  }
1444 }
1445 
1446 void BESDapResponseBuilder::intern_dap4_data_using_ce(DMR &dmr)
1447 {
1448  BESStopWatch sw;
1449  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1450  if (!d_dap4ce.empty()) {
1451  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data_using_ce() - expression constraint is not empty. " <<endl);
1452  D4ConstraintEvaluator parser(&dmr);
1453  bool parse_ok = parser.parse(d_dap4ce);
1454  if (!parse_ok) throw Error(malformed_expr, "Constraint Expression (" + d_dap4ce + ") failed to parse.");
1455  }
1456  // with an empty CE, send everything. Even though print_dap4() and serialize()
1457  // don't need this, other code may depend on send_p being set. This may change
1458  // if DAP4 has a separate function evaluation phase. jhrg 11/25/13
1459  else {
1460  dmr.root()->set_send_p(true);
1461  }
1462 
1463  if (dmr.response_limit() != 0 && (dmr.request_size(true) > dmr.response_limit())) {
1464  string msg = "The Request for " + long_to_string(dmr.request_size(true))
1465  + "KB is too large; requests for this server are limited to " + long_to_string(dmr.response_limit())
1466  + "KB.";
1467  throw Error(msg);
1468  }
1469 
1470 }
1471 
1472 void BESDapResponseBuilder::send_dap4_data(ostream &out, DMR &dmr, bool with_mime_headers)
1473 {
1474  // If a function was passed in with this request, evaluate it and use that DMR
1475  // for the remainder of this request.
1476  // TODO Add caching for these function invocations
1477  if (!d_dap4function.empty()) {
1478  D4BaseTypeFactory d4_factory;
1479  DMR function_result(&d4_factory, "function_results");
1480 
1481  // Function modules load their functions onto this list. The list is
1482  // part of libdap, not the BES.
1483  if (!ServerFunctionsList::TheList())
1484  throw Error(
1485  "The function expression could not be evaluated because there are no server functions defined on this server");
1486 
1487  D4FunctionEvaluator parser(&dmr, ServerFunctionsList::TheList());
1488  bool parse_ok = parser.parse(d_dap4function);
1489  if (!parse_ok) throw Error("Function Expression (" + d_dap4function + ") failed to parse.");
1490 
1491  parser.eval(&function_result);
1492 
1493  // Now use the results of running the functions for the remainder of the
1494  // send_data operation.
1495  send_dap4_data_using_ce(out, function_result, with_mime_headers);
1496  }
1497  else {
1498  send_dap4_data_using_ce(out, dmr, with_mime_headers);
1499  }
1500 }
1501 
1505 void BESDapResponseBuilder::serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers)
1506 {
1507  BESStopWatch sw;
1508  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1509 
1510  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
1511 
1512  if (with_mime_headers) set_mime_binary(out, dap4_data, x_plain, last_modified_time(d_dataset), dmr.dap_version());
1513 
1514  BESDEBUG(MODULE, prolog << "dmr.request_xml_base(): \"" << dmr.request_xml_base() << "\""<< endl);
1515 
1516  // Write the DMR
1517  XMLWriter xml;
1518  dmr.print_dap4(xml, !d_dap4ce.empty());
1519 
1520  // now make the chunked output stream; set the size to be at least chunk_size
1521  // but make sure that the whole of the xml plus the CRLF can fit in the first
1522  // chunk. (+2 for the CRLF bytes).
1523  chunked_ostream cos(out, max((unsigned int) CHUNK_SIZE, xml.get_doc_size() + 2));
1524 
1525  conditional_timeout_cancel();
1526 
1527  // using flush means that the DMR and CRLF are in the first chunk.
1528  cos << xml.get_doc() << CRLF << flush;
1529 
1530  // Write the data, chunked with checksums
1531  D4StreamMarshaller m(cos);
1532  dmr.root()->serialize(m, dmr, !d_dap4ce.empty());
1533 #ifdef CLEAR_LOCAL_DATA
1534  dmr.root()->clear_local_data();
1535 #endif
1536  cos << flush;
1537 
1538  BESDEBUG(MODULE, prolog << "END" << endl);
1539 }
1540 
1555 bool BESDapResponseBuilder::store_dap4_result(ostream &out, libdap::DMR &dmr)
1556 {
1557  if (get_store_result().length() != 0) {
1558  string serviceUrl = get_store_result();
1559 
1560  D4AsyncUtil d4au;
1561  XMLWriter xmlWrtr;
1562 
1563  // FIXME See above comment for store dap2 result
1564  bool found;
1565  string *stylesheet_ref = 0, ss_ref_value;
1566  TheBESKeys::TheKeys()->get_value(D4AsyncUtil::STYLESHEET_REFERENCE_KEY, ss_ref_value, found);
1567  if (found && ss_ref_value.length() > 0) {
1568  stylesheet_ref = &ss_ref_value;
1569  }
1570 
1572  if (resultCache == NULL) {
1573 
1579  string msg = "The Stored Result request cannot be serviced. ";
1580  msg += "Unable to acquire StoredResultCache instance. ";
1581  msg += "This is most likely because the StoredResultCache is not (correctly) configured.";
1582 
1583  BESDEBUG(MODULE, prolog << "[WARNING] " << msg << endl);
1584  d4au.writeD4AsyncResponseRejected(xmlWrtr, UNAVAILABLE, msg, stylesheet_ref);
1585  out << xmlWrtr.get_doc();
1586  out << flush;
1587  BESDEBUG(MODULE, prolog << "Sent AsyncRequestRejected" << endl);
1588 
1589  return true;
1590  }
1591 
1592  if (get_async_accepted().length() != 0) {
1593 
1597  BESDEBUG(MODULE, prolog << "serviceUrl="<< serviceUrl << endl);
1598 
1599  string storedResultId = "";
1600  storedResultId = resultCache->store_dap4_result(dmr, get_ce(), this);
1601 
1602  BESDEBUG(MODULE,prolog << "storedResultId='"<< storedResultId << "'" << endl);
1603 
1604  string targetURL = BESUtil::assemblePath(serviceUrl, storedResultId);
1605  BESDEBUG(MODULE, prolog << "targetURL='"<< targetURL << "'" << endl);
1606 
1607  d4au.writeD4AsyncAccepted(xmlWrtr, 0, 0, targetURL, stylesheet_ref);
1608  out << xmlWrtr.get_doc();
1609  out << flush;
1610  BESDEBUG(MODULE, prolog << "Sent AsyncAccepted" << endl);
1611 
1612  }
1613  else {
1618  d4au.writeD4AsyncRequired(xmlWrtr, 0, 0, stylesheet_ref);
1619  out << xmlWrtr.get_doc();
1620  out << flush;
1621  BESDEBUG(MODULE, prolog << "Sent AsyncAccepted" << endl);
1622  }
1623 
1624  return true;
1625  }
1626 
1627  return false;
1628 }
1629 
1630 
1631 
1650 libdap::DMR *
1652 {
1653  BESStopWatch sw;
1654  if (BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()) sw.start(prolog + "Timer", "");
1655  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() - BEGIN"<< endl);
1656 
1657  dhi.first_container();
1658 
1659 #if 0
1660  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(obj);
1661  if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
1662 
1663  DDS *dds = bdds->get_dds();
1664 
1665  set_dataset_name(dds->filename());
1666  set_ce(dhi.data[POST_CONSTRAINT]);
1667  set_async_accepted(dhi.data[ASYNC]);
1668  set_store_result(dhi.data[STORE_RESULT]);
1669 
1670 
1671  ConstraintEvaluator &eval = bdds->get_ce();
1672 #endif
1673  BESDMRResponse *bdmr = dynamic_cast<BESDMRResponse *>(obj);
1674  if (!bdmr) throw BESInternalFatalError("Expected a BESDMRResponse instance", __FILE__, __LINE__);
1675 
1676  DMR *dmr = bdmr->get_dmr();
1677 
1678  D4Group* root_grp=NULL;
1679  BESDEBUG("dap", "BESDapResponseBuilder::dmr filename - END"<< dmr->filename() <<endl);
1680 
1681  // Set the correct context by following intern_dap2_data()
1682  set_dataset_name(dmr->filename());
1683  set_dap4ce(dhi.data[DAP4_CONSTRAINT]);
1684  set_dap4function(dhi.data[DAP4_FUNCTION]);
1685  set_async_accepted(dhi.data[ASYNC]);
1686  set_store_result(dhi.data[STORE_RESULT]);
1687 
1688  // Following send_dap4_data(),KY 05/13/2020.
1689  // If a function was passed in with this request, evaluate it and use that DMR
1690  // for the remainder of this request.
1691  // TODO Add caching for these function invocations
1692  if (!d_dap4function.empty()) {
1693  D4BaseTypeFactory d4_factory;
1694  DMR function_result(&d4_factory, "function_results");
1695 
1696  // Function modules load their functions onto this list. The list is
1697  // part of libdap, not the BES.
1698  if (!ServerFunctionsList::TheList())
1699  throw Error(
1700  "The function expression could not be evaluated because there are no server functions defined on this server");
1701 
1702  D4FunctionEvaluator parser(dmr, ServerFunctionsList::TheList());
1703  bool parse_ok = parser.parse(d_dap4function);
1704  if (!parse_ok) throw Error("Function Expression (" + d_dap4function + ") failed to parse.");
1705 
1706  parser.eval(&function_result);
1707 
1708  // Now use the results of running the functions for the remainder of the
1709  // send_data operation.
1710  intern_dap4_data_using_ce(function_result);
1711  root_grp = function_result.root();
1712  }
1713  else {
1714  BESDEBUG("dap", "BESDapResponseBuilder:: going to the expression constraint. " <<endl);
1715  intern_dap4_data_using_ce(*dmr);
1716  root_grp = dmr->root();
1717  }
1718 
1719 
1720  // Iterate through the variables in the DataDDS and read
1721  // in the data if the variable has the send flag set.
1722  //D4Group* root_grp = dmr->root();
1723 
1724  for (D4Group::Vars_iter i = root_grp->var_begin(), e = root_grp->var_end(); i != e; ++i) {
1725  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() - "<< (*i)->name() <<endl);
1726  if ((*i)->send_p()) {
1727  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() Obtain data- "<< (*i)->name() <<endl);
1728 #if 0
1729  D4Attributes*d4_attrs = (*i)->attributes();
1730  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() number of attributes "<< d4_attrs <<endl);
1731  for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
1732  string name = (*ii)->name();
1733  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() attribute name is "<<name <<endl);
1734  }
1735 #endif
1736  (*i)->intern_data();
1737  }
1738  }
1739 
1740  for (D4Group::groupsIter gi = root_grp->grp_begin(), ge = root_grp->grp_end(); gi != ge; ++gi) {
1741  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() root group- "<< (*gi)->name() <<endl);
1742  intern_dap4_data_grp(*gi);
1743  }
1744  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() - END"<< endl);
1745 
1746  return dmr;
1747 }
1748 
1749 void BESDapResponseBuilder::intern_dap4_data_grp(libdap::D4Group* grp) {
1750  for (D4Group::Vars_iter i = grp->var_begin(), e = grp->var_end(); i != e; ++i) {
1751  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() - "<< (*i)->name() <<endl);
1752  if ((*i)->send_p()) {
1753  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() Obtain data- "<< (*i)->name() <<endl);
1754 #if 0
1755  D4Attributes*d4_attrs = (*i)->attributes();
1756  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() number of attributes "<< d4_attrs <<endl);
1757  for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
1758  string name = (*ii)->name();
1759  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() attribute name is "<<name <<endl);
1760  }
1761 #endif
1762  (*i)->intern_data();
1763  }
1764  }
1765 
1766  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1767  BESDEBUG("dap", "BESDapResponseBuilder::intern_dap4_data() group- "<< (*gi)->name() <<endl);
1768  intern_dap4_data_grp(*gi);
1769  }
1770 
1771 
1772 }
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Holds a DDS object within the BES.
void set_dds(libdap::DDS *ddsIn)
libdap::ConstraintEvaluator & get_ce()
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
Cache the results from server functions.
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual std::string get_dataset_name() const
Get the dataset name.
virtual libdap::DMR * intern_dap4_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
virtual std::string get_dap4function() const
Get the DAP4 server side function expression.
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual std::string get_ce() const
Get the constraint expression.
virtual void set_dap4ce(std::string _ce)
virtual void remove_timeout() const
Transmit data.
virtual libdap::DDS * process_dap2_dds(BESResponseObject *obj, BESDataHandlerInterface &dhi)
Process a DDS (i.e., apply a constraint) for a non-DAP transmitter.
virtual void serialize_dap4_data(std::ostream &out, libdap::DMR &dmr, bool with_mime_headers=true)
virtual libdap::DDS * intern_dap2_data(BESResponseObject *obj, BESDataHandlerInterface &dhi)
virtual void send_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool constrained=false, bool with_mime_headers=true)
Transmit a DDS.
virtual std::string get_dap4ce() const
Get the DAP4 constraint expression.
virtual void establish_timeout(std::ostream &stream) const
virtual bool store_dap4_result(ostream &out, libdap::DMR &dmr)
virtual void send_ddx(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool with_mime_headers=true)
virtual void serialize_dap2_data_dds(std::ostream &out, libdap::DDS **dds, libdap::ConstraintEvaluator &eval, bool ce_eval=true)
virtual void set_dap4function(std::string _func)
void set_timeout(int timeout=0)
virtual void set_ce(std::string _ce)
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:160
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
virtual string store_dap4_result(libdap::DMR &dmr, const string &constraint, BESDapResponseBuilder *rb)
static BESStoredDapResultCache * get_instance()
error thrown if there is a user syntax error in the request or any other user error
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
Definition: BESUtil.cc:821
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71