bes  Updated for version 3.20.8
NgapContainer.cc
1 // NgapContainer.cc
2 
3 // -*- mode: c++; c-basic-offset:4 -*-
4 
5 // This file is part of ngap_module, A C++ module that can be loaded in to
6 // the OPeNDAP Back-End Server (BES) and is able to handle remote requests.
7 
8 // Copyright (c) 2020 OPeNDAP, Inc.
9 // Author: Nathan Potter <ndp@opendap.org>
10 //
11 // This library is free software; you can redistribute it and/or
12 // modify it under the terms of the GNU Lesser General Public
13 // License as published by the Free Software Foundation; either
14 // version 2.1 of the License, or (at your option) any later version.
15 //
16 // This library is distributed in the hope that it will be useful,
17 // but WITHOUT ANY WARRANTY; without even the implied warranty of
18 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
19 // Lesser General Public License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 // Authors:
27 // ndp Nathan Potter <ndp@opendap.org>
28 
29 #include "config.h"
30 
31 #include <cstdio>
32 #include <map>
33 #include <sstream>
34 #include <string>
35 #include <fstream>
36 #include <streambuf>
37 #include <time.h>
38 
39 #include "BESStopWatch.h"
40 #include "BESLog.h"
41 #include "BESSyntaxUserError.h"
42 #include "BESNotFoundError.h"
43 #include "BESInternalError.h"
44 #include "BESDebug.h"
45 #include "BESUtil.h"
46 #include "TheBESKeys.h"
47 #include "AllowedHosts.h"
48 #include "BESContextManager.h"
49 #include "CurlUtils.h"
50 #include "HttpUtils.h"
51 #include "RemoteResource.h"
52 #include "url_impl.h"
53 
54 #include "NgapContainer.h"
55 #include "NgapApi.h"
56 #include "NgapNames.h"
57 
58 #define prolog std::string("NgapContainer::").append(__func__).append("() - ")
59 
60 using namespace std;
61 using namespace bes;
62 
63 #define UID_CONTEXT "uid"
64 #define AUTH_TOKEN_CONTEXT "edl_auth_token"
65 #define EDL_ECHO_TOKEN_CONTEXT "edl_echo_token"
66 
67 namespace ngap {
68 
79  NgapContainer::NgapContainer(const string &sym_name,
80  const string &real_name,
81  const string &type) :
82  BESContainer(sym_name, real_name, type),
83  d_dmrpp_rresource(0) {
84 
85  BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << endl);
86 
87  bool found;
88 
89  NgapApi ngap_api;
90  if (type.empty())
91  set_container_type("ngap");
92 
93  string uid = BESContextManager::TheManager()->get_context(UID_CONTEXT, found);
94  string access_token = BESContextManager::TheManager()->get_context(AUTH_TOKEN_CONTEXT, found);
95 
96  BESDEBUG(MODULE, prolog << "UID_CONTEXT(" << UID_CONTEXT << "): " << uid << endl);
97 
98  string data_access_url = ngap_api.convert_ngap_resty_path_to_data_access_url(real_name, uid);
99 
100  set_real_name(data_access_url);
101  // Because we know the name is really a URL, then we know the "relative_name" is meaningless
102  // So we set it to be the same as "name"
103  set_relative_name(data_access_url);
104  }
105 
109  NgapContainer::NgapContainer(const NgapContainer &copy_from) :
110  BESContainer(copy_from),
111  d_dmrpp_rresource(copy_from.d_dmrpp_rresource) {
112  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << " Copying from: " << (void *) &copy_from << endl);
113  // we can not make a copy of this container once the request has
114  // been made
115  if (d_dmrpp_rresource) {
116  string err = (string) "The Container has already been accessed, "
117  + "can not create a copy of this container.";
118  throw BESInternalError(err, __FILE__, __LINE__);
119  }
120  BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << endl);
121  }
122 
123  void NgapContainer::_duplicate(NgapContainer &copy_to) {
124  if (copy_to.d_dmrpp_rresource) {
125  string err = (string) "The Container has already been accessed, "
126  + "can not duplicate this resource.";
127  throw BESInternalError(err, __FILE__, __LINE__);
128  }
129  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << " Copying to: " << (void *) &copy_to << endl);
130  copy_to.d_dmrpp_rresource = d_dmrpp_rresource;
131  BESContainer::_duplicate(copy_to);
132  }
133 
134  BESContainer *
136  NgapContainer *container = new NgapContainer;
137  _duplicate(*container);
138  BESDEBUG(MODULE, prolog << "object address: "<< (void *) this << " to: " << (void *)container << endl);
139  return container;
140  }
141 
142  NgapContainer::~NgapContainer() {
143  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << endl);
144  if (d_dmrpp_rresource) {
145  release();
146  }
147  BESDEBUG(MODULE, prolog << "END object address: "<< (void *) this << endl);
148  }
149 
150 
151 #if 0
152 
153 
154  bool cache_terminal_urls(){
155  bool found;
156  string value;
157  TheBESKeys::TheKeys()->get_value(NGAP_CACHE_TERMINAL_URLS_KEY,value,found);
158  return found && BESUtil::lowercase(value)=="true";
159  }
160 
161 
162 
163  void cache_final_redirect_url(string data_access_url_str) {
164  // See if the data_access_url has already been processed into a terminal signed URL
165  // in TheBESKeys
166  bool found;
167  std::map<std::string,std::string> data_access_url_info;
168  TheBESKeys::TheKeys()->get_values(data_access_url_str, data_access_url_info, false, found);
169  if(found){
170  // Is it expired?
171  http::url target_url(data_access_url_info);
172  found = NgapApi::signed_url_is_expired(target_url);
173  }
174  // It not found or expired, reload.
175  if(!found){
176  string last_accessed_url_str;
177  curl::find_last_redirect(data_access_url_str, last_accessed_url_str);
178  BESDEBUG(MODULE, prolog << "last_accessed_url: " << last_accessed_url_str << endl);
179 
180  http::url last_accessed_url(last_accessed_url_str);
181  last_accessed_url.kvp(data_access_url_info);
182 
183  // Placing the last accessed URL information in TheBESKeys associated with the data_access_url as the
184  // key allows allows other modules, such as dmrpp_module to access the crucial last accessed URL
185  // information which eliminates any number of redirects during access operations.
186  TheBESKeys::TheKeys()->set_keys(data_access_url_str, data_access_url_info, false, false);
187  }
188  }
189 #endif
190 
197  BESDEBUG(MODULE, prolog << "BEGIN object address: "<< (void *) this << endl);
198 
199  // Since this the ngap we know that the real_name is a URL.
200  string data_access_url_str = get_real_name();
201 
202 #if 0
203  if(cache_terminal_urls()){
204  curl::cache_final_redirect_url(data_access_url_str, 0);
205  }
206 #endif
207 
208  // And we know that the dmr++ file should "right next to it" (side-car)
209  string dmrpp_url = data_access_url_str + ".dmrpp";
210 
211  BESDEBUG(MODULE, prolog << "data_access_url: " << data_access_url_str << endl);
212  BESDEBUG(MODULE, prolog << "dmrpp_url: " << dmrpp_url << endl);
213 
214  string type = get_container_type();
215  if (type == "ngap")
216  type = "";
217 
218  if (!d_dmrpp_rresource) {
219  BESDEBUG(MODULE, prolog << "Building new RemoteResource (dmr++)." << endl);
220  string replace_template;
221  string replace_value;
222  if (inject_data_url()) {
223  replace_template = DATA_ACCESS_URL_KEY;
224  replace_value = data_access_url_str;
225  }
226  {
227  d_dmrpp_rresource = new http::RemoteResource(dmrpp_url);
228  BESStopWatch besTimer;
229  if (BESISDEBUG(MODULE) || BESDebug::IsSet(TIMING_LOG_KEY) || BESLog::TheLog()->is_verbose()){
230  besTimer.start("DMR++ retrieval: "+ dmrpp_url);
231  }
232  d_dmrpp_rresource->retrieveResource(replace_template, replace_value);
233  }
234  }
235  BESDEBUG(MODULE, prolog << "Retrieved remote resource: " << dmrpp_url << endl);
236 
237  // TODO This file should be read locked before leaving this method.
238  string cachedResource = d_dmrpp_rresource->getCacheFileName();
239  BESDEBUG(MODULE, prolog << "Using local cache file: " << cachedResource << endl);
240 
241  type = d_dmrpp_rresource->getType();
242  set_container_type(type);
243  BESDEBUG(MODULE, prolog << "Type: " << type << endl);
244  BESDEBUG(MODULE, prolog << "Done retrieving: " << dmrpp_url << " returning cached file " << cachedResource << endl);
245  BESDEBUG(MODULE, prolog << "END" << endl);
246 
247  return cachedResource; // this should return the dmr++ file name from the NgapCache
248  }
249 
250 
258  // TODO The cache file (that will be) read locked in the access() method must be unlocked here.
259  if (d_dmrpp_rresource) {
260  BESDEBUG(MODULE, prolog << "Releasing RemoteResource" << endl);
261  delete d_dmrpp_rresource;
262  d_dmrpp_rresource = 0;
263  }
264 
265  BESDEBUG(MODULE, prolog << "Done releasing Ngap response" << endl);
266  return true;
267  }
268 
276  void NgapContainer::dump(ostream &strm) const {
277  strm << BESIndent::LMarg << "NgapContainer::dump - (" << (void *) this
278  << ")" << endl;
279  BESIndent::Indent();
280  BESContainer::dump(strm);
281  if (d_dmrpp_rresource) {
282  strm << BESIndent::LMarg << "RemoteResource.getCacheFileName(): " << d_dmrpp_rresource->getCacheFileName()
283  << endl;
284  strm << BESIndent::LMarg << "response headers: ";
285  vector<string> *hdrs = d_dmrpp_rresource->getResponseHeaders();
286  if (hdrs) {
287  strm << endl;
288  BESIndent::Indent();
289  vector<string>::const_iterator i = hdrs->begin();
290  vector<string>::const_iterator e = hdrs->end();
291  for (; i != e; i++) {
292  string hdr_line = (*i);
293  strm << BESIndent::LMarg << hdr_line << endl;
294  }
295  BESIndent::UnIndent();
296  } else {
297  strm << "none" << endl;
298  }
299  } else {
300  strm << BESIndent::LMarg << "response not yet obtained" << endl;
301  }
302  BESIndent::UnIndent();
303  }
304 
305  bool NgapContainer::inject_data_url(){
306  bool result = false;
307  bool found;
308  string key_value;
309  TheBESKeys::TheKeys()->get_value(NGAP_INJECT_DATA_URL_KEY, key_value, found);
310  if (found && key_value == "true") {
311  result = true;
312  }
313  BESDEBUG(MODULE, prolog << "NGAP_INJECT_DATA_URL_KEY(" << NGAP_INJECT_DATA_URL_KEY << "): " << result << endl);
314  return result;
315  }
316 }
A container is something that holds data. E.G., a netcdf file or a database entry.
Definition: BESContainer.h:65
void set_container_type(const std::string &type)
set the type of data that this container represents, such as cedar or netcdf.
Definition: BESContainer.h:161
virtual void dump(std::ostream &strm) const
dumps information about this object
Definition: BESContainer.cc:73
void set_real_name(const std::string &real_name)
set the real name for this container, such as a file name if reading a data file.
Definition: BESContainer.h:146
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
void set_relative_name(const std::string &relative)
Set the relative name of the object in this container.
Definition: BESContainer.h:152
void _duplicate(BESContainer &copy_to)
duplicate this instance into the passed container
Definition: BESContainer.cc:54
std::string get_real_name() const
retrieve the real name for this container, such as a file name.
Definition: BESContainer.h:180
virtual std::string get_context(const std::string &name, bool &found)
retrieve the value of the specified context from the BES
static bool IsSet(const std::string &flagName)
see if the debug context flagName is set to true
Definition: BESDebug.h:160
exception thrown if internal error encountered
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
Definition: TheBESKeys.cc:370
void set_keys(const std::string &key, const std::vector< std::string > &values, bool addto)
allows the user to set key/value pairs from within the application.
Definition: TheBESKeys.cc:235
std::string getCacheFileName()
std::vector< std::string > * getResponseHeaders()
std::string getType()
std::string convert_ngap_resty_path_to_data_access_url(const std::string &restified_path, const std::string &uid="")
Converts an NGAP restified granule path into a CMR metadata query for the granule.
Definition: NgapApi.cc:434
virtual void dump(std::ostream &strm) const
dumps information about this object
virtual std::string access()
access the remote target response by making the remote request
virtual BESContainer * ptr_duplicate()
pure abstract method to duplicate this instances of BESContainer
virtual bool release()
release the resources