bes  Updated for version 3.20.8
TheBESKeys.cc
1 // TheBESKeys.cc
2 
3 // This file is part of bes, A C++ back-end server implementation framework
4 // for the OPeNDAP Data Access Protocol.
5 
6 // Copyright (c) 2004-2009 University Corporation for Atmospheric Research
7 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
8 //
9 // This library is free software; you can redistribute it and/or
10 // modify it under the terms of the GNU Lesser General Public
11 // License as published by the Free Software Foundation; either
12 // version 2.1 of the License, or (at your option) any later version.
13 //
14 // This library is distributed in the hope that it will be useful,
15 // but WITHOUT ANY WARRANTY; without even the implied warranty of
16 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
17 // Lesser General Public License for more details.
18 //
19 // You should have received a copy of the GNU Lesser General Public
20 // License along with this library; if not, write to the Free Software
21 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
22 //
23 // You can contact University Corporation for Atmospheric Research at
24 // 3080 Center Green Drive, Boulder, CO 80301
25 
26 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
27 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
28 //
29 // Authors:
30 // pwest Patrick West <pwest@ucar.edu>
31 // jgarcia Jose Garcia <jgarcia@ucar.edu>
32 
33 #include "config.h"
34 
35 #if HAVE_UNISTD_H
36 #include <unistd.h>
37 #endif
38 
39 #include <cerrno>
40 #include <cstring>
41 
42 #include <string>
43 #include <vector>
44 #include <map>
45 #include <sstream>
46 
47 #include "BESDebug.h"
48 #include "TheBESKeys.h"
49 #include "kvp_utils.h"
50 #include "BESUtil.h"
51 #include "BESRegex.h"
52 #include "BESFSDir.h"
53 #include "BESFSFile.h"
54 #include "BESInternalFatalError.h"
55 #include "BESInternalError.h"
56 #include "BESSyntaxUserError.h"
57 #include "BESLog.h"
58 
59 #define BES_INCLUDE_KEY "BES.Include"
60 
61 using namespace std;
62 
63 #define MODULE "bes"
64 #define prolog std::string("TheBESKeys::").append(__func__).append("() - ")
65 
66 set<string> TheBESKeys::d_ingested_key_files;
67 
68 TheBESKeys *TheBESKeys::d_instance = 0;
69 string TheBESKeys::ConfigFile = "";
70 
72 {
73  if (d_instance) return d_instance;
74 
75  if (!TheBESKeys::ConfigFile.empty()) {
76  d_instance = new TheBESKeys(TheBESKeys::ConfigFile);
77  return d_instance;
78  }
79 
80  // d_instance is a nullptr and TheBESKeys::ConfigFile is ""
81  // so lets try some obvious places...
82 
83  string try_ini = "/usr/local/etc/bes/bes.conf";
84  if (access(try_ini.c_str(), R_OK) == 0) {
85  TheBESKeys::ConfigFile = try_ini;
86  d_instance = new TheBESKeys(TheBESKeys::ConfigFile);
87  return d_instance;
88  }
89 
90  try_ini = "/etc/bes/bes.conf";
91  if (access(try_ini.c_str(), R_OK) == 0) {
92  TheBESKeys::ConfigFile = try_ini;
93  d_instance = new TheBESKeys(TheBESKeys::ConfigFile);
94  return d_instance;
95  }
96 
97  try_ini = "/usr/etc/bes/bes.conf";
98  if (access(try_ini.c_str(), R_OK) == 0) {
99  TheBESKeys::ConfigFile = try_ini;
100  d_instance = new TheBESKeys(TheBESKeys::ConfigFile);
101  return d_instance;
102 }
103  throw BESInternalFatalError("Unable to locate a BES configuration file.", __FILE__, __LINE__);
104 }
105 
122 TheBESKeys::TheBESKeys(const string &keys_file_name) :
123  d_keys_file_name(keys_file_name), d_the_keys(0), d_the_original_keys(0), d_dynamic_config_in_use(false), d_own_keys(true)
124 {
125  d_the_keys = new map<string, vector<string> >;
126  d_the_original_keys = new map<string, vector<string> >;
127  initialize_keys();
128 }
129 
130 #if 0
131 TheBESKeys::TheBESKeys(const string &keys_file_name, map<string, vector<string> > *keys) :
132  d_keys_file_name(keys_file_name), d_the_keys(keys), d_the_original_keys(0), d_dynamic_config_in_use(false), d_own_keys(false)
133 {
134  initialize_keys();
135 }
136 #endif
137 
141 {
142  clean();
143 }
144 
145 void TheBESKeys::initialize_keys()
146 {
147  kvp::load_keys(d_keys_file_name, d_ingested_key_files, *d_the_keys);
148  *d_the_original_keys = *d_the_keys;
149  BESDEBUG(MODULE, prolog << " d_the_keys.size(): " << d_the_keys->size() << endl);
150  BESDEBUG(MODULE, prolog << "d_the_original_keys.size(): " << d_the_original_keys->size() << endl);
151 }
152 
153 void TheBESKeys::clean()
154 {
155 
156  if (d_the_keys && d_own_keys) {
157  delete d_the_keys;
158  d_the_keys = 0;
159  }
160  if(d_the_original_keys){
161  delete d_the_original_keys;
162  d_the_original_keys = 0;
163  }
164 }
165 
173 bool TheBESKeys::LoadedKeys(const string &key_file)
174 {
175 #if 0
176  vector<string>::const_iterator i = TheBESKeys::d_ingested_key_files.begin();
177  vector<string>::const_iterator e = TheBESKeys::d_ingested_key_files.end();
178  for (; i != e; i++) {
179  if ((*i) == key_file) {
180  return true;
181  }
182  }
183 #endif
184  set<string>::iterator it = d_ingested_key_files.find(key_file);
185 
186  return it != d_ingested_key_files.end();
187 }
188 
205 void TheBESKeys::set_key(const string &key, const string &val, bool addto)
206 {
207  map<string, vector<string> >::iterator i;
208  i = d_the_keys->find(key);
209  if (i == d_the_keys->end()) {
210  vector<string> vals;
211  (*d_the_keys)[key] = vals;
212  }
213  if (!addto) (*d_the_keys)[key].clear();
214  if (!val.empty()) {
215  (*d_the_keys)[key].push_back(val);
216  }
217 }
218 
235 void TheBESKeys::set_keys(const string &key, const vector<string> &values, bool addto)
236 {
237  map<string, vector<string> >::iterator i;
238  i = d_the_keys->find(key);
239  if (i == d_the_keys->end()) {
240  vector<string> vals;
241  (*d_the_keys)[key] = vals;
242  }
243  if (!addto) (*d_the_keys)[key].clear();
244 
245  size_t j;
246  for(j = 0; j!=values.size(); j++){
247  if (!values[j].empty()) {
248  (*d_the_keys)[key].push_back(values[j]);
249  }
250  }
251 }
252 
253 
272  const string &key,
273  const map<string, string> &values,
274  const bool case_insensitive_map_keys,
275  bool addto)
276 {
277  map<string, vector<string> >::iterator i;
278  i = d_the_keys->find(key);
279  if (i == d_the_keys->end()) {
280  vector<string> vals;
281  (*d_the_keys)[key] = vals;
282  }
283  if (!addto) {
284  (*d_the_keys)[key].clear();
285  }
286 
287  map<string, string>::const_iterator mit;
288  for(mit = values.begin(); mit!=values.end(); mit++){
289  string map_key = mit->first;
290  if(map_key.empty() ){
291  BESDEBUG(MODULE, prolog << "The map_key is empty. SKIPPING." << endl);
292  }
293  else {
294  if(case_insensitive_map_keys){
295  map_key = BESUtil::lowercase(map_key);
296  }
297  string map_record=map_key+":"+mit->second;
298  (*d_the_keys)[key].push_back(map_record);
299  }
300  }
301 }
302 
303 
304 
316 void TheBESKeys::set_key(const string &pair)
317 {
318  string key;
319  string val;
320  bool addto = false;
321  kvp::break_pair(pair.c_str(), key, val, addto);
322  set_key(key, val, addto);
323 }
324 
339 void TheBESKeys::get_value(const string &s, string &val, bool &found)
340 {
341  found = false;
342  map<string, vector<string> >::iterator i;
343  i = d_the_keys->find(s);
344  if (i != d_the_keys->end()) {
345  found = true;
346  if ((*i).second.size() > 1) {
347  string err = string("Multiple values for the key ") + s + " found, should only be one.";
348  throw BESInternalError(err, __FILE__, __LINE__);
349  }
350  if ((*i).second.size() == 1) {
351  val = (*i).second[0];
352  }
353  else {
354  val = "";
355  }
356  }
357 }
358 
370 void TheBESKeys::get_values(const string& s, vector<string> &vals, bool &found)
371 {
372  found = false;
373  map<string, vector<string> >::iterator i;
374  i = d_the_keys->find(s);
375  if (i != d_the_keys->end()) {
376  found = true;
377  vector<string>::iterator j;
378  for(j=(*i).second.begin(); j!=(*i).second.end(); j++){
379  vals.push_back(*j);
380  }
381  // vals = (*i).second; // BUT WHY NOT?
382  }
383 }
384 
397 bool TheBESKeys::read_bool_key(const string &key, bool default_value)
398 {
399  bool found = false;
400  string value;
401  TheBESKeys::TheKeys()->get_value(key, value, found);
402  // 'key' holds the string value at this point if key_found is true
403  if (found) {
404  value = BESUtil::lowercase(value);
405  return (value == "true" || value == "yes"|| value == "on");
406  }
407  else {
408  return default_value;
409  }
410  }
411 
422 string TheBESKeys::read_string_key(const string &key, const string &default_value)
423 {
424  bool found = false;
425  string value;
426  TheBESKeys::TheKeys()->get_value(key, value, found);
427  // 'value' holds the string value at this point if found is true
428  if (found) {
429  if (value[value.length() - 1] == '/') value.erase(value.length() - 1);
430  return value;
431  }
432  else {
433  return default_value;
434  }
435 }
436 
447 int TheBESKeys::read_int_key(const string &key, int default_value)
448 {
449  bool found = false;
450  string value;
451  TheBESKeys::TheKeys()->get_value(key, value, found);
452  // 'key' holds the string value at this point if found is true
453  if (found) {
454  std::istringstream iss(value);
455  int int_val;
456  iss >> int_val;
457  if (iss.eof() || iss.bad() || iss.fail())
458  return default_value;
459  else
460  return int_val;
461  }
462  else {
463  return default_value;
464  }
465 }
466 
473 void TheBESKeys::dump(ostream &strm) const
474 {
475  strm << dump();
476 }
477 
483 string TheBESKeys::dump() const
484 {
485  stringstream ss;
486  ss << BESIndent::LMarg << "BESKeys::dump - (" << (void *) this << ")" << endl;
487  BESIndent::Indent();
488  ss << BESIndent::LMarg << "key file:" << d_keys_file_name << endl;
489 
490 #if 0
491  if (_keys_file && *_keys_file) {
492  strm << BESIndent::LMarg << "key file is valid" << endl;
493  }
494  else {
495  strm << BESIndent::LMarg << "key file is NOT valid" << endl;
496  }
497 #endif
498 
499  if (d_the_keys && d_the_keys->size()) {
500  ss << BESIndent::LMarg << " keys:" << endl;
501  BESIndent::Indent();
502  Keys_citer i = d_the_keys->begin();
503  Keys_citer ie = d_the_keys->end();
504  for (; i != ie; i++) {
505  ss << BESIndent::LMarg << (*i).first << ": " /*<< endl*/;
506  // BESIndent::Indent();
507  vector<string>::const_iterator v = (*i).second.begin();
508  vector<string>::const_iterator ve = (*i).second.end();
509  for (; v != ve; v++) {
510  ss << (*v) << " "; //endl;
511  }
512  ss << endl;
513  //BESIndent::UnIndent();
514  }
515  BESIndent::UnIndent();
516  }
517  else {
518  ss << BESIndent::LMarg << "keys: none" << endl;
519  }
520  BESIndent::UnIndent();
521  return ss.str();
522 }
523 
524 
525 
526 #define MAP_SEPARATOR ":"
527 
528 bool parse_map_record(const string &map_record, const bool &case_insensitive_map_keys, string &key, string &value) {
529  int primary_index = map_record.find(MAP_SEPARATOR);
530  if (primary_index > 0) {
531  key = map_record.substr(0, primary_index);
532  if (case_insensitive_map_keys)
533  key = BESUtil::lowercase(key);
534  value = map_record.substr(primary_index + 1);
535  BESDEBUG(MODULE, prolog << "key: '" << key << "' value: " << value << endl);
536  return true;
537  }
538  return false;
539 }
540 
541 
550  const std::string &key,
551  std::map<std::string,std::string> &map_values,
552  const bool &case_insensitive_map_keys,
553  bool &found){
554 
555  vector<string> values;
556  get_values(key, values, found);
557  if(!found){
558  return;
559  }
560 
561  vector<string>::iterator it;
562  for(it=values.begin(); it!=values.end(); it++){
563  string map_key;
564  string map_value;
565  if(parse_map_record(*it,case_insensitive_map_keys,map_key,map_value)){
566  map_values.insert( std::pair<string,string>(map_key,map_value));
567  }
568  else {
569  BESDEBUG(MODULE, prolog << string("The configuration entry for the ") << key << " was not " <<
570  "formatted as a map record. The offending entry: " << *it << " HAS BEEN SKIPPED." << endl);
571  }
572  }
573 
574 }
575 
576 
585  const std::string &key,
586  std::map< std::string, std::map<std::string,std::vector<std::string> > > &primary_map,
587  const bool &case_insensitive_map_keys,
588  bool &found){
589 
590  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
591  vector<string> values;
592  get_values(key, values, found);
593  if(!found){
594  return;
595  }
596 
597  vector<string>::iterator it;
598  for(it=values.begin(); it!=values.end(); it++){
599  string map_record = *it;
600  string primary_map_key;
601  string primary_map_value;
602  if(parse_map_record(map_record,case_insensitive_map_keys,primary_map_key,primary_map_value)){
603  string secondary_key;
604  string secondary_value;
605  if(parse_map_record(primary_map_value,case_insensitive_map_keys,secondary_key,secondary_value)){
606  map<string, map<string,vector<string>>>::iterator pit;
607  pit = primary_map.find(primary_map_key);
608  if(pit!=primary_map.end()){
609  map<string,vector<string>>::iterator sit;
610  sit = pit->second.find(secondary_key);
611  if(sit!=pit->second.end()){
612  sit->second.push_back(secondary_value);
613  }
614  else {
615  // How to make a vector<string>> and poke in to the secondary_map??
616  vector<string> secondary_map_entry_values;
617  secondary_map_entry_values.push_back(secondary_value);
618  pit->second.insert(pair<string,vector<string>>(secondary_key,secondary_map_entry_values));
619  }
620  }
621  else {
622  // How to make a map<string,vector<string>> and poke in to the primary_map??
623  map<string,vector<string>> secondary_map_entry;
624  vector<string> secondary_map_entry_values;
625  secondary_map_entry_values.push_back(secondary_value);
626  secondary_map_entry.insert(pair<string,vector<string>>(secondary_key,secondary_map_entry_values));
627  primary_map.insert(pair<string, map<string,vector<string>>>(primary_map_key,secondary_map_entry));
628  }
629  }
630  else {
631  // Map entry improperly formatted.
632  BESDEBUG(MODULE, prolog << string("The configuration entry for the ") << key << " was not " <<
633  "formatted as a map record. The offending entry: " << map_record <<
634  " HAS BEEN SKIPPED." << endl);
635  }
636  }
637  else {
638  BESDEBUG(MODULE, prolog << string("The configuration entry for the ") << key << " was not " <<
639  "formatted as a map record. The offending entry: " << map_record <<
640  " HAS BEEN SKIPPED." << endl);
641  }
642  }
643  BESDEBUG(MODULE, prolog << "END" << endl);
644 
645 }
646 
647 bool TheBESKeys::using_dynamic_config(){
648  return d_dynamic_config_in_use;
649 }
650 
651 
656 void TheBESKeys::load_dynamic_config(const string name)
657 {
658 #if DYNAMIC_CONFIG_ENABLED
659 
660  BESDEBUG(MODULE, prolog << "BEGIN" << endl);
661 
662  // Clear the active keys and copy the original keys into
663  // the active keys (resets the keys to 'as read from config files')
664  if( d_dynamic_config_in_use ){
665  BESDEBUG(MODULE, prolog << "Unloading DynamicConfig." << endl);
666  d_the_keys->clear();
667  *d_the_keys = *d_the_original_keys;
668  d_dynamic_config_in_use = false;
669  }
670 
671  map<string, map<string, vector<string>>> dynamic_confg;
672  bool found;
673  get_values(DYNAMIC_CONFIG_KEY, dynamic_confg, true, found);
674  if(!found){
675  BESDEBUG(MODULE, prolog << "Unable to locate " << DYNAMIC_CONFIG_KEY
676  << " in the configuration keys." << endl);
677  return;
678  }
679  BESDEBUG(MODULE, prolog << "Found a " << DYNAMIC_CONFIG_KEY << " in TheBESKeys." << endl);
680 
681  string best_matching_config_name;
682  long longest_match=0;
683  map<string, map<string, vector<string>>>::iterator best_matching_config=dynamic_confg.end();
684 
685  map<string, map<string, vector<string>>>::iterator dcit;
686  for(dcit = dynamic_confg.begin(); dcit != dynamic_confg.end(); dcit++){
687  BESDEBUG(MODULE, prolog << "Processing " << DYNAMIC_CONFIG_KEY << "["<<dcit->first<< "]" << endl);
688 
689  map<string, vector<string>>::iterator rit;
690  rit = dcit->second.find(DC_REGEX_KEY);
691  if(rit==dcit->second.end()){
692  BESDEBUG(MODULE, prolog << "Could not find a " << DC_REGEX_KEY << " (regular expression) for the "
693  << DYNAMIC_CONFIG_KEY << " named: " << dcit->first << " SKIPPING!" << endl);
694  }
695  else {
696  BESDEBUG(MODULE, prolog << "Found " << DC_REGEX_KEY << " vector for "
697  << DYNAMIC_CONFIG_KEY << "["<< dcit->first << "]" << endl);
698  vector<string>::iterator vit;
699  for(vit = rit->second.begin(); vit != rit->second.end(); vit ++){ // For all the regex expressions
700  BESDEBUG(MODULE, prolog << "Processing " << DC_REGEX_KEY << " value '" << *vit << "'" << endl);
701  BESRegex regex((*vit).c_str()); // make BESRegex
702  long match_length = regex.match(name.c_str(),name.size(),0); // Eval match
703 
704  BESDEBUG(MODULE, prolog << "The name '"<< name << (match_length<0?"' does not match ":"' matches ")
705  << "the regular expression: '"<< *vit << "' (match_length: " << match_length << ")" << endl);
706  if(match_length>longest_match){ // Is is a better match?
707  BESDEBUG(MODULE, prolog << "match_length of " << match_length
708  << " is larger than the current longest_match of "<< longest_match << endl);
709 
710  map<string, vector<string>>::iterator cit;
711  cit = dcit->second.find(DC_CONFIG_KEY);
712  if(cit==dcit->second.end() || cit->second.empty()){ // does it have a config?
713  BESDEBUG(MODULE, prolog << "There were no " << DC_CONFIG_KEY
714  << " (configuration) values for the " << DYNAMIC_CONFIG_KEY << " named: "
715  << dcit->first << " SKIPPING!" << endl);
716  }
717  else {
718 
719  best_matching_config = dcit;
720  longest_match = match_length;
721  best_matching_config_name = dcit->first;
722  BESDEBUG(MODULE, prolog << "Found new best " << DYNAMIC_CONFIG_KEY << " match for '" << name
723  << "' " << DYNAMIC_CONFIG_KEY << ": " << best_matching_config_name << endl);
724  }
725  }
726  }
727  }
728  }
729 
730  if( longest_match==0 || best_matching_config==dynamic_confg.end() ){
731  BESDEBUG(MODULE, prolog << "None of the " << DYNAMIC_CONFIG_KEY
732  << " regex patterns matched the name: " << name << endl);
733  return;
734  }
735 
736  {
737  stringstream msg;
738  msg << prolog << "Using " << DYNAMIC_CONFIG_KEY << ":" << best_matching_config_name << " for: " << name << endl;
739  BESDEBUG(MODULE, msg.str());
740  LOG( msg.str());
741  }
742 
743  // Now load the specific keys from the dynamic config;
744  map<string, vector<string>>::iterator cit;
745  cit = best_matching_config->second.find(DC_CONFIG_KEY);
746  vector<string>::iterator vit;
747  for(vit=cit->second.begin(); vit != cit->second.end(); vit++){
748  // Each value of this vector should be a regular BESKeys kvp. i.e. "BES.LogName=./opendap.log"
749  // Which we just feed into the keys, since we just backed them up...
750  BESDEBUG(MODULE, prolog << "Adding dynamic configuration BES Key: " << *vit << endl);
751  set_key(*vit);
752  }
753  d_dynamic_config_in_use = true;
754 
755  BESDEBUG(MODULE, prolog << "END" << endl);
756 #endif
757 
758  BESDEBUG("bes:keys",dump());
759 
760 }
761 
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
int match(const char *s, int len, int pos=0)
Does the pattern match.
Definition: BESRegex.cc:107
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:200
mapping of key/value pairs defining different behaviors of an application.
Definition: TheBESKeys.h:85
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:339
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
void set_key(const std::string &key, const std::string &val, bool addto=false)
allows the user to set key/value pairs from within the application.
Definition: TheBESKeys.cc:205
int read_int_key(const std::string &key, int default_value)
Read an integer-valued key from the bes.conf file.
Definition: TheBESKeys.cc:447
void get_values(const std::string &s, std::vector< std::string > &vals, bool &found)
Retrieve the values of a given key, if set.
Definition: TheBESKeys.cc:370
void load_dynamic_config(std::string name)
Definition: TheBESKeys.cc:656
virtual std::string dump() const
dumps information about this object
Definition: TheBESKeys.cc:483
static std::string ConfigFile
Definition: TheBESKeys.h:184
bool read_bool_key(const std::string &key, bool default_value)
Read a boolean-valued key from the bes.conf file.
Definition: TheBESKeys.cc:397
std::string read_string_key(const std::string &key, const std::string &default_value)
Read a string-valued key from the bes.conf file.
Definition: TheBESKeys.cc:422
void set_keys(const std::string &key, const std::vector< std::string > &values, bool addto)
allows the user to set key/value pairs from within the application.
Definition: TheBESKeys.cc:235
virtual ~TheBESKeys()
cleans up the key/value pair mapping
Definition: TheBESKeys.cc:140