bes Updated for version 3.20.10
util_ff.cc
1
2// -*- mode: c++; c-basic-offset:4 -*-
3
4// This file is part of ff_handler a FreeForm API handler for the OPeNDAP
5// DAP2 data server.
6
7// Copyright (c) 2005 OPeNDAP, Inc.
8// Author: James Gallagher <jgallagher@opendap.org>
9//
10// This is free software; you can redistribute it and/or modify it under the
11// terms of the GNU Lesser General Public License as published by the Free
12// Software Foundation; either version 2.1 of the License, or (at your
13// option) any later version.
14//
15// This software is distributed in the hope that it will be useful, but
16// WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
17// or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
18// License for more details.
19//
20// You should have received a copy of the GNU Lesser General Public
21// License along with this library; if not, write to the Free Software
22// Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23//
24// You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
25
26// (c) COPYRIGHT URI/MIT 1997-99
27// Please read the full copyright statement in the file COPYRIGHT.
28//
29// Authors: reza (Reza Nekovei)
30
31// Utility functions for the FreeForm data server.
32//
33// jhrg 3/29/96
34
35#include "config_ff.h"
36
37static char rcsid[] not_used =
38 { "$Id$" };
39
40#ifndef WIN32
41#include <unistd.h> // for access
42#else
43#define F_OK 0
44#endif
45
46#include <iostream>
47#include <sstream>
48#include <fstream>
49#include <string>
50#include <vector>
51#include <cstdlib>
52
53#include <BESRegex.h>
54#include <BESDebug.h>
55
56#include <libdap/BaseType.h>
57#include <libdap/Byte.h>
58#include <libdap/Int16.h>
59#include <libdap/Int32.h>
60#include <libdap/UInt16.h>
61#include <libdap/UInt32.h>
62#include <libdap/Float32.h>
63#include <libdap/Float64.h>
64#include <libdap/InternalErr.h>
65#include <libdap/dods-limits.h>
66#include <libdap/util.h>
67#include <libdap/debug.h>
68
69#include "BESRegex.h"
70#include "BESDebug.h"
71
72#include "FFRequestHandler.h"
73#include "util_ff.h"
74
75using namespace std;
76
90static string &remove_paths(string &src)
91{
92 size_t p1 = src.find_first_of('/');
93 if (p1 == string::npos)
94 return src;
95 size_t p2 = src.find_last_of('/');
96 // The string has one '/', not a big deal
97 if (p2 == p1)
98 return src;
99
100 src.erase(p1, p2-p1+1);
101 return src;
102}
103
104// These two functions are defined in FFND/error.c. They were originally
105// static functions. I used them to read error strings since FreeForm
106// did not have a good way to get the error text. jhrg 9/11/12
107extern "C" FF_ERROR_PTR pull_error(void);
108extern "C" BOOLEAN is_a_warning(FF_ERROR_PTR error);
109
123static string freeform_error_message()
124{
125 FF_ERROR_PTR error = pull_error();
126 if (!error)
127 throw BESInternalError("Called the FreeForm error message code, but there was no error.", __FILE__, __LINE__);
128
129 ostringstream oss;
130 do {
131 if (is_a_warning(error))
132 oss << "Warning: ";
133 else
134 oss << "Error: ";
135
136 // if either of these contain a pathname, remove it
137 string problem = error->problem;
138 string message = error->message;
139 oss << remove_paths(problem) << ": " << remove_paths(message) << endl;
140
141 ff_destroy_error (error);
142 error = pull_error();
143 } while (error);
144
145 return oss.str();
146}
147
160long read_ff(const char *dataset, const char *if_file, const char *o_format, char *o_buffer, unsigned long bsize)
161{
162 FF_BUFSIZE_PTR newform_log = NULL;
163 FF_STD_ARGS_PTR std_args = NULL;
164
165 try {
166 std_args = ff_create_std_args();
167 if (!std_args)
168 throw BESInternalError("FreeForm could not allocate a 'stdargs' object.", __FILE__, __LINE__);
169
170 // set the std_arg structure values - cast away const for dataset, if_file,
171 // and o_format.
172 std_args->error_prompt = FALSE;
173 std_args->user.is_stdin_redirected = 0;
174 std_args->input_file = (char*) (dataset);
175 std_args->input_format_file = (char*) (if_file);
176 std_args->output_file = NULL;
177 std_args->output_format_buffer = (char*) (o_format);
178 std_args->log_file = (char *) "/dev/null";
179#if 0
180 // This just doesn't seem to work within the BES framework. jhrg 9/11/12
181 std_args->log_file = (char *)"/tmp/ffdods.log";
182#endif
183
184 // memory freed automatically on exit
185 vector<char> l_bufsz(sizeof(FF_BUFSIZE));
186 //bufsz = (FF_BUFSIZE *)&l_bufsz[0];
187 FF_BUFSIZE_PTR bufsz = reinterpret_cast<FF_BUFSIZE_PTR>(&l_bufsz[0]);
188
189 bufsz->usage = 1;
190 bufsz->buffer = o_buffer;
191 bufsz->total_bytes = (FF_BSS_t) bsize;
192 bufsz->bytes_used = (FF_BSS_t) 0;
193
194 std_args->output_bufsize = bufsz;
195
196 newform_log = ff_create_bufsize(SCRATCH_QUANTA);
197 if (!newform_log)
198 throw BESInternalError("FreeForm could not allocate a 'newform_log' object.", __FILE__, __LINE__);
199
200 // passing 0 for the FILE* param is a wash since a non-null
201 // value for newform_log selects that as the 'logging' sink.
202 // jhrg 9/11/12
203 int status = newform(std_args, newform_log, 0 /*stderr*/);
204
205 BESDEBUG("ff", "FreeForm: newform returns " << status << endl);
206
207 if (err_count()) {
208 string message = freeform_error_message();
209 BESDEBUG("ff", "FreeForm: error message " << message << endl);
210 throw BESError(message, BES_SYNTAX_USER_ERROR, __FILE__, __LINE__);
211 }
212
213 ff_destroy_bufsize(newform_log);
214 ff_destroy_std_args(std_args);
215
216 return bufsz->bytes_used;
217 }
218 catch (...) {
219 if (newform_log)
220 ff_destroy_bufsize(newform_log);
221 if (std_args)
222 ff_destroy_std_args(std_args);
223
224 throw;
225 }
226
227 return 0;
228}
229
236void free_ff_char_vector(char **v, int len)
237{
238 for (int i = 0; i < len; ++i)
239 if (v && v[i])
240 free (v[i]);
241 if (v && len > 0)
242 free (v);
243}
244
245// Given the name of a DODS data type, return the name of the corresponding
246// FreeForm data type.
247//
248// Returns: a const char * if the DODS type maps into a FreeForm type,
249// otherwise NULL.
250
251const string ff_types(Type dods_type)
252{
253 switch (dods_type) {
254 case dods_byte_c:
255 return "uint8";
256 case dods_int16_c:
257 return "int16";
258 case dods_uint16_c:
259 return "uint16";
260 case dods_int32_c:
261 return "int32";
262 case dods_uint32_c:
263 return "uint32";
264 case dods_float32_c:
265 return "float32";
266 case dods_float64_c:
267 return "float64";
268 case dods_str_c:
269 return "text";
270 case dods_url_c:
271 return "text";
272 default:
273 throw Error("ff_types: DODS type " + D2type_name(dods_type) + " does not map to a FreeForm type.");
274 }
275}
276
277// Given the name of a DODS data type, return the precision of the
278// corresponding FreeForm data type.
279//
280// Returns: a positive integer if the DODS type maps into a FreeForm type,
281// otherwise -1.
282
283int ff_prec(Type dods_type)
284{
285 switch (dods_type) {
286 case dods_byte_c:
287 case dods_int16_c:
288 case dods_uint16_c:
289 case dods_int32_c:
290 case dods_uint32_c:
291 return 0;
292 case dods_float32_c:
293 return DODS_FLT_DIG;
294 case dods_float64_c:
295 return DODS_DBL_DIG;
296 case dods_str_c:
297 case dods_url_c:
298 return 0;
299 default:
300 throw Error("ff_prec: DODS type " + D2type_name(dods_type) + " does not map to a FreeForm type.");
301 }
302}
303
309const string
310make_output_format(const string & name, Type type, const int width)
311{
312 ostringstream str;
313
314 str << "binary_output_data \"DODS binary output data\"" << endl;
315 str << name << " 1 " << width << " " << ff_types(type)
316 << " " << ff_prec(type) << endl;
317
318 return str.str();
319}
320
321// format for multi-dimension array
322const string
323makeND_output_format(const string & name, Type type, const int width,
324 int ndim, const long *start, const long *edge, const
325 long *stride, string * dname)
326{
327 ostringstream str;
328 str << "binary_output_data \"DODS binary output data\"" << endl;
329 str << name << " 1 " << width << " ARRAY";
330
331 for (int i = 0; i < ndim; i++)
332 str << "[" << "\"" << dname[i] << "\" " << start[i] + 1 << " to "
333 << start[i] + (edge[i] - 1) * stride[i] +
334 1 << " by " << stride[i] << " ]";
335
336 str << " of " << ff_types(type) << " " << ff_prec(type) << endl;
337
338 DBG(cerr << "ND output format: " << str.str() << endl);
339
340 return str.str();
341}
342
348const string & format_delimiter(const string & new_delimiter)
349{
350 static string delimiter = ".";
351
352 if (new_delimiter != "")
353 delimiter = new_delimiter;
354
355 return delimiter;
356}
357
364const string & format_extension(const string & new_extension)
365{
366 static string extension = ".fmt";
367
368 if (new_extension != "")
369 extension = new_extension;
370
371 return extension;
372}
373
376static bool
377cmp_array_conduit(FF_ARRAY_CONDUIT_PTR src_conduit,
378 FF_ARRAY_CONDUIT_PTR trg_conduit)
379{
380 if (src_conduit->input && trg_conduit->input)
381 return (bool) ff_format_comp(src_conduit->input->fd->format,
382 trg_conduit->input->fd->format);
383 else if (src_conduit->output && trg_conduit->output)
384 return (bool) ff_format_comp(src_conduit->output->fd->format,
385 trg_conduit->output->fd->format);
386 else
387 return false;
388}
389
390static int merge_redundant_conduits(FF_ARRAY_CONDUIT_LIST conduit_list)
391{
392 int error = 0;
393 error = list_replace_items((pgenobj_cmp_t) cmp_array_conduit,
394 conduit_list);
395 return (error);
396}
397
406int SetDodsDB(FF_STD_ARGS_PTR std_args, DATA_BIN_HANDLE dbin_h, char *Msgt)
407{
408 FORMAT_DATA_LIST format_data_list = NULL;
409 int error = 0;
410
411 assert(dbin_h);
412
413 if (!dbin_h) {
414 snprintf(Msgt, Msgt_size, "Error: NULL DATA_BIN_HANDLE in %s", ROUTINE_NAME);
415 return (ERR_API);
416 }
417
418 if (!*dbin_h) {
419 *dbin_h = db_make(std_args->input_file);
420
421 if (!*dbin_h) {
422 snprintf(Msgt, Msgt_size, "Error in Standard Data Bin");
423 return (ERR_MEM_LACK);
424 }
425 }
426
427 /* Now set the formats and the auxiliary files */
428
429 if (db_set(*dbin_h, DBSET_READ_EQV, std_args->input_file)) {
430 snprintf(Msgt, Msgt_size, "Error making name table for %s",
431 std_args->input_file);
432 return (DBSET_READ_EQV);
433 }
434
435 if (db_set(*dbin_h,
436 DBSET_INPUT_FORMATS,
437 std_args->input_file,
438 std_args->output_file,
439 std_args->input_format_file,
440 std_args->input_format_buffer,
441 std_args->input_format_title, &format_data_list)) {
442 if (format_data_list)
443 dll_free_holdings(format_data_list);
444
445 snprintf(Msgt, Msgt_size, "Error setting an input format for %s",
446 std_args->input_file);
447 return (DBSET_INPUT_FORMATS);
448 }
449
450 error =
451 db_set(*dbin_h, DBSET_CREATE_CONDUITS, std_args, format_data_list);
452 dll_free_holdings(format_data_list);
453 if (error) {
454 snprintf(Msgt, Msgt_size, "Error creating array information for %s",
455 std_args->input_file);
456 return (DBSET_CREATE_CONDUITS);
457 }
458
459 if (db_set(*dbin_h, DBSET_HEADER_FILE_NAMES, FFF_INPUT,
460 std_args->input_file)) {
461 snprintf(Msgt, Msgt_size, "Error determining input header file names for %s",
462 std_args->input_file);
463 return (DBSET_HEADER_FILE_NAMES);
464 }
465
466 if (db_set(*dbin_h, DBSET_HEADERS)) {
467 snprintf(Msgt, Msgt_size, "getting header file for %s", std_args->input_file);
468 return (DBSET_HEADERS);
469 }
470
471
472 if (db_set(*dbin_h, DBSET_INIT_CONDUITS, FFF_DATA,
473 std_args->records_to_read)) {
474 snprintf(Msgt, Msgt_size, "Error creating array information for %s",
475 std_args->input_file);
476 return (DBSET_INIT_CONDUITS);
477 }
478
479 error = merge_redundant_conduits((*dbin_h)->array_conduit_list);
480 if (error)
481 snprintf(Msgt, Msgt_size, "Error merging redundent conduits");
482
483 return (error);
484}
485
491long Records(const string &filename)
492{
493 int error = 0;
494 DATA_BIN_PTR dbin = NULL;
495 FF_STD_ARGS_PTR SetUps = NULL;
496 PROCESS_INFO_LIST pinfo_list = NULL;
497 PROCESS_INFO_PTR pinfo = NULL;
498 static char Msgt[255];
499
500 SetUps = ff_create_std_args();
501 if (!SetUps) {
502 return -1;
503 }
504
506 SetUps->user.is_stdin_redirected = 0;
507 SetUps->input_file = const_cast<char*>(filename.c_str());
508
509 SetUps->output_file = NULL;
510
511 error = SetDodsDB(SetUps, &dbin, Msgt);
512 if (error && error < ERR_WARNING_ONLY) {
513 ff_destroy_std_args(SetUps);
514 db_destroy(dbin);
515 return -1;
516 }
517
518 ff_destroy_std_args(SetUps);
519
520 error = db_ask(dbin, DBASK_PROCESS_INFO, FFF_INPUT | FFF_DATA, &pinfo_list);
521 if (error)
522 return (-1);
523
524 pinfo_list = dll_first(pinfo_list);
525
526 pinfo = ((PROCESS_INFO_PTR) (pinfo_list)->data.u.pi);
527
528 long num_records = PINFO_SUPER_ARRAY_ELS(pinfo);
529
530 ff_destroy_process_info_list(pinfo_list);
531 db_destroy(dbin);
532
533 return num_records;
534}
535
536
537bool file_exist(const char *filename)
538{
539 return access(filename, F_OK) == 0;
540}
541
554const string
555find_ancillary_rss_formats(const string & dataset, const string & /* delimiter */,
556 const string & /* extension */)
557{
558 string FormatFile;
559 string FormatPath = FFRequestHandler::get_RSS_format_files();
560 string BaseName;
561 string FileName;
562
563 // Separate the filename from the pathname, for both plain files
564 // and cached decompressed files (ones with '#' in their names).
565 size_t delim = dataset.rfind("#");
566 if (delim != string::npos)
567 FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
568 else {
569 delim = dataset.rfind("/");
570 if (delim != string::npos)
571 FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
572 else
573 FileName = dataset;
574 }
575
576 // The file/dataset name has to have an underscore...
577 delim = FileName.find("_");
578 if ( delim != string::npos ) {
579 BaseName = FileName.substr(0,delim+1);
580 }
581 else {
582 throw Error("Could not find input format for: " + dataset);
583 }
584
585 // Now determine if this is files holds averaged or daily data.
586 string DatePart = FileName.substr(delim+1, FileName.length()-delim+1);
587
588 if (FormatPath[FormatPath.length()-1] != '/')
589 FormatPath.append("/");
590
591 if ( (DatePart.find("_") != string::npos) || (DatePart.length() < 10) )
592 FormatFile = FormatPath + BaseName + "averaged.fmt";
593 else
594 FormatFile = FormatPath + BaseName + "daily.fmt";
595
596 return string(FormatFile);
597}
598
611const string
612find_ancillary_rss_das(const string & dataset, const string & /* delimiter */,
613 const string & /* extension */)
614{
615 string FormatFile;
616 string FormatPath = FFRequestHandler::get_RSS_format_files();
617 string BaseName;
618 string FileName;
619
620 size_t delim = dataset.rfind("#");
621 if (delim != string::npos)
622 FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
623 else {
624 delim = dataset.rfind("/");
625 if (delim != string::npos)
626 FileName = dataset.substr(delim + 1, dataset.length() - delim + 1);
627 else
628 FileName = dataset;
629 }
630
631 delim = FileName.find("_");
632 if ( delim != string::npos ) {
633 BaseName = FileName.substr(0,delim+1);
634 }
635 else {
636 string msg = "Could not find input format for: ";
637 msg += dataset;
638 throw InternalErr(msg);
639 }
640
641 string DatePart = FileName.substr(delim+1, FileName.length()-delim+1);
642
643 if (FormatPath[FormatPath.length()-1] != '/')
644 FormatPath.append("/");
645
646 if ( (DatePart.find("_") != string::npos) || (DatePart.length() < 10) )
647 FormatFile = FormatPath + BaseName + "averaged.das";
648 else
649 FormatFile = FormatPath + BaseName + "daily.das";
650
651 return string(FormatFile);
652}
653
654// These functions are used by the Date/Time Factory classes but they might
655// be generally useful in writing server-side functions. 1/21/2002 jhrg
656
657bool is_integer_type(BaseType * btp)
658{
659 switch (btp->type()) {
660 case dods_null_c:
661 return false;
662
663 case dods_byte_c:
664 case dods_int16_c:
665 case dods_uint16_c:
666 case dods_int32_c:
667 case dods_uint32_c:
668 return true;
669
670 case dods_float32_c:
671 case dods_float64_c:
672 case dods_str_c:
673 case dods_url_c:
674 case dods_array_c:
675 case dods_structure_c:
676 case dods_sequence_c:
677 case dods_grid_c:
678 default:
679 return false;
680 }
681}
682
683bool is_float_type(BaseType * btp)
684{
685 switch (btp->type()) {
686 case dods_null_c:
687 case dods_byte_c:
688 case dods_int16_c:
689 case dods_uint16_c:
690 case dods_int32_c:
691 case dods_uint32_c:
692 return false;
693
694 case dods_float32_c:
695 case dods_float64_c:
696 return true;
697
698 case dods_str_c:
699 case dods_url_c:
700 case dods_array_c:
701 case dods_structure_c:
702 case dods_sequence_c:
703 case dods_grid_c:
704 default:
705 return false;
706 }
707}
708
713dods_uint32 get_integer_value(BaseType * var) throw(InternalErr)
714{
715 if (!var)
716 return 0;
717
718 switch (var->type()) {
719 case dods_byte_c:
720 return static_cast<Byte*>(var)->value();
721
722 case dods_int16_c:
723 return static_cast<Int16*>(var)->value();
724
725 case dods_int32_c:
726 return static_cast<Int32*>(var)->value();
727
728 case dods_uint16_c:
729 return static_cast<UInt16*>(var)->value();
730
731 case dods_uint32_c:
732 return static_cast<UInt32*>(var)->value();
733
734 default:
735 throw InternalErr(__FILE__, __LINE__,
736 "Tried to get an integer value for a non-integer datatype!");
737 }
738}
739
740dods_float64 get_float_value(BaseType * var) throw(InternalErr)
741{
742 if (!var)
743 return 0.0;
744
745 switch (var->type()) {
746 case dods_int16_c:
747 case dods_uint16_c:
748 case dods_int32_c:
749 case dods_uint32_c:
750 return get_integer_value(var);
751
752 case dods_float32_c:
753 return static_cast<Float32*>(var)->value();
754
755 case dods_float64_c:
756 return static_cast<Float64*>(var)->value();
757
758 default:
759 throw InternalErr(__FILE__, __LINE__,
760 "Tried to get an float value for a non-numeric datatype!");
761 }
762}
763
764string get_Regex_format_file(const string & filename)
765{
766 string::size_type found = filename.find_last_of("/\\");
767 string base_name = filename.substr(found+1);
768 string retVal = "";
769 std::map<string,string> mapFF = FFRequestHandler::get_fmt_regex_map();
770 for (auto rgx = mapFF.begin(); rgx != mapFF.end(); ++ rgx) {
771 BESDEBUG("ff", "get_Regex_format_file() - filename: '" << filename << "'" <<
772 " regex: '" << (*rgx).first << "'" <<
773 " format: '" << (*rgx).second << "'" << endl);
774 BESRegex regex(((*rgx).first).c_str());
775 if ( (unsigned long) regex.match(base_name.c_str(), base_name.length()) == base_name.length() ){
776 retVal = string((*rgx).second);
777 break;
778 }
779 }
780 BESDEBUG("ff", "get_Regex_format_file() - returning format filename: '"<< retVal << "'" << endl);
781 return retVal;
782}
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
Regular expression matching.
Definition: BESRegex.h:53
Type
Type of JSON value.
Definition: rapidjson.h:664