#!/usr/bin/sh
set -e
#set -x;

GET_DMRPP_VERSION="get_dmrpp-3.20.10";

###############################################################################
#
# Default BES Configuration.
#
function mk_default_bes_conf() {
  BES_CONF_DOC=$(
    cat <<EOF
# produce DMR documents for use with build_dmrpp. The DAP, XML Command
# and HDF5 handler modules are the 'installed' ones (not the modules
# found in the build tree). jhrg 5/11/18

BES.ServerAdministrator=admin.email.address@your.domain.name

BES.User=user_name
BES.Group=group_name

BES.LogName=./bes.log
BES.LogVerbose=no

BES.modules=dap,cmd,h5,dmrpp,nc,fonc

BES.module.dap=/usr/lib64/bes/libdap_module.so
BES.module.cmd=/usr/lib64/bes/libdap_xml_module.so
BES.module.h5=/usr/lib64/bes/libhdf5_module.so
BES.module.dmrpp=/usr/lib64/bes/libdmrpp_module.so
BES.module.nc=/usr/lib64/bes/libnc_module.so
BES.module.fonc=/usr/lib64/bes/libfonc_module.so

# The value "@hdf5_root_directory@" is replaced at run time.
#BES.Catalog.catalog.RootDirectory=@hdf5_root_directory@
BES.Catalog.catalog.RootDirectory=${BES_DATA_ROOT}
BES.Data.RootDirectory=/dev/null

# The order is crucial here. The dmr++ files must be associated
# with the dmrpp_module while everything else is being
# gobbled up by the greedy regex for the hdf5_handler.
# If the dmrpp.TypeMatch doesn't come first then the
# hdf5_handler will claim the dmr++ files too.
BES.Catalog.catalog.TypeMatch=dmrpp:.*\.(dmrpp)$;
BES.Catalog.catalog.TypeMatch+=h5:.*(\.bz2|\.gz|\.Z)?$;

BES.Catalog.catalog.Include=;
BES.Catalog.catalog.Exclude=^\..*;

BES.FollowSymLinks=Yes
BES.Catalog.catalog.FollowSymLinks=Yes

BES.Info.Buffered=no
BES.Info.Type=xml

BES.UncompressCache.dir=/tmp/hyrax_ux
BES.UncompressCache.prefix=ux_
BES.UncompressCache.size=500
BES.Uncompress.Retry=2000
BES.Uncompress.NumTries=10

BES.Container.Persistence=strict

BES.Memory.GlobalArea.EmergencyPoolSize=1
BES.Memory.GlobalArea.MaximumHeapSize=20
BES.Memory.GlobalArea.Verbose=no
BES.Memory.GlobalArea.ControlHeap=no

BES.ProcessManagerMethod=multiple

BES.DefaultResponseMethod=POST

# Control the Metadata Response Store. Here, DAP metadata responses
# are stored/cached so that they can be returned by the server w/o
# having to touch the data files/objects. Setting the 'path' to null
# disables uses of the MDS. Setting 'size' to zero makes the MDS
# hold objects forever; setting a positive non-zero size makes the
# MDS behave like a cache, purging responses when the size is exceeded.

# FIXME This will need to be modified for the DMR++ responses when
# when enable Arch #2 since regenerating the DMR++ will be more
# expensive than regenerating other responses from files

#DAP.GlobalMetadataStore.path = /usr/share/modules/dmrpp_module/data/mds
#DAP.GlobalMetadataStore.prefix = mds
#DAP.GlobalMetadataStore.size = 0

# The MDS writes a ledger of additions and removals. By default the
# ledger is kept in 'mds_ledger.txt' in the directory used to start
# the BES.

#DAP.GlobalMetadataStore.ledger = /usr/share/modules/dmrpp_module/data/mds_ledger.txt

#-----------------------------------------------------------------------#
# FONc handler specific parameters:
#-----------------------------------------------------------------------#
#Make sure this follows the netCDF-4 enhanced model
FONc.ClassicModel=false
#Don't generate global attributes since we only
#care about the variable value information.
FONc.NoGlobalAttrs=true

#-----------------------------------------------------------------------#
# HDF5 handler specific parameters:
#-----------------------------------------------------------------------#
# EnableCF: Groom the HDF5 data to follow the CF conventions
#   (true,yes|false,no, defaults to false)
#
H5.EnableCF=false
H5.EnableCFDMR=true

H5.DefaultHandleDimension=true
H5.KeepVarLeadingUnderscore=false
H5.EnableCheckNameClashing=true
H5.EnableAddPathAttrs=true
H5.EnableDropLongString=true
H5.DisableStructMetaAttr=true
H5.EnableFillValueCheck=true
H5.CheckIgnoreObj=false
H5.MetaDataMemCacheEntries=300
H5.LargeDataMemCacheEntries=0
H5.SmallDataMemCacheEntries=0
#H5.CachePurgeLevel=0.2

H5.EnableDiskMetaDataCache=false
H5.EnableDiskDDSCache=false
H5.DiskMetaDataCachePath=/tmp

H5.EnableEOSGeoCacheFile=false
H5.Cache.latlon.path=/tmp/latlon
H5.Cache.latlon.prefix=l
H5.Cache.latlon.size=20000

H5.EnableDiskDataCache=false
H5.DiskCacheDataPath=/tmp
H5.DiskCacheFilePrefix=c
H5.DiskCacheSize=10000
H5.DiskCacheComp=true
H5.DiskCacheFloatOnlyComp=true
H5.DiskCacheCompThreshold=2.0
H5.DiskCacheCompVarSize=10000

AllowedHosts+=^https?:\/\/

EOF
  )

}
###############################################################################

###############################################################################
#
# Print the usage information.
#
show_usage() {
    echo ""
    echo " ["`version`"]"
    cat <<EOF

 Usage: $0 [options] <hdf5 file>

 Write the DMR++ for hdf5_file to stdout

 By default the BES Data Root directory is set to /tmp. This
 utility will add an entry to the bes.log specified in the
 configuration file. The DMR++ is built using the DMR as returned
 by the HDF5 handler, using options as set in the bes
 configuration file found here.

 -h: Show help
 -z: Show version information (Verbose got here first.)
 -v: Verbose: Print the DMR too
 -V: Very Verbose: print the DMR, the command and the configuration
     file used to build the DMR, and do not remove temporary files.
 -D: Just print the DMR that will be used to build the DMR++
 -u: The binary object URL for use in the DMR++ file. If option '-u' is
     not used; then dap4:Dataset/@dmrpp:href attribute will contain the template string
     OPeNDAP_DMRpp_DATA_ACCESS_URL which can be replaced at runtime.
 -b: The fully qualified path to the BES_DATA_ROOT directory. May not be "/" or "/etc".
     The default value is /tmp if a value is not provided
 -c: The path to the bes configuration file to use.
 -s: The path to an optional addendum configuration file which will be appended to the
     default BES configuration. Much like the site.conf file works for the full server
     deployment it will be loaded last and the settings there-in will have an override
     effect on the default configuration.
 -o: The name of the dmr++ file to create.
 -T: Run ALL hyrax tests on the resulting dmr++ file and compare the responses
     the ones generated by the source hdf5 file.
 -I: Run hyrax inventory tests on the resulting dmr++ file and compare the responses
     the ones generated by the source hdf5 file.
 -F: Run hyrax value probe tests on the resulting dmr++ file and compare the responses
     the ones generated by the source hdf5 file.
 -M: Create and merge missing CF coordinate domain variables into the dmrpp. If there are
     missing variables, a sidecar file named <input_file_name>.missing will be created
     in the same directory location as the input_data_file.
     If option 'p' is not used; missing variable chunk href will contain OPeNDAP_DMRpp_MISSING_DATA_ACCESS_URL.
     If option 'p' is selected; missing variable chunk href will contain the argument provided to that option
 -p: The value to use for each missing variable's dmrpp:chunk/@dmrpp:href attribute.  If option '-p' is
     not used; the missing variable dmrpp:chunk/@dmrpp:href attributes will contain the template string
     OPeNDAP_DMRpp_MISSING_DATA_ACCESS_URL which can be replaced at runtime.
 -r: The path to the file that contains missing variable information for sets of input data files that share
     common missing variables. The file will be created if it doesn't exist and the result may be used in subsequent
     invocations of get_dmrpp (using -r) to identify the missing variable file.

 Limitations:
 * The name of the hdf5 file must be expressed relative to the BES_DATA_ROOT

EOF
}

OPTIND=1 # Reset in case getopts has been used previously in this shell

verbose=
very_verbose=
just_dmr=
dmrpp_url=
bes_conf_file=
site_conf_file=
run_inventory_tests=
run_value_tests=
merge_missing_vars=
output_file=
reduce_missing_files=
MASTER_SHA256=
BES_DATA_ROOT="/tmp"

while getopts "h?vVDu:o:c:b:s:p:r:zTIFM" opt; do
  case "$opt" in
  h | \?)
    show_usage >&2
    exit 0
    ;;
  z)
    echo "${GET_DMRPP_VERSION}" >&2
    exit 0
    ;;
  v)
    verbose="yes"
    echo "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -"
    echo "${0} - BEGIN (verbose)"
    ;;
  V)
    very_verbose="-v"
    verbose="yes"
    echo "- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -"
    echo "${0} - BEGIN (very_verbose)"
    ;;
  c)
    bes_conf_file="$OPTARG"
    ;;
  s)
    site_conf_file="$OPTARG"
    ;;
  b)
    BES_DATA_ROOT="$OPTARG"
    ;;
  M)
    merge_missing_vars="yes"
    ;;
  p)
    MISSING_DATA_HREF="$OPTARG"
    ;;
  r)
    MASTER_SHA256="$OPTARG"
    reduce_missing_files="yes"
    ;;
  D)
    just_dmr="yes"
    ;;
  u)
    dmrpp_url="$OPTARG"
    ;;
  o)
    output_file="$OPTARG"
    ;;
  T)
    run_inventory_tests="yes"
    run_value_tests="yes"
    ;;
  I)
    run_inventory_tests="yes"
    ;;
  F)
    run_value_tests="yes"
    ;;
  esac
done

shift $((OPTIND - 1))
[ "$1" = "--" ] && shift

if test -n "$very_verbose"; then
  set -x
fi

input_data_file="${1}"

if test -n "${verbose}"; then
  echo "    OUTPUT_FILE: ${output_file}"
  echo -n "       just_dmr: "
  if test -n "$just_dmr"; then
    echo "true"
  else
    echo "false"
  fi
  echo "      dmrpp_url: ${dmrpp_url}"
fi

if test -z "${dmrpp_url}"; then
  dmrpp_url="OPeNDAP_DMRpp_DATA_ACCESS_URL"
  echo "Using Template Data Access URL String: '${dmrpp_url}'" >&2
fi

# If we are running tests then we need to have the output filename for the dmr++ content.
if [ -n "${run_inventory_tests}" ] || [ -n "${run_value_tests}" ]; then
  if [ -z "${output_file}" ]; then
    echo "" >&2
    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >&2
    echo "" >&2
    echo "OUCH! In order to run the tests the dmr++ output must be written to a file using the -o option. " >&2
    echo "" >&2
    exit 1
  fi
fi

# If we are merging missing variables then we need to have the output filename for the dmr++ content.
if [ -n "${merge_missing_vars}" ]; then
  if [ -z "${output_file}" ]; then
    echo "" >&2
    echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" >&2
    echo "" >&2
    echo "OUCH! In order to merge missing variables the dmr++ output must be written to a file using the -o option. " >&2
    echo "" >&2
    exit 1
  fi
fi

dap4ConstraintContainerElement=$(
  cat <<EOF
    <bes:container name="c">
      <bes:dap4constraint>${dap2_ce}</bes:dap4constraint>
    </bes:container>
EOF
)
dap2ConstraintContainerElement=$(
  cat <<EOF
    <bes:container name="c">
      <bes:constraint>${dap2_ce}</bes:constraint>
    </bes:container>
EOF
)
########################################################################################################################
# mkBesCmd()
# Creates a BES request document for the $dap_thing (dmr, ddx, etc)
# from the $source_data_path file and sticks the command in a temp file.
noConstraintContainerElement="<bes:container name=\"c\" />"

function mkDAP2BesCmd() {
  source_data_path="${1}"
  dap=${2}
  dap2_ce="${3}"
  output_file="${4}"

  if test -n "${dap2_ce}"; then
    #echo "Applying DAP2 CE: '${dap2_ce}'" >&2
    container_element=$(
      cat <<EOF
    <bes:container name="c">
        <bes:constraint>${dap2_ce}</bes:constraint>
    </bes:container>
EOF
    )
  else
    container_element="${noConstraintContainerElement}"
  fi
  #echo "Container Element: " >&2
  #echo "${container_element}" >&2

  if test -n "${output_file}"; then
    #echo "Generate MISSING_FILE: ${output_file}"
    besGET_ELEMENT=$(
      cat <<EOF
      </bes:define>
        <bes:get type="${dap}" definition="d" returnAs="netcdf-4" />
      </bes:request>
EOF
  )
  else
    besGET_ELEMENT=$(
      cat <<EOF
      </bes:define>
        <bes:get type="${dap}" definition="d" />
      </bes:request>
EOF
  )
  fi

  d4BESCmd=$(
    cat <<EOF
<?xml version="1.0" encoding="UTF-8"?>
<bes:request xmlns:bes="http://xml.opendap.org/ns/bes/1.0#" reqID="get_dmrpp.sh">
  <bes:setContext name="dap_explicit_containers">no</bes:setContext>
  <bes:setContext name="errors">xml</bes:setContext>
  <bes:setContext name="max_response_size">0</bes:setContext>

  <bes:setContainer name="c">${source_data_path}</bes:setContainer>

  <bes:define name="d" space="default">
${container_element}
${besGET_ELEMENT}

EOF
  )
  TEMP_FILE=$(mktemp -t dmr_XXXX)
  echo "${d4BESCmd}" >${TEMP_FILE}
  echo "${TEMP_FILE}"

  if test -n "$verbose"; then
    echo "      data_path: ${source_data_path}" >&2
    echo "        dap2_ce: '${dap2_ce}'" >&2
    echo "    ${dap}_request: ${TEMP_FILE} " >&2
    #echo " " >&2
  fi

  # ls -l ${TEMP_FILE}
  # cat ${TEMP_FILE}
}
########################################################################################################################

########################################################################################################################
# Prepare the bes.conf file - use external if provided otherwise use the here doc.
#
function make_bes_conf() {

  if test -n "${1}"; then
    if test -n "${verbose}"; then
      echo "Using BES Configuration: \"${1}\"" >&2
    fi
    BES_CONF_DOC=$(cat ${1})
  else
    mk_default_bes_conf
  fi

  TMP_CONF=$(mktemp -t conf_XXXX)
  echo "${BES_CONF_DOC}" > ${TMP_CONF}

  if test -n "${2}"; then
    if test -n "${verbose}"; then
      echo "Adding additional BES configuration from: ${2}" >&2
    fi
    cat "${2}" >> ${TMP_CONF}
  fi

  if test -n "$very_verbose"; then
    echo "TMP_CONF: ${TMP_CONF}" >&2
    ls -l ${TMP_CONF} >&2
    cat ${TMP_CONF} >&2
  fi

  if test -n "$verbose"; then
    if test -n "${2}"; then
      echo "      site_conf: ${2}" >&2
    fi
    echo "       bes_conf: ${TMP_CONF}" >&2
  fi

  echo "${TMP_CONF}"
  unset TMP_CONF
}
########################################################################################################################

########################################################################################################################
# getDap()
# Gets the passed $dap response (dmr, ddx, etc) for the file $data_path and places the reponse in a temp file.
#
function mkDapRequest() {
  data_path="${1}"
  dap=${2}
  dap_ce=${3}
  output_file=${4}

  if test -n "${dap_ce}"; then
    DAP_CMD=$(mkDAP2BesCmd ${data_path} ${dap} ${dap_ce} ${output_file})
  else
    DAP_CMD=$(mkDAP2BesCmd ${data_path} ${dap})
  fi
  if test -n "${verbose}"; then echo "        DAP_CMD: ${DAP_CMD}" >&2; fi

  if test -n "${output_file}"; then
    DAP_RESPONSE=$(echo "${BES_DATA_ROOT}"/"${output_file}")
  else
    DAP_RESPONSE=$(mktemp -t ${dap}_XXXX)
  fi
  if test -n "${verbose}"; then echo "   DAP_RESPONSE: ${DAP_RESPONSE}" >&2; fi

  besstandalone -c ${BES_CONF} -i ${DAP_CMD} >${DAP_RESPONSE}
  status=$?
  if [ ${status} -ne 0 ]; then
    echo ""
    echo ""
    echo "!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!-!"
    echo "ERROR! The besstandalone program failed to build a ${dap} file!."
    echo "       The configuration is here: ${BES_CONF}"
    echo "             The command is here: ${DAP_CMD}"
    echo "The besstandalone output is here: ${DAP_RESPONSE}"
    exit 2
  fi

  if test -n "$very_verbose"; then
    echo "${dap}_response: ${DAP_RESPONSE} data_path: ${data_path}" >&2
  fi

  if test -n "$very_verbose"; then
    echo "DAP_RESPONSE: " >&2
    if test -z "${output_file}"; then
      cat ${DAP_RESPONSE} >&2
    fi
  fi

  # This is the return value of the function!
  echo "${DAP_RESPONSE}"

#----------------------------------------------------------
# I commented this out because it was screwing things up
# to delete the configuration after the DMR was created but
# before build_dmrpp had been run. ndp 11/01/21
#
  # Clean up temporary files if not in debug mode
  #if test -z "${verbose}"; then
    #echo "DAP_CMD: ${DAP_CMD}"
    #echo "BES_CONF: ${BES_CONF}"
    # rm -f ${DAP_CMD} ${BES_CONF}
  #  ;
  #fi
#----------------------------------------------------------

}
########################################################################################################################

########################################################################################################################
# mk_dmrpp()
# Uses build_dmrpp application to build the requested dmr++ file.
#
function mk_dmrpp() {
  datafile="${1}"
  full_data_path="${BES_DATA_ROOT}/${datafile}"

  if test -n "$verbose"; then
    echo "  BES_DATA_ROOT: ${BES_DATA_ROOT}"
    echo "       datafile: ${datafile}"
    echo " full_data_path: ${full_data_path}"
    echo "       bes.conf: ${BES_CONF} "
    echo "       dmr_file: ${SOURCE_DMR}"
    echo "      dmrpp_url: ${dmrpp_url}"
    echo "    output_file: ${output_file}"
  fi

  if test -z "${just_dmr}"; then
    prms="${very_verbose}"
    prms="${prms} -c ${BES_CONF} "
    prms="${prms} -f ${full_data_path} "
    prms="${prms} -r ${SOURCE_DMR} "
    prms="${prms} -u ${dmrpp_url} "
    prms="${prms} -M " # Adds the production metadata (version numbers and configuration) to the dmr++ result.
    if test -n "$verbose"; then echo "   build_params: ${prms}" >&2; fi
    if test -n "$output_file"; then
      build_dmrpp ${prms} >"${output_file}"
    else
      build_dmrpp ${prms}
    fi
  else
    echo "The just_dmr flag is set, skipping dmr++ construction."
  fi

  #if test ! -n "$very_verbose" && test ! -n "$verbose"; then
    #echo "  mk_dmrpp:BES_CONF: ${BES_CONF}"
    #echo "mk_dmrpp:SOURCE_DMR: ${SOURCE_DMR}"
    #rm -f ${BES_CONF} ${SOURCE_DMR}
  #fi

  # TODO Use trap to ensure these are really removed
  # rm $DMR_CMD $SOURCE_DMR $TMP_CONF
}

#######################################################################################################################
# check_dmrpp()
# Uses check_dmrpp application to identify missing variables in the requested dmr++ file.
#
function ck_dmrpp() {

  missing_vars_file=$(mktemp -t missvars_XXXX)

  if test -n "$very_verbose"; then
    echo "     dmrpp_file: ${output_file}"
    echo "     mvars_file: ${missing_vars_file}"
  fi

  ck_prms="${ck_prms} ${output_file} "
  ck_prms="${ck_prms} ${missing_vars_file} "
  if test -n "$verbose"; then echo "   build_params: ${ck_prms}"; fi
  if test -n "$missing_vars_file"; then
    check_dmrpp ${ck_prms}
  else
    echo "The missing_vars file wasn't specified, skipping missing variable construction."
  fi

  # TODO Use trap to ensure these are really removed
  # rm $DMR_CMD $SOURCE_DMR $TMP_CONF
}
########################################################################################################################

###############################################################################
#
# BES Configuration elements to generate missing variable HDF5 sidecar file.
#

function make_missing_conf() {
  MISSING_CONF_ELEMENTS=$(
    cat <<EOF

#Make sure this follows the netCDF-4 enhanced model
FONc.ClassicModel=false

#Don't generate global attributes since we only
#care about the variable value information.
FONc.NoGlobalAttrs=true

H5.ForceFlattenNDCoorAttr=true
H5.RmConventionAttrPath=true
#We don't map 64-bit integer to DMR
#to make it consistent with data value reading.
H5.EnableDMR64bitInt=false
#When the HDF5 storage size is zero, Fullnamepath
#attribute should not be generated for DMRPP to work.
H5.NoZeroSizeFullnameAttr=true

EOF
  )

  if test -n "${1}"; then
    if test -n "${verbose}"; then
      echo "Using BES Configuration: \"${1}\""
    fi
    MISSING_CONF_DOC=$(cat ${1})
  else
    mk_default_bes_conf
  fi

  MISSING_CONF=$(mktemp -t conf_XXXX)

  echo "${BES_CONF_DOC}" "${MISSING_CONF_ELEMENTS}" >${MISSING_CONF}

  if test -n "$very_verbose"; then
    echo "MISSING_CONF: ${MISSING_CONF}" >&2
    ls -l ${MISSING_CONF} >&2
    cat ${MISSING_CONF} >&2
  fi

  echo "${MISSING_CONF}"
  unset MISSING_CONF

}
###############################################################################

#######################################################################################################################
# merge_dmrpp()
# Uses merge_dmrpp application to identify missing variables in the requested dmr++ file.
#
function mg_dmrpp() {

  if test -n "$verbose"; then
    echo "  missing_dmrpp: ${missing_dmrpp_file}"
    echo "          dmrpp: ${master_dmrpp_file}"
    echo "    target_path: ${target_path}"
    echo "   missing_vars: ${missing_vars_file}"
  fi

  mg_prms="${mg_prms} ${missing_dmrpp_file} "
  mg_prms="${mg_prms} ${master_dmrpp_file} "
  mg_prms="${mg_prms} ${target_path}"
  mg_prms="${mg_prms} ${missing_vars_file} "
  if test -n "$verbose"; then echo "   build_params: ${mg_prms}"; fi
  if test -n "$missing_vars_file"; then
    merge_dmrpp ${mg_prms}
  else
    echo "The missing_vars file doesn't exist, there are no missing variables to merge."
  fi

  # TODO Use trap to ensure these are really removed
  # rm $DMR_CMD $SOURCE_DMR $TMP_CONF
}
########################################################################################################################

#######################################################################################################################
# merge_dmrpp()
# Uses merge_dmrpp application to identify missing variables in the requested dmr++ file.
#
function reduce_dmrpp() {
  datafile="${1}"
  full_data_path="${BES_DATA_ROOT}/${datafile}"

  if test -n "$verbose"; then
    echo "  missing_dmrpp: ${missing_dmrpp_file}"
    echo "   missing_vars: ${full_data_path}"
    echo "  master_sha256: ${MASTER_SHA256}"
    echo "      tempstore: ${tempstore_file}"
  fi

  rd_prms="${rd_prms} ${missing_dmrpp_file} "
  rd_prms="${rd_prms} ${full_data_path} "
  rd_prms="${rd_prms} ${MASTER_SHA256}"
  rd_prms="${rd_prms} ${tempstore_file} "
  if test -n "$verbose"; then echo "   build_params: ${rd_prms}"; fi
  if test -n "${missing_dmrpp_file}"; then
    reduce_mdf ${rd_prms}
    exit_status=$?
    #echo ${exit_status}
  else
    echo "The missing_vars_dmrpp file doesn't exist, there are no missing variables to merge."
  fi

  # TODO Use trap to ensure these are really removed
  # rm $DMR_CMD $SOURCE_DMR $TMP_CONF
}
########################################################################################################################

########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################

########################################################################################################################
# dap2_vars()
# Prints a list of DAP2 DDX variable declarations found in the file $1
# The leading and trailing whitespace is trimmed from each line, and the result is sorted.
#
function dap2_vars() {
  grep -e "<Byte" \
    -e "<Int" \
    -e "<UInt" \
    -e "<Float" \
    -e "<Float" \
    -e "<String" \
    -e "<URL" \
    -e "<Grid" \
    -e "<Sequence" \
    -e "<Structure" \
    -e "<Array" \
    "${1}" | sed 's/ *$//g' | sort
}
########################################################################################################################

########################################################################################################################
# dap2_attributes()
# Prints a list of DAP2 DDX Attribute declarations found in the file $1
# The leading and trailing whitespace is trimmed from each line, and the result is sorted.
#
function dap2_attributes() {
  grep -e "<Attribute name=" -e "<Value>" "${1}" | sed 's/ *$//g' | sort
}
########################################################################################################################

########################################################################################################################
# dap4_attributes()
# Prints a list of DAP4 DMR Attribute declarations found in the file $1
# The leading and trailing whitespace is trimmed from each line, and the result is sorted.
#
function dap4_attributes() {
  grep -e "<Attribute name=" -e "<Value>" "${1}" | sed 's/ *$//g' | sort
}
########################################################################################################################

########################################################################################################################
# dap4_vars()
# Prints a list of DAP4 DMR variable declarations found in the file $1
# The leading and trailing whitespace is trimmed from each line, and the result is sorted.
#
function dap4_vars() {
  grep \
    -e "<Int8" \
    -e "<UInt8" \
    -e "<Byte" \
    -e "<Char" \
    -e "<Int" \
    -e "<UInt" \
    -e "<Float" \
    -e "<String" \
    -e "<URI" \
    -e "<Enumeration" \
    -e "<Sequence" \
    -e "<Structure" \
    -e "<Array" \
    -e "<Group" \
    -e "<Dimension" \
    "${1}" | sed 's/ *$//g' | sort
}
########################################################################################################################

########################################################################################################################
# dap4_vars()
# Prints a list of DAP4 DMR variable declarations found in the file $1
# The leading and trailing whitespace is trimmed from each line, and the result is sorted.
#
function dap4_array_counter() {
  dmr_file=${1}
  type=${2}

  #echo "    dmr_file: ${dmr_file}" >&2;
  #echo "   DAP4 type: ${type}" >&2;

  cat "${dmr_file}" | awk -v type=${type} 'BEGIN{
            inVar = 0;
            typeMatchString = "^ *<"type;
            closeMatchString = "^ *</"type;
            match_count=0;
       }
        {
        if( match($0,typeMatchString)==1){
            inVar=1;
            match_count++;
        }
        if( inVar>0){
            if( match($0,closeMatchString)==1){
                inVar=0;
            }
        }
    }END{print match_count;}' -

}
########################################################################################################################

function dap4_array_dim_counter() {
  dmr_file=${1}
  type=${2}
  match_num="${3}"
  awk_params="-v type=${type} -v match_num=${match_num}"

  cat "${dmr_file}" | awk ${awk_params} 'BEGIN{
            inVar = 0;
            typeMatchString = "^ *<"type;
            # print "typeMatchString: " typeMatchString;
            closeMatchString = "^ *</"type
            # print "closeMatchString: " closeMatchString;
            target_dim_count=0;
            match_count=0;
            target_var_name="not found";
        }
        {
        if( match($0,typeMatchString)==1){
            inVar=1;
            match_count++;
            if(match_count==match_num){
                # <Float32 name="ClrOLR_A">
                target_var_name=$2;
                sub("name=\"","",target_var_name);
                sub("\">","",target_var_name);
            }
        }
        if( inVar>0){
            if(match_count==match_num &&  match($0,"^ *<Dim")==1  ){
                target_dim_count++;
            }
            else if( match($0,closeMatchString)==1){
                inVar=0;
            }
        }
    }END{ print target_var_name, target_dim_count; }' -

}
########################################################################################################################

########################################################################################################################
# dap4_vars()
# Prints a list of DAP4 DMR variable declarations found in the file $1
# The leading and trailing whitespace is trimmed from each line, and the result is sorted.
#
function dap4_array_test() {
  dmr_file=${1}
  type=${2}
  #echo "    dmr_file: ${dmr_file}" >&2;
  #echo "   DAP4 type: ${type}" >&2;

  var_count=$(dap4_array_counter ${dmr_file} ${type})
  if test -n "${verbose}"; then echo "Found ${var_count} variables of type ${type} in DMR: ${dmr_file}" >&2; fi
  if [ ${var_count} -lt 1 ]; then
    echo "ERROR! No variables found in DMR!" >&2
    return 1
  fi

  pick_var=$(echo "${var_count}*${RANDOM}/32767" | bc)
  if test -n "${verbose}"; then echo "Picked variable: ${pick_var}" >&2; fi

  name_and_dim=$(dap4_array_dim_counter ${dmr_file} ${type} ${pick_var})
  if test -n "${very_verbose}"; then echo "name_and_dim: ${name_and_dim}" >&2; fi

  var_name=$(echo "${name_and_dim}" | awk '{print $1;}' -)
  if test -n "${very_verbose}"; then echo "var_name: ${var_name}" >&2; fi

  var_dims=$(echo "${name_and_dim}" | awk '{print $2;}' -)
  if test -n "${very_verbose}"; then echo "var_dims: ${var_dims}" >&2; fi

  query_string="${var_name}"
  for ((i = 1; i <= ${var_dims}; i++)); do
    query_string=${query_string}"[0]"
  done
  if test -n "${verbose}"; then echo "   query_string: ${query_string}" >&2; fi

  bes_cmd_file=$(mkDAP2BesCmd "${input_data_file}" "dods" "${query_string}")
  #if test -n "${verbose}"; then echo "bes_cmd_file: ${bes_cmd_file}" >&2; fi
  dap2_source_response=$(mktemp -t d4_array_test_XXXX)
  besstandalone -c ${BES_CONF} -i ${bes_cmd_file} >${dap2_source_response}
  if test -n "${verbose}"; then echo "dap2_source_response: ${dap2_source_response}" >&2; fi
  if test -n "${verbose}"; then getdap -D -M ${dap2_source_response} >&2; fi

  if test -n "$very_verbose"; then
    echo "dap2_source_response: " >&2
    cat ${dap2_source_response} >&2
  fi

  bes_cmd_file=$(mkDAP2BesCmd "${output_file}" "dods" "${query_string}")
  #if test -n "$verbose"; then echo "bes_cmd_file: ${bes_cmd_file}" >&2; fi
  dap2_dmrpp_response=$(mktemp -t d4_array_test_XXXX)
  besstandalone -c ${BES_CONF} -i ${bes_cmd_file} >${dap2_dmrpp_response}
  if test -n "$verbose"; then echo "dap2_dmrpp_response: ${dap2_dmrpp_response}" >&2; fi
  if test -n "$verbose"; then getdap -D -M ${dap2_dmrpp_response} >&2; fi

  if test -n "$verbose"; then echo "Comparing binary responses..." >&2; fi
  cmp ${dap2_source_response} ${dap2_dmrpp_response} >&2
  if [ $? -eq 0 ]; then
    if test -n "$verbose"; then echo "Responses are the same. w00t" >&2; fi
    return 0
  else
    if test -n "$verbose"; then echo "ERROR! Responses differ." >&2; fi
    return 100
  fi

}

########################################################################################################################

########################################################################################################################
# compare_files()
# Compares two files by first applying the check_function to each file and then comparing the results of the
# check_function using diff.
#
function compare_files() {
  check_function=${1}
  first_file=${2}
  second_file=${3}
  success="${4}"
  failure="${5}"
  if test -n "$very_verbose"; then
    echo "compare_files() check_function: ${check_function}" >&2
    echo "compare_files()     first_file: ${first_file}" >&2
    echo "compare_files()    second_file: ${second_file}" >&2
  fi

  first_result=$(mktemp -t dmr_XXXX)
  if test -n "$very_verbose"; then
    echo "compare_files()   first_result: ${first_result}" >&2
  fi
  f_stuff=$(${check_function} ${first_file})
  echo "${f_stuff}" >${first_result}

  second_result=$(mktemp -t dmr_XXXX)
  if test -n "$very_verbose"; then
    echo "compare_files()  second_result: ${second_result}" >&2
  fi
  s_stuff=$(${check_function} ${second_file})
  echo "${s_stuff}" >${second_result}

  retval=0
  diff --ignore-space-change --brief ${first_result} ${second_result}
  if [ $? -eq 0 ]; then
    if test -n "${verbose}"; then
      echo "${success}" >&2
    fi
  else
    echo "${failure}" >&2
    retval=1
  fi
  return ${retval}
}
########################################################################################################################

########################################################################################################################
# dmrpp_inventory_test()
#
function dmrpp_inventory_test() {

  if [[ ${output_file} == ${BES_DATA_ROOT}* ]]; then
    target_file=$(echo "${output_file}" | sed -e "s+${BES_DATA_ROOT}++")
  else
    test_file="${BES_DATA_ROOT}/"$(basename ${output_file})
    if test ! -f ${testfile}; then
      cp ${output_file} ${BES_DATA_ROOT}
    fi
    target_file=$(basename ${output_file})
  fi

  if test -n "${verbose}"; then
    echo "       BES_CONF: ${BES_CONF}" >&2
    echo "SOURCE_DATAFILE: ${input_data_file}" >&2
    echo "TARGET_DATAFILE: ${target_file}" >&2
  fi

  SOURCE_DDX=$(mkDapRequest "${input_data_file}" ddx)
  TARGET_DDX=$(mkDapRequest "${target_file}" ddx)

  # SOURCE_DMR=$(mkDapRequest "${input_data_file}" dmr) # This is done before the dmr++ file is built.
  TARGET_DMR=$(mkDapRequest "${target_file}" dmr)

  retval=0
  # echo "dmrpp_inventory_test: retval: ${retval}"

  compare_files dap4_vars ${SOURCE_DMR} ${TARGET_DMR} " DAP4 variables: Matched." "ERROR! DAP4 variables mismatch!"
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "dmrpp_inventory_test: retval: ${retval}"

  compare_files dap4_attributes ${SOURCE_DMR} ${TARGET_DMR} "DAP4 Attributes: Matched." "ERROR! DAP4 Attributes mismatch!"
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "dmrpp_inventory_test: retval: ${retval}"

  compare_files dap2_vars ${SOURCE_DDX} ${TARGET_DDX} " DAP2 variables: Matched." "ERROR! DAP2 variables mismatch!"
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "dmrpp_inventory_test: retval: ${retval}"

  compare_files dap2_attributes ${SOURCE_DDX} ${TARGET_DDX} "DAP2 Attributes: Matched." "ERROR! DAP2 Attributes mismatch!"
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "dmrpp_inventory_test: retval: ${retval}"

  return ${retval}
}

function dmrpp_value_test() {
  if test -n "${very_verbose}"; then
    echo "dvt        BES_CONF: ${BES_CONF}" >&2
    echo "dvt SOURCE_DATAFILE: ${input_data_file}" >&2
    echo "dvt TARGET_DATAFILE: ${output_file}" >&2
    echo "dvt      SOURCE_DMR: ${SOURCE_DMR}" >&2
  fi
  dap4_array_test "${SOURCE_DMR}" "Float32"
  return $?
}
########################################################################################################################
########################################################################################################################
########################################################################################################################
########################################################################################################################

BES_CONF=$(make_bes_conf "${bes_conf_file}" "${site_conf_file}")
SOURCE_DMR=$(mkDapRequest "${input_data_file}" dmr)

# If 'just_dmr' is set redirect SOURCE_DMR to either <stdout> or $output_file
if [ -n "${just_dmr}" ]; then
  if [ -z "${output_file}" ]; then
    cat ${SOURCE_DMR} >&2
  else
    cat ${SOURCE_DMR} >${output_file};
  fi
  if test -z "${verbose}"; then
      rm -f ${SOURCE_DMR}
  fi
  exit 0
fi


mk_dmrpp "${input_data_file}"
retval=$?
#echo "retval: ${retval}";

if test -z "${verbose}"; then
    if test -f "${BES_CONF}" ; then rm -f "${BES_CONF}"; fi
    if test -f "${DAP_CMD}" ; then rm -f "${DAP_CMD}"; fi
fi

if test -n "${merge_missing_vars}"; then

  # use $PATH/check_dmrpp to query dmrpp for missing CF domain variables [requires ${output_file}; -o option]
  ck_dmrpp
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "retval: ${retval}";

  # check-dmrpp will create missing_vars_file if there are any missing variables.
  if test -s ${missing_vars_file}; then
    MISSING_VARS=$(cat ${missing_vars_file})
    if test -n "$very_verbose"; then echo "The missing vars: ${MISSING_VARS}" >&2; fi

    # augment bes_conf_file with additional H5 parameters necessary to synthesize missing_variables from input_data_file.
    BES_CONF=$(make_missing_conf "${bes_conf_file}")

    # insert '_missing' into input_data_file name and use for sidecar_missing_file name
    sidecar_missing_file="${input_data_file}.missing"
    if test -n "$very_verbose"; then echo "  MISSING_FILE: ${sidecar_missing_file}" >&2; fi

    # use besstandalone to generate sidecar_missing_file
    MISSING_FILE_BESCMD=$(mkDapRequest "${input_data_file}" dods "${MISSING_VARS}" "${sidecar_missing_file}")
    source_data_path="${sidecar_missing_file}"

    BES_CONF=$(make_bes_conf "${bes_conf_file}" "${site_conf_file}")  # reset bes_conf_file; removing additional H5 parameters

    SOURCE_DMR=$(mkDapRequest "${sidecar_missing_file}" dmr)  # use besstandalone to generate dmr for sidecar_missing_file
    master_dmrpp_file="${output_file}"  # save original dmrpp filename

    if test -n "$reduce_missing_files"; then
      if test ! -s ${MASTER_SHA256}; then
        output_file="${BES_DATA_ROOT}/${sidecar_missing_file}.dmrpp"
      else
        output_file=$(mktemp -t dmrpp_XXXX) # generate temporary dmrpp filename for sidecar_missing_file
      fi
    else
      output_file=$(mktemp -t dmrpp_XXXX) # generate temporary dmrpp filename for sidecar_missing_file
    fi

    mk_dmrpp "${sidecar_missing_file}"  # use $PATH/build_dmrpp to generate dmrpp for sidecar_missing_file
    retval=$?
    #echo "retval: ${retval}";

    missing_dmrpp_file="${output_file}"

    if test -n "$reduce_missing_files"; then
      tempstore_file=$(mktemp -t tmpstore_XXXX) # generate temporary filename for tempstore to hold missing_dmrpp info
      if test -n "$very_verbose"; then
        echo "missing_dmrpp_file: ${missing_dmrpp_file}" >&2;
        echo " missing_vars_file: ${sidecar_missing_file}" >&2;
        echo "     master_sha256: ${MASTER_SHA256}" >&2;
        echo "         tempstore: ${tempstore_file}" >&2;
      fi

      reduce_dmrpp "${sidecar_missing_file}" # use $PATH/reduce_mdf to merge sidecar_missing_file dmrpp with master_dmrpp_file
      retval=$?
      #echo "reduce_dmrpp: ${retval}"

      if test -s ${tempstore_file}; then # if tempstore_file is not empty then there will be temp that need to be removed.
        temp_sidecar_dmrpp=${missing_dmrpp_file}
        temp_sidecar_file="${BES_DATA_ROOT}/${sidecar_missing_file}"
        missing_dmrpp_file=$(echo `awk -F'[ ]' '{print $2}' "${tempstore_file}"`)
      fi

    fi

    if test -n "$very_verbose"; then
      echo "missing_dmrpp_file: ${missing_dmrpp_file}" >&2;
      echo " master_dmrpp_file: ${master_dmrpp_file}" >&2;
      echo "       target_path: ${target_path}" >&2;
      echo " missing_vars_file: ${missing_vars_file}" >&2;
    fi

    # set 'href' content to use when merging chunks from sidecar_missing_file dmrpp into master_dmrpp_file
    if test -n "${MISSING_DATA_HREF}"; then
      target_path=$(echo "${MISSING_DATA_HREF}")
    else
      target_path=$(echo "OPeNDAP_DMRpp_MISSING_DATA_ACCESS_URL")
    fi

    mg_dmrpp  # use $PATH/merge_dmrpp to merge sidecar_missing_file dmrpp with master_dmrpp_file
    retval=$(echo "${retval} + ${?}" | bc)
    #echo "retval: ${retval}"

  else
    echo " There are no missing variables to merge."
  fi
fi

if test -n "${run_inventory_tests}"; then
  dmrpp_inventory_test
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "retval: ${retval}";
fi

if test -n "${run_value_tests}"; then
  dmrpp_value_test
  retval=$(echo "${retval} + ${?}" | bc)
  #echo "retval: ${retval}";
fi

if test -z "${verbose}"; then
  if test -n "${merge_missing_vars}"; then
    echo "    main:MISSING_VARS_FILE: ${missing_vars_file}"
    echo " main:SIDECAR_MISSING_FILE: ${temp_sidecar_file}"
    echo "main:SIDECAR_MISSING_DMRPP: ${temp_sidecar_dmrpp}"
    echo "       main:TEMPSTORE_FILE: ${tempstore_file}"
    rm -f ${missing_vars_file} ${temp_sidecar_file} ${temp_sidecar_dmrpp} ${tempstore_file}
  fi
  #echo " main:SOURCE_DMR: ${SOURCE_DMR}"
  rm -f ${SOURCE_DMR}
fi

#echo "retval: ${retval}";
exit $retval

# rm -f ${SOURCE_DMR} ${SOURCE_DDX} ${TARGET_DDX} ${TARGET_DMR} ${DDX_CMD} ${DMR_CMD} ${BES_CONF}

#./get_dmrpp -u "http://test.opendap.org/data/dmrpp/AIRS.2015.01.01.L3.RetStd_IR001.v6.0.11.0.G15013155825.nc.h5" -o airs.h5.dmrpp -T -v  airs.h5
#./get_dmrpp -u "http://test.opendap.org/data/dmrpp/SMAP_L3_SM_P_20150406_R14010_001.h5" -o smap.h5.dmrpp -T -v  smap.h5
