file standalone/manual_hdf5_combine.cpp

[No description available] More…


intmain(int argc, char * argv[])

Detailed Description

Author: Ben Farmer (

Date: 2018 Mar

This is code for a stand-alone binary which performs the end-of-run combination of temporary HDF5 files from a GAMBIT scan.

It is essentially the same code that GAMBIT runs automatically in the HDF5 printer, just moved into a stand-alone tool so that the combination can be done “manually” if needed.

Authors (add name and date if you modify):

Functions Documentation

function usage

void usage()

function main

int main(
    int argc,
    char * argv[]

Source code

//   GAMBIT: Global and Modular BSM Inference Tool
//   *********************************************
///  \file
///  This is code for a stand-alone binary which
///  performs the end-of-run combination of
///  temporary HDF5 files from a GAMBIT scan.
///  It is essentially the same code that GAMBIT
///  runs automatically in the HDF5 printer, just
///  moved into a stand-alone tool so that the
///  combination can be done "manually" if needed.
///  *********************************************
///  Authors (add name and date if you modify):
///  \author Ben Farmer
///          (
///  \date 2018 Mar
///  *********************************************

#include <stdlib.h>
#include <getopt.h>
#include <vector>
#include <sstream>
#include <utility>
#include <string>
#include <sys/stat.h>
#include <chrono>

// GAMBIT headers
#include "gambit/Printers/printers/hdf5printer/hdf5_combine_tools.hpp"
#include "gambit/Utils/stream_overloads.hpp"
#include "gambit/Utils/util_functions.hpp"

// Annoying other things we need due to mostly unwanted dependencies
#include "gambit/Utils/static_members.hpp"

using namespace Gambit;
using namespace Printers;

void usage()
    std::cout << "\nusage: hdf5combine <filename> <group> [options] "
          "\n "
          "\n  filename - Base name of the temporary files to be combined" 
          "\n               e.g. /path/to/my/hdf5file.hdf5 "
          "\n  group    - location inside the HDF5 files which contains the datasets to be combined"
          "\n   -h/--help             Display this usage information"
          "\n   -f/--force            Attempt combination while ignoring missing temporary files"
          "\n   -c/--cleanup          Delete temporary files after successful combination"
          "\n   -o/--out <path>       Set output folder (default is same folder as temp files)"
          "\n                         (note, this first creates the output in the default place,"
          "\n                         and then just moves it)"
          "\n   -i/--in <file_with_list>"
          "\n                         Uses named files as input instead of automatically inferring"
          "\n                         them from <filename>. Filenames are to be supplied in a single"
          "\n                         ascii file, on separate lines."
          "\n                         In this mode <filename> will only be used for naming the output"
          "\n                         file (Note: WITHOUT the usual addition of '_temp_combined'!!!)."
          "\n                         All files must have the same <group>."
          "\n                         Note: Auxilliary ('RA') datasets will be IGNORED! These should"
          "\n                         be combined during 'normal' combination of results from a single"
          "\n                         run. This 'custom' mode is intended for combining results from"
          "\n                         DIFFERENT runs."

int main(int argc, char* argv[])
   // Process command line options
   // Must at least have two arguments.
   if (argc < 3)

   // First, the required options (folder and temporary file base name)
   std::string finalfile  = argv[1];
   std::string group      = argv[2];
   std::string outpath;

   const struct option primary_options[] =
     {"force", no_argument, 0, 'f'},
     {"help",  no_argument, 0, 'h'},
     {"cleanup", no_argument, 0 , 'c'},
     {"in", required_argument, 0, 'i'},
     {"out", required_argument, 0, 'o'},

   int index;
   int iarg=0;
   std::string filename;
   std::vector<std::string> input_files;
   bool custom_mode = false;
   std::string tmp_comb_file;
   bool error_if_inconsistent = true;
   bool do_cleanup = false;
   bool move_output = false;
   size_t num;
   bool combined_file_exists;

   while(iarg != -1)
     iarg = getopt_long(argc, argv, "fhcoi:", primary_options, &index);
     switch (iarg)
       case 'h':
         // Display usage information and exit
       case 'f':
         // Ignore missing temporary files
         error_if_inconsistent = false;
       case 'c':
         do_cleanup = true;
       case 'o':
         outpath = optarg; 
         move_output = true;
       case 'i':
            std::ifstream file_list(optarg);
            std::string line; 
            while(std::getline(file_list, line))
         custom_mode = true;
         combined_file_exists = false;
         num = input_files.size();
       case '?':
         // display usage message and quit (also happens on unrecognised options)
   // Begining timing of operation
   std::chrono::time_point<std::chrono::system_clock> startT = std::chrono::system_clock::now();

      // Custom specification of input files
      std::cout <<"  Running combine code in custom filename mode" <<std::endl;
      std::cout <<"  "<<num<<" files to be combined:"<<std::endl;
      for(std::vector<std::string>::iterator it=input_files.begin(); it!=input_files.end(); ++it)
        std::cout << "    " << *it << std::endl;
        std::ostringstream errmsg;
        errmsg << "  ERROR! Less than two files parsed from '--in' argument! At least two files are required for combination to do anything!";
        std::cerr << errmsg.str() << std::endl;
      HDF5::combine_hdf5_files(finalfile, "", group, num, false, false, true, input_files);
      // Report what we are going to do
      std::cout <<"  Searching for temporary files to combine" <<std::endl;
      std::cout <<"     with base filename: "<<finalfile<<std::endl;

      // Search for the temporary files to be combined
      size_t max_i = 0;
      std::pair<std::vector<std::string>,std::vector<size_t>> out = HDF5::find_temporary_files(finalfile, max_i);
      std::vector<std::string> tmp_files = out.first;
      std::vector<size_t> missing = out.second;

      std::cout <<"  Found "<<tmp_files.size()<<" temporary files to be combined"<<std::endl;
      // Check if all temporary files found (i.e. check if output from some rank is missing)
        std::cout <<"  WARNING! Temporary files missing from the following ranks: "<<missing<<std::endl; 
          std::ostringstream errmsg;
          errmsg << "  Could not locate all the expected temporary output files (found "<<tmp_files.size()<<" temporary files, but are missing the files from the ranks reported above)! If you want to attempt the combination anyway, please add the -f flag to the command line invocation.";
          std::cerr << errmsg.str() << std::endl;

      // Name of temporary combined file, if one exists
      std::ostringstream name;
      name << finalfile << "_temp_combined";
      tmp_comb_file = name.str();

      combined_file_exists = Utils::file_exists(tmp_comb_file);
         std::cout<<"  Found pre-existing (temporary) combined file"<<std::endl;
         std::cout<<"  No pre-existing (temporary) combined file found"<<std::endl;  

      num = tmp_files.size() + missing.size();
      // We don't actually use their names here, Greg's code assumes that they
      // follow a fixed format. Previous all files had to exist in increasing
      // numerical order, however now, if you use the -f flag, it is allowed
      // for some to just be missing. These will just be ignored if they fail
      // to open.
      HDF5::combine_hdf5_files(tmp_comb_file, finalfile, group, num, combined_file_exists, do_cleanup, !error_if_inconsistent);

   std::cout<<"  Combination finished successfully!"<<std::endl;
      std::string fname(Utils::base_name(tmp_comb_file));
      std::string outname(outpath+"/"+fname);
      std::cout<<"  Moving output from "<<std::endl
               <<"  "<<tmp_comb_file<<std::endl
               <<"  to"<<std::endl
               <<"  "<<outname<<std::endl;
      int status = std::system(("mv " + tmp_comb_file+ " " + outname).c_str());
         std::cerr << "  FAILED to move output to requested directory!! It should still be located at"<<std::endl
                   << "    "<<tmp_comb_file<<std::endl;

   // End timing of operation
   std::chrono::time_point<std::chrono::system_clock> endT = std::chrono::system_clock::now();
   std::chrono::duration<double> interval = endT - startT;
   long runtime = std::chrono::duration_cast<std::chrono::seconds>(interval).count(); 
   long mins = runtime / 60;
   long secs = runtime % 60;
   std::cout<<"  Operation took "<<mins<<" minutes and "<<secs<<" seconds."<<std::endl;

   // Finished!

Updated on 2023-06-26 at 21:36:54 +0000