statistics.hpp

// FEAT3: Finite Element Analysis Toolbox, Version 3
// Copyright (C) 2010 by Stefan Turek & the FEAT group
// FEAT3 is released under the GNU General Public License version 3,
// see the file 'copyright.txt' in the top level directory for details.
 
#pragma once
 
#include <kernel/base_header.hpp>
#include <kernel/util/string.hpp>
#include <kernel/util/exception.hpp>
#include <kernel/util/kahan_accumulator.hpp>
#include <kernel/solver/expression.hpp>
 
#include <list>
#include <map>
#include <time.h>
#include <iostream>
#include <fstream>
#include <memory>
 
namespace FEAT
{
  class Statistics
  {
    private:
 
      static Index _flops;
 
      static KahanAccumulator<double> _time_reduction;
 
      static KahanAccumulator<double> _time_blas2;
 
      static KahanAccumulator<double> _time_blas3;
 
      static KahanAccumulator<double> _time_axpy;
 
      static KahanAccumulator<double> _time_precon;
 
      static KahanAccumulator<double> _time_mpi_execute_reduction;
 
      static KahanAccumulator<double> _time_mpi_execute_blas2;
 
      static KahanAccumulator<double> _time_mpi_execute_blas3;
 
      static KahanAccumulator<double> _time_mpi_execute_collective;
 
      static KahanAccumulator<double> _time_mpi_wait_reduction;
 
      static KahanAccumulator<double> _time_mpi_wait_blas2;
 
      static KahanAccumulator<double> _time_mpi_wait_blas3;
 
      static KahanAccumulator<double> _time_mpi_wait_collective;
 
      static std::map<String, std::list<std::shared_ptr<Solver::ExpressionBase>>> _solver_expressions;
 
      static std::map<String, String> _formatted_solver_trees;
 
      static std::map<String, std::list<double>> _overall_toe;
      static std::map<String, std::list<Index>> _overall_iters;
      static std::map<String, std::list<double>> _overall_mpi_execute_reduction;
      static std::map<String, std::list<double>> _overall_mpi_execute_blas2;
      static std::map<String, std::list<double>> _overall_mpi_execute_blas3;
      static std::map<String, std::list<double>> _overall_mpi_execute_collective;
      static std::map<String, std::list<double>> _overall_mpi_wait_reduction;
      static std::map<String, std::list<double>> _overall_mpi_wait_blas2;
      static std::map<String, std::list<double>> _overall_mpi_wait_blas3;
      static std::map<String, std::list<double>> _overall_mpi_wait_collective;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_toe;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_execute_reduction;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_execute_blas2;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_execute_blas3;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_execute_collective;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_wait_reduction;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_wait_blas2;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_wait_blas3;
      static std::map<String, std::list<std::vector<double>>> _outer_mg_mpi_wait_collective;
      static std::map<String, std::list<double>> _outer_schwarz_toe;
      static std::map<String, std::list<Index>> _outer_schwarz_iters;
 
      /*static String _format_solver_statistics(String branch, SolverStatistics & stat)
      {
        String result;
 
        result += "\n" + branch + "\n";
        result +="Iteration Defects:\n";
 
        for (Index i(0) ; i < stat.defect.size() ; ++i)
        {
          if (i == 0 && stat.defect.at(i) == double(-1))
            continue;
          if (stat.defect.at(i) == double(-1))
            result += "------\n";
          else
            result += stringify_fp_sci(stat.defect.at(i)) + "\n";
        }
 
        result +="Iteration Timings [s]:\n";
        for (Index i(0) ; i < stat.toe.size() ; ++i)
        {
          if (i == 0 && stat.toe.at(i) == double(-1))
            continue;
          if (stat.toe.at(i) == double(-1))
            result += "------\n";
          else
            result += stringify_fp_sci(stat.toe.at(i)) + "\n";
        }
 
        result +="MPI Execution Iteration Timings [s]:\n";
        for (Index i(0) ; i < stat.mpi_execute.size() ; ++i)
        {
          if (i == 0 && stat.mpi_execute.at(i) == double(-1))
            continue;
          if (stat.mpi_execute.at(i) == double(-1))
            result += "------\n";
          else
            result += stringify_fp_sci(stat.mpi_execute.at(i)) + "\n";
        }
 
        result +="MPI Wait Iteration Timings [s]:\n";
        for (Index i(0) ; i < stat.mpi_wait.size() ; ++i)
        {
          if (i == 0 && stat.mpi_wait.at(i) == double(-1))
            continue;
          if (stat.mpi_wait.at(i) == double(-1))
            result += "------\n";
          else
            result += stringify_fp_sci(stat.mpi_wait.at(i)) + "\n";
        }
 
        return result;
      }*/
 
      static String _generate_formatted_solver_tree(String target);
 
    public:
 
      static bool enable_solver_expressions;
      static double toe_partition;
      static double toe_assembly;
      static double toe_solve;
      static String expression_target;
 
 
      static void reset()
      {
        reset_flops();
        reset_times();
        _solver_expressions.clear();
        _overall_toe.clear();
        _overall_iters.clear();
        _overall_mpi_execute_reduction.clear();
        _overall_mpi_execute_blas2.clear();
        _overall_mpi_execute_blas3.clear();
        _overall_mpi_execute_collective.clear();
        _overall_mpi_wait_reduction.clear();
        _overall_mpi_wait_blas2.clear();
        _overall_mpi_wait_blas3.clear();
        _overall_mpi_wait_collective.clear();
        _outer_mg_toe.clear();
        _outer_mg_mpi_execute_reduction.clear();
        _outer_mg_mpi_execute_blas2.clear();
        _outer_mg_mpi_execute_blas3.clear();
        _outer_mg_mpi_execute_collective.clear();
        _outer_mg_mpi_wait_reduction.clear();
        _outer_mg_mpi_wait_blas2.clear();
        _outer_mg_mpi_wait_blas3.clear();
        _outer_mg_mpi_wait_collective.clear();
        _outer_schwarz_toe.clear();
        _outer_schwarz_iters.clear();
      }
 
      static void add_flops(Index flops)
      {
        _flops += flops;
      }
 
      static Index get_flops()
      {
        return _flops;
      }
 
      static String get_formatted_flops(double seconds, int ranks = 1)
      {
        double flops((double)_flops);
        flops /= seconds;
        flops /= 1000.; // kilo
        flops /= 1000.; // mega
        flops /= 1000.; // giga
        return stringify(flops) + " GFlop/s [" + stringify(flops * double(ranks)) + " GFlops/s]";
      }
 
      static void reset_flops()
      {
        _flops = Index(0);
      }
 
      inline static void add_time_reduction(double seconds)
      {
        _time_reduction += seconds;
      }
      inline static void add_time_blas2(double seconds)
      {
        _time_blas2 += seconds;
      }
      inline static void add_time_blas3(double seconds)
      {
        _time_blas3 += seconds;
      }
      inline static void add_time_axpy(double seconds)
      {
        _time_axpy += seconds;
      }
      inline static void add_time_precon(double seconds)
      {
        _time_precon += seconds;
      }
      inline static void add_time_mpi_execute_reduction(double seconds)
      {
        _time_mpi_execute_reduction += seconds;
      }
      inline static void add_time_mpi_execute_blas2(double seconds)
      {
        _time_mpi_execute_blas2 += seconds;
      }
      inline static void add_time_mpi_execute_blas3(double seconds)
      {
        _time_mpi_execute_blas3 += seconds;
      }
      inline static void add_time_mpi_execute_collective(double seconds)
      {
        _time_mpi_execute_collective += seconds;
      }
      inline static void add_time_mpi_wait_reduction(double seconds)
      {
        _time_mpi_wait_reduction += seconds;
      }
      inline static void add_time_mpi_wait_blas2(double seconds)
      {
        _time_mpi_wait_blas2 += seconds;
      }
      inline static void add_time_mpi_wait_blas3(double seconds)
      {
        _time_mpi_wait_blas3 += seconds;
      }
      inline static void add_time_mpi_wait_collective(double seconds)
      {
        _time_mpi_wait_collective += seconds;
      }
 
      inline static double get_time_reduction()
      {
        return _time_reduction;
      }
      inline static double get_time_blas2()
      {
        return _time_blas2.value;
      }
      inline static double get_time_blas3()
      {
        return _time_blas3.value;
      }
      inline static double get_time_axpy()
      {
        return _time_axpy.value;
      }
      inline static double get_time_precon()
      {
        return _time_precon.value;
      }
      inline static double get_time_mpi_execute_reduction()
      {
        return _time_mpi_execute_reduction.value;
      }
      inline static double get_time_mpi_execute_blas2()
      {
        return _time_mpi_execute_blas2.value;
      }
      inline static double get_time_mpi_execute_blas3()
      {
        return _time_mpi_execute_blas3.value;
      }
      inline static double get_time_mpi_execute_collective()
      {
        return _time_mpi_execute_collective.value;
      }
      inline static double get_time_mpi_wait_reduction()
      {
        return _time_mpi_wait_reduction.value;
      }
      inline static double get_time_mpi_wait_blas2()
      {
        return _time_mpi_wait_blas2.value;
      }
      inline static double get_time_mpi_wait_blas3()
      {
        return _time_mpi_wait_blas3.value;
      }
      inline static double get_time_mpi_wait_collective()
      {
        return _time_mpi_wait_collective.value;
      }
 
      inline static void add_solver_expression(std::shared_ptr<Solver::ExpressionBase> expression)
      {
        if(enable_solver_expressions)
          _solver_expressions[expression_target].push_back(expression);
      }
 
      static const std::list<std::shared_ptr<Solver::ExpressionBase>> & get_solver_expressions()
      {
        return _solver_expressions.at(expression_target);
      }
 
      static String get_formatted_solver_tree(String target = "default")
      {
        if (_formatted_solver_trees.count(target) == 0)
          compress_solver_expressions();
        if (_formatted_solver_trees.count(target) == 0)
          return String("no solver tree for target '") + target + "' found";
        else
          return _formatted_solver_trees.at(target);
      }
 
      static void print_solver_expressions();
 
      static void compress_solver_expressions();
 
      static inline std::list<double> & get_time_toe(String target)
      {
        return _overall_toe.at(target);
      }
 
      static inline std::list<Index> & get_iters(String target)
      {
        return _overall_iters.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_execute_reduction(String target)
      {
        return _overall_mpi_execute_reduction.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_execute_blas2(String target)
      {
        return _overall_mpi_execute_blas2.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_execute_blas3(String target)
      {
        return _overall_mpi_execute_blas3.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_execute_collective(String target)
      {
        return _overall_mpi_execute_collective.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_wait_reduction(String target)
      {
        return _overall_mpi_wait_reduction.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_wait_blas2(String target)
      {
        return _overall_mpi_wait_blas2.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_wait_blas3(String target)
      {
        return _overall_mpi_wait_blas3.at(target);
      }
 
      static inline std::list<double> & get_time_mpi_wait_collective(String target)
      {
        return _overall_mpi_wait_collective.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg(String target)
      {
        return _outer_mg_toe.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_execute_reduction(String target)
      {
        return _outer_mg_mpi_execute_reduction.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_execute_blas2(String target)
      {
        return _outer_mg_mpi_execute_blas2.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_execute_blas3(String target)
      {
        return _outer_mg_mpi_execute_blas3.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_execute_collective(String target)
      {
        return _outer_mg_mpi_execute_collective.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_wait_reduction(String target)
      {
        return _outer_mg_mpi_wait_reduction.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_wait_blas2(String target)
      {
        return _outer_mg_mpi_wait_blas2.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_wait_blas3(String target)
      {
        return _outer_mg_mpi_wait_blas3.at(target);
      }
 
      static inline std::list<std::vector<double>> & get_time_mg_mpi_wait_collective(String target)
      {
        return _outer_mg_mpi_wait_collective.at(target);
      }
 
      static inline std::list<double> & get_time_schwarz(String target)
      {
        return _outer_schwarz_toe.at(target);
      }
 
      static inline std::list<Index> & get_iters_schwarz(String target)
      {
        return _outer_schwarz_iters.at(target);
      }
 
      static String get_formatted_times(double total_time);
 
      static String get_formatted_solver_internals(String target = "default");
 
      static void reset_times()
      {
        _time_reduction.clear();
        _time_blas2.clear();
        _time_blas3.clear();
        _time_axpy.clear();
        _time_precon.clear();
        _time_mpi_execute_reduction.clear();
        _time_mpi_execute_blas2.clear();
        _time_mpi_execute_blas3.clear();
        _time_mpi_execute_collective.clear();
        _time_mpi_wait_reduction.clear();
        _time_mpi_wait_blas2.clear();
        _time_mpi_wait_blas3.clear();
        _time_mpi_wait_collective.clear();
      }
 
      /*
      static void write_out_solver_statistics_scheduled(Index rank, size_t la_bytes, size_t domain_bytes, size_t mpi_bytes, Index cells, Index dofs, Index nzes, String filename = "solver_stats");
 
      static void write_out_solver_statistics(Index rank, size_t la_bytes, size_t domain_bytes, size_t mpi_bytes, Index cells, Index dofs, Index nzes, String filename = "solver_stats")
      {
        filename += ".";
        filename += stringify(rank);
 
        std::ofstream file(filename.c_str(), std::ofstream::out);
        if (! file.is_open())
          XABORTM("Unable to open statistics file " + filename);
 
        {
          time_t t = time(nullptr);
          struct tm * ts = localtime(&t);
 
          file << "timestamp " << stringify(ts->tm_mon + 1) << "/" << stringify(ts->tm_mday) << "/" << stringify(ts->tm_year + 1900) << " " << stringify(ts->tm_hour)
            << ":" << stringify(ts->tm_min) << ":" << stringify(ts->tm_sec) << "\n";
        }
 
        file << "\n";
 
        file << "#global time\n";
        file << "partition " << stringify(toe_partition) << "\n";
        file << "assembly " << stringify(toe_assembly) << "\n";
        file << "solve " << stringify(toe_solve) << "\n";
 
        file << "\n";
 
        file << "#solver time\n";
        file << "reduction " << stringify(get_time_reduction()) << "\n";
        file << "axpy " << stringify(get_time_axpy()) << "\n";
        file << "blas2 " << stringify(get_time_blas2()) << "\n";
        file << "precon " << stringify(get_time_precon()) << "\n";
        file << "mpi " << stringify(get_time_mpi_execute()) << "\n";
        file << "wait reduction" << stringify(get_time_mpi_wait_reduction()) << "\n";
        file << "wait blas2" << stringify(get_time_mpi_wait_blas2()) << "\n";
 
        file << "\n";
 
        file << "#global size\n";
        file << "domain " << stringify(domain_bytes) << "\n";
        file << "mpi " << stringify(mpi_bytes) << "\n";
        file << "la " << stringify(la_bytes) << "\n";
 
        file << "\n";
 
        file <<"#problem size\n";
        file << "cells " << stringify(cells) << "\n";
        file << "dofs " << stringify(dofs) << "\n";
        file << "nzes " << stringify(nzes) << "\n";
 
        file << "\n";
 
        file << "#solver statistics\n";
        for (auto stat : _solver_statistics)
        {
          file << stat.first << "\n";
 
          file << "#defects\n";
          for (Index i(0) ; i < stat.second.defect.size() ; ++i)
          {
            if (i == 0 && stat.second.defect.at(i) == double(-1))
              continue;
            if (stat.second.defect.at(i) == double(-1))
              file << "-\n";
            else
              file << stringify_fp_sci(stat.second.defect.at(i)) << "\n";
          }
 
          file << "#toe\n";
          for (Index i(0) ; i < stat.second.toe.size() ; ++i)
          {
            if (i == 0 && stat.second.toe.at(i) == double(-1))
              continue;
            if (stat.second.toe.at(i) == double(-1))
              file << "-\n";
            else
              file << stringify_fp_sci(stat.second.toe.at(i)) << "\n";
          }
 
          file << "#mpi_execute\n";
          for (Index i(0) ; i < stat.second.mpi_execute.size() ; ++i)
          {
            if (i == 0 && stat.second.mpi_execute.at(i) == double(-1))
              continue;
            if (stat.second.mpi_execute.at(i) == double(-1))
              file << "-\n";
            else
              file << stringify_fp_sci(stat.second.mpi_execute.at(i)) << "\n";
          }
 
          file << "#mpi_wait\n";
          for (Index i(0) ; i < stat.second.mpi_wait.size() ; ++i)
          {
            if (i == 0 && stat.second.mpi_wait.at(i) == double(-1))
              continue;
            if (stat.second.mpi_wait.at(i) == double(-1))
              file << "-\n";
            else
              file << stringify_fp_sci(stat.second.mpi_wait.at(i)) << "\n";
          }
        }
 
        file.close();
      }*/
  };
} // namespace FEAT