dist.hpp

// FEAT3: Finite Element Analysis Toolbox, Version 3
// Copyright (C) 2010 by Stefan Turek & the FEAT group
// FEAT3 is released under the GNU General Public License version 3,
// see the file 'copyright.txt' in the top level directory for details.
 
#pragma once
 
// includes, FEAT
#include <kernel/base_header.hpp>
#include <kernel/util/assertion.hpp>
#include <kernel/util/exception.hpp>
#include <kernel/util/binary_stream.hpp>
#include <kernel/util/type_traits.hpp>
 
// includes, system
#include <vector>
#include <sstream>
 
// includes, MPI
#ifdef FEAT_HAVE_MPI
#include <mpi.h>
#endif // FEAT_HAVE_MPI
 
namespace FEAT
{
  namespace Dist
  {
    bool initialize(int& argc, char**& argv);
 
    void finalize();
 
    struct Datatype
    {
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
      MPI_Datatype dt;
      std::size_t bytes;
 
      explicit Datatype(MPI_Datatype dt_, std::size_t bytes_) : dt(dt_), bytes(bytes_) {}
#else
    private:
      int dt;
      std::size_t bytes;
 
    public:
      explicit Datatype(int dt_, std::size_t bytes_) : dt(dt_), bytes(bytes_) {}
#endif // FEAT_HAVE_MPI
 
    public:
      bool operator==(const Datatype& other) const
      {
        return this->dt == other.dt;
      }
 
      bool operator!=(const Datatype& other) const
      {
        return this->dt != other.dt;
      }
 
      std::size_t size() const
      {
        return bytes;
      }
    }; // class Datatype
 
    extern const Datatype dt_byte;
    extern const Datatype dt_char;
    extern const Datatype dt_wchar;
    extern const Datatype dt_signed_char;
    extern const Datatype dt_signed_short;
    extern const Datatype dt_signed_int;
    extern const Datatype dt_signed_long;
    extern const Datatype dt_signed_long_long;
    extern const Datatype dt_unsigned_char;
    extern const Datatype dt_unsigned_short;
    extern const Datatype dt_unsigned_int;
    extern const Datatype dt_unsigned_long;
    extern const Datatype dt_unsigned_long_long;
    extern const Datatype dt_float;
    extern const Datatype dt_double;
    extern const Datatype dt_long_double;
    extern const Datatype dt_signed_int8;
    extern const Datatype dt_signed_int16;
    extern const Datatype dt_signed_int32;
    extern const Datatype dt_signed_int64;
    extern const Datatype dt_unsigned_int8;
    extern const Datatype dt_unsigned_int16;
    extern const Datatype dt_unsigned_int32;
    extern const Datatype dt_unsigned_int64;
 
#if defined(FEAT_HAVE_QUADMATH) || defined(DOXYGEN)
    extern const Datatype dt__float128;
#endif
 
#if defined(FEAT_HAVE_HALFMATH) || defined(DOXYGEN)
    extern const Datatype dt__half;
#endif
 
    template<typename T_> const Datatype& autotype();
 
    template<> inline const Datatype& autotype<char>()              {return dt_char;}
    template<> inline const Datatype& autotype<wchar_t>()           {return dt_wchar;}
    template<> inline const Datatype& autotype<signed char>()       {return dt_signed_char;}
    template<> inline const Datatype& autotype<signed short>()      {return dt_signed_short;}
    template<> inline const Datatype& autotype<signed int>()        {return dt_signed_int;}
    template<> inline const Datatype& autotype<signed long>()       {return dt_signed_long;}
    template<> inline const Datatype& autotype<signed long long>()  {return dt_signed_long_long;}
    template<> inline const Datatype& autotype<unsigned char>()     {return dt_unsigned_char;}
    template<> inline const Datatype& autotype<unsigned short>()    {return dt_unsigned_short;}
    template<> inline const Datatype& autotype<unsigned int>()      {return dt_unsigned_int;}
    template<> inline const Datatype& autotype<unsigned long>()     {return dt_unsigned_long;}
    template<> inline const Datatype& autotype<unsigned long long>(){return dt_unsigned_long_long;}
    template<> inline const Datatype& autotype<float>()             {return dt_float;}
    template<> inline const Datatype& autotype<double>()            {return dt_double;}
    template<> inline const Datatype& autotype<long double>()       {return dt_long_double;}
 
#if defined(FEAT_HAVE_QUADMATH) || defined(DOXYGEN)
    template<> inline const Datatype& autotype<__float128>()        {return dt__float128;}
#endif
 
#if defined(FEAT_HAVE_HALFMATH) || defined(DOXYGEN)
    template<> inline const Datatype& autotype<__half>()  {return dt__half;}
#endif
 
 
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
 
    struct Operation
    {
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
      MPI_Op op;
 
      explicit Operation(MPI_Op op_) : op(op_) {}
#else
    private:
      int op;
    public:
      explicit Operation(int op_) : op(op_) {}
#endif // FEAT_HAVE_MPI
 
    public:
      bool operator==(const Operation& other) const
      {
        return this->op == other.op;
      }
 
      bool operator!=(const Operation& other) const
      {
        return this->op != other.op;
      }
    }; // class Operation
 
    extern const Operation op_sum;
    extern const Operation op_max;
    extern const Operation op_min;
 
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
 
    class Status
    {
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
    public:
      MPI_Status status;
 
      Status() :
        status()
      {
        status.MPI_SOURCE = MPI_PROC_NULL;
      }
 
      Status(const Status&) = default;
      Status& operator=(const Status&) = default;
 
      MPI_Status* mpi_status()
      {
        return &status;
      }
 
      bool is_null() const
      {
        return status.MPI_SOURCE == MPI_PROC_NULL;
      }
 
      int source() const
      {
        return status.MPI_SOURCE;
      }
 
      int tag() const
      {
        return status.MPI_TAG;
      }
 
      int error() const
      {
        return status.MPI_ERROR;
      }
 
      std::size_t get_count(const Datatype& datatype) const
      {
        int c(0);
        MPI_Get_count(&status, datatype.dt, &c);
        return std::size_t(c);
      }
 
      std::size_t get_size() const
      {
        int c(0);
        MPI_Get_count(&status, MPI_BYTE, &c);
        return std::size_t(c);
      }
#else // non-MPI implementation
    public:
      int status;
 
      Status() :
        status(0)
      {
      }
#endif // FEAT_HAVE_MPI
    }; // class Status
 
#ifdef FEAT_HAVE_MPI
    // Ensure that 'Status' is a POD container for 'MPI_Status',
    // as this is required by the RequestVector class.
    static_assert(sizeof(Status) == sizeof(MPI_Status), "invalid Status class size; class Status must be a POD");
#endif
 
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
 
    class Request
    {
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
    public:
      MPI_Request request;
#else
    private:
      int request;
#endif // FEAT_HAVE_MPI
 
    public:
      Request();
 
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
      explicit Request(MPI_Request req_);
 
      MPI_Request* mpi_request()
      {
        return &request;
      }
#endif // FEAT_HAVE_MPI
 
      Request(const Request&) = delete;
      Request& operator=(const Request&) = delete;
 
      Request(Request&& other);
 
      Request& operator=(Request&& other);
 
      ~Request()
      {
        XASSERTM(is_null(), "You are trying to destroy an active request!");
      }
 
      bool is_null() const;
 
      void free();
 
      void cancel();
 
      bool test(Status& status);
 
      bool test()
      {
        Status status;
        return test(status);
      }
 
      bool wait(Status& status);
 
      bool wait()
      {
        Status status;
        return wait(status);
      }
    }; // class Request
 
#ifdef FEAT_HAVE_MPI
    // Ensure that 'Request' is a POD container for 'MPI_Request',
    // as this is required by the RequestVector class.
    static_assert(sizeof(Request) == sizeof(MPI_Request), "invalid Request class size; class Request must be a POD");
#endif
 
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
 
    class RequestVector
    {
    private:
      std::vector<Request> _reqs;
      std::vector<Status> _stats;
 
#if defined(FEAT_HAVE_MPI)
      int _isize() const
      {
        return int(_reqs.size());
      }
 
      MPI_Request* _reqs_array()
      {
        return reinterpret_cast<MPI_Request*>(_reqs.data());
      }
 
      MPI_Status* _stats_array()
      {
        return reinterpret_cast<MPI_Status*>(_stats.data());
      }
#endif // FEAT_HAVE_MPI
 
    public:
      RequestVector() :
        _reqs(),
        _stats()
      {
      }
 
      explicit RequestVector(std::size_t size_) :
        _reqs(size_),
        _stats(size_)
      {
      }
 
      RequestVector(const RequestVector&) = delete;
      RequestVector& operator=(const RequestVector&) = delete;
 
      RequestVector(RequestVector&& other) :
        _reqs(std::forward<std::vector<Request>>(other._reqs)),
        _stats(std::forward<std::vector<Status>>(other._stats))
      {
        other.clear();
      }
 
      RequestVector& operator=(RequestVector&& other)
      {
        if(this != &other)
        {
          XASSERT(is_null());
          _reqs = std::forward<std::vector<Request>>(other._reqs);
          _stats = std::forward<std::vector<Status>>(other._stats);
          other.clear();
        }
        return *this;
      }
 
      virtual ~RequestVector()
      {
        XASSERTM(is_null(), "You are trying to destroy an active request vector!");
      }
 
      Request& get_request(std::size_t idx)
      {
        XASSERT(idx < _reqs.size());
        return _reqs[idx];
      }
 
      const Request& get_request(std::size_t idx) const
      {
        XASSERT(idx < _reqs.size());
        return _reqs[idx];
      }
 
      Status& get_status(std::size_t idx)
      {
        XASSERT(idx < _stats.size());
        return _stats[idx];
      }
 
      const Status& get_status(std::size_t idx) const
      {
        XASSERT(idx < _stats.size());
        return _stats[idx];
      }
 
      Request& operator[](std::size_t idx)
      {
        XASSERT(idx < _reqs.size());
        return _reqs[idx];
      }
 
      const Request& operator[](std::size_t idx) const
      {
        XASSERT(idx < _reqs.size());
        return _reqs[idx];
      }
 
      void reserve(std::size_t size_)
      {
        if(size_ <= _reqs.size())
          return;
        _reqs.reserve(size_);
        _stats.reserve(size_);
      }
 
      void resize(std::size_t size_)
      {
        // make sure that all to-be-removed requests are null
        for(std::size_t i(size_); i < _reqs.size(); ++i)
        {
          XASSERT(_reqs.at(i).is_null());
        }
        _reqs.resize(size_);
        _stats.resize(size_);
      }
 
      std::size_t compress()
      {
        std::size_t i(0), j(0), n = _reqs.size();
        while(i+j < n)
        {
          if(_reqs[i+j].is_null())
            ++j;
          else
          {
            _reqs[i] = std::move(_reqs[i+j]);
            _stats[i] = std::move(_stats[i+j]);
            ++i;
          }
        }
        _reqs.resize(n-j);
        _stats.resize(n-j);
        return n-j;
      }
 
      void push_back(Request&& request)
      {
        _reqs.emplace_back(std::forward<Request>(request));
        _stats.emplace_back(Status());
      }
 
      std::size_t size() const
      {
        return _reqs.size();
      }
 
      std::size_t active_count() const
      {
        std::size_t count(0);
        for(const auto& r : _reqs)
        {
          if(!r.is_null())
            ++count;
        }
        return count;
      }
 
      bool empty() const
      {
        return _reqs.empty();
      }
 
      bool is_null() const
      {
        for(const auto& r : _reqs)
        {
          if(!r.is_null())
            return false;
        }
        return true;
      }
 
      void free()
      {
        for(auto& r : _reqs)
        {
          if(!r.is_null())
            r.free();
        }
      }
 
      void cancel()
      {
        for(auto& r : _reqs)
        {
          if(!r.is_null())
            r.cancel();
        }
      }
 
      void clear()
      {
        XASSERT(is_null());
        _reqs.clear();
        _stats.clear();
      }
 
      bool test_for(std::size_t idx)
      {
        return get_request(idx).test(get_status(idx));
      }
 
      bool test_for(std::size_t idx, Status& status)
      {
        return get_request(idx).test(status);
      }
 
      bool test_all();
 
      bool test_any(std::size_t& idx, Status& status);
 
      bool test_any(std::size_t& idx)
      {
        Status status;
        if(!test_any(idx, status))
          return false;
        _stats[idx] = status;
        return true;
      }
 
      bool test_any(Status& status)
      {
        std::size_t idx;
        return test_any(idx, status);
      }
 
      bool wait_for(std::size_t idx)
      {
        return get_request(idx).wait(get_status(idx));
      }
 
      bool wait_for(std::size_t idx, Status& status)
      {
        return get_request(idx).wait(status);
      }
 
      void wait_all();
 
      bool wait_any(std::size_t& idx, Status& status);
 
      bool wait_any(std::size_t& idx)
      {
        Status status;
        if(!wait_any(idx, status))
          return false;
        _stats[idx] = status;
        return true;
      }
 
      bool wait_any(Status& status)
      {
        std::size_t idx;
        return wait_any(idx, status);
      }
    }; // class RequestVector
 
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
    /* ************************************************************************************************************* */
 
    class Comm
    {
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
    public:
      MPI_Comm comm;
#endif // FEAT_HAVE_MPI
 
    protected:
      int _rank;
      int _size;
 
#if !defined(FEAT_HAVE_MPI) && !defined(DOXYGEN)
      explicit Comm(int);
#endif
 
    public:
      Comm();
 
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
      explicit Comm(MPI_Comm comm_);
#endif // FEAT_HAVE_MPI
 
      Comm(const Comm&) = delete;
      Comm& operator=(const Comm&) = delete;
 
      Comm(Comm&& other);
 
      Comm& operator=(Comm&& other);
 
      virtual ~Comm();
 
#if defined(FEAT_HAVE_MPI) || defined(DOXYGEN)
      const MPI_Comm& mpi_comm() const
      {
        return comm;
      }
 
      MPI_Comm& mpi_comm()
      {
        return comm;
      }
#endif //FEAT_HAVE_MPI
 
      static Comm world();
 
      static Comm self();
 
      static Comm null();
 
      bool is_world() const;
 
      bool is_self() const;
 
      bool is_null() const;
 
      int rank() const
      {
        return _rank;
      }
 
      int size() const
      {
        return _size;
      }
 
 
      Comm comm_dup() const;
 
      Comm comm_create_range_incl(int count, int first = 0, int stride = 1) const;
 
      Comm comm_create_incl(int n, const int* ranks) const;
 
      Comm comm_split(int color, int key) const;
 
 
 
      void barrier() const;
 
      Request ibarrier() const;
 
      // end of barrier synchronization group
 
 
      void send(const void* buffer, std::size_t count, const Datatype& datatype, int dest, int tag = 0) const;
 
      template<typename T_>
      void send(const T_* buffer, std::size_t count, int dest, int tag = 0) const
      {
        send(buffer, count, autotype<T_>(), dest, tag);
      }
 
      void recv(void* buffer, std::size_t count, const Datatype& datatype, int source, int tag, Status& status) const;
 
      void recv(void* buffer, std::size_t count, const Datatype& datatype, int source, int tag = 0) const
      {
        Status status;
        recv(buffer, count, datatype, source, tag, status);
      }
 
      template<typename T_>
      void recv(T_* buffer, std::size_t count, int source, int tag, Status& status) const
      {
        recv(buffer, count, autotype<T_>(), source, tag, status);
      }
 
      template<typename T_>
      void recv(T_* buffer, std::size_t count, int source, int tag = 0) const
      {
        Status status;
        recv(buffer, count, autotype<T_>(), source, tag, status);
      }
 
      // end of blocking point-to-point group
 
 
      Request isend(const void* buffer, std::size_t count, const Datatype& datatype, int dest, int tag = 0) const;
 
      template<typename T_>
      Request isend(const T_* buffer, std::size_t count, int dest, int tag = 0) const
      {
        return isend(buffer, count, autotype<T_>(), dest, tag);
      }
 
      Request irecv(void* buffer, std::size_t count, const Datatype& datatype, int source, int tag = 0) const;
 
      template<typename T_>
      Request irecv(T_* buffer, std::size_t count, int source, int tag = 0) const
      {
        return irecv(buffer, count, autotype<T_>(), source, tag);
      }
 
      // end of nonblocking point-to-point group
 
 
      void bcast(void* buffer, std::size_t count, const Datatype& datatype, int root) const;
 
      template<typename T_>
      void bcast(T_* buffer, std::size_t count, int root) const
      {
        bcast(buffer, count, autotype<T_>(), root);
      }
 
      Request ibcast(void* buffer, std::size_t count, const Datatype& datatype, int root) const;
 
      template<typename T_>
      Request ibcast(T_* buffer, std::size_t count, int root) const
      {
        return ibcast(buffer, count, autotype<T_>(), root);
      }
 
      // end of broadcast group
 
 
      void gather(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype, int root) const;
 
      template<typename ST_, typename RT_>
      void gather(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount, int root) const
      {
        gather(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>(), root);
      }
 
      Request igather(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype, int root) const;
 
      template<typename ST_, typename RT_>
      Request igather(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount, int root) const
      {
        return igather(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>(), root);
      }
 
      void scatter(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype, int root) const;
 
      template<typename ST_, typename RT_>
      void scatter(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount, int root) const
      {
        scatter(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>(), root);
      }
 
      Request iscatter(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype, int root) const;
 
      template<typename ST_, typename RT_>
      Request iscatter(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount, int root) const
      {
        return iscatter(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>(), root);
      }
 
      void allgather(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype) const;
 
      template<typename ST_, typename RT_>
      void allgather(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount) const
      {
        allgather(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>());
      }
 
      Request iallgather(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype) const;
 
      template<typename ST_, typename RT_>
      Request iallgather(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount) const
      {
        return iallgather(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>());
      }
 
      void allgatherv(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, const int* recvcounts, const int* displs, const Datatype& recvtype) const;
 
      template<typename ST_, typename RT_>
      void allgatherv(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, const int* recvcounts, const int* displs) const
      {
        allgatherv(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcounts, displs, autotype<RT_>());
      }
 
      void alltoall(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype) const;
 
      template<typename ST_, typename RT_>
      void alltoall(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount) const
      {
        alltoall(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>());
      }
 
      Request ialltoall(const void* sendbuf, std::size_t sendcount, const Datatype& sendtype, void* recvbuf, std::size_t recvcount, const Datatype& recvtype) const;
 
      template<typename ST_, typename RT_>
      Request ialltoall(const ST_* sendbuf, std::size_t sendcount, RT_* recvbuf, std::size_t recvcount) const
      {
        return ialltoall(sendbuf, sendcount, autotype<ST_>(), recvbuf, recvcount, autotype<RT_>());
      }
 
      void alltoallv(const void* sendbuf, const int* sendcounts, const int* sdispls, const Datatype& sendtype, void* recvbuf, const int* recvcounts, const int* rdispls, const Datatype& recvtype) const;
 
      template<typename ST_, typename RT_>
      void alltoallv(const ST_* sendbuf, const int* sendcounts, const int* sdispls, RT_* recvbuf, const int* recvcounts, const int* rdispls) const
      {
        alltoallv(sendbuf, sendcounts, sdispls, autotype<ST_>(), recvbuf, recvcounts, rdispls, autotype<RT_>());
      }
 
      // end of gather/scatter group
 
 
      void reduce(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op, int root) const;
 
      template<typename T_>
      void reduce(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op, int root) const
      {
        reduce(sendbuf, recvbuf, count, autotype<T_>(), op, root);
      }
 
      Request ireduce(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op, int root) const;
 
      template<typename T_>
      Request ireduce(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op, int root) const
      {
        return ireduce(sendbuf, recvbuf, count, autotype<T_>(), op, root);
      }
 
      void allreduce(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op) const;
 
      template<typename T_>
      void allreduce(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op) const
      {
        allreduce(sendbuf, recvbuf, count, autotype<T_>(), op);
      }
 
      Request iallreduce(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op) const;
 
      template<typename T_>
      Request iallreduce(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op) const
      {
        return iallreduce(sendbuf, recvbuf, count, autotype<T_>(), op);
      }
 
      void scan(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op) const;
 
      template<typename T_>
      void scan(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op) const
      {
        scan(sendbuf, recvbuf, count, autotype<T_>(), op);
      }
 
      /*Request iscan(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op) const;
 
      template<typename T_>
      Request iscan(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op) const
      {
        return iscan(sendbuf, recvbuf, count, autotype<T_>(), op);
      }*/
 
      void exscan(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op) const;
 
      template<typename T_>
      void exscan(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op) const
      {
        exscan(sendbuf, recvbuf, count, autotype<T_>(), op);
      }
 
      /*Request iexscan(const void* sendbuf, void* recvbuf, std::size_t count, const Datatype& datatype, const Operation& op) const;
 
      template<typename T_>
      Request iexscan(const T_* sendbuf, T_* recvbuf, std::size_t count, const Operation& op) const
      {
        return iexscan(sendbuf, recvbuf, count, autotype<T_>(), op);
      }*/
 
      // end of reductions group
 
 
      void bcast_stringstream(std::stringstream& stream, int root = 0) const;
 
      void bcast_binarystream(BinaryStream& stream, int root = 0) const;
 
      void print(std::ostream& os, const String& msg, int root = 0) const;
 
      void print(const String& msg, int root = 0) const
      {
        print(std::cout, msg, root);
      }
 
      void allprint(std::ostream& os, const String& msg, int root = 0) const;
 
      void allprint(const String& msg, int root = 0) const
      {
        allprint(std::cout, msg, root);
      }
 
      void print_flush(std::ostream& os, int root = 0) const;
 
      void print_flush(int root = 0) const
      {
        print_flush(std::cout, root);
      }
 
      // end of extended comm group
    }; // class Comm
  } // namespace Dist
} // namespace FEAT