Tangle: Update files

bench/test_main.cxx

@@ -20,8 +20,26 @@ int main(int argc, char** argv) {
   constexpr double elem_to_gb = 8.0 / 1024.0 / 1024.0 / 1024.0;
   const int no(hauta::option<int>(argc, argv, "--no"))
           , nv(hauta::option<int>(argc, argv, "--nv"))
+          , itMod(hauta::option<int>(argc, argv, "--mod", 100))
           ;
 
+  const bool nochrono(hauta::option<bool>(argc, argv, "--nochrono", false))
+           , barrier(hauta::option<bool>(argc, argv, "--barrier", false))
+           ;
+  const std::string tuplesDistributionString
+    = hauta::option<std::string>(argc, argv, "--dist", "naive");
+
+  atrip::Atrip::Input::TuplesDistribution tuplesDistribution;
+  { using atrip::Atrip;
+    if (tuplesDistributionString == "naive") {
+      tuplesDistribution = Atrip::Input::TuplesDistribution::NAIVE;
+    } else if (tuplesDistributionString == "group") {
+      tuplesDistribution = Atrip::Input::TuplesDistribution::GROUP_AND_SORT;
+    } else {
+      std::cout << "--dist should be either naive or group\n";
+      exit(1);
+    }
+  }
 
   std::vector<int> symmetries(4, NS)
                  , vo({nv, no})
@@ -53,9 +71,7 @@ int main(int argc, char** argv) {
   Vppph.fill_random(0, 1);
 
   atrip::Atrip::init();
-  atrip::Atrip::Input in;
+  const auto in = atrip::Atrip::Input()
-
-  in
     // Tensors
     .with_epsilon_i(&ei)
     .with_epsilon_a(&ea)
@@ -65,8 +81,10 @@ int main(int argc, char** argv) {
     .with_Vijka(&Vhhhp)
     .with_Vabci(&Vppph)
     // some options
-    .with_barrier(false)
+    .with_barrier(barrier)
-    .with_iterationMod(100)
+    .with_chrono(!nochrono)
+    .with_iterationMod(itMod)
+    .with_tuplesDistribution(tuplesDistribution)
     ;
 
   auto out = atrip::Atrip::run(in);

include/atrip/Atrip.hpp

@@ -7,6 +7,13 @@
 
 #include <ctf.hpp>
 
+#define ADD_ATTRIBUTE(_type, _name, _default)   \
+  _type _name = _default;                       \
+  Input& with_ ## _name(_type i) {              \
+    _name = i;                                  \
+    return *this;                               \
+  }
+
 namespace atrip {
 
   struct Atrip {
@@ -24,9 +31,6 @@ namespace atrip {
                         , *Vhhhp = nullptr
                         , *Vppph = nullptr
                         ;
-      int maxIterations = 0, iterationMod = -1;
-      bool barrier = false;
-      bool chrono = false;
       Input& with_epsilon_i(CTF::Tensor<double> * t) { ei = t; return *this; }
       Input& with_epsilon_a(CTF::Tensor<double> * t) { ea = t; return *this; }
       Input& with_Tai(CTF::Tensor<double> * t) { Tph = t; return *this; }
@@ -34,10 +38,19 @@ namespace atrip {
       Input& with_Vabij(CTF::Tensor<double> * t) { Vpphh = t; return *this; }
       Input& with_Vijka(CTF::Tensor<double> * t) { Vhhhp = t; return *this; }
       Input& with_Vabci(CTF::Tensor<double> * t) { Vppph = t; return *this; }
-      Input& with_maxIterations(int i) { maxIterations = i; return *this; }
+
-      Input& with_iterationMod(int i) { iterationMod = i; return *this; }
+      enum TuplesDistribution {
-      Input& with_barrier(bool i) { barrier = i; return *this; }
+        NAIVE,
-      Input& with_chrono(bool i) { chrono = i; return *this; }
+        GROUP_AND_SORT,
+      };
+
+      ADD_ATTRIBUTE(bool, chrono, false)
+      ADD_ATTRIBUTE(bool, barrier, false)
+      ADD_ATTRIBUTE(int, maxIterations, 0)
+      ADD_ATTRIBUTE(int, iterationMod, -1)
+      ADD_ATTRIBUTE(TuplesDistribution, tuplesDistribution, NAIVE)
+
+
     };
 
     struct Output {
@@ -47,4 +60,6 @@ namespace atrip {
   };
 
 }
+
+#undef ADD_ATTRIBUTE
 // Atrip:1 ends here

include/atrip/Debug.hpp

@@ -2,12 +2,17 @@
 #pragma once
 #define ATRIP_BENCHMARK
 //#define ATRIP_DONT_SLICE
-#define ATRIP_DEBUG 1
 //#define ATRIP_WORKLOAD_DUMP
 #define ATRIP_USE_DGEMM
 //#define ATRIP_PRINT_TUPLES
 
+#ifndef ATRIP_DEBUG
+#define ATRIP_DEBUG 1
+#endif
+
+#ifndef LOG
 #define LOG(level, name) if (Atrip::rank == 0) std::cout << name << ": "
+#endif
 
 #if ATRIP_DEBUG == 4
 #  pragma message("WARNING: You have OCD debugging ABC triples "\
@@ -45,7 +50,7 @@
 #  define WITH_CRAZY_DEBUG if (false)
 #  define WITH_DBG
 #  define DBG(...) dbg(__VA_ARGS__)
-#elif ATRIP_DEBUG == 1
+#else
 #  define OCD_Barrier(com)
 #  define WITH_OCD if (false)
 #  define WITH_ROOT if (false)
@@ -54,7 +59,5 @@
 #  define WITH_DBG if (false)
 #  define WITH_CRAZY_DEBUG if (false)
 #  define DBG(...)
-#else
-#  error("ATRIP_DEBUG is not defined!")
 #endif
 // Debug:1 ends here

include/atrip/Tuples.hpp

@@ -77,10 +77,16 @@ getABCRange(size_t np, size_t rank, ABCTuples const& tuplesList) {
 
   auto const& it = n_tuples_per_rank.begin();
 
-  return
+  std::pair<size_t, size_t> const
-    { std::accumulate(it, it + rank    , 0)
+    range = { std::accumulate(it, it + rank    , 0)
-    , std::accumulate(it, it + rank + 1, 0)
+            , std::accumulate(it, it + rank + 1, 0) - 1
-    };
+            };
+
+  WITH_RANK << "range = "
+            << range.first << " -> " << range.second
+            << std::endl;
+
+  return range;
 
 }
 // Naive list:2 ends here
@@ -92,12 +98,23 @@ struct NaiveDistribution : public TuplesDistribution {
     MPI_Comm_rank(universe, &rank);
     MPI_Comm_size(universe, &np);
     auto const all = getTuplesList(Nv);
-    auto const range = getABCRange((size_t)np, (size_t)rank, all);
+    const size_t
+      tuplesPerRank
+        = all.size() / np
+        + size_t(all.size() % np != 0)
+        ;
+    //auto const range = getABCRange((size_t)np, (size_t)rank, all);
+
+    std::pair<size_t, size_t> const
+      range = { tuplesPerRank * rank
+              , tuplesPerRank * (rank + 1) - 1
+              };
+
     std::vector<ABCTuple> result(range.second - range.first, FAKE_TUPLE);
     std::copy(all.begin() + range.first,
               range.second >= all.size()
-              ? all.end()
+                ? all.end()
-              : all.begin() + range.first + range.second,
+                : all.begin() + range.first + range.second,
               result.begin());
     return result;
   }
@@ -226,7 +243,6 @@ specialDistribution(Info info, std::vector<ABCTuple> const& allTuples) {
   size_t nNodes(info.nNodes);
   size_t np(info.np);
   size_t N(allTuples.size());
-  size_t tuplePerNode( ceil( ((double)N) / nNodes) );
 
   //      nodeid          tuple list
   std::map<size_t, std::vector<ABCTuple> > container1d;

include/atrip/Utils.hpp

@@ -6,6 +6,7 @@
 #include <chrono>
 
 #include <ctf.hpp>
+#include <atrip/Debug.hpp>
 
 namespace atrip {
 // Prolog:1 ends here

src/atrip/Atrip.cxx

@@ -83,36 +83,36 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
   // all tensors
   std::vector< SliceUnion* > unions = {&taphh, &hhha, &abph, &abhh, &tabhh};
 
-  //CONSTRUCT TUPLE LIST ==============================================={{{1
+  // get tuples for the current rank
-  LOG(0,"Atrip") << "BUILD TUPLE LIST\n";
+  TuplesDistribution *distribution;
-  const auto tuplesList = std::move(getTuplesList(Nv));
-  WITH_RANK << "tupList.size() = " << tuplesList.size() << "\n";
 
-  // GET ABC INDEX RANGE FOR RANK ======================================{{{1
+  if (in.tuplesDistribution == Atrip::Input::TuplesDistribution::NAIVE) {
-  auto abcIndex = getABCRange(np, rank, tuplesList);
+    LOG(0,"Atrip") << "Using the naive distribution\n";
-  size_t nIterations = abcIndex.second - abcIndex.first;
+    distribution = new NaiveDistribution();
-
+  } else {
-#ifdef ATRIP_BENCHMARK
+    LOG(0,"Atrip") << "Using the group-and-sort distribution\n";
-  { const size_t maxIterations = in.maxIterations;
+    distribution = new group_and_sort::Distribution();
-    if (maxIterations != 0) {
-      abcIndex.second = abcIndex.first + maxIterations % (nIterations + 1);
-      nIterations = maxIterations % (nIterations + 1);
-    }
   }
-#endif
 
-  WITH_RANK << "abcIndex = " << pretty_print(abcIndex) << "\n";
+  LOG(0,"Atrip") << "BUILDING TUPLE LIST\n";
-  LOG(0,"Atrip") << "#iterations: " << nIterations << "\n";
+  WITH_CHRONO(chrono["tuples:build"],
-
+    auto const tuplesList = distribution->getTuples(Nv, universe);
-  // first abc
+    )
-  const ABCTuple firstAbc = tuplesList[abcIndex.first];
+  size_t nIterations = tuplesList.size();
-
+  {
-
+    const size_t _all_tuples = Nv * (Nv + 1) * (Nv + 2) / 6 - Nv;
-  double energy(0.);
+    LOG(0,"Atrip") << "#iterations: "
+                  << nIterations
+                  << "/"
+                  << _all_tuples
+                  << "\n";
+  }
 
 
   auto const isFakeTuple
-    = [&tuplesList](size_t const i) { return i >= tuplesList.size(); };
+    = [&tuplesList, distribution](size_t const i) {
+      return distribution->tupleIsFake(tuplesList[i]);
+    };
 
 
   auto communicateDatabase
@@ -255,10 +255,10 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
 
   // START MAIN LOOP ======================================================{{{1
 
-  Slice::Database db;
+  double energy(0.);
 
-  for ( size_t i = abcIndex.first, iteration = 1
+  for ( size_t i = 0, iteration = 1
-      ; i < abcIndex.second
+      ; i < tuplesList.size()
       ; i++, iteration++
       ) {
     chrono["iterations"].start();
@@ -267,13 +267,10 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
     chrono["start:stop"].start(); chrono["start:stop"].stop();
 
     // check overhead of doing a barrier at the beginning
-    chrono["oneshot-mpi:barrier"].start();
+    WITH_CHRONO(chrono["oneshot-mpi:barrier"],
-    chrono["mpi:barrier"].start();
+    WITH_CHRONO(chrono["mpi:barrier"],
-    // TODO: REMOVE
+      if (in.barrier) MPI_Barrier(universe);
-    if (in.barrier == 1)
+    ))
-    MPI_Barrier(universe);
-    chrono["mpi:barrier"].stop();
-    chrono["oneshot-mpi:barrier"].stop();
 
     if (iteration % in.iterationMod == 0) {
       LOG(0,"Atrip")
@@ -297,7 +294,7 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
     const ABCTuple abc = isFakeTuple(i)
                        ? tuplesList[tuplesList.size() - 1]
                        : tuplesList[i]
-                 , *abcNext = i == (abcIndex.second - 1)
+                 , *abcNext = i == (tuplesList.size() - 1)
                             ? nullptr
                             : isFakeTuple(i + 1)
                             ? &tuplesList[tuplesList.size() - 1]
@@ -314,12 +311,12 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
 
 
     // COMM FIRST DATABASE ================================================{{{1
-    if (i == abcIndex.first) {
+    if (i == 0) {
       WITH_RANK << "__first__:first database ............ \n";
-      const auto __db = communicateDatabase(abc, universe);
+      const auto db = communicateDatabase(abc, universe);
       WITH_RANK << "__first__:first database communicated \n";
       WITH_RANK << "__first__:first database io phase \n";
-      doIOPhase(__db);
+      doIOPhase(db);
       WITH_RANK << "__first__:first database io phase DONE\n";
       WITH_RANK << "__first__::::Unwrapping all slices for first database\n";
       for (auto& u: unions) u->unwrapAll(abc);
@@ -331,8 +328,7 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
     if (abcNext) {
       WITH_RANK << "__comm__:" << iteration << "th communicating database\n";
       chrono["db:comm"].start();
-      //const auto db = communicateDatabase(*abcNext, universe);
+      const auto db = communicateDatabase(*abcNext, universe);
-      db = communicateDatabase(*abcNext, universe);
       chrono["db:comm"].stop();
       chrono["db:io"].start();
       doIOPhase(db);