diff --git a/include/atrip/DatabaseCommunicator.hpp b/include/atrip/DatabaseCommunicator.hpp new file mode 100644 index 0000000..9c5b4c2 --- /dev/null +++ b/include/atrip/DatabaseCommunicator.hpp @@ -0,0 +1,20 @@ +#pragma once +#include +#include +#include +#include + +namespace atrip { + + template + using Unions = std::vector*>; + + template + typename Slice::Database + naiveDatabase(Unions &unions, + size_t nv, + size_t np, + size_t iteration, + MPI_Comm const& c); + +} // namespace atrip diff --git a/src/Makefile.am b/src/Makefile.am index 27f3ba2..a489e31 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -7,7 +7,7 @@ AM_CPPFLAGS = $(CTF_CPPFLAGS) lib_LIBRARIES = libatrip.a libatrip_a_CPPFLAGS = -I$(top_srcdir)/include/ -libatrip_a_SOURCES = ./atrip/Blas.cxx ./atrip/Tuples.cxx +libatrip_a_SOURCES = ./atrip/Blas.cxx ./atrip/Tuples.cxx ./atrip/DatabaseCommunicator.cxx NVCC_FILES = ./atrip/Equations.cxx ./atrip/Complex.cxx ./atrip/Atrip.cxx if WITH_CUDA diff --git a/src/atrip/Atrip.cxx b/src/atrip/Atrip.cxx index 95e2e11..9987d64 100644 --- a/src/atrip/Atrip.cxx +++ b/src/atrip/Atrip.cxx @@ -21,6 +21,7 @@ #include #include #include +#include using namespace atrip; #if defined(HAVE_CUDA) @@ -299,9 +300,16 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { using Database = typename Slice::Database; auto communicateDatabase = [ &unions + , &in + , Nv , np - ] (ABCTuple const& abc, MPI_Comm const& c) -> Database { + ] (ABCTuple const& abc, MPI_Comm const& c, size_t iteration) -> Database { + if (in.tuplesDistribution == Atrip::Input::TuplesDistribution::NAIVE) { + + return naiveDatabase(unions, Nv, np, iteration, c); + + } else { WITH_CHRONO("db:comm:type:do", auto MPI_LDB_ELEMENT = Slice::mpi::localDatabaseElement(); ) @@ -334,6 +342,8 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { WITH_CHRONO("db:comm:type:free", MPI_Type_free(&MPI_LDB_ELEMENT);) return db; + } + }; auto doIOPhase @@ -564,7 +574,7 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { // COMM FIRST DATABASE ================================================{{{1 if (i == first_iteration) { WITH_RANK << "__first__:first database ............ \n"; - const auto db = communicateDatabase(abc, universe); + const auto db = communicateDatabase(abc, universe, i); WITH_RANK << "__first__:first database communicated \n"; WITH_RANK << "__first__:first database io phase \n"; doIOPhase(db); @@ -579,7 +589,7 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { if (abcNext) { WITH_RANK << "__comm__:" << iteration << "th communicating database\n"; WITH_CHRONO("db:comm", - const auto db = communicateDatabase(*abcNext, universe); + const auto db = communicateDatabase(*abcNext, universe, i); ) WITH_CHRONO("db:io", doIOPhase(db); diff --git a/src/atrip/DatabaseCommunicator.cxx b/src/atrip/DatabaseCommunicator.cxx new file mode 100644 index 0000000..490b505 --- /dev/null +++ b/src/atrip/DatabaseCommunicator.cxx @@ -0,0 +1,167 @@ +#include +#include + + +namespace atrip { + + static + ABCTuples get_nth_naive_tuples(size_t Nv, size_t np) { + + const size_t + // total number of tuples for the problem + n = Nv * (Nv + 1) * (Nv + 2) / 6 - Nv + + // all ranks should have the same number of tuples_per_rank + , tuples_per_rank = n / np + size_t(n % np != 0) + ; + + + ABCTuples result(np); + + for (size_t a(0), g(0); a < Nv; a++) + for (size_t b(a); b < Nv; b++) + for (size_t c(b); c < Nv; c++){ + if ( a == b && b == c ) continue; + for (size_t rank = 0; rank < np; rank++) { + + const size_t + // start index for the global tuples list + start = tuples_per_rank * rank + + // end index for the global tuples list + , end = tuples_per_rank * (rank + 1) + ; + + if ( start <= g && g < end) result[rank] = {a, b, c}; + + } + g++; + } + + return result; + + } + + + template + static + typename Slice::LocalDatabase + build_local_database_fake(ABCTuple const& abc_prev, + ABCTuple const& abc, + size_t rank, + SliceUnion* u) { + + typename Slice::LocalDatabase result; + + // vector of type x tuple + auto const needed = u->neededSlices(abc); + auto const needed_prev = u->neededSlices(abc_prev); + + for (auto const& pair: needed) { + auto const type = pair.first; + auto const tuple = pair.second; + auto const from = u->rankMap.find(abc, type); + + // Try to find in the previously needed slices + // one that exactly matches the tuple. + // Not necessarily has to match the type. + // + // If we find it, then it means that the fake rank + // will mark it as recycled. This covers + // the finding of Ready slices and Recycled slices. + { + auto const& it + = std::find_if(needed_prev.begin(), needed_prev.end(), + [&tuple, &type](typename Slice::Ty_x_Tu const& o) { + return o.second == tuple; + }); + + if (it != needed_prev.end()) { + typename Slice::Info info; + info.tuple = tuple; + info.type = type; + info.from = from; + info.state = Slice::Recycled; + result.push_back({u->name, info}); + continue; + } + } + + { + typename Slice::Info info; + info.type = type; + info.tuple = tuple; + info.from = from; + + // Handle self sufficiency + info.state = rank == from.rank + ? Slice::SelfSufficient + : Slice::Fetch + ; + result.push_back({u->name, info}); + continue; + } + + } + + return result; + + } + + + + template + typename Slice::Database + naiveDatabase(Unions &unions, + size_t nv, + size_t np, + size_t iteration, + MPI_Comm const& c) { + + using Database = typename Slice::Database; + Database db; + const auto tuples = get_nth_naive_tuples(nv, np); + const auto prev_tuples = get_nth_naive_tuples(nv, np); + + for (size_t rank = 0; rank < np; rank++) { + auto abc = tuples[rank]; + typename Slice::LocalDatabase ldb; + + for (auto const& tensor: unions) { + if (rank == Atrip::rank) { + auto const& tensorDb = tensor->buildLocalDatabase(abc); + ldb.insert(ldb.end(), tensorDb.begin(), tensorDb.end()); + } else { + auto const& tensorDb + = build_local_database_fake(prev_tuples[rank], + abc, + rank, + tensor); + ldb.insert(ldb.end(), tensorDb.begin(), tensorDb.end()); + } + } + + db.insert(db.end(), ldb.begin(), ldb.end()); + + } + + return db; + } + + template + typename Slice::Database + naiveDatabase(Unions &unions, + size_t nv, + size_t np, + size_t iteration, + MPI_Comm const& c); + + template + typename Slice::Database + naiveDatabase(Unions &unions, + size_t nv, + size_t np, + size_t iteration, + MPI_Comm const& c); + +} // namespace atrip