From fed19ff52c54b854766e098d6ad5c63542dbb6ff Mon Sep 17 00:00:00 2001 From: Alejandro Gallo Date: Tue, 5 Apr 2022 17:16:11 +0200 Subject: [PATCH] Implement deleting Vppph --- atrip.org | 47 +++++----- bench/test_main.cxx | 53 ++++++----- include/atrip/Atrip.hpp | 15 +-- include/atrip/Slice.hpp | 196 +++++++++++++++++++-------------------- include/atrip/Tuples.hpp | 44 ++++----- include/atrip/Utils.hpp | 2 +- src/atrip/Atrip.cxx | 20 ++-- 7 files changed, 196 insertions(+), 181 deletions(-) diff --git a/atrip.org b/atrip.org index 41327dd..8b03d7a 100644 --- a/atrip.org +++ b/atrip.org @@ -2935,13 +2935,13 @@ namespace atrip { template struct Input { CTF::Tensor *ei = nullptr - , *ea = nullptr - , *Tph = nullptr - , *Tpphh = nullptr - , *Vpphh = nullptr - , *Vhhhp = nullptr - , *Vppph = nullptr - ; + , *ea = nullptr + , *Tph = nullptr + , *Tpphh = nullptr + , *Vpphh = nullptr + , *Vhhhp = nullptr + , *Vppph = nullptr + ; Input& with_epsilon_i(CTF::Tensor * t) { ei = t; return *this; } Input& with_epsilon_a(CTF::Tensor * t) { ea = t; return *this; } Input& with_Tai(CTF::Tensor * t) { Tph = t; return *this; } @@ -2955,6 +2955,7 @@ namespace atrip { GROUP_AND_SORT, }; + ADD_ATTRIBUTE(bool, deleteVppph, false) ADD_ATTRIBUTE(bool, rankRoundRobin, false) ADD_ATTRIBUTE(bool, chrono, false) ADD_ATTRIBUTE(bool, barrier, false) @@ -3063,14 +3064,6 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { MPI_Comm_size(child_comm, &child_size); } - - // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1 - WITH_CHRONO("nv-slices", - LOG(0,"Atrip") << "BUILD NV-SLICES\n"; - TAPHH taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); - HHHA hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); - ) - // BUILD SLICES PARAMETRIZED BY NV x NV =============================={{{1 WITH_CHRONO("nv-nv-slices", LOG(0,"Atrip") << "BUILD NV x NV-SLICES\n"; @@ -3079,6 +3072,18 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { TABHH tabhh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); ) + // delete the Vppph so that we don't have a HWM situation for the NV slices + if (in.deleteVppph) { + delete in.Vppph; + } + + // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1 + WITH_CHRONO("nv-slices", + LOG(0,"Atrip") << "BUILD NV-SLICES\n"; + TAPHH taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); + HHHA hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); + ) + // all tensors std::vector< SliceUnion* > unions = {&taphh, &hhha, &abph, &abhh, &tabhh}; @@ -3251,12 +3256,12 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { const double doublesFlops = double(No) - * double(No) - * double(No) - * (double(No) + double(Nv)) - * 2.0 - * (traits::isComplex() ? 2.0 : 1.0) - * 6.0 + ,* double(No) + ,* double(No) + ,* (double(No) + double(Nv)) + ,* 2.0 + ,* (traits::isComplex() ? 2.0 : 1.0) + ,* 6.0 / 1e9 ; diff --git a/bench/test_main.cxx b/bench/test_main.cxx index ea13004..4f9bab5 100644 --- a/bench/test_main.cxx +++ b/bench/test_main.cxx @@ -17,13 +17,15 @@ int main(int argc, char** argv) { MPI_Init(&argc, &argv); int no(10), nv(10), itMod(-1), percentageMod(10); - bool nochrono(false), barrier(false), rankRoundRobin(false); + bool nochrono(false), barrier(false), rankRoundRobin(false), + keepVppph(false); std::string tuplesDistributionString = "naive"; CLI::App app{"Main bench for atrip"}; app.add_option("--no", no, "Occupied orbitals"); app.add_option("--nv", nv, "Virtual orbitals"); app.add_option("--mod", itMod, "Iteration modifier"); + app.add_flag("--keep-vppph", keepVppph, "Do not delete Vppph"); app.add_flag("--nochrono", nochrono, "Do not print chrono"); app.add_flag("--rank-round-robin", rankRoundRobin, "Do rank round robin"); app.add_flag("--barrier", barrier, "Use the first barrier"); @@ -38,12 +40,13 @@ int main(int argc, char** argv) { constexpr double elem_to_gb = 8.0 / 1024.0 / 1024.0 / 1024.0; // USER PRINTING TEST BEGIN - const double doublesFlops = no * no * no - * (no + nv) - * 2.0 - * 6.0 - / 1.0e9 - ; + const double doublesFlops + = no * no * no + * (no + nv) + * 2.0 + * 6.0 + / 1.0e9 + ; double lastElapsedTime = 0; bool firstHeaderPrinted = false; atrip::registerIterationDescriptor @@ -110,23 +113,25 @@ int main(int argc, char** argv) { Vppph.fill_random(0, 1); atrip::Atrip::init(); - const auto in = atrip::Atrip::Input() - // Tensors - .with_epsilon_i(&ei) - .with_epsilon_a(&ea) - .with_Tai(&Tph) - .with_Tabij(&Tpphh) - .with_Vabij(&Vpphh) - .with_Vijka(&Vhhhp) - .with_Vabci(&Vppph) - // some options - .with_barrier(barrier) - .with_chrono(!nochrono) - .with_rankRoundRobin(rankRoundRobin) - .with_iterationMod(itMod) - .with_percentageMod(percentageMod) - .with_tuplesDistribution(tuplesDistribution) - ; + const auto in + = atrip::Atrip::Input() + // Tensors + .with_epsilon_i(&ei) + .with_epsilon_a(&ea) + .with_Tai(&Tph) + .with_Tabij(&Tpphh) + .with_Vabij(&Vpphh) + .with_Vijka(&Vhhhp) + .with_Vabci(&Vppph) + // some options + .with_deleteVppph(!keepVppph) + .with_barrier(barrier) + .with_chrono(!nochrono) + .with_rankRoundRobin(rankRoundRobin) + .with_iterationMod(itMod) + .with_percentageMod(percentageMod) + .with_tuplesDistribution(tuplesDistribution) + ; auto out = atrip::Atrip::run(in); diff --git a/include/atrip/Atrip.hpp b/include/atrip/Atrip.hpp index 6a4e437..ce3399d 100644 --- a/include/atrip/Atrip.hpp +++ b/include/atrip/Atrip.hpp @@ -40,13 +40,13 @@ namespace atrip { template struct Input { CTF::Tensor *ei = nullptr - , *ea = nullptr - , *Tph = nullptr - , *Tpphh = nullptr - , *Vpphh = nullptr - , *Vhhhp = nullptr - , *Vppph = nullptr - ; + , *ea = nullptr + , *Tph = nullptr + , *Tpphh = nullptr + , *Vpphh = nullptr + , *Vhhhp = nullptr + , *Vppph = nullptr + ; Input& with_epsilon_i(CTF::Tensor * t) { ei = t; return *this; } Input& with_epsilon_a(CTF::Tensor * t) { ea = t; return *this; } Input& with_Tai(CTF::Tensor * t) { Tph = t; return *this; } @@ -60,6 +60,7 @@ namespace atrip { GROUP_AND_SORT, }; + ADD_ATTRIBUTE(bool, deleteVppph, false) ADD_ATTRIBUTE(bool, rankRoundRobin, false) ADD_ATTRIBUTE(bool, chrono, false) ADD_ATTRIBUTE(bool, barrier, false) diff --git a/include/atrip/Slice.hpp b/include/atrip/Slice.hpp index ff6d982..260e8c9 100644 --- a/include/atrip/Slice.hpp +++ b/include/atrip/Slice.hpp @@ -44,36 +44,36 @@ struct Slice { // Prolog:1 ends here // [[file:../../atrip.org::*Location][Location:1]] -struct Location { size_t rank; size_t source; }; + struct Location { size_t rank; size_t source; }; // Location:1 ends here // [[file:../../atrip.org::*Type][Type:1]] -enum Type - { A = 10 - , B - , C - // Two-parameter slices - , AB = 20 - , BC - , AC - // for abci and the doubles - , CB - , BA - , CA - // The non-typed slice - , Blank = 404 - }; + enum Type + { A = 10 + , B + , C + // Two-parameter slices + , AB = 20 + , BC + , AC + // for abci and the doubles + , CB + , BA + , CA + // The non-typed slice + , Blank = 404 + }; // Type:1 ends here // [[file:../../atrip.org::*State][State:1]] -enum State { - Fetch = 0, - Dispatched = 2, - Ready = 1, - SelfSufficient = 911, - Recycled = 123, - Acceptor = 405 -}; + enum State { + Fetch = 0, + Dispatched = 2, + Ready = 1, + SelfSufficient = 911, + Recycled = 123, + Acceptor = 405 + }; // State:1 ends here // [[file:../../atrip.org::*The Info structure][The Info structure:1]] @@ -101,25 +101,25 @@ using Ty_x_Tu = std::pair< Type, PartialTuple >; // The Info structure:1 ends here // [[file:../../atrip.org::*Name][Name:1]] -enum Name - { TA = 100 - , VIJKA = 101 - , VABCI = 200 - , TABIJ = 201 - , VABIJ = 202 - }; + enum Name + { TA = 100 + , VIJKA = 101 + , VABCI = 200 + , TABIJ = 201 + , VABIJ = 202 + }; // Name:1 ends here // [[file:../../atrip.org::*Database][Database:1]] -struct LocalDatabaseElement { - Slice::Name name; - Slice::Info info; -}; + struct LocalDatabaseElement { + Slice::Name name; + Slice::Info info; + }; // Database:1 ends here // [[file:../../atrip.org::*Database][Database:2]] -using LocalDatabase = std::vector; -using Database = LocalDatabase; + using LocalDatabase = std::vector; + using Database = LocalDatabase; // Database:2 ends here // [[file:../../atrip.org::*MPI Types][MPI Types:1]] @@ -359,91 +359,91 @@ static Slice& findByInfo(std::vector> &slices, // Static utilities:6 ends here // [[file:../../atrip.org::*Attributes][Attributes:1]] -Info info; + Info info; // Attributes:1 ends here // [[file:../../atrip.org::*Attributes][Attributes:2]] -F *data; + F *data; // Attributes:2 ends here // [[file:../../atrip.org::*Attributes][Attributes:3]] -MPI_Request request; + MPI_Request request; // Attributes:3 ends here // [[file:../../atrip.org::*Attributes][Attributes:4]] -const size_t size; + const size_t size; // Attributes:4 ends here // [[file:../../atrip.org::*Member functions][Member functions:1]] -void markReady() noexcept { - info.state = Ready; - info.recycling = Blank; -} + void markReady() noexcept { + info.state = Ready; + info.recycling = Blank; + } // Member functions:1 ends here // [[file:../../atrip.org::*Member functions][Member functions:2]] -bool isUnwrapped() const noexcept { - return info.state == Ready - || info.state == SelfSufficient - ; -} + bool isUnwrapped() const noexcept { + return info.state == Ready + || info.state == SelfSufficient + ; + } // Member functions:2 ends here // [[file:../../atrip.org::*Member functions][Member functions:3]] -bool isUnwrappable() const noexcept { - return isUnwrapped() - || info.state == Recycled - || info.state == Dispatched - ; -} + bool isUnwrappable() const noexcept { + return isUnwrapped() + || info.state == Recycled + || info.state == Dispatched + ; + } -inline bool isDirectlyFetchable() const noexcept { - return info.state == Ready || info.state == Dispatched; -} + inline bool isDirectlyFetchable() const noexcept { + return info.state == Ready || info.state == Dispatched; + } -void free() noexcept { - info.tuple = {0, 0}; - info.type = Blank; - info.state = Acceptor; - info.from = {0, 0}; - info.recycling = Blank; - data = nullptr; -} + void free() noexcept { + info.tuple = {0, 0}; + info.type = Blank; + info.state = Acceptor; + info.from = {0, 0}; + info.recycling = Blank; + data = nullptr; + } -inline bool isFree() const noexcept { - return info.tuple == PartialTuple{0, 0} - && info.type == Blank - && info.state == Acceptor - && info.from.rank == 0 - && info.from.source == 0 - && info.recycling == Blank - && data == nullptr - ; -} + inline bool isFree() const noexcept { + return info.tuple == PartialTuple{0, 0} + && info.type == Blank + && info.state == Acceptor + && info.from.rank == 0 + && info.from.source == 0 + && info.recycling == Blank + && data == nullptr + ; + } // Member functions:3 ends here // [[file:../../atrip.org::*Member functions][Member functions:4]] -inline bool isRecyclable() const noexcept { - return ( info.state == Dispatched - || info.state == Ready - || info.state == Fetch - ) - && hasValidDataPointer() - ; -} + inline bool isRecyclable() const noexcept { + return ( info.state == Dispatched + || info.state == Ready + || info.state == Fetch + ) + && hasValidDataPointer() + ; + } // Member functions:4 ends here // [[file:../../atrip.org::*Member functions][Member functions:5]] -inline bool hasValidDataPointer() const noexcept { - return data != nullptr - && info.state != Acceptor - && info.type != Blank - ; -} + inline bool hasValidDataPointer() const noexcept { + return data != nullptr + && info.state != Acceptor + && info.type != Blank + ; + } // Member functions:5 ends here // [[file:../../atrip.org::*Member functions][Member functions:6]] -void unwrapAndMarkReady() { + void unwrapAndMarkReady() { if (info.state == Ready) return; if (info.state != Dispatched) throw @@ -475,14 +475,14 @@ void unwrapAndMarkReady() { // Member functions:6 ends here // [[file:../../atrip.org::*Epilog][Epilog:1]] -Slice(size_t size_) - : info({}) - , data(nullptr) - , size(size_) - {} + Slice(size_t size_) + : info({}) + , data(nullptr) + , size(size_) + {} -}; // struct Slice + }; // struct Slice // Epilog:1 ends here // [[file:../../atrip.org::*Debug][Debug:1]] diff --git a/include/atrip/Tuples.hpp b/include/atrip/Tuples.hpp index 26c9d2b..e1691c8 100644 --- a/include/atrip/Tuples.hpp +++ b/include/atrip/Tuples.hpp @@ -467,31 +467,31 @@ std::vector main(MPI_Comm universe, size_t Nv) { // Main:1 ends here // [[file:../../atrip.org::*Main][Main:2]] -size_t const - tuplesPerRankLocal - = nodeTuples.size() / nodeInfos[rank].ranksPerNode - + size_t(nodeTuples.size() % nodeInfos[rank].ranksPerNode != 0) - ; + size_t const + tuplesPerRankLocal + = nodeTuples.size() / nodeInfos[rank].ranksPerNode + + size_t(nodeTuples.size() % nodeInfos[rank].ranksPerNode != 0) + ; -size_t tuplesPerRankGlobal; + size_t tuplesPerRankGlobal; -MPI_Reduce(&tuplesPerRankLocal, - &tuplesPerRankGlobal, - 1, - MPI_UINT64_T, - MPI_MAX, - 0, - universe); + MPI_Reduce(&tuplesPerRankLocal, + &tuplesPerRankGlobal, + 1, + MPI_UINT64_T, + MPI_MAX, + 0, + universe); -MPI_Bcast(&tuplesPerRankGlobal, - 1, - MPI_UINT64_T, - 0, - universe); + MPI_Bcast(&tuplesPerRankGlobal, + 1, + MPI_UINT64_T, + 0, + universe); -LOG(1,"Atrip") << "Tuples per rank: " << tuplesPerRankGlobal << "\n"; -LOG(1,"Atrip") << "ranks per node " << nodeInfos[rank].ranksPerNode << "\n"; -LOG(1,"Atrip") << "#nodes " << nNodes << "\n"; + LOG(1,"Atrip") << "Tuples per rank: " << tuplesPerRankGlobal << "\n"; + LOG(1,"Atrip") << "ranks per node " << nodeInfos[rank].ranksPerNode << "\n"; + LOG(1,"Atrip") << "#nodes " << nNodes << "\n"; // Main:2 ends here // [[file:../../atrip.org::*Main][Main:3]] @@ -531,7 +531,7 @@ if (computeDistribution) { // Main:4 ends here // [[file:../../atrip.org::*Main][Main:5]] -return result; + return result; } // Main:5 ends here diff --git a/include/atrip/Utils.hpp b/include/atrip/Utils.hpp index 39c462f..a7ba187 100644 --- a/include/atrip/Utils.hpp +++ b/include/atrip/Utils.hpp @@ -33,7 +33,7 @@ namespace atrip { // Prolog:1 ends here // [[file:../../atrip.org::*Pretty printing][Pretty printing:1]] -template + template std::string pretty_print(T&& value) { std::stringstream stream; #if ATRIP_DEBUG > 2 diff --git a/src/atrip/Atrip.cxx b/src/atrip/Atrip.cxx index e10600b..ac87966 100644 --- a/src/atrip/Atrip.cxx +++ b/src/atrip/Atrip.cxx @@ -96,14 +96,6 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { MPI_Comm_size(child_comm, &child_size); } - - // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1 - WITH_CHRONO("nv-slices", - LOG(0,"Atrip") << "BUILD NV-SLICES\n"; - TAPHH taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); - HHHA hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); - ) - // BUILD SLICES PARAMETRIZED BY NV x NV =============================={{{1 WITH_CHRONO("nv-nv-slices", LOG(0,"Atrip") << "BUILD NV x NV-SLICES\n"; @@ -112,6 +104,18 @@ Atrip::Output Atrip::run(Atrip::Input const& in) { TABHH tabhh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); ) + // delete the Vppph so that we don't have a HWM situation for the NV slices + if (in.deleteVppph) { + delete in.Vppph; + } + + // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1 + WITH_CHRONO("nv-slices", + LOG(0,"Atrip") << "BUILD NV-SLICES\n"; + TAPHH taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); + HHHA hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe); + ) + // all tensors std::vector< SliceUnion* > unions = {&taphh, &hhha, &abph, &abhh, &tabhh};