Implement deleting Vppph

2022-04-05 17:16:11 +02:00 · 2022-04-05 17:16:11 +02:00 · fed19ff52c
commit fed19ff52c
parent 7d37bd9f8b
7 changed files with 196 additions and 181 deletions
--- a/atrip.org
+++ b/atrip.org
@ -2955,6 +2955,7 @@ namespace atrip {
        GROUP_AND_SORT,
      };

+      ADD_ATTRIBUTE(bool, deleteVppph, false)
      ADD_ATTRIBUTE(bool, rankRoundRobin, false)
      ADD_ATTRIBUTE(bool, chrono, false)
      ADD_ATTRIBUTE(bool, barrier, false)
@ -3063,14 +3064,6 @@ Atrip::Output Atrip::run(Atrip::Input<F> const& in) {
    MPI_Comm_size(child_comm, &child_size);
  }

-
-  // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1
-  WITH_CHRONO("nv-slices",
-    LOG(0,"Atrip") << "BUILD NV-SLICES\n";
-    TAPHH<F> taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
-    HHHA<F>  hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
-  )
-
  // BUILD SLICES PARAMETRIZED BY NV x NV =============================={{{1
  WITH_CHRONO("nv-nv-slices",
    LOG(0,"Atrip") << "BUILD NV x NV-SLICES\n";
@ -3079,6 +3072,18 @@ Atrip::Output Atrip::run(Atrip::Input<F> const& in) {
    TABHH<F> tabhh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
  )

+  // delete the Vppph so that we don't have a HWM situation for the NV slices
+  if (in.deleteVppph) {
+    delete in.Vppph;
+  }
+
+  // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1
+  WITH_CHRONO("nv-slices",
+    LOG(0,"Atrip") << "BUILD NV-SLICES\n";
+    TAPHH<F> taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
+    HHHA<F>  hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
+  )
+
  // all tensors
  std::vector< SliceUnion<F>* > unions = {&taphh, &hhha, &abph, &abhh, &tabhh};

@ -3251,12 +3256,12 @@ Atrip::Output Atrip::run(Atrip::Input<F> const& in) {

  const double doublesFlops
    = double(No)
-    * double(No)
-    * double(No)
-    * (double(No) + double(Nv))
-    * 2.0
-    * (traits::isComplex<F>() ? 2.0 : 1.0)
-    * 6.0
+    ,* double(No)
+    ,* double(No)
+    ,* (double(No) + double(Nv))
+    ,* 2.0
+    ,* (traits::isComplex<F>() ? 2.0 : 1.0)
+    ,* 6.0
    / 1e9
    ;

--- a/bench/test_main.cxx
+++ b/bench/test_main.cxx
@ -17,13 +17,15 @@ int main(int argc, char** argv) {
  MPI_Init(&argc, &argv);

  int no(10), nv(10), itMod(-1), percentageMod(10);
-  bool nochrono(false), barrier(false), rankRoundRobin(false);
+  bool nochrono(false), barrier(false), rankRoundRobin(false),
+       keepVppph(false);
  std::string tuplesDistributionString = "naive";

  CLI::App app{"Main bench for atrip"};
  app.add_option("--no", no, "Occupied orbitals");
  app.add_option("--nv", nv, "Virtual orbitals");
  app.add_option("--mod", itMod, "Iteration modifier");
+  app.add_flag("--keep-vppph", keepVppph, "Do not delete Vppph");
  app.add_flag("--nochrono", nochrono, "Do not print chrono");
  app.add_flag("--rank-round-robin", rankRoundRobin, "Do rank round robin");
  app.add_flag("--barrier", barrier, "Use the first barrier");
@ -38,7 +40,8 @@ int main(int argc, char** argv) {
  constexpr double elem_to_gb = 8.0 / 1024.0 / 1024.0 / 1024.0;

  // USER PRINTING TEST BEGIN
-  const double doublesFlops = no * no * no
+  const double doublesFlops
+       = no * no * no
       * (no + nv)
       * 2.0
       * 6.0
@ -110,7 +113,8 @@ int main(int argc, char** argv) {
  Vppph.fill_random(0, 1);

  atrip::Atrip::init();
-  const auto in = atrip::Atrip::Input<double>()
+  const auto in
+       = atrip::Atrip::Input<double>()
       // Tensors
       .with_epsilon_i(&ei)
       .with_epsilon_a(&ea)
@ -120,6 +124,7 @@ int main(int argc, char** argv) {
       .with_Vijka(&Vhhhp)
       .with_Vabci(&Vppph)
       // some options
+       .with_deleteVppph(!keepVppph)
       .with_barrier(barrier)
       .with_chrono(!nochrono)
       .with_rankRoundRobin(rankRoundRobin)
--- a/include/atrip/Atrip.hpp
+++ b/include/atrip/Atrip.hpp
@ -60,6 +60,7 @@ namespace atrip {
        GROUP_AND_SORT,
      };

+      ADD_ATTRIBUTE(bool, deleteVppph, false)
      ADD_ATTRIBUTE(bool, rankRoundRobin, false)
      ADD_ATTRIBUTE(bool, chrono, false)
      ADD_ATTRIBUTE(bool, barrier, false)
--- a/include/atrip/Slice.hpp
+++ b/include/atrip/Slice.hpp
@ -44,11 +44,11 @@ struct Slice {
 // Prolog:1 ends here

 // [[file:../../atrip.org::*Location][Location:1]]
-struct Location { size_t rank; size_t source; };
+  struct Location { size_t rank; size_t source; };
 // Location:1 ends here

 // [[file:../../atrip.org::*Type][Type:1]]
-enum Type
+  enum Type
    { A = 10
    , B
    , C
@ -66,14 +66,14 @@ enum Type
 // Type:1 ends here

 // [[file:../../atrip.org::*State][State:1]]
-enum State {
+  enum State {
    Fetch = 0,
    Dispatched = 2,
    Ready = 1,
    SelfSufficient = 911,
    Recycled = 123,
    Acceptor = 405
-};
+  };
 // State:1 ends here

 // [[file:../../atrip.org::*The Info structure][The Info structure:1]]
@ -101,7 +101,7 @@ using Ty_x_Tu = std::pair< Type, PartialTuple >;
 // The Info structure:1 ends here

 // [[file:../../atrip.org::*Name][Name:1]]
-enum Name
+  enum Name
    { TA    = 100
    , VIJKA = 101
    , VABCI = 200
@ -111,15 +111,15 @@ enum Name
 // Name:1 ends here

 // [[file:../../atrip.org::*Database][Database:1]]
-struct LocalDatabaseElement {
+  struct LocalDatabaseElement {
    Slice<F>::Name name;
    Slice<F>::Info info;
-};
+  };
 // Database:1 ends here

 // [[file:../../atrip.org::*Database][Database:2]]
-using LocalDatabase = std::vector<LocalDatabaseElement>;
-using Database = LocalDatabase;
+  using LocalDatabase = std::vector<LocalDatabaseElement>;
+  using Database = LocalDatabase;
 // Database:2 ends here

 // [[file:../../atrip.org::*MPI Types][MPI Types:1]]
@ -359,58 +359,58 @@ static Slice<F>& findByInfo(std::vector<Slice<F>> &slices,
 // Static utilities:6 ends here

 // [[file:../../atrip.org::*Attributes][Attributes:1]]
-Info info;
+    Info info;
 // Attributes:1 ends here

 // [[file:../../atrip.org::*Attributes][Attributes:2]]
-F  *data;
+    F  *data;
 // Attributes:2 ends here

 // [[file:../../atrip.org::*Attributes][Attributes:3]]
-MPI_Request request;
+    MPI_Request request;
 // Attributes:3 ends here

 // [[file:../../atrip.org::*Attributes][Attributes:4]]
-const size_t size;
+    const size_t size;
 // Attributes:4 ends here

 // [[file:../../atrip.org::*Member functions][Member functions:1]]
-void markReady() noexcept {
+    void markReady() noexcept {
      info.state = Ready;
      info.recycling = Blank;
-}
+    }
 // Member functions:1 ends here

 // [[file:../../atrip.org::*Member functions][Member functions:2]]
-bool isUnwrapped() const noexcept {
+    bool isUnwrapped() const noexcept {
      return info.state == Ready
          || info.state == SelfSufficient
          ;
-}
+    }
 // Member functions:2 ends here

 // [[file:../../atrip.org::*Member functions][Member functions:3]]
-bool isUnwrappable() const noexcept {
+    bool isUnwrappable() const noexcept {
      return isUnwrapped()
          || info.state == Recycled
          || info.state == Dispatched
          ;
-}
+    }

-inline bool isDirectlyFetchable() const noexcept {
+    inline bool isDirectlyFetchable() const noexcept {
      return info.state == Ready || info.state == Dispatched;
-}
+    }

-void free() noexcept {
+    void free() noexcept {
      info.tuple      = {0, 0};
      info.type       = Blank;
      info.state      = Acceptor;
      info.from       = {0, 0};
      info.recycling  = Blank;
      data            = nullptr;
-}
+    }

-inline bool isFree() const noexcept {
+    inline bool isFree() const noexcept {
      return info.tuple       == PartialTuple{0, 0}
          && info.type        == Blank
          && info.state       == Acceptor
@ -419,31 +419,31 @@ inline bool isFree() const noexcept {
          && info.recycling   == Blank
          && data             == nullptr
           ;
-}
+    }
 // Member functions:3 ends here

 // [[file:../../atrip.org::*Member functions][Member functions:4]]
-inline bool isRecyclable() const noexcept {
+    inline bool isRecyclable() const noexcept {
      return (  info.state == Dispatched
             || info.state == Ready
             || info.state == Fetch
             )
          && hasValidDataPointer()
          ;
-}
+    }
 // Member functions:4 ends here

 // [[file:../../atrip.org::*Member functions][Member functions:5]]
-inline bool hasValidDataPointer() const noexcept {
+    inline bool hasValidDataPointer() const noexcept {
      return data       != nullptr
          && info.state != Acceptor
          && info.type  != Blank
          ;
-}
+    }
 // Member functions:5 ends here

 // [[file:../../atrip.org::*Member functions][Member functions:6]]
-void unwrapAndMarkReady() {
+    void unwrapAndMarkReady() {
      if (info.state == Ready) return;
      if (info.state != Dispatched)
        throw
@ -475,14 +475,14 @@ void unwrapAndMarkReady() {
 // Member functions:6 ends here

 // [[file:../../atrip.org::*Epilog][Epilog:1]]
-Slice(size_t size_)
+    Slice(size_t size_)
      : info({})
      , data(nullptr)
      , size(size_)
      {}


-}; // struct Slice
+  }; // struct Slice
 // Epilog:1 ends here

 // [[file:../../atrip.org::*Debug][Debug:1]]
--- a/include/atrip/Tuples.hpp
+++ b/include/atrip/Tuples.hpp
@ -467,15 +467,15 @@ std::vector<ABCTuple> main(MPI_Comm universe, size_t Nv) {
 // Main:1 ends here

 // [[file:../../atrip.org::*Main][Main:2]]
-size_t const
+  size_t const
    tuplesPerRankLocal
       = nodeTuples.size() / nodeInfos[rank].ranksPerNode
       + size_t(nodeTuples.size() % nodeInfos[rank].ranksPerNode != 0)
       ;

-size_t tuplesPerRankGlobal;
+  size_t tuplesPerRankGlobal;

-MPI_Reduce(&tuplesPerRankLocal,
+  MPI_Reduce(&tuplesPerRankLocal,
             &tuplesPerRankGlobal,
             1,
             MPI_UINT64_T,
@ -483,15 +483,15 @@ MPI_Reduce(&tuplesPerRankLocal,
             0,
             universe);

-MPI_Bcast(&tuplesPerRankGlobal,
+  MPI_Bcast(&tuplesPerRankGlobal,
            1,
            MPI_UINT64_T,
            0,
            universe);

-LOG(1,"Atrip") << "Tuples per rank: " << tuplesPerRankGlobal << "\n";
-LOG(1,"Atrip") << "ranks per node " << nodeInfos[rank].ranksPerNode << "\n";
-LOG(1,"Atrip") << "#nodes " << nNodes << "\n";
+  LOG(1,"Atrip") << "Tuples per rank: " << tuplesPerRankGlobal << "\n";
+  LOG(1,"Atrip") << "ranks per node " << nodeInfos[rank].ranksPerNode << "\n";
+  LOG(1,"Atrip") << "#nodes " << nNodes << "\n";
 // Main:2 ends here

 // [[file:../../atrip.org::*Main][Main:3]]
@ -531,7 +531,7 @@ if (computeDistribution) {
 // Main:4 ends here

 // [[file:../../atrip.org::*Main][Main:5]]
-return result;
+  return result;

 }
 // Main:5 ends here
--- a/include/atrip/Utils.hpp
+++ b/include/atrip/Utils.hpp
@ -33,7 +33,7 @@ namespace atrip {
 // Prolog:1 ends here

 // [[file:../../atrip.org::*Pretty printing][Pretty printing:1]]
-template <typename T>
+  template <typename T>
  std::string pretty_print(T&& value) {
    std::stringstream stream;
 #if ATRIP_DEBUG > 2
--- a/src/atrip/Atrip.cxx
+++ b/src/atrip/Atrip.cxx
@ -96,14 +96,6 @@ Atrip::Output Atrip::run(Atrip::Input<F> const& in) {
    MPI_Comm_size(child_comm, &child_size);
  }

-
-  // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1
-  WITH_CHRONO("nv-slices",
-    LOG(0,"Atrip") << "BUILD NV-SLICES\n";
-    TAPHH<F> taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
-    HHHA<F>  hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
-  )
-
  // BUILD SLICES PARAMETRIZED BY NV x NV =============================={{{1
  WITH_CHRONO("nv-nv-slices",
    LOG(0,"Atrip") << "BUILD NV x NV-SLICES\n";
@ -112,6 +104,18 @@ Atrip::Output Atrip::run(Atrip::Input<F> const& in) {
    TABHH<F> tabhh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
  )

+  // delete the Vppph so that we don't have a HWM situation for the NV slices
+  if (in.deleteVppph) {
+    delete in.Vppph;
+  }
+
+  // BUILD SLICES PARAMETRIZED BY NV ==================================={{{1
+  WITH_CHRONO("nv-slices",
+    LOG(0,"Atrip") << "BUILD NV-SLICES\n";
+    TAPHH<F> taphh(*in.Tpphh, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
+    HHHA<F>  hhha(*in.Vhhhp, (size_t)No, (size_t)Nv, (size_t)np, child_comm, universe);
+  )
+
  // all tensors
  std::vector< SliceUnion<F>* > unions = {&taphh, &hhha, &abph, &abhh, &tabhh};