Update sources
This commit is contained in:
parent
7e5feccca9
commit
1d6d14c398
@ -22,6 +22,8 @@ namespace atrip {
|
||||
static int rank;
|
||||
static int np;
|
||||
static Timings chrono;
|
||||
static size_t networkSend;
|
||||
static size_t localSend;
|
||||
static void init();
|
||||
|
||||
struct Input {
|
||||
@ -46,6 +48,7 @@ namespace atrip {
|
||||
GROUP_AND_SORT,
|
||||
};
|
||||
|
||||
ADD_ATTRIBUTE(bool, rankRoundRobin, false)
|
||||
ADD_ATTRIBUTE(bool, chrono, false)
|
||||
ADD_ATTRIBUTE(bool, barrier, false)
|
||||
ADD_ATTRIBUTE(int, maxIterations, 0)
|
||||
|
||||
@ -5,22 +5,36 @@
|
||||
#include <algorithm>
|
||||
|
||||
#include <atrip/Slice.hpp>
|
||||
#include <atrip/Tuples.hpp>
|
||||
|
||||
namespace atrip {
|
||||
struct RankMap {
|
||||
|
||||
static bool RANK_ROUND_ROBIN;
|
||||
std::vector<size_t> const lengths;
|
||||
size_t const np, size;
|
||||
ClusterInfo const clusterInfo;
|
||||
|
||||
RankMap(std::vector<size_t> lens, size_t np_)
|
||||
RankMap(std::vector<size_t> lens, size_t np_, MPI_Comm comm)
|
||||
: lengths(lens)
|
||||
, np(np_)
|
||||
, size(std::accumulate(lengths.begin(), lengths.end(),
|
||||
1UL, std::multiplies<size_t>()))
|
||||
, clusterInfo(getClusterInfo(comm))
|
||||
{ assert(lengths.size() <= 2); }
|
||||
|
||||
size_t find(Slice::Location const& p) const noexcept {
|
||||
if (RANK_ROUND_ROBIN) {
|
||||
return p.source * np + p.rank;
|
||||
} else {
|
||||
const size_t
|
||||
rankPosition = p.source * clusterInfo.ranksPerNode
|
||||
+ clusterInfo.rankInfos[p.rank].localRank
|
||||
;
|
||||
return rankPosition * clusterInfo.nNodes
|
||||
+ clusterInfo.rankInfos[p.rank].nodeId
|
||||
;
|
||||
}
|
||||
}
|
||||
|
||||
size_t nSources() const noexcept {
|
||||
@ -40,8 +54,9 @@ namespace atrip {
|
||||
}
|
||||
|
||||
Slice::Location
|
||||
find(ABCTuple const& abc, Slice::Type sliceType) const noexcept {
|
||||
find(ABCTuple const& abc, Slice::Type sliceType) const {
|
||||
// tuple = {11, 8} when abc = {11, 8, 9} and sliceType = AB
|
||||
// tuple = {11, 0} when abc = {11, 8, 9} and sliceType = A
|
||||
const auto tuple = Slice::subtupleBySlice(abc, sliceType);
|
||||
|
||||
const size_t index
|
||||
@ -49,9 +64,51 @@ namespace atrip {
|
||||
+ tuple[1] * (lengths.size() > 1 ? lengths[0] : 0)
|
||||
;
|
||||
|
||||
size_t rank, source;
|
||||
|
||||
if (RANK_ROUND_ROBIN) {
|
||||
|
||||
rank = index % np;
|
||||
source = index / np;
|
||||
|
||||
} else {
|
||||
|
||||
size_t const
|
||||
|
||||
// the node that will be assigned to
|
||||
nodeId = index % clusterInfo.nNodes
|
||||
|
||||
// how many times it has been assigned to the node
|
||||
, s_n = index / clusterInfo.nNodes
|
||||
|
||||
// which local rank in the node should be
|
||||
, localRank = s_n % clusterInfo.ranksPerNode
|
||||
|
||||
// and the local source (how many times we chose this local rank)
|
||||
, localSource = s_n / clusterInfo.ranksPerNode
|
||||
;
|
||||
|
||||
// find the localRank-th entry in clusterInfo
|
||||
auto const& it =
|
||||
std::find_if(clusterInfo.rankInfos.begin(),
|
||||
clusterInfo.rankInfos.end(),
|
||||
[nodeId, localRank](RankInfo const& ri) {
|
||||
return ri.nodeId == nodeId
|
||||
&& ri.localRank == localRank
|
||||
;
|
||||
});
|
||||
if (it == clusterInfo.rankInfos.end()) {
|
||||
throw "FATAL! Error in node distribution of the slices";
|
||||
}
|
||||
|
||||
rank = (*it).globalRank;
|
||||
source = localSource;
|
||||
|
||||
}
|
||||
|
||||
return
|
||||
{ index % np
|
||||
, index / np
|
||||
{ rank
|
||||
, source
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@ -178,8 +178,14 @@ namespace atrip {
|
||||
if (blank.info.state == Slice::SelfSufficient) {
|
||||
blank.data = sources[from.source].data();
|
||||
} else {
|
||||
if (freePointers.size() == 0)
|
||||
throw std::domain_error("No more free pointers!");
|
||||
if (freePointers.size() == 0) {
|
||||
std::stringstream stream;
|
||||
stream << "No more free pointers "
|
||||
<< "for type " << type
|
||||
<< " and name " << name
|
||||
;
|
||||
throw std::domain_error(stream.str());
|
||||
}
|
||||
auto dataPointer = freePointers.begin();
|
||||
freePointers.erase(dataPointer);
|
||||
blank.data = *dataPointer;
|
||||
@ -332,7 +338,7 @@ namespace atrip {
|
||||
, Slice::Name name_
|
||||
, size_t nSliceBuffers = 4
|
||||
)
|
||||
: rankMap(paramLength, np)
|
||||
: rankMap(paramLength, np, global_world)
|
||||
, world(child_world)
|
||||
, universe(global_world)
|
||||
, sliceLength(sliceLength_)
|
||||
@ -419,16 +425,27 @@ namespace atrip {
|
||||
* \brief Send asynchronously only if the state is Fetch
|
||||
*/
|
||||
void send( size_t otherRank
|
||||
, Slice::Info const& info
|
||||
, Slice::LocalDatabaseElement const& el
|
||||
, size_t tag) const noexcept {
|
||||
MPI_Request request;
|
||||
bool sendData_p = false;
|
||||
auto const& info = el.info;
|
||||
|
||||
if (info.state == Slice::Fetch) sendData_p = true;
|
||||
// TODO: remove this because I have SelfSufficient
|
||||
if (otherRank == info.from.rank) sendData_p = false;
|
||||
if (!sendData_p) return;
|
||||
|
||||
switch (el.name) {
|
||||
case Slice::Name::TA:
|
||||
case Slice::Name::VIJKA:
|
||||
if (otherRank / 48 == Atrip::rank / 48) {
|
||||
Atrip::localSend++;
|
||||
} else {
|
||||
Atrip::networkSend++;
|
||||
}
|
||||
}
|
||||
|
||||
MPI_Isend( sources[info.from.source].data()
|
||||
, sources[info.from.source].size()
|
||||
, MPI_DOUBLE /* TODO: adapt this with traits */
|
||||
|
||||
@ -85,26 +85,30 @@ struct RankInfo {
|
||||
const size_t ranksPerNode;
|
||||
};
|
||||
|
||||
template <typename A>
|
||||
std::vector<A> unique(std::vector<A> const &xs) {
|
||||
auto result = xs;
|
||||
std::sort(result.begin(), result.end());
|
||||
auto const& last = std::unique(result.begin(), result.end());
|
||||
result.erase(last, result.end());
|
||||
return result;
|
||||
}
|
||||
|
||||
std::vector<RankInfo>
|
||||
getNodeInfos(std::vector<string> const& nodeNames) {
|
||||
std::vector<RankInfo> result;
|
||||
auto uniqueNames = nodeNames;
|
||||
{
|
||||
std::sort(uniqueNames.begin(), uniqueNames.end());
|
||||
auto const& last = std::unique(uniqueNames.begin(), uniqueNames.end());
|
||||
uniqueNames.erase(last, uniqueNames.end());
|
||||
}
|
||||
const auto index = [&uniqueNames](std::string const& s) {
|
||||
auto const uniqueNames = unique(nodeNames);
|
||||
auto const index = [&uniqueNames](std::string const& s) {
|
||||
auto const& it = std::find(uniqueNames.begin(), uniqueNames.end(), s);
|
||||
return std::distance(uniqueNames.begin(), it);
|
||||
};
|
||||
std::vector<size_t> localRanks(uniqueNames.size(), 0);
|
||||
size_t rank = 0;
|
||||
size_t globalRank = 0;
|
||||
for (auto const& name: nodeNames) {
|
||||
const size_t nodeId = index(name);
|
||||
result.push_back({name,
|
||||
nodeId,
|
||||
rank++,
|
||||
globalRank++,
|
||||
localRanks[nodeId]++,
|
||||
std::count(nodeNames.begin(),
|
||||
nodeNames.end(),
|
||||
@ -113,6 +117,25 @@ getNodeInfos(std::vector<string> const& nodeNames) {
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
struct ClusterInfo {
|
||||
const size_t nNodes, np, ranksPerNode;
|
||||
const std::vector<RankInfo> rankInfos;
|
||||
};
|
||||
|
||||
ClusterInfo
|
||||
getClusterInfo(MPI_Comm comm) {
|
||||
auto const names = getNodeNames(comm);
|
||||
auto const rankInfos = getNodeInfos(names);
|
||||
|
||||
return ClusterInfo {
|
||||
unique(names).size(),
|
||||
names.size(),
|
||||
rankInfos[0].ranksPerNode,
|
||||
rankInfos
|
||||
};
|
||||
|
||||
}
|
||||
// Node information:2 ends here
|
||||
|
||||
// [[file:~/atrip/atrip.org::*Naive%20list][Naive list:1]]
|
||||
@ -189,8 +212,6 @@ size_t isOnNode(size_t tuple, size_t nodes) { return tuple % nodes; }
|
||||
|
||||
struct Info {
|
||||
size_t nNodes;
|
||||
size_t Nv;
|
||||
size_t np;
|
||||
size_t nodeId;
|
||||
};
|
||||
|
||||
@ -212,28 +233,33 @@ std::vector<size_t> getTupleNodes(ABCTuple t, size_t nNodes) {
|
||||
// Utils:1 ends here
|
||||
|
||||
// [[file:~/atrip/atrip.org::*Distribution][Distribution:1]]
|
||||
std::vector<ABCTuple>
|
||||
specialDistribution(Info info, std::vector<ABCTuple> const& allTuples) {
|
||||
ABCTuples specialDistribution(Info const& info, ABCTuples const& allTuples) {
|
||||
|
||||
std::vector<ABCTuple> nodeTuples;
|
||||
size_t nNodes(info.nNodes);
|
||||
size_t np(info.np);
|
||||
size_t N(allTuples.size());
|
||||
ABCTuples nodeTuples;
|
||||
size_t const nNodes(info.nNodes);
|
||||
|
||||
// nodeid tuple list
|
||||
std::map<size_t, std::vector<ABCTuple> > container1d;
|
||||
std::map<size_t, std::vector<ABCTuple> > container2d;
|
||||
std::map<size_t, std::vector<ABCTuple> > container3d;
|
||||
std::map< size_t /* nodeId */, ABCTuples >
|
||||
container1d, container2d, container3d;
|
||||
|
||||
// build container-n-d's
|
||||
for (auto t: allTuples) {
|
||||
for (auto const& t: allTuples) {
|
||||
// one which node(s) are the tuple elements located...
|
||||
// put them into the right container
|
||||
auto nt = getTupleNodes(t, nNodes);
|
||||
if ( nt.size() == 1) container1d[nt[0]].push_back(t);
|
||||
if ( nt.size() == 2) container2d[nt[0] + nNodes*nt[1]].push_back(t);
|
||||
if ( nt.size() == 3)
|
||||
container3d[nt[0] + nNodes*nt[1] + nNodes*nNodes*nt[2]].push_back(t);
|
||||
auto const _nodes = getTupleNodes(t, nNodes);
|
||||
switch (_nodes.size()) {
|
||||
case 1:
|
||||
container1d[_nodes[0]].push_back(t);
|
||||
case 2:
|
||||
container2d[ _nodes[0]
|
||||
+ nNodes * _nodes[1]
|
||||
].push_back(t);
|
||||
case 3:
|
||||
container3d[ _nodes[0]
|
||||
+ nNodes * _nodes[1]
|
||||
+ nNodes * nNodes * _nodes[2]
|
||||
].push_back(t);
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if (info.nodeId == 0)
|
||||
@ -241,97 +267,114 @@ specialDistribution(Info info, std::vector<ABCTuple> const& allTuples) {
|
||||
// DISTRIBUTE 1-d containers
|
||||
// every tuple which is only located at one node belongs to this node
|
||||
{
|
||||
auto const& tuplesVec = container1d[info.nodeId];
|
||||
nodeTuples.resize(tuplesVec.size());
|
||||
std::copy(tuplesVec.begin(), tuplesVec.end(), nodeTuples.begin());
|
||||
auto const& _tuplesVec = container1d[info.nodeId];
|
||||
nodeTuples.resize(_tuplesVec.size());
|
||||
std::copy(_tuplesVec.begin(), _tuplesVec.end(), nodeTuples.begin());
|
||||
}
|
||||
|
||||
if (info.nodeId == 0)
|
||||
std::cout << "\tBuilding 2-d containers\n";
|
||||
// DISTRIBUTE 2-d containers
|
||||
//the tuples which are located at two nodes are half/half given to these nodes
|
||||
for (auto &m: container2d) {
|
||||
size_t idx = m.first%nNodes;
|
||||
size_t idy = m.first/nNodes;
|
||||
size_t myNode = idx;
|
||||
for (auto const& m: container2d) {
|
||||
|
||||
// either idx or idy is my node
|
||||
if (idx != info.nodeId && idy != info.nodeId) continue;
|
||||
if (idy == info.nodeId) myNode = idy;
|
||||
auto const& _tuplesVec = m.second;
|
||||
const
|
||||
size_t idx = m.first % nNodes
|
||||
// remeber: m.first = idy * nNodes + idx
|
||||
, idy = m.first / nNodes
|
||||
, n_half = _tuplesVec.size() / 2
|
||||
, size = nodeTuples.size()
|
||||
;
|
||||
|
||||
auto tuplesVec = m.second;
|
||||
auto n = tuplesVec.size() / 2;
|
||||
auto size = nodeTuples.size();
|
||||
if (myNode == idx) {
|
||||
nodeTuples.resize(size + n);
|
||||
std::copy(tuplesVec.begin(),
|
||||
tuplesVec.begin() + n,
|
||||
nodeTuples.begin() + size);
|
||||
size_t nextra, nbegin, nend;
|
||||
if (info.nodeId == idx) {
|
||||
nextra = n_half;
|
||||
nbegin = 0 * n_half;
|
||||
nend = n_half;
|
||||
} else if (info.nodeId == idy) {
|
||||
nextra = _tuplesVec.size() - n_half;
|
||||
nbegin = 1 * n_half;
|
||||
nend = _tuplesVec.size();
|
||||
} else {
|
||||
auto ny = tuplesVec.size() - n;
|
||||
nodeTuples.resize(size + ny);
|
||||
std::copy(tuplesVec.begin() + n,
|
||||
tuplesVec.end(),
|
||||
nodeTuples.begin() + size);
|
||||
// either idx or idy is my node
|
||||
continue;
|
||||
}
|
||||
|
||||
nodeTuples.resize(size + nextra);
|
||||
std::copy(_tuplesVec.begin() + nbegin,
|
||||
_tuplesVec.begin() + nend,
|
||||
nodeTuples.begin() + size);
|
||||
|
||||
}
|
||||
|
||||
if (info.nodeId == 0)
|
||||
std::cout << "\tBuilding 3-d containers\n";
|
||||
// DISTRIBUTE 3-d containers
|
||||
// similar game for the tuples which belong to three different nodes
|
||||
for (auto m: container3d){
|
||||
auto tuplesVec = m.second;
|
||||
auto idx = m.first%nNodes;
|
||||
auto idy = (m.first/nNodes)%nNodes;
|
||||
auto idz = m.first/nNodes/nNodes;
|
||||
if (idx != info.nodeId && idy != info.nodeId && idz != info.nodeId) continue;
|
||||
for (auto const& m: container3d){
|
||||
auto const& _tuplesVec = m.second;
|
||||
|
||||
size_t nx = tuplesVec.size() / 3;
|
||||
size_t n, nbegin, nend;
|
||||
const
|
||||
size_t idx = m.first % nNodes
|
||||
, idy = (m.first / nNodes) % nNodes
|
||||
// remember: m.first = idx + idy * nNodes + idz * nNodes^2
|
||||
, idz = m.first / nNodes / nNodes
|
||||
, n_third = _tuplesVec.size() / 3
|
||||
, size = nodeTuples.size()
|
||||
;
|
||||
|
||||
size_t nextra, nbegin, nend;
|
||||
if (info.nodeId == idx) {
|
||||
n = nx;
|
||||
nbegin = 0;
|
||||
nend = n;
|
||||
nextra = n_third;
|
||||
nbegin = 0 * n_third;
|
||||
nend = nextra;
|
||||
} else if (info.nodeId == idy) {
|
||||
n = nx;
|
||||
nbegin = n;
|
||||
nend = n + n;
|
||||
nextra = n_third;
|
||||
nbegin = 1 * n_third;
|
||||
nend = 2 * nextra;
|
||||
} else if (info.nodeId == idz) {
|
||||
nextra = _tuplesVec.size() - 2 * n_third;
|
||||
nbegin = 2 * n_third;
|
||||
nend = _tuplesVec.size();
|
||||
} else {
|
||||
n = tuplesVec.size() - 2 * nx;
|
||||
nbegin = 2 * nx;
|
||||
nend = 2 * nx + n;
|
||||
// either idx or idy or idz is my node
|
||||
continue;
|
||||
}
|
||||
|
||||
auto size = nodeTuples.size();
|
||||
nodeTuples.resize(size + n);
|
||||
std::copy(tuplesVec.begin() + nbegin,
|
||||
tuplesVec.begin() + nend,
|
||||
nodeTuples.resize(size + nextra);
|
||||
std::copy(_tuplesVec.begin() + nbegin,
|
||||
_tuplesVec.begin() + nend,
|
||||
nodeTuples.begin() + size);
|
||||
|
||||
}
|
||||
|
||||
|
||||
if (info.nodeId == 0)
|
||||
std::cout << "\tsorting...\n";
|
||||
// sort part of group-and-sort algorithm
|
||||
// every tuple on a given node is sorted in a way that
|
||||
// the 'home elements' are the fastest index.
|
||||
// 1:yyy 2:yyn(x) 3:yny(x) 4:ynn(x) 5:nyy 6:nyn(x) 7:nny 8:nnn
|
||||
size_t n = info.nodeId;
|
||||
if (info.nodeId == 0) std::cout << "\tswapping tuples...\n";
|
||||
/*
|
||||
* sort part of group-and-sort algorithm
|
||||
* every tuple on a given node is sorted in a way that
|
||||
* the 'home elements' are the fastest index.
|
||||
* 1:yyy 2:yyn(x) 3:yny(x) 4:ynn(x) 5:nyy 6:nyn(x) 7:nny 8:nnn
|
||||
*/
|
||||
for (auto &nt: nodeTuples){
|
||||
if ( isOnNode(nt[0], nNodes) == n ){ // 1234
|
||||
if ( isOnNode(nt[2], nNodes) != n ){ // 24
|
||||
size_t x(nt[0]); nt[0] = nt[2]; nt[2] = x; // switch first and last
|
||||
if ( isOnNode(nt[0], nNodes) == info.nodeId ){ // 1234
|
||||
if ( isOnNode(nt[2], nNodes) != info.nodeId ){ // 24
|
||||
size_t const x(nt[0]);
|
||||
nt[0] = nt[2]; // switch first and last
|
||||
nt[2] = x;
|
||||
}
|
||||
else if ( isOnNode(nt[1], nNodes) != n){ // 3
|
||||
size_t x(nt[0]); nt[0] = nt[1]; nt[1] = x; // switch first two
|
||||
else if ( isOnNode(nt[1], nNodes) != info.nodeId){ // 3
|
||||
size_t const x(nt[0]);
|
||||
nt[0] = nt[1]; // switch first two
|
||||
nt[1] = x;
|
||||
}
|
||||
} else {
|
||||
if ( isOnNode(nt[1], nNodes) == n // 56
|
||||
&& isOnNode(nt[2], nNodes) != n){ // 6
|
||||
size_t x(nt[1]); nt[1] = nt[2]; nt[2] = x; // switch last two
|
||||
if ( isOnNode(nt[1], nNodes) == info.nodeId // 56
|
||||
&& isOnNode(nt[2], nNodes) != info.nodeId
|
||||
) { // 6
|
||||
size_t const x(nt[1]);
|
||||
nt[1] = nt[2]; // switch last two
|
||||
nt[2] = x;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -358,29 +401,19 @@ std::vector<ABCTuple> main(MPI_Comm universe, size_t Nv) {
|
||||
|
||||
std::vector<ABCTuple> result;
|
||||
|
||||
const auto nodeNames(getNodeNames(universe));
|
||||
auto nodeNamesUnique(nodeNames);
|
||||
{
|
||||
const auto& last = std::unique(nodeNamesUnique.begin(),
|
||||
nodeNamesUnique.end());
|
||||
nodeNamesUnique.erase(last, nodeNamesUnique.end());
|
||||
}
|
||||
// we pick one rank from every node
|
||||
auto const nodeNames(getNodeNames(universe));
|
||||
size_t const nNodes = unique(nodeNames).size();
|
||||
auto const nodeInfos = getNodeInfos(nodeNames);
|
||||
size_t const nNodes = nodeNamesUnique.size();
|
||||
|
||||
// We want to construct a communicator which only contains of one
|
||||
// element per node
|
||||
bool const makeDistribution
|
||||
bool const computeDistribution
|
||||
= nodeInfos[rank].localRank == 0;
|
||||
|
||||
std::vector<ABCTuple>
|
||||
nodeTuples = makeDistribution
|
||||
? specialDistribution(Info{ nNodes
|
||||
, Nv
|
||||
, np
|
||||
, nodeInfos[rank].nodeId
|
||||
},
|
||||
nodeTuples
|
||||
= computeDistribution
|
||||
? specialDistribution(Info{nNodes, nodeInfos[rank].nodeId},
|
||||
getAllTuplesList(Nv))
|
||||
: std::vector<ABCTuple>()
|
||||
;
|
||||
@ -400,7 +433,7 @@ std::vector<ABCTuple> main(MPI_Comm universe, size_t Nv) {
|
||||
// Main:1 ends here
|
||||
|
||||
// [[file:~/atrip/atrip.org::*Main][Main:2]]
|
||||
const size_t
|
||||
size_t const
|
||||
tuplesPerRankLocal
|
||||
= nodeTuples.size() / nodeInfos[rank].ranksPerNode
|
||||
+ size_t(nodeTuples.size() % nodeInfos[rank].ranksPerNode != 0)
|
||||
@ -431,7 +464,8 @@ LOG(1,"Atrip") << "#nodes " << nNodes << "\n";
|
||||
size_t const totalTuples
|
||||
= tuplesPerRankGlobal * nodeInfos[rank].ranksPerNode;
|
||||
|
||||
if (makeDistribution) {
|
||||
if (computeDistribution) {
|
||||
// pad with FAKE_TUPLEs
|
||||
nodeTuples.insert(nodeTuples.end(),
|
||||
totalTuples - nodeTuples.size(),
|
||||
FAKE_TUPLE);
|
||||
@ -462,13 +496,13 @@ if (makeDistribution) {
|
||||
}
|
||||
// Main:4 ends here
|
||||
|
||||
// [[file:~/atrip/atrip.org::*Main][Main:6]]
|
||||
// [[file:~/atrip/atrip.org::*Main][Main:5]]
|
||||
LOG(1,"Atrip") << "scattering tuples \n";
|
||||
|
||||
return result;
|
||||
|
||||
}
|
||||
// Main:6 ends here
|
||||
// Main:5 ends here
|
||||
|
||||
// [[file:~/atrip/atrip.org::*Interface][Interface:1]]
|
||||
struct Distribution : public TuplesDistribution {
|
||||
|
||||
@ -57,7 +57,7 @@ namespace atrip {
|
||||
, child_world
|
||||
, global_world
|
||||
, Slice::TA
|
||||
, 4) {
|
||||
, 5) {
|
||||
init(sourceTensor);
|
||||
}
|
||||
|
||||
@ -94,7 +94,7 @@ namespace atrip {
|
||||
, child_world
|
||||
, global_world
|
||||
, Slice::VIJKA
|
||||
, 4) {
|
||||
, 5) {
|
||||
init(sourceTensor);
|
||||
}
|
||||
|
||||
|
||||
@ -9,13 +9,18 @@
|
||||
|
||||
using namespace atrip;
|
||||
|
||||
bool RankMap::RANK_ROUND_ROBIN;
|
||||
int Atrip::rank;
|
||||
int Atrip::np;
|
||||
Timings Atrip::chrono;
|
||||
size_t Atrip::networkSend;
|
||||
size_t Atrip::localSend;
|
||||
|
||||
void Atrip::init() {
|
||||
MPI_Comm_rank(MPI_COMM_WORLD, &Atrip::rank);
|
||||
MPI_Comm_size(MPI_COMM_WORLD, &Atrip::np);
|
||||
Atrip::networkSend = 0;
|
||||
Atrip::localSend = 0;
|
||||
}
|
||||
|
||||
Atrip::Output Atrip::run(Atrip::Input const& in) {
|
||||
@ -43,6 +48,15 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
|
||||
in.ea->read_all(epsa.data());
|
||||
in.Tph->read_all(Tai.data());
|
||||
|
||||
RankMap::RANK_ROUND_ROBIN = in.rankRoundRobin;
|
||||
if (RankMap::RANK_ROUND_ROBIN) {
|
||||
LOG(0,"Atrip") << "Doing rank round robin slices distribution" << "\n";
|
||||
} else {
|
||||
LOG(0,"Atrip")
|
||||
<< "Doing node > local rank round robin slices distribution" << "\n";
|
||||
}
|
||||
|
||||
|
||||
// COMMUNICATOR CONSTRUCTION ========================================={{{1
|
||||
//
|
||||
// Construct a new communicator living only on a single rank
|
||||
@ -224,7 +238,7 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
|
||||
;
|
||||
|
||||
WITH_CHRONO("db:io:send",
|
||||
u.send(otherRank, el.info, sendTag);
|
||||
u.send(otherRank, el, sendTag);
|
||||
)
|
||||
|
||||
} // send phase
|
||||
@ -268,6 +282,25 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
|
||||
))
|
||||
|
||||
if (iteration % in.iterationMod == 0) {
|
||||
|
||||
size_t networkSend;
|
||||
MPI_Reduce(&Atrip::networkSend,
|
||||
&networkSend,
|
||||
1,
|
||||
MPI_UINT64_T,
|
||||
MPI_SUM,
|
||||
0,
|
||||
universe);
|
||||
|
||||
size_t localSend;
|
||||
MPI_Reduce(&Atrip::localSend,
|
||||
&localSend,
|
||||
1,
|
||||
MPI_UINT64_T,
|
||||
MPI_SUM,
|
||||
0,
|
||||
universe);
|
||||
|
||||
LOG(0,"Atrip")
|
||||
<< "iteration " << iteration
|
||||
<< " [" << 100 * iteration / nIterations << "%]"
|
||||
@ -275,7 +308,12 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
|
||||
<< "GF)"
|
||||
<< " (" << doublesFlops * iteration / Atrip::chrono["iterations"].count()
|
||||
<< "GF)"
|
||||
<< " ===========================\n";
|
||||
<< " :net " << networkSend
|
||||
<< " :loc " << localSend
|
||||
<< " :loc/net " << (double(localSend) / double(networkSend))
|
||||
//<< " ===========================\n"
|
||||
<< "\n";
|
||||
|
||||
|
||||
// PRINT TIMINGS
|
||||
if (in.chrono)
|
||||
@ -415,6 +453,7 @@ Atrip::Output Atrip::run(Atrip::Input const& in) {
|
||||
|
||||
}
|
||||
|
||||
// TODO: remove this
|
||||
if (isFakeTuple(i)) {
|
||||
// fake iterations should also unwrap whatever they got
|
||||
WITH_RANK << iteration
|
||||
|
||||
Loading…
Reference in New Issue
Block a user