1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
|
// Copyright (C) 2011 Júlio Hoffimann.
// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// Message Passing Interface 1.1 -- Section 4.5. Gatherv
#ifndef BOOST_MPI_GATHERV_HPP
#define BOOST_MPI_GATHERV_HPP
#include <boost/mpi/exception.hpp>
#include <boost/mpi/datatype.hpp>
#include <vector>
#include <boost/mpi/packed_oarchive.hpp>
#include <boost/mpi/packed_iarchive.hpp>
#include <boost/mpi/detail/point_to_point.hpp>
#include <boost/mpi/communicator.hpp>
#include <boost/mpi/environment.hpp>
#include <boost/assert.hpp>
namespace boost { namespace mpi {
namespace detail {
// We're gathering at the root for a type that has an associated MPI
// datatype, so we'll use MPI_Gatherv to do all of the work.
template<typename T>
void
gatherv_impl(const communicator& comm, const T* in_values, int in_size,
T* out_values, const int* sizes, const int* displs, int root, mpl::true_)
{
MPI_Datatype type = get_mpi_datatype<T>(*in_values);
BOOST_MPI_CHECK_RESULT(MPI_Gatherv,
(const_cast<T*>(in_values), in_size, type,
out_values, const_cast<int*>(sizes), const_cast<int*>(displs),
type, root, comm));
}
// We're gathering from a non-root for a type that has an associated MPI
// datatype, so we'll use MPI_Gatherv to do all of the work.
template<typename T>
void
gatherv_impl(const communicator& comm, const T* in_values, int in_size, int root,
mpl::true_)
{
MPI_Datatype type = get_mpi_datatype<T>(*in_values);
BOOST_MPI_CHECK_RESULT(MPI_Gatherv,
(const_cast<T*>(in_values), in_size, type,
0, 0, 0, type, root, comm));
}
// We're gathering at the root for a type that does not have an
// associated MPI datatype, so we'll need to serialize
// it. Unfortunately, this means that we cannot use MPI_Gatherv, so
// we'll just have all of the non-root nodes send individual
// messages to the root.
template<typename T>
void
gatherv_impl(const communicator& comm, const T* in_values, int in_size,
T* out_values, const int* sizes, const int* displs, int root, mpl::false_)
{
int tag = environment::collectives_tag();
int nprocs = comm.size();
for (int src = 0; src < nprocs; ++src) {
if (src == root)
// Our own values will never be transmitted: just copy them.
std::copy(in_values, in_values + in_size, out_values + displs[src]);
else {
// comm.recv(src, tag, out_values + displs[src], sizes[src]);
// Receive archive
packed_iarchive ia(comm);
MPI_Status status;
detail::packed_archive_recv(comm, src, tag, ia, status);
for (int i = 0; i < sizes[src]; ++i)
ia >> out_values[ displs[src] + i ];
}
}
}
// We're gathering at a non-root for a type that does not have an
// associated MPI datatype, so we'll need to serialize
// it. Unfortunately, this means that we cannot use MPI_Gatherv, so
// we'll just have all of the non-root nodes send individual
// messages to the root.
template<typename T>
void
gatherv_impl(const communicator& comm, const T* in_values, int in_size, int root,
mpl::false_)
{
int tag = environment::collectives_tag();
// comm.send(root, tag, in_values, in_size);
packed_oarchive oa(comm);
for (int i = 0; i < in_size; ++i)
oa << in_values[i];
detail::packed_archive_send(comm, root, tag, oa);
}
} // end namespace detail
template<typename T>
void
gatherv(const communicator& comm, const T* in_values, int in_size,
T* out_values, const std::vector<int>& sizes, const std::vector<int>& displs,
int root)
{
if (comm.rank() == root)
detail::gatherv_impl(comm, in_values, in_size,
out_values, &sizes[0], &displs[0],
root, is_mpi_datatype<T>());
else
detail::gatherv_impl(comm, in_values, in_size, root, is_mpi_datatype<T>());
}
template<typename T>
void
gatherv(const communicator& comm, const std::vector<T>& in_values,
T* out_values, const std::vector<int>& sizes, const std::vector<int>& displs,
int root)
{
::boost::mpi::gatherv(comm, &in_values[0], in_values.size(), out_values, sizes, displs, root);
}
template<typename T>
void gatherv(const communicator& comm, const T* in_values, int in_size, int root)
{
BOOST_ASSERT(comm.rank() != root);
detail::gatherv_impl(comm, in_values, in_size, root, is_mpi_datatype<T>());
}
template<typename T>
void gatherv(const communicator& comm, const std::vector<T>& in_values, int root)
{
BOOST_ASSERT(comm.rank() != root);
detail::gatherv_impl(comm, &in_values[0], in_values.size(), root, is_mpi_datatype<T>());
}
///////////////////////
// common use versions
///////////////////////
template<typename T>
void
gatherv(const communicator& comm, const T* in_values, int in_size,
T* out_values, const std::vector<int>& sizes, int root)
{
int nprocs = comm.size();
std::vector<int> displs( nprocs );
for (int rank = 0, aux = 0; rank < nprocs; ++rank) {
displs[rank] = aux;
aux += sizes[rank];
}
::boost::mpi::gatherv(comm, in_values, in_size, out_values, sizes, displs, root);
}
template<typename T>
void
gatherv(const communicator& comm, const std::vector<T>& in_values,
T* out_values, const std::vector<int>& sizes, int root)
{
::boost::mpi::gatherv(comm, &in_values[0], in_values.size(), out_values, sizes, root);
}
} } // end namespace boost::mpi
#endif // BOOST_MPI_GATHERV_HPP
|
