boost/mpi/nonblocking.hpp
// Copyright (C) 2006 Douglas Gregor <doug.gregor -at- gmail.com>.
// Use, modification and distribution is subject to the Boost Software
// License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
/** @file nonblocking.hpp
*
* This header defines operations for completing non-blocking
* communication requests.
*/
#ifndef BOOST_MPI_NONBLOCKING_HPP
#define BOOST_MPI_NONBLOCKING_HPP
#include <boost/mpi/config.hpp>
#include <vector>
#include <iterator> // for std::iterator_traits
#include <boost/optional.hpp>
#include <utility> // for std::pair
#include <algorithm> // for iter_swap, reverse
#include <boost/static_assert.hpp>
#include <boost/mpi/request.hpp>
#include <boost/mpi/status.hpp>
#include <boost/mpi/exception.hpp>
namespace boost { namespace mpi {
/**
* @brief Wait until any non-blocking request has completed.
*
* This routine takes in a set of requests stored in the iterator
* range @c [first,last) and waits until any of these requests has
* been completed. It provides functionality equivalent to
* @c MPI_Waitany.
*
* @param first The iterator that denotes the beginning of the
* sequence of request objects.
*
* @param last The iterator that denotes the end of the sequence of
* request objects. This may not be equal to @c first.
*
* @returns A pair containing the status object that corresponds to
* the completed operation and the iterator referencing the completed
* request.
*/
template<typename ForwardIterator>
std::pair<status, ForwardIterator>
wait_any(ForwardIterator first, ForwardIterator last)
{
using std::advance;
BOOST_ASSERT(first != last);
typedef typename std::iterator_traits<ForwardIterator>::difference_type
difference_type;
bool all_trivial_requests = true;
difference_type n = 0;
ForwardIterator current = first;
while (true) {
// Check if we have found a completed request. If so, return it.
if (current->m_requests[0] != MPI_REQUEST_NULL &&
current->m_requests[1] != MPI_REQUEST_NULL)
if (optional<status> result = current->test())
return std::make_pair(*result, current);
// Check if this request (and all others before it) are "trivial"
// requests, e.g., they can be represented with a single
// MPI_Request.
all_trivial_requests =
all_trivial_requests
&& !current->m_handler
&& current->m_requests[1] == MPI_REQUEST_NULL;
// Move to the next request.
++n;
if (++current == last) {
// We have reached the end of the list. If all requests thus far
// have been trivial, we can call MPI_Waitany directly, because
// it may be more efficient than our busy-wait semantics.
if (all_trivial_requests) {
std::vector<MPI_Request> requests;
requests.reserve(n);
for (current = first; current != last; ++current)
requests.push_back(current->m_requests[0]);
// Let MPI wait until one of these operations completes.
int index;
status stat;
BOOST_MPI_CHECK_RESULT(MPI_Waitany,
(n, &requests[0], &index, &stat.m_status));
// We don't have a notion of empty requests or status objects,
// so this is an error.
if (index == MPI_UNDEFINED)
boost::throw_exception(exception("MPI_Waitany", MPI_ERR_REQUEST));
// Find the iterator corresponding to the completed request.
current = first;
advance(current, index);
current->m_requests[0] = requests[index];
return std::make_pair(stat, current);
}
// There are some nontrivial requests, so we must continue our
// busy waiting loop.
n = 0;
current = first;
all_trivial_requests = true;
}
}
// We cannot ever get here
BOOST_ASSERT(false);
}
/**
* @brief Test whether any non-blocking request has completed.
*
* This routine takes in a set of requests stored in the iterator
* range @c [first,last) and tests whether any of these requests has
* been completed. This routine is similar to @c wait_any, but will
* not block waiting for requests to completed. It provides
* functionality equivalent to @c MPI_Testany.
*
* @param first The iterator that denotes the beginning of the
* sequence of request objects.
*
* @param last The iterator that denotes the end of the sequence of
* request objects.
*
* @returns If any outstanding requests have completed, a pair
* containing the status object that corresponds to the completed
* operation and the iterator referencing the completed
* request. Otherwise, an empty @c optional<>.
*/
template<typename ForwardIterator>
optional<std::pair<status, ForwardIterator> >
test_any(ForwardIterator first, ForwardIterator last)
{
for (ForwardIterator current = first; first != last; ++first) {
// Check if we have found a completed request. If so, return it.
if (optional<status> result = current->test())
return std::make_pair(*result, current);
}
// We found nothing
return optional<std::pair<status, ForwardIterator> >();
}
/**
* @brief Wait until all non-blocking requests have completed.
*
* This routine takes in a set of requests stored in the iterator
* range @c [first,last) and waits until all of these requests have
* been completed. It provides functionality equivalent to
* @c MPI_Waitall.
*
* @param first The iterator that denotes the beginning of the
* sequence of request objects.
*
* @param last The iterator that denotes the end of the sequence of
* request objects.
*
* @param out If provided, an output iterator through which the
* status of each request will be emitted. The @c status objects are
* emitted in the same order as the requests are retrieved from
* @c [first,last).
*
* @returns If an @p out parameter was provided, the value @c out
* after all of the @c status objects have been emitted.
*/
template<typename ForwardIterator, typename OutputIterator>
OutputIterator
wait_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
{
typedef typename std::iterator_traits<ForwardIterator>::difference_type
difference_type;
using std::distance;
difference_type num_outstanding_requests = distance(first, last);
std::vector<status> results(num_outstanding_requests);
std::vector<bool> completed(num_outstanding_requests);
while (num_outstanding_requests > 0) {
bool all_trivial_requests = true;
difference_type idx = 0;
for (ForwardIterator current = first; current != last; ++current, ++idx) {
if (!completed[idx]) {
if (optional<status> stat = current->test()) {
// This outstanding request has been completed. We're done.
results[idx] = *stat;
completed[idx] = true;
--num_outstanding_requests;
all_trivial_requests = false;
} else {
// Check if this request (and all others before it) are "trivial"
// requests, e.g., they can be represented with a single
// MPI_Request.
all_trivial_requests =
all_trivial_requests
&& !current->m_handler
&& current->m_requests[1] == MPI_REQUEST_NULL;
}
}
}
// If we have yet to fulfill any requests and all of the requests
// are trivial (i.e., require only a single MPI_Request to be
// fulfilled), call MPI_Waitall directly.
if (all_trivial_requests
&& num_outstanding_requests == (difference_type)results.size()) {
std::vector<MPI_Request> requests;
requests.reserve(num_outstanding_requests);
for (ForwardIterator current = first; current != last; ++current)
requests.push_back(current->m_requests[0]);
// Let MPI wait until all of these operations completes.
std::vector<MPI_Status> stats(num_outstanding_requests);
BOOST_MPI_CHECK_RESULT(MPI_Waitall,
(num_outstanding_requests, &requests[0],
&stats[0]));
for (std::vector<MPI_Status>::iterator i = stats.begin();
i != stats.end(); ++i, ++out) {
status stat;
stat.m_status = *i;
*out = stat;
}
return out;
}
all_trivial_requests = false;
}
return std::copy(results.begin(), results.end(), out);
}
/**
* \overload
*/
template<typename ForwardIterator>
void
wait_all(ForwardIterator first, ForwardIterator last)
{
typedef typename std::iterator_traits<ForwardIterator>::difference_type
difference_type;
using std::distance;
difference_type num_outstanding_requests = distance(first, last);
std::vector<bool> completed(num_outstanding_requests);
while (num_outstanding_requests > 0) {
bool all_trivial_requests = true;
difference_type idx = 0;
for (ForwardIterator current = first; current != last; ++current, ++idx) {
if (!completed[idx]) {
if (optional<status> stat = current->test()) {
// This outstanding request has been completed.
completed[idx] = true;
--num_outstanding_requests;
all_trivial_requests = false;
} else {
// Check if this request (and all others before it) are "trivial"
// requests, e.g., they can be represented with a single
// MPI_Request.
all_trivial_requests =
all_trivial_requests
&& !current->m_handler
&& current->m_requests[1] == MPI_REQUEST_NULL;
}
}
}
// If we have yet to fulfill any requests and all of the requests
// are trivial (i.e., require only a single MPI_Request to be
// fulfilled), call MPI_Waitall directly.
if (all_trivial_requests
&& num_outstanding_requests == (difference_type)completed.size()) {
std::vector<MPI_Request> requests;
requests.reserve(num_outstanding_requests);
for (ForwardIterator current = first; current != last; ++current)
requests.push_back(current->m_requests[0]);
// Let MPI wait until all of these operations completes.
BOOST_MPI_CHECK_RESULT(MPI_Waitall,
(num_outstanding_requests, &requests[0],
MPI_STATUSES_IGNORE));
// Signal completion
num_outstanding_requests = 0;
}
}
}
/**
* @brief Tests whether all non-blocking requests have completed.
*
* This routine takes in a set of requests stored in the iterator
* range @c [first,last) and determines whether all of these requests
* have been completed. However, due to limitations of the underlying
* MPI implementation, if any of the requests refers to a
* non-blocking send or receive of a serialized data type, @c
* test_all will always return the equivalent of @c false (i.e., the
* requests cannot all be finished at this time). This routine
* performs the same functionality as @c wait_all, except that this
* routine will not block. This routine provides functionality
* equivalent to @c MPI_Testall.
*
* @param first The iterator that denotes the beginning of the
* sequence of request objects.
*
* @param last The iterator that denotes the end of the sequence of
* request objects.
*
* @param out If provided and all requests hav been completed, an
* output iterator through which the status of each request will be
* emitted. The @c status objects are emitted in the same order as
* the requests are retrieved from @c [first,last).
*
* @returns If an @p out parameter was provided, the value @c out
* after all of the @c status objects have been emitted (if all
* requests were completed) or an empty @c optional<>. If no @p out
* parameter was provided, returns @c true if all requests have
* completed or @c false otherwise.
*/
template<typename ForwardIterator, typename OutputIterator>
optional<OutputIterator>
test_all(ForwardIterator first, ForwardIterator last, OutputIterator out)
{
std::vector<MPI_Request> requests;
for (; first != last; ++first) {
// If we have a non-trivial request, then no requests can be
// completed.
if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
return optional<OutputIterator>();
requests.push_back(first->m_requests[0]);
}
int flag = 0;
int n = requests.size();
std::vector<MPI_Status> stats(n);
BOOST_MPI_CHECK_RESULT(MPI_Testall, (n, &requests[0], &flag, &stats[0]));
if (flag) {
for (int i = 0; i < n; ++i, ++out) {
status stat;
stat.m_status = stats[i];
*out = stat;
}
return out;
} else {
return optional<OutputIterator>();
}
}
/**
* \overload
*/
template<typename ForwardIterator>
bool
test_all(ForwardIterator first, ForwardIterator last)
{
std::vector<MPI_Request> requests;
for (; first != last; ++first) {
// If we have a non-trivial request, then no requests can be
// completed.
if (first->m_handler || first->m_requests[1] != MPI_REQUEST_NULL)
return false;
requests.push_back(first->m_requests[0]);
}
int flag = 0;
int n = requests.size();
BOOST_MPI_CHECK_RESULT(MPI_Testall,
(n, &requests[0], &flag, MPI_STATUSES_IGNORE));
return flag != 0;
}
/**
* @brief Wait until some non-blocking requests have completed.
*
* This routine takes in a set of requests stored in the iterator
* range @c [first,last) and waits until at least one of the requests
* has completed. It then completes all of the requests it can,
* partitioning the input sequence into pending requests followed by
* completed requests. If an output iterator is provided, @c status
* objects will be emitted for each of the completed requests. This
* routine provides functionality equivalent to @c MPI_Waitsome.
*
* @param first The iterator that denotes the beginning of the
* sequence of request objects.
*
* @param last The iterator that denotes the end of the sequence of
* request objects. This may not be equal to @c first.
*
* @param out If provided, the @c status objects corresponding to
* completed requests will be emitted through this output iterator.
* @returns If the @p out parameter was provided, a pair containing
* the output iterator @p out after all of the @c status objects have
* been written through it and an iterator referencing the first
* completed request. If no @p out parameter was provided, only the
* iterator referencing the first completed request will be emitted.
*/
template<typename BidirectionalIterator, typename OutputIterator>
std::pair<OutputIterator, BidirectionalIterator>
wait_some(BidirectionalIterator first, BidirectionalIterator last,
OutputIterator out)
{
using std::advance;
if (first == last)
return std::make_pair(out, first);
typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
difference_type;
bool all_trivial_requests = true;
difference_type n = 0;
BidirectionalIterator current = first;
BidirectionalIterator start_of_completed = last;
while (true) {
// Check if we have found a completed request.
if (optional<status> result = current->test()) {
using std::iter_swap;
// Emit the resulting status object
*out++ = *result;
// We're expanding the set of completed requests
--start_of_completed;
if (current == start_of_completed) {
// If we have hit the end of the list of pending
// requests. Finish up by fixing the order of the completed
// set to match the order in which we emitted status objects,
// then return.
std::reverse(start_of_completed, last);
return std::make_pair(out, start_of_completed);
}
// Swap the request we just completed with the last request that
// has not yet been tested.
iter_swap(current, start_of_completed);
continue;
}
// Check if this request (and all others before it) are "trivial"
// requests, e.g., they can be represented with a single
// MPI_Request.
all_trivial_requests =
all_trivial_requests
&& !current->m_handler
&& current->m_requests[1] == MPI_REQUEST_NULL;
// Move to the next request.
++n;
if (++current == start_of_completed) {
if (start_of_completed != last) {
// We have satisfied some requests. Make the order of the
// completed requests match that of the status objects we've
// already emitted and we're done.
std::reverse(start_of_completed, last);
return std::make_pair(out, start_of_completed);
}
// We have reached the end of the list. If all requests thus far
// have been trivial, we can call MPI_Waitsome directly, because
// it may be more efficient than our busy-wait semantics.
if (all_trivial_requests) {
std::vector<MPI_Request> requests;
std::vector<int> indices(n);
std::vector<MPI_Status> stats(n);
requests.reserve(n);
for (current = first; current != last; ++current)
requests.push_back(current->m_requests[0]);
// Let MPI wait until some of these operations complete.
int num_completed;
BOOST_MPI_CHECK_RESULT(MPI_Waitsome,
(n, &requests[0], &num_completed, &indices[0],
&stats[0]));
// Translate the index-based result of MPI_Waitsome into a
// partitioning on the requests.
int current_offset = 0;
current = first;
for (int index = 0; index < num_completed; ++index, ++out) {
using std::iter_swap;
// Move "current" to the request object at this index
advance(current, indices[index] - current_offset);
current_offset = indices[index];
// Emit the status object
status stat;
stat.m_status = stats[index];
*out = stat;
// Finish up the request and swap it into the "completed
// requests" partition.
current->m_requests[0] = requests[indices[index]];
--start_of_completed;
iter_swap(current, start_of_completed);
}
// We have satisfied some requests. Make the order of the
// completed requests match that of the status objects we've
// already emitted and we're done.
std::reverse(start_of_completed, last);
return std::make_pair(out, start_of_completed);
}
// There are some nontrivial requests, so we must continue our
// busy waiting loop.
n = 0;
current = first;
}
}
// We cannot ever get here
BOOST_ASSERT(false);
}
/**
* \overload
*/
template<typename BidirectionalIterator>
BidirectionalIterator
wait_some(BidirectionalIterator first, BidirectionalIterator last)
{
using std::advance;
if (first == last)
return first;
typedef typename std::iterator_traits<BidirectionalIterator>::difference_type
difference_type;
bool all_trivial_requests = true;
difference_type n = 0;
BidirectionalIterator current = first;
BidirectionalIterator start_of_completed = last;
while (true) {
// Check if we have found a completed request.
if (optional<status> result = current->test()) {
using std::iter_swap;
// We're expanding the set of completed requests
--start_of_completed;
// If we have hit the end of the list of pending requests, we're
// done.
if (current == start_of_completed)
return start_of_completed;
// Swap the request we just completed with the last request that
// has not yet been tested.
iter_swap(current, start_of_completed);
continue;
}
// Check if this request (and all others before it) are "trivial"
// requests, e.g., they can be represented with a single
// MPI_Request.
all_trivial_requests =
all_trivial_requests
&& !current->m_handler
&& current->m_requests[1] == MPI_REQUEST_NULL;
// Move to the next request.
++n;
if (++current == start_of_completed) {
// If we have satisfied some requests, we're done.
if (start_of_completed != last)
return start_of_completed;
// We have reached the end of the list. If all requests thus far
// have been trivial, we can call MPI_Waitsome directly, because
// it may be more efficient than our busy-wait semantics.
if (all_trivial_requests) {
std::vector<MPI_Request> requests;
std::vector<int> indices(n);
requests.reserve(n);
for (current = first; current != last; ++current)
requests.push_back(current->m_requests[0]);
// Let MPI wait until some of these operations complete.
int num_completed;
BOOST_MPI_CHECK_RESULT(MPI_Waitsome,
(n, &requests[0], &num_completed, &indices[0],
MPI_STATUSES_IGNORE));
// Translate the index-based result of MPI_Waitsome into a
// partitioning on the requests.
int current_offset = 0;
current = first;
for (int index = 0; index < num_completed; ++index) {
using std::iter_swap;
// Move "current" to the request object at this index
advance(current, indices[index] - current_offset);
current_offset = indices[index];
// Finish up the request and swap it into the "completed
// requests" partition.
current->m_requests[0] = requests[indices[index]];
--start_of_completed;
iter_swap(current, start_of_completed);
}
// We have satisfied some requests, so we are done.
return start_of_completed;
}
// There are some nontrivial requests, so we must continue our
// busy waiting loop.
n = 0;
current = first;
}
}
// We cannot ever get here
BOOST_ASSERT(false);
}
/**
* @brief Test whether some non-blocking requests have completed.
*
* This routine takes in a set of requests stored in the iterator
* range @c [first,last) and tests to see if any of the requests has
* completed. It completes all of the requests it can, partitioning
* the input sequence into pending requests followed by completed
* requests. If an output iterator is provided, @c status objects
* will be emitted for each of the completed requests. This routine
* is similar to @c wait_some, but does not wait until any requests
* have completed. This routine provides functionality equivalent to
* @c MPI_Testsome.
*
* @param first The iterator that denotes the beginning of the
* sequence of request objects.
*
* @param last The iterator that denotes the end of the sequence of
* request objects. This may not be equal to @c first.
*
* @param out If provided, the @c status objects corresponding to
* completed requests will be emitted through this output iterator.
* @returns If the @p out parameter was provided, a pair containing
* the output iterator @p out after all of the @c status objects have
* been written through it and an iterator referencing the first
* completed request. If no @p out parameter was provided, only the
* iterator referencing the first completed request will be emitted.
*/
template<typename BidirectionalIterator, typename OutputIterator>
std::pair<OutputIterator, BidirectionalIterator>
test_some(BidirectionalIterator first, BidirectionalIterator last,
OutputIterator out)
{
BidirectionalIterator current = first;
BidirectionalIterator start_of_completed = last;
while (current != start_of_completed) {
// Check if we have found a completed request.
if (optional<status> result = current->test()) {
using std::iter_swap;
// Emit the resulting status object
*out++ = *result;
// We're expanding the set of completed requests
--start_of_completed;
// Swap the request we just completed with the last request that
// has not yet been tested.
iter_swap(current, start_of_completed);
continue;
}
// Move to the next request.
++current;
}
// Finish up by fixing the order of the completed set to match the
// order in which we emitted status objects, then return.
std::reverse(start_of_completed, last);
return std::make_pair(out, start_of_completed);
}
/**
* \overload
*/
template<typename BidirectionalIterator>
BidirectionalIterator
test_some(BidirectionalIterator first, BidirectionalIterator last)
{
BidirectionalIterator current = first;
BidirectionalIterator start_of_completed = last;
while (current != start_of_completed) {
// Check if we have found a completed request.
if (optional<status> result = current->test()) {
using std::iter_swap;
// We're expanding the set of completed requests
--start_of_completed;
// Swap the request we just completed with the last request that
// has not yet been tested.
iter_swap(current, start_of_completed);
continue;
}
// Move to the next request.
++current;
}
return start_of_completed;
}
} } // end namespace boost::mpi
#endif // BOOST_MPI_NONBLOCKING_HPP