boost/test/utils/iterator/token_iterator.hpp
// (C) Copyright Gennadiy Rozental 2004-2008.
// Distributed under the Boost Software License, Version 1.0.
// (See accompanying file LICENSE_1_0.txt or copy at
// http://www.boost.org/LICENSE_1_0.txt)
// See http://www.boost.org/libs/test for the library home page.
//
// File : $RCSfile$
//
// Version : $Revision: 49312 $
//
// Description : token iterator for string and range tokenization
// ***************************************************************************
#ifndef BOOST_TOKEN_ITERATOR_HPP_071894GER
#define BOOST_TOKEN_ITERATOR_HPP_071894GER
// Boost
#include <boost/config.hpp>
#include <boost/detail/workaround.hpp>
#include <boost/iterator/iterator_categories.hpp>
#include <boost/iterator/iterator_traits.hpp>
#include <boost/test/utils/iterator/input_iterator_facade.hpp>
#include <boost/test/utils/basic_cstring/basic_cstring.hpp>
#include <boost/test/utils/named_params.hpp>
#include <boost/test/utils/foreach.hpp>
// STL
#include <iosfwd>
#include <cctype>
#include <boost/test/detail/suppress_warnings.hpp>
//____________________________________________________________________________//
#ifdef BOOST_NO_STDC_NAMESPACE
namespace std{ using ::ispunct; using ::isspace; }
#endif
namespace boost {
namespace unit_test {
// ************************************************************************** //
// ************** ti_delimeter_type ************** //
// ************************************************************************** //
enum ti_delimeter_type {
dt_char, // character is delimeter if it among explicit list of some characters
dt_ispunct, // character is delimeter if it satisfies ispunct functor
dt_isspace, // character is delimeter if it satisfies isspace functor
dt_none // no character is delimeter
};
namespace ut_detail {
// ************************************************************************** //
// ************** default_char_compare ************** //
// ************************************************************************** //
template<typename CharT>
class default_char_compare {
public:
bool operator()( CharT c1, CharT c2 )
{
#ifdef BOOST_CLASSIC_IOSTREAMS
return std::string_char_traits<CharT>::eq( c1, c2 );
#else
return std::char_traits<CharT>::eq( c1, c2 );
#endif
}
};
// ************************************************************************** //
// ************** delim_policy ************** //
// ************************************************************************** //
template<typename CharT,typename CharCompare>
class delim_policy {
typedef basic_cstring<CharT const> cstring;
public:
// Constructor
explicit delim_policy( ti_delimeter_type t = dt_char, cstring d = cstring() )
: m_type( t )
{
set_delimeters( d );
}
void set_delimeters( ti_delimeter_type t ) { m_type = t; }
template<typename Src>
void set_delimeters( Src d )
{
nfp::optionally_assign( m_delimeters, d );
if( !m_delimeters.is_empty() )
m_type = dt_char;
}
bool operator()( CharT c )
{
switch( m_type ) {
case dt_char: {
BOOST_TEST_FOREACH( CharT, delim, m_delimeters )
if( CharCompare()( delim, c ) )
return true;
return false;
}
case dt_ispunct:
return (std::ispunct)( c ) != 0;
case dt_isspace:
return (std::isspace)( c ) != 0;
case dt_none:
return false;
}
return false;
}
private:
// Data members
cstring m_delimeters;
ti_delimeter_type m_type;
};
// ************************************************************************** //
// ************** token_assigner ************** //
// ************************************************************************** //
template<typename TraversalTag>
struct token_assigner {
#if BOOST_WORKAROUND( BOOST_DINKUMWARE_STDLIB, < 306 )
template<typename Iterator, typename C, typename T>
static void assign( Iterator b, Iterator e, std::basic_string<C,T>& t )
{ for( ; b != e; ++b ) t += *b; }
template<typename Iterator, typename C>
static void assign( Iterator b, Iterator e, basic_cstring<C>& t ) { t.assign( b, e ); }
#else
template<typename Iterator, typename Token>
static void assign( Iterator b, Iterator e, Token& t ) { t.assign( b, e ); }
#endif
template<typename Iterator, typename Token>
static void append_move( Iterator& b, Token& ) { ++b; }
};
//____________________________________________________________________________//
template<>
struct token_assigner<single_pass_traversal_tag> {
template<typename Iterator, typename Token>
static void assign( Iterator b, Iterator e, Token& t ) {}
template<typename Iterator, typename Token>
static void append_move( Iterator& b, Token& t ) { t += *b; ++b; }
};
} // namespace ut_detail
// ************************************************************************** //
// ************** modifiers ************** //
// ************************************************************************** //
namespace {
nfp::keyword<struct dropped_delimeters_t > dropped_delimeters;
nfp::keyword<struct kept_delimeters_t > kept_delimeters;
nfp::typed_keyword<bool,struct keep_empty_tokens_t > keep_empty_tokens;
nfp::typed_keyword<std::size_t,struct max_tokens_t > max_tokens;
}
// ************************************************************************** //
// ************** token_iterator_base ************** //
// ************************************************************************** //
template<typename Derived,
typename CharT,
typename CharCompare = ut_detail::default_char_compare<CharT>,
typename ValueType = basic_cstring<CharT const>,
typename Reference = basic_cstring<CharT const>,
typename Traversal = forward_traversal_tag>
class token_iterator_base
: public input_iterator_facade<Derived,ValueType,Reference,Traversal> {
typedef basic_cstring<CharT const> cstring;
typedef ut_detail::delim_policy<CharT,CharCompare> delim_policy;
typedef input_iterator_facade<Derived,ValueType,Reference,Traversal> base;
protected:
// Constructor
explicit token_iterator_base()
: m_is_dropped( dt_isspace )
, m_is_kept( dt_ispunct )
, m_keep_empty_tokens( false )
, m_tokens_left( (std::size_t)-1 )
, m_token_produced( false )
{
}
template<typename Modifier>
void
apply_modifier( Modifier const& m )
{
if( m.has( dropped_delimeters ) )
m_is_dropped.set_delimeters( m[dropped_delimeters] );
if( m.has( kept_delimeters ) )
m_is_kept.set_delimeters( m[kept_delimeters] );
if( m.has( keep_empty_tokens ) )
m_keep_empty_tokens = true;
nfp::optionally_assign( m_tokens_left, m, max_tokens );
}
template<typename Iter>
bool get( Iter& begin, Iter end )
{
typedef ut_detail::token_assigner<BOOST_DEDUCED_TYPENAME iterator_traversal<Iter>::type> Assigner;
Iter check_point;
this->m_value.clear();
if( !m_keep_empty_tokens ) {
while( begin != end && m_is_dropped( *begin ) )
++begin;
if( begin == end )
return false;
check_point = begin;
if( m_tokens_left == 1 )
while( begin != end )
Assigner::append_move( begin, this->m_value );
else if( m_is_kept( *begin ) )
Assigner::append_move( begin, this->m_value );
else
while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
Assigner::append_move( begin, this->m_value );
--m_tokens_left;
}
else { // m_keep_empty_tokens is true
check_point = begin;
if( begin == end ) {
if( m_token_produced )
return false;
m_token_produced = true;
}
if( m_is_kept( *begin ) ) {
if( m_token_produced )
Assigner::append_move( begin, this->m_value );
m_token_produced = !m_token_produced;
}
else if( !m_token_produced && m_is_dropped( *begin ) )
m_token_produced = true;
else {
if( m_is_dropped( *begin ) )
check_point = ++begin;
while( begin != end && !m_is_dropped( *begin ) && !m_is_kept( *begin ) )
Assigner::append_move( begin, this->m_value );
m_token_produced = true;
}
}
Assigner::assign( check_point, begin, this->m_value );
return true;
}
private:
// Data members
delim_policy m_is_dropped;
delim_policy m_is_kept;
bool m_keep_empty_tokens;
std::size_t m_tokens_left;
bool m_token_produced;
};
// ************************************************************************** //
// ************** basic_string_token_iterator ************** //
// ************************************************************************** //
template<typename CharT,
typename CharCompare = ut_detail::default_char_compare<CharT> >
class basic_string_token_iterator
: public token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> {
typedef basic_cstring<CharT const> cstring;
typedef token_iterator_base<basic_string_token_iterator<CharT,CharCompare>,CharT,CharCompare> base;
public:
explicit basic_string_token_iterator() {}
explicit basic_string_token_iterator( cstring src )
: m_src( src )
{
this->init();
}
template<typename Src, typename Modifier>
basic_string_token_iterator( Src src, Modifier const& m )
: m_src( src )
{
this->apply_modifier( m );
this->init();
}
private:
friend class input_iterator_core_access;
// input iterator implementation
bool get()
{
typename cstring::iterator begin = m_src.begin();
bool res = base::get( begin, m_src.end() );
m_src.assign( begin, m_src.end() );
return res;
}
// Data members
cstring m_src;
};
typedef basic_string_token_iterator<char> string_token_iterator;
typedef basic_string_token_iterator<wchar_t> wstring_token_iterator;
// ************************************************************************** //
// ************** range_token_iterator ************** //
// ************************************************************************** //
template<typename Iter,
typename CharCompare = ut_detail::default_char_compare<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
typename ValueType = std::basic_string<BOOST_DEDUCED_TYPENAME iterator_value<Iter>::type>,
typename Reference = ValueType const&>
class range_token_iterator
: public token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> {
typedef basic_cstring<typename ValueType::value_type> cstring;
typedef token_iterator_base<range_token_iterator<Iter,CharCompare,ValueType,Reference>,
typename iterator_value<Iter>::type,CharCompare,ValueType,Reference> base;
public:
explicit range_token_iterator() {}
explicit range_token_iterator( Iter begin, Iter end = Iter() )
: m_begin( begin ), m_end( end )
{
this->init();
}
range_token_iterator( range_token_iterator const& rhs )
: base( rhs )
{
if( this->m_valid ) {
m_begin = rhs.m_begin;
m_end = rhs.m_end;
}
}
template<typename Modifier>
range_token_iterator( Iter begin, Iter end, Modifier const& m )
: m_begin( begin ), m_end( end )
{
this->apply_modifier( m );
this->init();
}
private:
friend class input_iterator_core_access;
// input iterator implementation
bool get()
{
return base::get( m_begin, m_end );
}
// Data members
Iter m_begin;
Iter m_end;
};
// ************************************************************************** //
// ************** make_range_token_iterator ************** //
// ************************************************************************** //
template<typename Iter>
inline range_token_iterator<Iter>
make_range_token_iterator( Iter begin, Iter end = Iter() )
{
return range_token_iterator<Iter>( begin, end );
}
//____________________________________________________________________________//
template<typename Iter,typename Modifier>
inline range_token_iterator<Iter>
make_range_token_iterator( Iter begin, Iter end, Modifier const& m )
{
return range_token_iterator<Iter>( begin, end, m );
}
//____________________________________________________________________________//
} // namespace unit_test
} // namespace boost
//____________________________________________________________________________//
#include <boost/test/detail/enable_warnings.hpp>
#endif // BOOST_TOKEN_ITERATOR_HPP_071894GER