libs/spirit/example/lex/word_count_functor.cpp
// Copyright (c) 2001-2010 Hartmut Kaiser // // Distributed under the Boost Software License, Version 1.0. (See accompanying // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt) // This example is the equivalent to the following flex program: /* //[wcf_flex_version %{ #define ID_WORD 1000 #define ID_EOL 1001 #define ID_CHAR 1002 int c = 0, w = 0, l = 0; %} %% [^ \t\n]+ { return ID_WORD; } \n { return ID_EOL; } . { return ID_CHAR; } %% bool count(int tok) { switch (tok) { case ID_WORD: ++w; c += yyleng; break; case ID_EOL: ++l; ++c; break; case ID_CHAR: ++c; break; default: return false; } return true; } void main() { int tok = EOF; do { tok = yylex(); if (!count(tok)) break; } while (EOF != tok); printf("%d %d %d\n", l, w, c); } //] */ // Its purpose is to do the word count function of the wc command in UNIX. It // prints the number of lines, words and characters in a file. // // This examples shows how to use the tokenize() function together with a // simple functor, which gets executed whenever a token got matched in the // input sequence. // #define BOOST_SPIRIT_LEXERTL_DEBUG #include <boost/config/warning_disable.hpp> //[wcf_includes #include <boost/spirit/include/lex_lexertl.hpp> #include <boost/bind.hpp> #include <boost/ref.hpp> //] #include <iostream> #include <string> #include "example.hpp" //[wcf_namespaces namespace lex = boost::spirit::lex; //] /////////////////////////////////////////////////////////////////////////////// // Token id definitions /////////////////////////////////////////////////////////////////////////////// //[wcf_token_ids enum token_ids { ID_WORD = 1000, ID_EOL, ID_CHAR }; //] //[wcf_token_definition /*` The template `word_count_tokens` defines three different tokens: `ID_WORD`, `ID_EOL`, and `ID_CHAR`, representing a word (anything except a whitespace or a newline), a newline character, and any other character (`ID_WORD`, `ID_EOL`, and `ID_CHAR` are enum values representing the token ids, but could be anything else convertible to an integer as well). The direct base class of any token definition class needs to be the template `lex::lexer<>`, where the corresponding template parameter (here: `lex::lexertl::lexer<BaseIterator>`) defines which underlying lexer engine has to be used to provide the required state machine functionality. In this example we use the Lexertl based lexer engine as the underlying lexer type. */ template <typename Lexer> struct word_count_tokens : lex::lexer<Lexer> { word_count_tokens() { // define tokens (the regular expression to match and the corresponding // token id) and add them to the lexer this->self.add ("[^ \t\n]+", ID_WORD) // words (anything except ' ', '\t' or '\n') ("\n", ID_EOL) // newline characters (".", ID_CHAR) // anything else is a plain character ; } }; //] //[wcf_functor /*` In this example the struct 'counter' is used as a functor counting the characters, words and lines in the analyzed input sequence by identifying the matched tokens as passed from the /Spirit.Lex/ library. */ struct counter { //<- this is an implementation detail specific to boost::bind and doesn't show // up in the documentation typedef bool result_type; //-> // the function operator gets called for each of the matched tokens // c, l, w are references to the counters used to keep track of the numbers template <typename Token> bool operator()(Token const& t, std::size_t& c, std::size_t& w, std::size_t& l) const { switch (t.id()) { case ID_WORD: // matched a word // since we're using a default token type in this example, every // token instance contains a `iterator_range<BaseIterator>` as its token // attribute pointing to the matched character sequence in the input ++w; c += t.value().size(); break; case ID_EOL: // matched a newline character ++l; ++c; break; case ID_CHAR: // matched something else ++c; break; } return true; // always continue to tokenize } }; //] /////////////////////////////////////////////////////////////////////////////// //[wcf_main /*` The main function simply loads the given file into memory (as a `std::string`), instantiates an instance of the token definition template using the correct iterator type (`word_count_tokens<char const*>`), and finally calls `lex::tokenize`, passing an instance of the counter function object. The return value of `lex::tokenize()` will be `true` if the whole input sequence has been successfully tokenized, and `false` otherwise. */ int main(int argc, char* argv[]) { // these variables are used to count characters, words and lines std::size_t c = 0, w = 0, l = 0; // read input from the given file std::string str (read_from_file(1 == argc ? "word_count.input" : argv[1])); // create the token definition instance needed to invoke the lexical analyzer word_count_tokens<lex::lexertl::lexer<> > word_count_functor; // tokenize the given string, the bound functor gets invoked for each of // the matched tokens char const* first = str.c_str(); char const* last = &first[str.size()]; bool r = lex::tokenize(first, last, word_count_functor, boost::bind(counter(), _1, boost::ref(c), boost::ref(w), boost::ref(l))); // print results if (r) { std::cout << "lines: " << l << ", words: " << w << ", characters: " << c << "\n"; } else { std::string rest(first, last); std::cout << "Lexical analysis failed\n" << "stopped at: \"" << rest << "\"\n"; } return 0; } //]