Question

[Replaced code by a complete program and versions update]

The code below fails under Windows with the following message (sorry manualy translated from
French, I don’t know how to force Visual C++ to work in English). This is
under Windows. It works under Linux. You can compile with
Linux: gcc LimaTokenizerParser.cpp -o LimaTokenizerPars -lboost_system-mt -lstdc++
Windows(fails): cl LimaTokenizerParser.cpp /Ic:\boost\path\include /EHsc

I use Boost 1.50 with Visual C++ 2010 under Windows and Boost 1.48
and gcc 4.6.3 under Linux.

The assertion in boost grammar.hpp is accompanied by this comment:

// If you see the assertion below failing then the start rule
// passed to the constructor of the grammar is not compatible with
// the grammar (i.e. it uses different template parameters).

but I don’t understand how to understand it in my case…

Any idea ?

Regards,

Gaël

Code:

LimaTokenizerParser.cpp

#include "SpiritTokenizerParser.hpp"

#include <iostream>
#include <fstream>

void readStream(std::istream &is,std::string &dest)
{
  while (is.good() && !is.eof())
  {
    const int bufferSize = 10240;
    char buffer[bufferSize];
    is.read(buffer,bufferSize);
    const int size = is.gcount();
    if (size>0)
      dest.insert(dest.end(),buffer,buffer+size);
  }
}

int main(int argc, char* argv[])
{
  if (argc != 2)
  {
    std::cerr << "Needs exactly one argument" << std::endl;
    return 1;
  }
  namespace qi = boost::spirit::qi;
  namespace ascii = boost::spirit::ascii;
  using ascii::space;
  typedef std::string::const_iterator iterator_type;
  typedef tokenizer<iterator_type> tokenizer;

  // @ERROR DOES NOT WORK ON WINDOWS PORT
 tokenizer tokenizer_parser; 
  std::string str;
  std::ifstream file(argv[1]);
  readStream(file, str);

  std::string::const_iterator iter = str.begin();
  std::string::const_iterator end = str.end();

  tokenizer_automaton automaton;
  bool r = false;
  // @ERROR DOES NOT WORK ON WINDOWS PORT
  r = phrase_parse(iter, end, tokenizer_parser, skipper, automaton);

  if (r && iter == end)
  {
      std::cout << "Parsing succeeded: "<<automaton.size()<<" states" << std::endl;
  }
  else
  {
      std::string rest(iter, end);
      std::cout << "Parsing failed. Stopped at: \": " << rest << "\"\n";
  }

  return 0;
}

SpiritTokenizerParser.hpp

#ifndef SPIRITTOKENIZERPARSER_HPP
#define SPIRITTOKENIZERPARSER_HPP

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/spirit/include/phoenix_core.hpp>
#include <boost/spirit/include/phoenix_operator.hpp>
#include <boost/spirit/include/phoenix_object.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <boost/fusion/include/io.hpp>

#include <string>

namespace qi = boost::spirit::qi;
namespace ascii = boost::spirit::ascii;

#define skipper qi::space | ascii::char_('#') >> *(ascii::char_ - qi::eol) >> qi::eol
typedef BOOST_TYPEOF(skipper) skipper_type;

enum transitions {
  STORE,
  FLUSH,
  TOKEN,
  EXIT,
  LTOKEN
};

typedef std::vector<std::string> tokenizer_precondition;
typedef std::vector<std::string> tokenizer_postcondition;
struct tokenizer_transition
{
  std::vector<tokenizer_precondition> preconditions;
  std::vector<std::string> event;
  std::vector<tokenizer_postcondition> postconditions;
  transitions transition;
  std::string target;
  std::vector<std::string> statuses;
};

struct tokenizer_state
{
  std::string id;
  std::vector<tokenizer_transition> transitions;
};

typedef std::vector<tokenizer_state> tokenizer_automaton;

BOOST_FUSION_ADAPT_STRUCT(
                          tokenizer_transition,
                          (std::vector<tokenizer_precondition>, preconditions)
                          (std::vector<std::string>, event)
                          (std::vector<tokenizer_postcondition>, postconditions)
                          (transitions, transition)
                          (std::string, target)
                          (std::vector<std::string>, statuses)
                          )

BOOST_FUSION_ADAPT_STRUCT(
                          tokenizer_state,
                          (std::string, id)
                          (std::vector<tokenizer_transition>, transitions)
                          )

using ascii::space_type;

template <typename Iterator>
struct tokenizer : qi::grammar<Iterator, tokenizer_automaton(), skipper_type>
{
  struct transitionsymbol_ : qi::symbols<char, unsigned>
  {
    // > is +1 store
    // / is +1 flush (forget the current token)
    // = is +1 token
    // ^ is exit

    transitionsymbol_()
    {
      add
      (">", STORE)
      ("/", FLUSH)
      ("=", TOKEN)
      ("^", EXIT)
      ;
    }

  } transitionsymbol;

  tokenizer() : tokenizer::base_type(start)
  {
    using qi::alnum;
    using qi::lexeme;
    using ascii::char_;

    start %= *state ;
    state %= '(' >> identifier >> ')' >> '{' >> *transition >> '}';
    transition %= '-' >> *precondition >> event >> *postcondition >> transitionsymbol >> identifier >> -('(' >> identifier % ',' >> ')');
    identifier %= lexeme[+(alnum | char_('_'))];
    precondition %= '[' >> (identifier % '|') >> ']';
    event %= identifier % '|';
    postcondition %= identifier % '|';
  }

  qi::rule<Iterator, tokenizer_automaton(), skipper_type> start;
  qi::rule<Iterator, tokenizer_state(), skipper_type> state;
  qi::rule<Iterator, tokenizer_transition(), skipper_type> transition;
  qi::rule<Iterator, std::string(), skipper_type> identifier;
  qi::rule<Iterator, std::vector<std::string>(), skipper_type> precondition;
  qi::rule<Iterator, std::vector<std::string>(), skipper_type> event;
  qi::rule<Iterator, std::vector<std::string>(), skipper_type> postcondition;
};

//]

#endif // SPIRITTOKENIZERPARSER_HPP

Messages:

LimaTokenizerParser.cpp
c:\Program Files\boost\boost_1_50\include\boost/spirit/home/qi/nonterminal/grammar.hpp(77) : error C2664: 'boost::mpl::assertion_failed'<FF>: can not convert parameter 1 from 'boost::mpl::failed ************(__cdecl boost::spirit::qi::grammar<Iterator,T1,T2>::{ctor}::incompatible_start_rule::* ***********)(boost::spirit::qi::rule<Iterator,T1,boost::spirit::qi::alternative<Elements>>)' en 'boost::mpl::assert<false>::type'
        with
        [
            Iterator=iterator_type,
            T1=tokenizer_automaton (void),
            T2=skipper_type,
            Elements=boost::fusion::cons<boost::spirit::qi::char_class<boost::spirit::tag::char_code<boost::spirit::tag::space,boost::spirit::char_encoding::standard>>,boost::fusion::cons<boost::spirit::qi::sequence<boost::fusion::cons<boost::spirit::qi::literal_char<boost::spirit::char_encoding::ascii,false,false>,boost::fusion::cons<boost::spirit::qi::kleene<boost::spirit::qi::difference<boost::spirit::qi::char_class<boost::spirit::tag::char_code<boost::spirit::tag::char_,boost::spirit::char_encoding::ascii>>,boost::spirit::qi::eol_parser>>,boost::fusion::cons<boost::spirit::qi::eol_parser,boost::fusion::nil>>>>,boost::fusion::nil>>
        ]
        No constructor could take the source type, or the constructor overload resolution was ambiguous
        e:\projets\amose\sources\lima_linguisticprocessing\src\linguisticprocessing\core\flattokenizer\SpiritTokenizerParser.hpp(89)<FF>: see reference to function template instantiation 'boost::spirit::qi::grammar<Iterator,T1,T2>::grammar<Iterator,tokenizer_automaton(void),boost::spirit::qi::alternative<Elements>,boost::spirit::unused_type,boost::spirit::unused_type>(const boost::spirit::qi::rule<Iterator,T1,boost::spirit::qi::alternative<Elements>> &,const std::string &)' en cours de compilation
        with
        [
            Iterator=iterator_type,
            T1=tokenizer_automaton (void),
            T2=skipper_type,
            Elements=boost::fusion::cons<boost::spirit::qi::char_class<boost::spirit::tag::char_code<boost::spirit::tag::space,boost::spirit::char_encoding::standard>>,boost::fusion::cons<boost::spirit::qi::sequence<boost::fusion::cons<boost::spirit::qi::literal_char<boost::spirit::char_encoding::ascii,false,false>,boost::fusion::cons<boost::spirit::qi::kleene<boost::spirit::qi::difference<boost::spirit::qi::char_class<boost::spirit::tag::char_code<boost::spirit::tag::char_,boost::spirit::char_encoding::ascii>>,boost::spirit::qi::eol_parser>>,boost::fusion::cons<boost::spirit::qi::eol_parser,boost::fusion::nil>>>>,boost::fusion::nil>>
        ]

TokenizerParser.hpp(88)<FF>: during the compilation of the member function ':: tokenizer tokenizer <iterator> (void)' model class
        with
        [
            Iterator=iterator_type
        ]
        LimaTokenizerParser.cpp(33)<FF>: see reference to class template instantiation 'tokenizer <iterator>' being compiled
        with
        [
            Iterator=iterator_type
        ]

Answer 1

When posting to the Spirit list (yep, I saw this posted there), it is best to provide something minimal. Making something minimal helps a lot. It avoids the visual clutter that tends to waste time when understanding the code. It is just a matter of elimination anyway that anyone asking for support could do. Attached is what I mean by minimal. If I’ve seen something like this instead, it would have taken me just a few secs, instead of 10s of minutes to understand the problem.

Anyway, here’s the code distilled and as minimal as it can be while still exhibiting the problem:

#include <boost/config/warning_disable.hpp>
#include <boost/spirit/include/qi.hpp>
#include <boost/fusion/include/adapt_struct.hpp>
#include <string>

namespace qi = boost::spirit::qi;
typedef qi::space_type skipper_type;

template <typename Iterator>
struct tokenizer : qi::grammar<Iterator, skipper_type>
{
  tokenizer() : tokenizer::base_type(start)
  {
  }

  qi::rule<Iterator, skipper_type> start;
};



int main()
{
  typedef std::string::const_iterator iterator_type;
  typedef tokenizer<iterator_type> tokenizer;

  // @ERROR DOES NOT WORK ON WINDOWS PORT
  tokenizer tokenizer_parser;
  return 0;
}

I notice that if I rename “skipper_type” to “skipper_type_” (note trailing underscore). Then it compiles fine! This leads me to believe that it is an MSVC bug.

Solution: just change the name skipper_type.