Question

Is there a way to make Boost tokenizer to split below string without splitting quoted part?

string s = "1st 2nd \"3rd with some comment\" 4th";

Exptected output:
1st
2nd
3rd with some comment
4th

Answer 1

Try this code and this way you can avoid using Boost.Tokenizer and Boost.Spirit libs

#include <vector>
#include <string>
#include <iostream>

const char Separators[] = { ' ', 9 };

bool Str_IsSeparator( const char Ch )
{
    for ( size_t i = 0; i != sizeof( Separators ); i++ )
    {
        if ( Separators[i] == Ch ) { return true; }
    }

    return false;
}

void SplitLine( size_t FromToken, size_t ToToken, const std::string& Str, std::vector<std::string>& Components /*, bool ShouldTrimSpaces*/ )
{
    size_t TokenNum = 0;
    size_t Offset   = FromToken - 1;

    const char* CStr  = Str.c_str();
    const char* CStrj = Str.c_str();

    while ( *CStr )
    {
        // bypass spaces & delimiting chars
        while ( *CStr && Str_IsSeparator( *CStr ) ) { CStr++; }

        if ( !*CStr ) { return; }

        bool InsideQuotes = ( *CStr == '\"' );

        if ( InsideQuotes )
        {
            for ( CStrj = ++CStr; *CStrj && *CStrj != '\"'; CStrj++ );
        }
        else
        {
            for ( CStrj = CStr; *CStrj && !Str_IsSeparator( *CStrj ); CStrj++ );
        }

        // extract token
        if ( CStr != CStrj )
        {
            TokenNum++;

            // store each token found
            if ( TokenNum >= FromToken )
            {
                  Components[ TokenNum-Offset ].assign( CStr, CStrj );
                  // if ( ShouldTrimSpaces ) { Str_TrimSpaces( &Components[ TokenNum-Offset ] ); }
                  // proceed to next token
                  if ( TokenNum >= ToToken ) { return; }
            }
            CStr = CStrj;

            // exclude last " from token, handle EOL
            if ( *CStr ) { CStr++; }
        }
    }
}

int main()
{
    std::string test = "1st 2nd \"3rd with some comment\" 4th";
    std::vector<std::string> Out;

    Out.resize(5);
    SplitLine(1, 4, test, Out);

    for(size_t j = 0 ; j != Out.size() ; j++) { std::cout << Out[j] << std::endl; }

    return 0;
}

It uses a preallocated string array (it is not zero-based, but that’s easily fixable) and it’s pretty simple.