Question

When I read in a file I am getting three characters being read first which aren’t in the file.

They are: “”

The file contains:

<root>
  <tag>
    <anothertag>Text</anothertag>
  </tag>
  <tag>
    <anothertag>Text</anothertag>
  </tag>
</root>

My code:

//scanner.h
enum TOKEN_TYPE {
    OPEN, CLOSE, TEXT
};

struct Token {
    int type;
    std::string value;
};

class Scanner {
public:
    Scanner(std::string path);

    void scan(std::string path);

private:
    std::vector<Token> tokens;

    Token readToken(std::ifstream& in);
};

std::string read_to(std::istream& in, char delim);

//scanner.cpp
Scanner::Scanner(std::string path) {
    scan(path);
}

void Scanner::scan(std::string path) {
    std::ifstream in(path, std::ios_base::beg);

    if(!in.is_open()) {
        return;
    }

    if(!tokens.empty()) {
        tokens.erase(tokens.begin(), tokens.end());
    }

    std::ofstream out("output.txt");
    while(!in.eof()) {
        tokens.push_back(readToken(in));

        if(tokens.back().value.empty()) {
            tokens.pop_back();
            continue;
        }

        out << tokens.size() << "\t" << tokens.back().value << "\t" << tokens.back().value.length() << std::endl;
    }

    in.close();

    out << "\n";
    for(int i = 0; i < tokens.size(); ++i) {
        if(tokens[i].type == TOKEN_TYPE::TEXT) {
            out << tokens[i].value << std::endl;
        }
    }

    out.close();
}

Token Scanner::readToken(std::ifstream& in) {
    Token token;

    char c = in.get();
    while(c == '\n' || c == '\t' || c == ' ') {
        c = in.get();
    }

    if(c == '<') {
        c = in.get();

        if(c == '/') {
            token.type = TOKEN_TYPE::CLOSE;
            token.value = read_to(in, '>');
        } else {
            in.unget();

            token.type = TOKEN_TYPE::OPEN;
            token.value = read_to(in, '>');
        }
    } else {
        in.unget();

        token.type = TOKEN_TYPE::TEXT;
        token.value = read_to(in, '<');

        in.unget();
    }

    return token;
}

inline std::string read_to(std::istream& in, char delim) {
    std::stringstream str;

    char c;
    while((c = in.get()) != delim && in.good()) {
        str << c;
    }

    return str.str();
}


int main(int argc, char** argv) {
    Scanner scanner("test.xml");
    return 0;
}

Why is the stream reading these characters that aren’t actually there?

Answer 1

“” is a UTF-8 BOM. You have to skip it manually, or remove it from the file (UTF-8 doesn’t really need a BOM).