Question

I am currently doing some testing with a new addition to the ICU dictionary-based break iterator.
I have code that allows me to test the word-breaking on a text document but when the text document is too large it gives the error: bash: ./a.out: Argument list too long

I am not sure how to edit the code to break-up the argument list when it gets too long so that a file of any size can be run through the code. The original code author is quite busy, would someone be willing to help out?

I tried removing the printing of what is being examined to see if that would help, but I still get the error on large files (printing what is being examined isn’t necessary – I just need the result).

If the code could be modified to read the source text file line by line and export the results line by line to another text file (ending up with all the lines when it is done), that would be perfect.

The code is as follows:

/*
Written by George Rhoten to test how word segmentation works.
Code inspired by the break ICU sample.

Here is an example to run this code under Cygwin.

PATH=$PATH:icu-test/source/lib ./a.exe "`cat input.txt`" > output.txt

Encode input.txt as UTF-8.
The output text is UTF-8.
*/

#include <stdio.h>
#include <unicode/brkiter.h>
#include <unicode/ucnv.h>

#define ZW_SPACE "\xE2\x80\x8B"

void printUnicodeString(const UnicodeString &s) {
    int32_t len = s.length() * U8_MAX_LENGTH + 1;
    char *charBuf = new char[len];
    len = s.extract(0, s.length(), charBuf, len, NULL);
    charBuf[len] = 0;
    printf("%s", charBuf);
    delete charBuf;
}

/* Creating and using text boundaries */
int main(int argc, char **argv)
{
    ucnv_setDefaultName("UTF-8");
    UnicodeString stringToExamine("Aaa bbb ccc. Ddd eee fff.");
    printf("Examining: ");
    if (argc > 1) {
        // Override the default charset.
        stringToExamine = UnicodeString(argv[1]);
        if (stringToExamine.charAt(0) == 0xFEFF) {
            // Remove the BOM
            stringToExamine = UnicodeString(stringToExamine, 1);
        }
    }
    printUnicodeString(stringToExamine);
    puts("");

    //print each sentence in forward and reverse order
    UErrorCode status = U_ZERO_ERROR;
    BreakIterator* boundary = BreakIterator::createWordInstance(NULL, status);
    if (U_FAILURE(status)) {
        printf("Failed to create sentence break iterator. status = %s", 
            u_errorName(status));
        exit(1);
    }

    printf("Result:    ");
    //print each word in order
    boundary->setText(stringToExamine);
    int32_t start = boundary->first();
    int32_t end = boundary->next();
    while (end != BreakIterator::DONE) {
        if (start != 0) {
            printf(ZW_SPACE);
        }
        printUnicodeString(UnicodeString(stringToExamine, start, end-start));
        start = end;
        end = boundary->next();
    }

    delete boundary;

    return 0;
}

Thanks so much!
-Nathan

Answer 1

The code below reads the content of a file whos name is given as the first parameter on the command-line and places it in a str::buffer. Then, instead of calling the function UnicodeString with argv[1], use that buffer instead.

#include<iostream>
#include<fstream>

using namespace std;

int main(int argc, char **argv)
{
    std::string buffer;

    if(argc > 1) {
        std::ifstream t;
        t.open(argv[1]);
        std::string line;
        while(t){
            std::getline(t, line);
            buffer += line + '\n';
        }
    }
    cout << buffer;
    return 0;
}

Update:

Input to UnicodeString should be char*. The function GetFileIntoCharPointer does that.
Note that only the most rudimentary error checking is implemented below!

#include<iostream>
#include<fstream>

using namespace std;

char * GetFileIntoCharPointer(char *pFile, long &lRet)
{
    FILE * fp = fopen(pFile,"rb");
    if (fp == NULL) return 0;

    fseek(fp, 0, SEEK_END);
    long size = ftell(fp);
    fseek(fp, 0, SEEK_SET);

    char *pData = new char[size + 1];
    lRet = fread(pData, sizeof(char), size, fp);

    fclose(fp);

    return pData;
}

int main(int argc, char **argv)
{
    long Len;
    char * Data = GetFileIntoCharPointer(argv[1], Len);
    std::cout << Data << std::endl;

    if (Data != NULL)
        delete [] Data;

    return 0;
}