Question

I’m trying to sort an array whose elements are read from a file which is approximately 5 GB in size and contains approximately 500000 data elements.

After a data size of 300.000.000, the program gives an error during sorting due to segmentation fault and terminates.

I think the problem occurs due to insufficient memory space allocated to program. How can I change it in my C code?

Could you help me about this? Thank you.

int arraysize = atoi(argv[1]);
int* array    = malloc(sizeof(int)*arraysize);
int* temp     = malloc(sizeof(int)*arraysize);
int i;

FILE *fi;
char buffer[20];
fi = fopen("DATASET.dat", "r");

for(i=0; i<arraysize; i++){
  fgets(buffer, 20, fi);
  array[i] = atoi(buffer);
}

fclose(fi);

//function is called to perform the sorting
mergesort_array(array, arraysize, temp);

Answer 1

int* array = malloc(sizeof(int)arraysize);
int temp = malloc(sizeof(int)*arraysize);

In general, whenever you allocate memory, check that the allocation succeeded:

int *array = NULL, *temp = NULL;

if (NULL == (array = malloc(sizeof(int)*arraysize)))
{
    fprintf(stderr, "Out of memory allocating %d bytes\n", sizeof(int)*arraysize);
    abort();
}
if (NULL == (temp = malloc(sizeof(int)*arraysize)))
{
    fprintf(stderr, "Out of memory allocating %d bytes\n", sizeof(int)*arraysize);
    abort();
}

Then, a possibility would be to implement mergesorting on disk, using a file of integers (you can mmap() the file, too).

But I find it strange that an allocation of 300000 integers on the heap – 4.8 megabytes at the most, using 64-bit integers – can cause an allocation error, so I think this is something in the mergesort implementation; maybe something having to do with a recursive implementation.

I’d start with compiling the program with full debug information, and checking the core dump with gdb.

A “simple” malloc problem

Having to handle a very large array of ASCII strings representing numbers, you could start by first converting it to a file of integers.

FILE *fi, *fo, *ft;
char buffer[20];
int  array[4096], b = 0;

fi = fopen("DATASET.dat", "r");
if (NULL == fi)
{
    fprintf(stderr, "Cannot open input file\n");
    abort();
}
fo = fopen("INTEGER.dat", "w");
if (NULL == fo)
{
    fprintf(stderr, "Cannot open output file\n");
    abort();
}
ft = fopen("TEMP.dat", "w");
if (NULL == ft)
{
    fprintf(stderr, "Cannot open output file\n");
    abort();
}
for(i=0; i<arraysize; i++){
   fgets(buffer, 20, fi);
   array[b++] = atoi(buffer);
   if (4096 == b)
   {
       if (b != fwrite(buffer, sizeof(int), b, fo))
       {
           fprintf(stderr, "write error\n");
           abort();
       }
       if (b != fwrite(buffer, sizeof(int), b, ft))
       {
           fprintf(stderr, "write error\n");
           abort();
       }
       b = 0;
   }
}
if (b)
{
    if (b != fwrite(buffer, sizeof(int), b, fo))
    {
        fprintf(stderr, "write error\n");
        abort();
    }
    if (b != fwrite(buffer, sizeof(int), b, ft))
    {
        fprintf(stderr, "write error\n");
        abort();
    }
}
fclose(fi); fi = NULL;
fclose(fo); fo = NULL;
fclose(ft); ft = NULL;

Now you have an INTEGER.dat file which is made of integers of fixed size. It is, to all intents and purposes, a file copy of an array in memory. Same goes for the temporary array.

And you can tell the system to treat that file as if it was an array in memory.

int *sort = NULL;
int *temp = NULL;

// Temp is not shown -- identical treatment as sort

fd = open ("INTEGERS.dat", O_RDWR);
if (fd == -1)
{
    fprintf(stderr, "cannot reopen output\n");
    abort();
}
if (MAP_FAILED == (sort = mmap (0, arraysize*sizeof(int), PROT_READ|PROT_WRITE, MAP_SHARED, fd, 0)))
{
    fprintf(stderr, "mmap error\n");
    abort();
}
if (-1 == close (fd))
{
    fprintf(stderr, "error closing output file\n");
    return 1;
}

do_sort(sort, temp, arraysize);

if (-1 == munmap (sort, arraysize*sizeof(int)))
{
    fprintf(stderr, "error releasing mmap for %s\n", "sort");
    abort();
}