Question

I am implementing one half of a stream where you should be able to jump X integers back or forth. On top of that when I open a file it should preserve the data in the file. In this implementation I try to use mmap. My problem is when I open a file a second time the mapped data is purely zeroes. My open method is called createStream and looks like this:

void MapperOutForward::createStream(const char* filename)
{
    this->filename = filename;
    pFile = open(filename, O_RDWR | O_CREAT | O_SYNC, S_IRWXG|S_IRWXO|S_IRWXU);
    if ( pFile ==  -1) 
    {
        int errsv = errno;
        cout << "MappedStreamOut createStream open error: " << strerror( errno ) << endl;
    }

    //getting file size and making the file big enough
    struct stat filestatus;
    stat( filename, &filestatus );
    if(filestatus.st_size < mapSize + 1)
    {
        lseek (pFile, mapSize + 1, SEEK_SET);
        write (pFile, "", 1);
    }
    lseek (pFile, 0, SEEK_SET);

    map = (int*) mmap (0, mapSize, PROT_WRITE | PROT_READ, MAP_PRIVATE, pFile, 0);
    close (pFile);

    if ( map == (int*) -1) 
    {
        cout << "MappedStreamOut createStream mmap error: " << strerror(errno) << endl;
    }

    numberOfOffsets = 1;
    numberOfElementsInMap = 0;

    cout << "Kopirer 5 før og 4 efter i Create" << endl;
    for(int i = 0; i < numberOfElementsInMap + 5; i++)
    {
        cout << map[i] << endl;
    }
    cout << "Create numberOfOffsets:numberOfElementsInMap " << numberOfOffsets << ":" << numberOfElementsInMap << endl; 
}

The output at the end tells me that when I am done opening I am at the beginning of the file and the first 5 integers are 0.

The O_CREAT flag in the open command should only create a file if it is not existing, and I can se that the file keeps its size after I call this command a second time. man open

The PROT_WRITE | PROT_READ flags in the mmap command seems only to activate some security and not changing anything in the file. man mmap

The rest of the class is here:

#include "MapperOutForward.h"

using namespace std;

#include <cstdlib>
#include <iostream>

#include <fcntl.h>
#include <sys/stat.h>
#include <cmath>

MapperOutForward::MapperOutForward(int mapSize) //number of pages
{ 
    pageSize = getpagesize();
    this->mapSize = mapSize * pageSize; //we work in pages
    numberOfElementsInMap = 0;
    numberOfOffsets = 0;
}

MapperOutForward::~MapperOutForward() 
{ 
    munmap(map, mapSize);
}

void MapperOutForward::createStream(const char* filename)
{
    this->filename = filename;
    pFile = open(filename, O_RDWR | O_CREAT | O_SYNC, S_IRWXG|S_IRWXO|S_IRWXU);
    if ( pFile ==  -1) 
    {
        int errsv = errno;
        cout << "MappedStreamOut createStream open error: " << strerror( errno ) << endl;
    }

    //getting file size and making the file big enough
    struct stat filestatus;
    stat( filename, &filestatus );
    if(filestatus.st_size < mapSize + 1)
    {
        lseek (pFile, mapSize + 1, SEEK_SET);
        write (pFile, "", 1);
    }
    lseek (pFile, 0, SEEK_SET);

    map = (int*) mmap (0, mapSize, PROT_WRITE | PROT_READ, MAP_PRIVATE, pFile, 0);
    close (pFile);

    if ( map == (int*) -1) 
    {
        cout << "MappedStreamOut createStream mmap error: " << strerror(errno) << endl;
    }

    numberOfOffsets = 1;
    numberOfElementsInMap = 0;

    cout << "Kopirer 5 før og 4 efter i Create" << endl;
    for(int i = 0; i < numberOfElementsInMap + 5; i++)
    {
        cout << map[i] << endl;
    }
    cout << "Create numberOfOffsets:numberOfElementsInMap " << numberOfOffsets << ":" << numberOfElementsInMap << endl; 
}

void MapperOutForward::writeNext(int* data, int numberOfElements)
{
    for(int i = 0; i < numberOfElements; i++)
    {
        writeHelper(data[i]);
    }   

    cout << "Kopirer 5 før og 4 efter i writeNext" << endl;
    for(int i = numberOfElementsInMap - 5; i < numberOfElementsInMap + 5; i++)
    {
        cout << map[i] << endl;
    }
    //cout << "WriteNext numberOfOffsets:numberOfElementsInMap " << numberOfOffsets << ":" << numberOfElementsInMap << endl;
}

void MapperOutForward::writeHelper(int data)
{
    int sizeOfInt = 4; //bytes
    if(numberOfElementsInMap >= mapSize / sizeOfInt) //We need to create the next part of the file
    {
        munmap(map, mapSize);

        pFile = open(filename, O_RDWR | O_CREAT | O_SYNC, S_IRWXG|S_IRWXO|S_IRWXU);

        if ( pFile ==  -1) 
        {
            std::cout<<"filename: "<<filename<<std::endl;
            std::cout<<"mapSize: "<<mapSize<<std::endl;

            int errsv = errno;
            cout << "MappedStreamOut writeNext open error: " << strerror( errno ) << endl;
        }

        numberOfOffsets++;

        //Make the file bigger 
        lseek (pFile, mapSize * numberOfOffsets, SEEK_SET);
        write (pFile, "", 1);
        lseek (pFile, 0, SEEK_SET);

        map = (int*) mmap (0, mapSize, PROT_WRITE | PROT_READ, MAP_PRIVATE, pFile, mapSize * (numberOfOffsets - 1));
        close (pFile);

        if ( map == (int*) -1) 
        {
            cout << "MappedStreamOut writeNext mmap error: " << strerror(errno) << endl;
        }

        numberOfElementsInMap = 0;
    }

    map[numberOfElementsInMap] = data;
    numberOfElementsInMap++;
}

void MapperOutForward::jumpTo(int offset)
{
    int sizeOfInt = 4; //bytes
    //cout << "numberOfElementsInMap + offset " << numberOfElementsInMap + offset << endl;
    //cout << "mapSize / sizeOfInt " << mapSize / sizeOfInt << endl;
    if(numberOfElementsInMap + offset >= mapSize / sizeOfInt)
    {
        munmap(map, mapSize);

        pFile = open(filename, O_RDWR | O_CREAT | O_SYNC, S_IRWXG|S_IRWXO|S_IRWXU);

        if ( pFile ==  -1) 
        {
            std::cout<<"filename: "<<filename<<std::endl;
            std::cout<<"mapSize: "<<mapSize<<std::endl;

            int errsv = errno;
            cout << "MappedStreamOut writeNext open error: " << strerror( errno ) << endl;
        }

        numberOfOffsets += (int) ceil( (double) (numberOfElementsInMap + offset - (mapSize / sizeOfInt)) / (mapSize / sizeOfInt)); // rundet op ((resterende antal ints ud over mapSize) / (Antal ints i en mapsize)) = Antal offsets der skal springes

        //Make the file bigger 
        lseek (pFile, mapSize * numberOfOffsets, SEEK_SET);
        write (pFile, "", 1);
        lseek (pFile, 0, SEEK_SET);

        map = (int*) mmap (0, mapSize, PROT_WRITE | PROT_READ, MAP_PRIVATE, pFile, mapSize * (numberOfOffsets - 1));
        close (pFile);

        if ( map == (int*) -1) 
        {
            cout << "MappedStreamOut writeNext mmap error: " << strerror(errno) << endl;
        }

        numberOfElementsInMap = (numberOfElementsInMap + offset - (mapSize / sizeOfInt)) % (mapSize / sizeOfInt); //(resterende antal ints ud over mapSize) % (Antal ints i en mapsize) = Antal elementer ind i offsettet
        //cout << "JumpTo offsetForward numberOfOffsets:numberOfElementsInMap " << numberOfOffsets << ":" << numberOfElementsInMap << endl;
    }
    else if(numberOfElementsInMap + offset < 0)
    {
        munmap(map, mapSize);

        pFile = open(filename, O_RDWR | O_CREAT | O_SYNC, S_IRWXG|S_IRWXO|S_IRWXU);

        if ( pFile ==  -1) 
        {
            std::cout<<"filename: "<<filename<<std::endl;
            std::cout<<"mapSize: "<<mapSize<<std::endl;

            int errsv = errno;
            cout << "MappedStreamOut writeNext open error: " << strerror( errno ) << endl;
        }

        numberOfOffsets += (int) floor( (double) (numberOfElementsInMap + offset) / (mapSize / sizeOfInt)); // rundet op ((resterende antal ints ud over mapSize) / (Antal ints i en mapsize)) = Antal offsets der skal springes

        //Make the file bigger 
        lseek (pFile, mapSize * numberOfOffsets, SEEK_SET);
        write (pFile, "", 1);
        lseek (pFile, 0, SEEK_SET);

        map = (int*) mmap (0, mapSize, PROT_WRITE | PROT_READ, MAP_PRIVATE, pFile, mapSize * (numberOfOffsets - 1));
        close (pFile);

        if ( map == (int*) -1) 
        {
            cout << "MappedStreamOut writeNext mmap error: " << strerror(errno) << endl;
        }

        numberOfElementsInMap = (mapSize + (numberOfElementsInMap + offset)) % (mapSize / sizeOfInt); //(resterende antal ints ud over mapSize) % (Antal ints i en mapsize) = Antal elementer ind i offsettet
        //cout << "JumpTo offsetBacward numberOfOffsets:numberOfElementsInMap " << numberOfOffsets << ":" << numberOfElementsInMap << endl;
    }
    else
    {
        numberOfElementsInMap += offset;
        //cout << "JumpTo else numberOfOffsets:numberOfElementsInMap " << numberOfOffsets << ":" << numberOfElementsInMap << endl;
    }
}

void MapperOutForward::closeStream()
{
    munmap(map, mapSize);
}

And the main file is here:

#include "MapperInForward.h"
#include "MapperOutForward.h"
//#include "MapperOutBackward.h"
//#include "MapperInBackward.h"
//#include "MapperStreamFactory.h"
#include <functional>

int main( int argc, const char* argv[] )
{
/*********************Tester Buffer Stream*************************************/
    close(open((char*) "mapperTest", O_WRONLY | O_CREAT | O_SYNC | O_TRUNC, S_IRWXG|S_IRWXO|S_IRWXU));
    //BufferStreamFactory* bsf = new BufferStreamFactory(3);

    MapperOutForward* mso = new MapperOutForward(1);
    mso->createStream((char*) "mapperTest");

    int data[4000];

    for(int i = 0; i < 4000; i++)
    {
        data[i] = i + 1;
    }

    mso->writeNext(data, 4000);
    mso->closeStream();

    MapperOutForward* mso1 = new MapperOutForward(1);
    mso1->createStream((char*) "mapperTest");

    data[0] = 9999;

    mso1->jumpTo(0);

    mso1->jumpTo(100);
    mso1->writeNext(data, 1);

    mso1->jumpTo(1000);
    mso1->writeNext(data, 1);

    mso1->jumpTo(-900);
    mso1->writeNext(data, 1);

    mso1->closeStream();
}

It is all running on Ubuntu Linux and compiled with g++.

Thanks for the time used to read all this and I hope you can help me.

Answer 1

You probably need MAP_SHARED instead of MAP_PRIVATE or the changes won’t be written to the file, as specified by the standard.

MAP_SHARED and MAP_PRIVATE describe the disposition of write
references to the memory object. If MAP_SHARED is specified, write
references shall change the underlying object. If MAP_PRIVATE is
specified, modifications to the mapped data by the calling process
shall be visible only to the calling process and shall not change the
underlying object.