Question

I am inputting a 200mb file in my application and due to a very strange reason the memory usage of my application is more than 600mb. I have tried vector and deque, as well as std::string and char * with no avail. I need the memory usage of my application to be almost the same as the file I am reading, any suggestions would be extremely helpful. Is there a bug that causes so much memory consumption? Could you pinpoint the problem or should I rewrite the whole thing?

Windows Vista SP1 x64, Microsoft Visual Studio 2008 SP1, 32Bit Release Version, Intel CPU

The whole application until now:

#include <string> #include <vector> #include <iostream> #include <iomanip> #include <fstream> #include <sstream> #include <iterator> #include <algorithm> #include <time.h>    static unsigned int getFileSize (const char *filename) {     std::ifstream fs;     fs.open (filename, std::ios::binary);     fs.seekg(0, std::ios::beg);     const std::ios::pos_type start_pos = fs.tellg();     fs.seekg(0, std::ios::end);     const std::ios::pos_type end_pos = fs.tellg();     const unsigned int ret_filesize (static_cast<unsigned int>(end_pos - start_pos));     fs.close();     return ret_filesize; } void str2Vec (std::string &str, std::vector<std::string> &vec) {     int newlineLastIndex(0);     for (int loopVar01 = str.size(); loopVar01 > 0; loopVar01--)     {         if (str[loopVar01]=='\n')         {             newlineLastIndex = loopVar01;             break;         }     }     int remainder(str.size()-newlineLastIndex);      std::vector<int> indexVec;     indexVec.push_back(0);     for (unsigned int lpVar02 = 0; lpVar02 < (str.size()-remainder); lpVar02++)     {         if (str[lpVar02] == '\n')         {             indexVec.push_back(lpVar02);         }     }     int memSize(0);     for (int lpVar03 = 0; lpVar03 < (indexVec.size()-1); lpVar03++)     {         memSize = indexVec[(lpVar03+1)] - indexVec[lpVar03];         std::string tempStr (memSize,'0');         memcpy(&tempStr[0],&str[indexVec[lpVar03]],memSize);         vec.push_back(tempStr);     } } void readFile(const std::string &fileName, std::vector<std::string> &vec) {     static unsigned int fileSize = getFileSize(fileName.c_str());     static std::ifstream fileStream;     fileStream.open (fileName.c_str(),std::ios::binary);     fileStream.clear();     fileStream.seekg (0, std::ios::beg);     const int chunks(1000);      int singleChunk(fileSize/chunks);     int remainder = fileSize - (singleChunk * chunks);     std::string fileStr (singleChunk, '0');     int fileIndex(0);     for (int lpVar01 = 0; lpVar01 < chunks; lpVar01++)     {         fileStream.read(&fileStr[0], singleChunk);         str2Vec(fileStr, vec);     }     std::string remainderStr(remainder, '0');     fileStream.read(&remainderStr[0], remainder);     str2Vec(fileStr, vec);       } int main (int argc, char *argv[]) {            std::vector<std::string> vec;         std::string inFile(argv[1]);         readFile(inFile, vec); } 

Answer 1

Your memory is being fragmented.

Try something like this :

  HANDLE heaps[1025];   DWORD nheaps = GetProcessHeaps((sizeof(heaps) / sizeof(HANDLE)) - 1, heaps);    for (DWORD i = 0; i < nheaps; ++i)    {     ULONG  HeapFragValue = 2;     HeapSetInformation(heaps[i],                        HeapCompatibilityInformation,                        &HeapFragValue,                        sizeof(HeapFragValue));   }