Question

Considering a really huge file(maybe more than 4GB) on disk,I want to scan through this file and calculate the times of a specific binary pattern occurs.

My thought is:

1. Use memory-mapped file(CreateFileMap
   or boost mapped_file) to load the
   file to the virtual memory.

2. For each 100MB mapped-memory,create one thread to scan and calculate the result.

Is this feasible?Are there any better method to do so?

Update:
Memory-mapped file would be a good choice,for scaning through a 1.6GB file could be handled within 11s.

thanks.

Answer 1

Multithreading is only going to make this go slower unless you want to scan multiple files with each on a different hard drive. Otherwise you are just going to seek.

I wrote a simple test function using memory mapped files, with a single thread a 1.4 Gb file took about 20 seconds to scan. With two threads, each taking half the file (even 1MB chunks to one thread, odd to the other), it took more than 80 seconds.

• 1 thread: 20015 milliseconds
• 2 threads: 83985 milliseconds

That’s right, 2 threads was Four times slower than 1 thread!

Here’s the code I used, this is the single threaded version, I used a 1 byte scan pattern, so the code to locate matches that straddle map boundaries is untested.

HRESULT ScanForPattern(LPCTSTR pszFilename, LPBYTE pbPattern, UINT cbPattern, LONGLONG * pcFound)
{
   HRESULT hr = S_OK;

   *pcFound = 0;
   if ( ! pbPattern || ! cbPattern)
      return E_INVALIDARG;

   //  Open the file
   //
   HANDLE hf = CreateFile(pszFilename,
                          GENERIC_READ,
                          FILE_SHARE_READ, NULL,
                          OPEN_EXISTING,
                          FILE_FLAG_SEQUENTIAL_SCAN,
                          NULL);

   if (INVALID_HANDLE_VALUE == hf)
      {
      hr = HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND);
      // catch an open file that exists but is in use
      if (ERROR_SHARING_VIOLATION == GetLastError())
         hr = HRESULT_FROM_WIN32(ERROR_SHARING_VIOLATION);
      return hr;
      }

   // get the file length
   //
   ULARGE_INTEGER  uli;
   uli.LowPart = GetFileSize(hf, &uli.HighPart);
   LONGLONG cbFileSize = uli.QuadPart;
   if (0 == cbFileSize)
      {
      CloseHandle (hf);
      return S_OK;
      }

   const LONGLONG cbStride = 1 * 1024 * 1024; // 1 MB stride.
   LONGLONG cFound  = 0;
   LPBYTE   pbGap = (LPBYTE) malloc(cbPattern * 2);

   //  Create a mapping of the file.
   //
   HANDLE hmap = CreateFileMapping(hf, NULL, PAGE_READONLY, 0, 0, NULL);
   if (NULL != hmap)
      {
      for (LONGLONG ix = 0; ix < cbFileSize; ix += cbStride)
         {
         uli.QuadPart = ix;
         UINT cbMap = (UINT) min(cbFileSize - ix, cbStride);
         LPCBYTE pb = (LPCBYTE) MapViewOfFile(hmap, FILE_MAP_READ, uli.HighPart, uli.LowPart, cbMap);
         if ( ! pb)
            {
            hr = HRESULT_FROM_WIN32(GetLastError());
            break;
            }
         // handle pattern scanning over the gap.
         if (cbPattern > 1 && ix > 0)
            {
            CopyMemory(pbGap + cbPattern - 1, &pb[0], cbPattern - 1);
            for (UINT ii = 1; ii < cbPattern; ++ii)
               {
               if (pb[ii] == pbPattern[0] && 0 == memcmp(&pb[ii], pbPattern, cbPattern))
                  {
                  ++cFound; 
                  // advance by cbPattern-1 to avoid detecting overlapping patterns
                  }
               }
            }

         for (UINT ii = 0; ii < cbMap - cbPattern + 1; ++ii)
            {
            if (pb[ii] == pbPattern[0] && 
                ((cbPattern == 1) || 0 == memcmp(&pb[ii], pbPattern, cbPattern)))
               {
               ++cFound; 
               // advance by cbPattern-1 to avoid detecting overlapping patterns
               }
            }
         if (cbPattern > 1 && cbMap >= cbPattern)
            {
            // save end of the view in our gap buffer so we can detect map-straddling patterns
            CopyMemory(pbGap, &pb[cbMap - cbPattern + 1], cbPattern - 1);
            }
         UnmapViewOfFile(pb);
         }

      CloseHandle (hmap);
      }
   CloseHandle (hf);

   *pcFound = cFound;
   return hr;
}