Question

I’ve created this program that can encrypt a found file and then it can be decrypted later via the CryptDecrypt function. The function succeeds but instead of decrypting the file back to plain text it makes the file look even more encrypted.

I’ve put both the CryptEncrypt function and CryptDecrypt function so you can have a more less view what I’m doing wrong. One more thing I’m using the Win32 API, no MFC or ATL.

if (LOWORD(wParam) == WORD(decrypt_id))
  {
   wchar_t filepath[256];
   GetWindowTextW(hWnd, filepath, (int)256);
   _wstat(filepath, &info4); 

   const long bytesize = info4.st_size;
   unsigned char *buffer = new unsigned char[bytesize];
   file = _wfopen(filepath, L"r");
   size_t readsize = fread(buffer, sizeof(char), info4.st_size , file);
   BOOL returnn = CryptAcquireContext(&hCryptProv, NULL, MS_ENHANCED_PROV,  PROV_RSA_FULL, 0);
   BOOL rvalue1 = CryptGenKey(hCryptProv, CALG_RC4, KEYLENGTH | CRYPT_EXPORTABLE, &hkey);
   DWORD val = GetLastError();
   DWORD datalength = info4.st_size;
   BOOL rvalue3 = CryptDecrypt(hkey, NULL, FALSE, NULL, buffer, &datalength);
   file2 = _wfopen(filepath, L"w");
   size_t writesize = fwrite(buffer, sizeof(char), sizeof(buffer), file2);
   free(buffer);
   CryptReleaseContext(hCryptProv, 0);
   CryptDestroyKey(hkey);
   if (rvalue3 == 0)
   {
    DWORD result = GetLastError();
    wchar_t dest[256] = L"Decryptor Failed To Decrypt File!";
    wcscat_s(dest, L"\n");
    wcscat_s(dest, L"Error Code: ");
    wchar_t code[256];
    swprintf_s(code, L"%d", result);
    wcscat_s(dest, code);
    wcscat_s(dest, L"\n");
    wcscat_s(dest, L"Error Code Information at: http://msdn.microsoft.com/en-us/library/ms681381(v=VS.85).aspx");
    MessageBoxW(hWnd, dest, L"Error", MB_ICONERROR | MB_OK);
    ShowWindow(encrypt_button, SW_HIDE);
   }
   else
   {
    MessageBox(hWnd, L"Successfully Decrypted The File!", L"", MB_OK | MB_ICONINFORMATION);
    ShowWindow(encrypt_button, SW_HIDE);
   }
  }
  if (LOWORD(wParam) == WORD(encrypt_id))
  {
   wchar_t filepath[256];
   GetWindowTextW(hWnd, filepath, (int)256);
   _wstat(filepath, &info4); 
   const long bytesize = info4.st_size;
   unsigned char *buffer = new unsigned char[bytesize];
   file = _wfopen(filepath, L"r");
   size_t readsize = fread(buffer, sizeof(char), info4.st_size , file);
   BOOL returnn = CryptAcquireContext(&hCryptProv, NULL, MS_ENHANCED_PROV,  PROV_RSA_FULL, 0);
   BOOL rvalue1 = CryptGenKey(hCryptProv, CALG_RC4, KEYLENGTH | CRYPT_EXPORTABLE, &hkey);
   DWORD val = GetLastError();
   DWORD datalength = info4.st_size;
   BOOL rvalue3 = CryptEncrypt(hkey, NULL, FALSE, NULL, buffer, &datalength, datalength);
   file2 = _wfopen(filepath, L"w");
   size_t writesize = fwrite(buffer, sizeof(char), sizeof(buffer), file2);
   free(buffer);
      CryptDestroyKey(hkey);
   CryptReleaseContext(hCryptProv, 0);
   if (rvalue3 == 0)
   {
    DWORD result = GetLastError();
    wchar_t dest[256] = L"Encryptor Failed To Encrypt File!";
    wcscat_s(dest, L"\n");
    wcscat_s(dest, L"Error Code: ");
    wchar_t code[256];
    swprintf_s(code, L"%d", result);
    wcscat_s(dest, code);
    wcscat_s(dest, L"\n");
    wcscat_s(dest, L"Error Code Information at: http://msdn.microsoft.com/en-us/library/ms681381(v=VS.85).aspx");
    MessageBoxW(hWnd, dest, L"Error", MB_ICONERROR | MB_OK);
    ShowWindow(encrypt_button, SW_HIDE);
   }
   else
   {
    MessageBox(hWnd, L"Successfully Encrypted The File!", L"", MB_OK | MB_ICONINFORMATION);
    ShowWindow(encrypt_button, SW_HIDE);
   }
  }

Answer 1

It looks like before encrypting or decrypting, you’re generating a random key with CryptGenKey. This means that you will use a different key for encryption and decryption, so your file will not decrypt correctly.

You will need to use the same key for encryption or decryption. Either by exporting and importing the key, or possibly by using CryptDeriveKey to derive the key from a shared password.