Question

Summary

I am creating a simple application that allows the user to select a process that contains a top-level window. The user first types the path of a native DLL (not a managed DLL). Then the user types the name of the method that will be called inside the hook procedure. The method must not return a value and must be parameter-less. Then their is a button that does the hooking.

The Problem

I am able to retrieve the module handle of the library that does the hooking. In addition, I am also able to get the module handle library the user wants to use that contains the method that he/she wants to hook. Plus, I am able to receive the procedures address both the method that user wants to hook and etc.

In other words their is not on invalid handle being returned. Other than the handle to the hook (HHOOK)

But SetWindowsHookEx returns a NULL HHOOK and GetLastError returns error code 126 (ERROR_NO_MOD_FOUND).

Possible Theories On Why I Get ERROR_MOD_NOT_FOUND

• There is a limitation on the number of global hooks because I read somewhere with someone who has had that same error when hooking.
• I am not getting the correct module handle of the DLL that will do the hooking. But then again I have tried many different method to get the libraries HMODULE/HINSTANCE but all fail with the same error.

WinHooker.cpp

#include "winhooker.h"
HMODULE GetCurrentModule()
{ 
  HMODULE hModule = NULL;
  GetModuleHandleEx(
    GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS,
    (LPCTSTR)GetCurrentModule,
    &hModule);

  return hModule;
}
LRESULT (WINAPI hookProc)(int nCode, WPARAM wParam, LPARAM lParam);
void recievedCallback(WIN32Hooker* hooker);
WIN32Hooker* current;

            WIN32Hooker::WIN32Hooker(HINSTANCE* hInstance, Callback callback)
            {
                if (!hInstance)
                {
                    HMODULE hModule = GetCurrentModule();
                    hInstance = &hModule;
                }
                this->callback = callback;
                this->hInstance = hInstance;
                return;
            }
            void WIN32Hooker::Hook(DWORD threadToHook)
            {
                recievedCallback(this);
                this->hhk = SetWindowsHookEx(WH_CALLWNDPROC, hookProc, *this->hInstance, threadToHook);
                DWORD errorCode = GetLastError(); // 126 ERROR_MOD_NOT_FOUND
                return;
            }
            void WIN32Hooker::NextHook(int nCode, WPARAM wParam, LPARAM lParam)
            {
                callback();
                CallNextHookEx(this->hhk, nCode, wParam, lParam);
                return;
            }
            void WIN32Hooker::Free()
            {
                UnhookWindowsHookEx(this->hhk);
                return;
            }



LRESULT (WINAPI hookProc)(int nCode, WPARAM wParam, LPARAM lParam)
{
    current->NextHook(nCode, wParam, lParam);
    return 0;
}
void recievedCallback(WIN32Hooker* hooker)
{
    current = hooker;
}
extern "C" WIN32Hooker* hookerMalloc(HINSTANCE* hInstance, Callback callback)
{
    return new WIN32Hooker(hInstance, callback);
}

Test.cpp

#include <Windows.h>
extern "C" void sendMessage(void)
{
    MessageBox(NULL, L"Test", L"Test", MB_ICONINFORMATION);
}

Main Code

// Window Hooker Bytes.cpp : Defines the entry point for the application.
//

#include "stdafx.h"
#include "Window Hooker Bytes.h"
#include "processes.h"
#include "button.h"
#include "edit.h"
#include "listbox.h"
#include "/Users/FatalisProgrammer/Documents/Visual Studio 2010/Projects/Window Hooker Bytes/WindowHookerLib/winhooker.h"
#define MAX_LOADSTRING 100
using namespace Processes;
using namespace Controls;
void Delete(); // Delete proto-type
void windowListCallback(map<HWND, wstring>* list); // Callback proto-type
// Global Variables:
HINSTANCE hInst;                                // current instance
TCHAR szTitle[MAX_LOADSTRING];                  // The title bar text
TCHAR szWindowClass[MAX_LOADSTRING];            // the main window class name
Button* hookButton;
Edit* dllPathEdit;
Edit* methodNameEdit;
ListBox* windowList;
ProcessEnumerator* processEnumerator;
map<HWND, wstring> mapList;
// Forward declarations of functions included in this code module:
ATOM                MyRegisterClass(HINSTANCE hInstance);
BOOL                InitInstance(HINSTANCE, int);
LRESULT CALLBACK    WndProc(HWND, UINT, WPARAM, LPARAM);
INT_PTR CALLBACK    About(HWND, UINT, WPARAM, LPARAM);

int APIENTRY _tWinMain(HINSTANCE hInstance,
                     HINSTANCE hPrevInstance,
                     LPTSTR    lpCmdLine,
                     int       nCmdShow)
{
    UNREFERENCED_PARAMETER(hPrevInstance);
    UNREFERENCED_PARAMETER(lpCmdLine);

    // TODO: Place code here.
    MSG msg;
    HACCEL hAccelTable;

    // Initialize global strings
    LoadString(hInstance, IDS_APP_TITLE, szTitle, MAX_LOADSTRING);
    LoadString(hInstance, IDC_WINDOWHOOKERBYTES, szWindowClass, MAX_LOADSTRING);
    MyRegisterClass(hInstance);

    // Perform application initialization:
    if (!InitInstance (hInstance, nCmdShow))
    {

        return FALSE;
    }

    hAccelTable = LoadAccelerators(hInstance, MAKEINTRESOURCE(IDC_WINDOWHOOKERBYTES));

    // Main message loop:
    while (GetMessage(&msg, NULL, 0, 0))
    {
        if (!TranslateAccelerator(msg.hwnd, hAccelTable, &msg))
        {
            TranslateMessage(&msg);
            DispatchMessage(&msg);
        }
    }

    return (int) msg.wParam;
}



//
//  FUNCTION: MyRegisterClass()
//
//  PURPOSE: Registers the window class.
//
//  COMMENTS:
//
//    This function and its usage are only necessary if you want this code
//    to be compatible with Win32 systems prior to the 'RegisterClassEx'
//    function that was added to Windows 95. It is important to call this function
//    so that the application will get 'well formed' small icons associated
//    with it.
//
ATOM MyRegisterClass(HINSTANCE hInstance)
{
    WNDCLASSEX wcex;

    wcex.cbSize = sizeof(WNDCLASSEX);

    wcex.style          = CS_HREDRAW | CS_VREDRAW;
    wcex.lpfnWndProc    = WndProc;
    wcex.cbClsExtra     = 0;
    wcex.cbWndExtra     = 0;
    wcex.hInstance      = hInstance;
    wcex.hIcon          = LoadIcon(hInstance, MAKEINTRESOURCE(IDI_WINDOWHOOKERBYTES));
    wcex.hCursor        = LoadCursor(NULL, IDC_ARROW);
    wcex.hbrBackground  = (HBRUSH)(COLOR_WINDOW+1);
    wcex.lpszMenuName   = MAKEINTRESOURCE(IDC_WINDOWHOOKERBYTES);
    wcex.lpszClassName  = szWindowClass;
    wcex.hIconSm        = LoadIcon(wcex.hInstance, MAKEINTRESOURCE(IDI_SMALL));

    return RegisterClassEx(&wcex);
}

//
//   FUNCTION: InitInstance(HINSTANCE, int)
//
//   PURPOSE: Saves instance handle and creates main window
//
//   COMMENTS:
//
//        In this function, we save the instance handle in a global variable and
//        create and display the main program window.
//
BOOL InitInstance(HINSTANCE hInstance, int nCmdShow)
{
   HWND hWnd;

   hInst = hInstance; // Store instance handle in our global variable

   hWnd = CreateWindow(szWindowClass, szTitle, WS_MINIMIZEBOX | WS_CAPTION | WS_SYSMENU, CW_USEDEFAULT, 0, 800 / 2, 480 / 2, NULL, NULL, hInstance, NULL);
   hookButton = new Button(hWnd, 120, L"Hook Method From DLL!", 5, (480 / 2) - 24 - 24 - 24 - 6, (800 / 2) - 15, 24);
   hookButton->show(hInst);
   dllPathEdit = new Edit(hWnd, 169, 5, 5, (800 / 2) - 15, 24);
   dllPathEdit->show(hInst);
   dllPathEdit->setWatermarkText(L"Enter Path Of A Native DLL.");
   methodNameEdit = new Edit(hWnd, 256, 5, (5 * 2) + 24, (800 / 2) - 15, 24);
   methodNameEdit->show(hInst);
   methodNameEdit->setWatermarkText(L"Enter Method (Must Return Void And Be Parameterless)");
   methodNameEdit->setFont(L"Times", 16);
   dllPathEdit->setFont(L"Times", 16);
   hookButton->setFont(L"Times", 16);
   windowList = new ListBox(hWnd, 333, 5, (5 * 8) + 24, (800 / 2) - 15, (124 / 2) + 24);
   windowList->show(hInst);
   windowList->setFont(L"Times", 16);
   hookButton->setUACShield();
   if (!hWnd)
   {
      return FALSE;
   }
   processEnumerator = new ProcessEnumerator();
   processEnumerator->enumerateProcesses(windowListCallback);
   ShowWindow(hWnd, nCmdShow);
   UpdateWindow(hWnd);

   return TRUE;
}

//
//  FUNCTION: WndProc(HWND, UINT, WPARAM, LPARAM)
//
//  PURPOSE:  Processes messages for the main window.
//
//  WM_COMMAND  - process the application menu
//  WM_PAINT    - Paint the main window
//  WM_DESTROY  - post a quit message and return
//
//
LRESULT CALLBACK WndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    int wmId, wmEvent;
    PAINTSTRUCT ps;
    HDC hdc;

    switch (message)
    {
    case WM_COMMAND:
        wmId    = LOWORD(wParam);
        wmEvent = HIWORD(wParam);

        if (wmId == hookButton->getID())
        {
            HMODULE libraryModule = LoadLibrary(L"WindowHookerLib.dll");
            typedef WIN32Hooker* (*HookerMalloc)(HINSTANCE* hInstance, Callback callback);
            HookerMalloc hookerMalloc =  (HookerMalloc)GetProcAddress(libraryModule, "hookerMalloc");
            DWORD errorCode = GetLastError();
            HMODULE libraryToHookModule = LoadLibrary(dllPathEdit->getText());
            if (!libraryToHookModule || !libraryModule)
            {
                MessageBox(hWnd, L"Error: Library That Contains The Method To Hook Or The WindowHookerLib.dll Does Not Exist!\nMake Sure WindowHookerLib.dll Is In The Current Working Directory!", L"Error In Application!", MB_ICONERROR);
            }
            else
            {
                typedef void (__stdcall *Method)(void);
                char* methodName = new char[wcslen(methodNameEdit->getText()) * 2];
                wcstombs(methodName, methodNameEdit->getText(), wcslen(methodNameEdit->getText()) * 2);
                Method method = (Method)GetProcAddress(libraryToHookModule, methodName);
                if (!hookerMalloc || !method)
                {
                    MessageBox(hWnd, L"Error: The Method To Hook Does Not Exist Or The Method To Initiate The Hooker Is Not Available!", L"Error In Application", MB_ICONERROR);
                }
                else
                {
                    WIN32Hooker* hooker = hookerMalloc(NULL, method);
                    DWORD pID;
                    int selectedItemIndex = windowList->getSelectedIndex();
                    vector<HWND> v;
                    for(map<HWND,wstring>::iterator it = mapList.begin(); it != mapList.end(); ++it) 
                    {
                        v.push_back(it->first);
                    }
                    GetWindowThreadProcessId(v[selectedItemIndex], &pID);
                    if (pID >= 0)
                    {

                        hooker->Hook(pID);
                    }
                    else
                    {
                        MessageBox(hWnd, L"Error Could Not Retrieve Process ID!", L"Error In Application", MB_ICONERROR);
                    }

                }
                delete methodName;
            }
        }

        // Parse the menu selections:
        switch (wmId)
        {

        case IDM_ABOUT:
            DialogBox(hInst, MAKEINTRESOURCE(IDD_ABOUTBOX), hWnd, About);
            break;
        case IDM_EXIT:
            DestroyWindow(hWnd);
            break;
        default:
            return DefWindowProc(hWnd, message, wParam, lParam);
        }
        break;
    case WM_PAINT:
        hdc = BeginPaint(hWnd, &ps);
        // TODO: Add any drawing code here...
        EndPaint(hWnd, &ps);
        break;
    case WM_DESTROY:
        Delete();
        PostQuitMessage(0);
        break;
    default:
        return DefWindowProc(hWnd, message, wParam, lParam);
    }
    return 0;
}

// Message handler for about box.
INT_PTR CALLBACK About(HWND hDlg, UINT message, WPARAM wParam, LPARAM lParam)
{
    UNREFERENCED_PARAMETER(lParam);
    switch (message)
    {
    case WM_INITDIALOG:
        return (INT_PTR)TRUE;

    case WM_COMMAND:
        if (LOWORD(wParam) == IDOK || LOWORD(wParam) == IDCANCEL)
        {
            EndDialog(hDlg, LOWORD(wParam));
            return (INT_PTR)TRUE;
        }
        break;
    }
    return (INT_PTR)FALSE;
}

void Delete()
{
    delete dllPathEdit;
    delete hookButton;
    delete methodNameEdit;
    delete windowList;
    processEnumerator->free();
    delete processEnumerator;
    return;
}
void windowListCallback(map<HWND, wstring>* list)
{
    mapList = *list;
    vector<wstring> v;
    for(map<HWND,wstring>::iterator it = mapList.begin(); it != mapList.end(); ++it) 
    {
        v.push_back(it->second);
    }
    for each(wstring item in v)
    {
        windowList->addItem(item);
    }
    return;
}

Conclusion

Any help would be appreciated. If I am doing something wrong, feel free to point it out. This project is for learning purposes, so tell me what needs to be fixed, or any tips would be great.

Answer 1

In this code:

HMODULE hModule = GetCurrentModule();
hInstance = &hModule;

you are assigning the address of a local variable to hInstance. When this is dereferenced in the call to SetWindowsHookEx you’ll get a bogus value.

Throughout the code, don’t use a pointer to an HINSTANCE, just use a plain HINSTANCE.