Question

Trying to make a Fraps type program. See comment for where it fails.

#include "precompiled.h"

typedef IDirect3D9* (STDMETHODCALLTYPE* Direct3DCreate9_t)(UINT SDKVersion);
Direct3DCreate9_t RealDirect3DCreate9 = NULL;

typedef HRESULT (STDMETHODCALLTYPE* CreateDevice_t)(UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow,
    DWORD BehaviorFlags, D3DPRESENT_PARAMETERS* pPresentationParameters,
    IDirect3DDevice9** ppReturnedDeviceInterface);
CreateDevice_t RealD3D9CreateDevice = NULL;

HRESULT STDMETHODCALLTYPE HookedD3D9CreateDevice(UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow,
    DWORD BehaviorFlags, D3DPRESENT_PARAMETERS* pPresentationParameters,
    IDirect3DDevice9** ppReturnedDeviceInterface)
{
    // this call makes it jump to HookedDirect3DCreate9 and crashes. i'm doing something wrong
    HRESULT ret = RealD3D9CreateDevice(Adapter, DeviceType, hFocusWindow, BehaviorFlags,
        pPresentationParameters, ppReturnedDeviceInterface);

    return ret;
}

IDirect3D9* STDMETHODCALLTYPE HookedDirect3DCreate9(UINT SDKVersion)
{
    MessageBox(0, L"Creating d3d", L"", 0);

    IDirect3D9* d3d = RealDirect3DCreate9(SDKVersion);

    UINT_PTR* pVTable = (UINT_PTR*)(*((UINT_PTR*)d3d));
    RealD3D9CreateDevice = (CreateDevice_t)pVTable[16];

    DetourTransactionBegin();
    DetourUpdateThread(GetCurrentThread());
    DetourAttach(&(PVOID&)RealD3D9CreateDevice, HookedD3D9CreateDevice);
    if (DetourTransactionCommit() != ERROR_SUCCESS)
    {
        MessageBox(0, L"failed to create createdev hook", L"", 0);
    }

    return d3d;
}

bool APIENTRY DllMain(HINSTANCE hModule, DWORD fdwReason, LPVOID lpReserved)
{
    if (fdwReason == DLL_PROCESS_ATTACH)
    {
        MessageBox(0, L"", L"", 0);

        RealDirect3DCreate9 = (Direct3DCreate9_t)GetProcAddress(GetModuleHandle(L"d3d9.dll"), "Direct3DCreate9");

        DetourTransactionBegin();
        DetourUpdateThread(GetCurrentThread());
        DetourAttach(&(PVOID&)RealDirect3DCreate9, HookedDirect3DCreate9);
        DetourTransactionCommit();
    }

    // TODO detach hooks

    return true;
}

Answer 1

The signature for the C interface of IDirect3D9::CreateDevice is:

STDMETHOD(CreateDevice)(
    THIS_ 
    UINT Adapter,D3DDEVTYPE DeviceType,HWND hFocusWindow,
    DWORD BehaviorFlags,D3DPRESENT_PARAMETERS* pPresentationParameters,
    IDirect3DDevice9** ppReturnedDeviceInterface) PURE;

Which expands to:

typedef HRESULT (STDMETHODCALLTYPE* CreateDevice_t)(
    IDirect3D9 FAR *This, // you forgot this.
    UINT Adapter, D3DDEVTYPE DeviceType, HWND hFocusWindow, 
    DWORD BehaviorFlags, D3DPRESENT_PARAMETERS* pPresentationParameters, 
    IDirect3DDevice9** ppReturnedDeviceInterface); 

In other words, you declared the thunk for CreateDevice incorrectly.

Also, instead of directly indexing into the IDirect3D9 vtable, you might just want to #define CINTERFACE and access the function you want to override through d3d->lpVtbl->CreateDevice.