Empty WH_CALLWNDPROC hook crashes program right after WM_WINDOWPOSCHANGING message

Hi,

I'm trying to hook into firefox.exe to capture a few messages. Which seems to work fine (messages are received), however when I give the window focus (by clicking on it in the task bar) Firefox crashes. It always happens right after the hook receives the WM_WINDOWPOSCHANGING message.

My DLL code is:

Code:

FIREFOXDLL_API LRESULT GetMsgProc(
  int code,
  WPARAM wParam,
  LPARAM lParam
)
{
  return CallNextHookEx(0, code, wParam, lParam);
}

I've previously had some file writing code here, to output the message data, which is how I know it always crashes after the WM_WINDOWPOSCHANGING message.

The main program has the following function:

Code:

// Called using EnumWindows(enumMainWindows, processID);
// hkprcSysMsg and hinstDLL are correctly initialized
BOOL CALLBACK enumMainWindows(HWND hwnd, LPARAM lParam)
{
  DWORD pid;
  GetWindowThreadProcessId(hwnd, &pid);
  if (pid == lParam)
  {
    DWORD tid = GetWindowThreadProcessId(hwnd, NULL);
    SetWindowsHookEx( 
                          WH_CALLWNDPROC,
                          hkprcSysMsg,
                          hinstDLL,
                          tid); 
    return false;
  }
  return true;
}

If anyone could share any insight they might have regarding what can be causing this, it would be much appreciated.

Originally Posted by Waterbottle

Hi,

I'm trying to hook into firefox.exe to capture a few messages. Which seems to work fine (messages are received), however when I give the window focus (by clicking on it in the task bar) Firefox crashes. It always happens right after the hook receives the WM_WINDOWPOSCHANGING message.

My DLL code is:

Code:

FIREFOXDLL_API LRESULT GetMsgProc(
  int code,
  WPARAM wParam,
  LPARAM lParam
)
{
  return CallNextHookEx(0, code, wParam, lParam);
}

I've previously had some file writing code here, to output the message data, which is how I know it always crashes after the WM_WINDOWPOSCHANGING message.

The main program has the following function:

Code:

// Called using EnumWindows(enumMainWindows, processID);
// hkprcSysMsg and hinstDLL are correctly initialized
BOOL CALLBACK enumMainWindows(HWND hwnd, LPARAM lParam)
{
  DWORD pid;
  GetWindowThreadProcessId(hwnd, &pid);
  if (pid == lParam)
  {
    DWORD tid = GetWindowThreadProcessId(hwnd, NULL);
    SetWindowsHookEx( 
                          WH_CALLWNDPROC,
                          hkprcSysMsg,
                          hinstDLL,
                          tid); 
    return false;
  }
  return true;
}

If anyone could share any insight they might have regarding what can be causing this, it would be much appreciated.

Did you try moving your enumMainWindows function into the DLL itself...
From SetWindowsHookEx function (Windows)

Originally Posted by MSDN

The global hooks are a shared resource, and installing one affects all applications in the same desktop as the calling thread. All global hook functions must be in libraries. Global hooks should be restricted to special-purpose applications or to use as a development aid during application debugging. Libraries that no longer need a hook should remove its hook procedure.

Thank you for responding. I tried that now, same problem. Maybe I need to call the DLL function some other way? Sorry for my ignorance, not very familiar with the Windows API, or DLLs.

.EXE:

Code:

typedef int (__cdecl *MYPROC)(HINSTANCE); 

static MYPROC  hkprcSysMsg;

  hkprcSysMsg = (MYPROC)GetProcAddress(hinstDLL, "?mainfunc@@YAHPAUHINSTANCE__@@@Z"); 
  if (!hkprcSysMsg)
    return 2;

  (hkprcSysMsg)(hinstDLL);

DLL:

Code:

HINSTANCE self;

FIREFOXDLL_API int mainfunc(HINSTANCE inst)
{
  self = inst;
  int id = MYPROCESSID;
  EnumWindows(enumMainWindows, id);
  return 0;
}

FIREFOXDLL_API BOOL CALLBACK enumMainWindows(HWND hwnd, LPARAM lParam)
{
  DWORD pid;
  GetWindowThreadProcessId(hwnd, &pid);
  if (pid == lParam)
  {
    DWORD tid = GetWindowThreadProcessId(hwnd, NULL);
    SetWindowsHookEx( 
                          WH_CALLWNDPROC,
                          (HOOKPROC)GetMsgProc,
                          self,
                          tid); 
    return false;
  }
  return true;
}

FIREFOXDLL_API LRESULT GetMsgProc(
  int code,
  WPARAM wParam,
  LPARAM lParam
)
{
  return CallNextHookEx(0, code, wParam, lParam);
}

Originally Posted by Waterbottle

Code:

    SetWindowsHookEx( 
                          WH_CALLWNDPROC,
                          (HOOKPROC)GetMsgProc,
                          self,
                          tid);

Compilers chew you out for bad code for a reason, that cast is the equivalent of putting your hands over your ears and going "la la la, I'm not listening." This code is bad GetMsgProc has the wrong calling convention attributed to it. Like the enumMainWindows function, there needs to a CALLBACK inserted between the return type and the name of a function to fix it.

The standard rule is, if you have to cast a function pointer, the code is broken.

I don't know how much help this will be but this is a complete live DLL with a hook in it...
It sends you a windows message when an application starts or stops and allows you to retrieve the window handle and filename of a newly launched application.


Helper.c

Code:

#include "Helper.h"


/////////////////////////////////////////////////////////////////
// Shared data segment

#pragma data_seg("Hook")
HINSTANCE   hdll    = NULL;               // dll handle
HWND        HWind   = NULL;               // handle of RMServer window
HHOOK       RMHook  = NULL;               // message hook handle
LAUNCHINFO  LInfo   = {0};                // info about program launch
#pragma data_seg()
#pragma comment(linker,"/SECTION:Hook,S")

/////////////////////////////////////////////////////////////////////////////////////////
// Export
//

// return launch info to server
MMAPI VOID HelperGetInfo(pLAUNCHINFO Info)
  { memcpy(Info,&LInfo,sizeof(LInfo)); }



/////////////////////////////////////////////////////////////////////////////////////////
// gather information
//

// get information about launched program
VOID HookGetInfo(HWND wParm)
  { TCHAR   cmdl[MAX_PATH] = {0};   // command line
    TCHAR   pgm[MAX_PATH] = {0};    // program name
    PTCHAR  args;                   // arguments
    // purge old data
    memset(&LInfo,0,sizeof(LAUNCHINFO));
    // set program window handle
    LInfo.Handle = wParm;    
    // get program info
    GetModuleFileName(NULL,pgm,MAX_PATH);
    _wsplitpath(pgm,NULL,NULL,LInfo.Name,LInfo.Ext);
    // get launched file info
    wcsncpy(cmdl,GetCommandLine(),MAX_PATH);
    args = PathGetArgs(cmdl);
    if(wcslen(args) < 1)
      return;
    args = wcsrchr(args,L'.');
    if (args)
      { wcscpy(LInfo.Type,args);
        // fix trailing quote
        args = wcsrchr(LInfo.Type,L'\"');
        if (args)
          *args = 0; } }



/////////////////////////////////////////////////////////////////////////////////////////
// Handle Shell Hook 
//

// hook tosser
MMAPI LRESULT CALLBACK AppCatcher(INT Code, WPARAM wParm, LPARAM lParm)
  { switch (Code)
      { case HSHELL_WINDOWCREATED   :         // new window
          // gather launch info
          HookGetInfo((HWND) wParm);
          // send message to RMServer, wparam = Handle               
          PostMessage(HWind,UM_HELPERRUN,wParm,0); 
          break;
        case HSHELL_WINDOWDESTROYED :         // window closing
          // send messate to RMServer
          PostMessage(HWind,UM_HELPEREXIT,wParm,0);
          break; }
    return CallNextHookEx(RMHook,Code,wParm,lParm); } 



/////////////////////////////////////////////////////////////////
// Set the shell hook
// Returns hook status 1 = active, 0 = not
//

// apply the hook to launched programs
MMAPI BOOL WINAPI SetHook(HWND hWind)
  { HWind = hWind;   // handle of window to receive message
    RMHook = SetWindowsHookEx(WH_SHELL,&AppCatcher,hdll,0);
    return (RMHook != NULL); }      



// remove the hook from launched programs
MMAPI BOOL WINAPI FreeHook(void)
  { return UnhookWindowsHookEx(RMHook); }



/////////////////////////////////////////////////////////////////
//  DLL entry point
//
BOOL APIENTRY DllMain(HINSTANCE hinst, DWORD reason, LPVOID reserved)
  { if (reason == DLL_PROCESS_ATTACH)  // save instance handle
          hdll = hinst;
    return 1; }

the helper.h file...

Code:

#ifndef HELPER_H
#define HELPER_H

// for the compiler
#define UNICODE
#define _UNICODE
#define WIN32_LEAN_AND_MEAN
#define _WIN32_WINNT 0x0502
#define _X86_

// project global headers
#include <shlwapi.h>
#include <wchar.h>

#pragma lib "Helper.lib"

#define MMAPI __declspec(dllexport)

// messages sent by helper dll
#define UM_HELPERRUN  WM_APP + 444
#define UM_HELPEREXIT WM_APP + 445 

// launch information
#pragma pack(1)
typedef struct tLAUNCHINFO
  { TCHAR Name[MAX_PROGRAMNAME]; // short program name
    TCHAR Ext[MAX_TYPENAME];     // program type
    TCHAR Type[MAX_TYPENAME];    // file type
    HWND  Handle; }              // program window handle  
  LAUNCHINFO, *pLAUNCHINFO;
#pragma pack()



// retrieve the command line
MMAPI VOID HelperGetInfo(pLAUNCHINFO Info);

// set the hook
MMAPI BOOL WINAPI SetHook(HWND hWind);

// release the hook
MMAPI BOOL WINAPI FreeHook(void);


#endif // HELPER_H