Question

I want to leverage machine learning to model a user’s intent and potentially automate commonly performed tasks. To do this I would like to have access to a fire-hose of information about user actions and the machine state. To this end, it is my current thinking that getting access to the stream of windows messages is probably the way forward.

I would like to have as much information as is possible, filtering the information to that which is pertinent I would like to leave to the machine learning tool.

How would this be accomplished? (Preferably in C#).

Please assume that I know how to manage and use this large influx of data.

Any help would be gratefully appreciated.

Answer 1

You can use SetWindowsHookEx to set low level hooks to catch (specific) windows messages.
Specifically these hook-ids might be interesting for monitoring:

WH_CALLWNDPROC (4) Installs a hook procedure that monitors messages
before the system sends them to the destination window procedure. For
more information, see the CallWndProc hook procedure.

WH_CALLWNDPROCRET(12) Installs a hook procedure that monitors
messages after they have been processed by the destination window
procedure. For more information, see the CallWndRetProc hook
procedure.

It’s been a while since I’ve implemented it, but as an example I’ve posted the base class I use to hook specific messages. (For example, I’ve used it in a global mousewheel trapper, that makes sure my winforms apps behave the same as internet explorer: scroll the control underneath the cursor, instead of the active control).

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Runtime.InteropServices;
using Subro.Win32;

namespace Subro
{
    /// <summary>
    /// Base class to relatively safely register global windows hooks
    /// </summary>
    public abstract class GlobalHookTrapper : FinalizerBase
    {
        [DllImport("user32", EntryPoint = "SetWindowsHookExA")]
        static extern IntPtr SetWindowsHookEx(int idHook, Delegate lpfn, IntPtr hmod, IntPtr dwThreadId);

        [DllImport("user32", EntryPoint = "UnhookWindowsHookEx")]
        private static extern int UnhookWindowsHookEx(IntPtr hHook);

        [DllImport("user32", EntryPoint = "CallNextHookEx")]
        static extern int CallNextHook(IntPtr hHook, int ncode, IntPtr wParam, IntPtr lParam);

        [DllImport("kernel32.dll")]
        static extern IntPtr GetCurrentThreadId();

        IntPtr hook;
        public readonly int HookId;
        public readonly GlobalHookTypes HookType;

        public GlobalHookTrapper(GlobalHookTypes Type):this(Type,false)
        {
        }

        public GlobalHookTrapper(GlobalHookTypes Type, bool OnThread)
        {
            this.HookType = Type;
            this.HookId = (int)Type;
            del = ProcessMessage;
            if (OnThread)
                hook = SetWindowsHookEx(HookId, del, IntPtr.Zero, GetCurrentThreadId());
            else
            {
                var hmod = IntPtr.Zero; // Marshal.GetHINSTANCE(GetType().Module);
                hook = SetWindowsHookEx(HookId, del, hmod, IntPtr.Zero);
            }

            if (hook == IntPtr.Zero)
            {
                int err = Marshal.GetLastWin32Error();
                if (err != 0)
                    OnHookFailed(err);
            }
        }

        protected virtual void OnHookFailed(int Error)
        {
            throw Win32Functions.TranslateError(Error);
        }

        private const int HC_ACTION = 0;

        [MarshalAs(UnmanagedType.FunctionPtr)]
        private MessageDelegate del;

        private delegate int MessageDelegate(int code, IntPtr wparam, IntPtr lparam);

        private int ProcessMessage(int hookcode, IntPtr wparam, IntPtr lparam)
        {
            if (HC_ACTION == hookcode)
            {
                try
                {
                    if (Handle(wparam, lparam)) return 1;
                }
                catch { }
            }
            return CallNextHook(hook, hookcode, wparam, lparam);
        }

        protected abstract bool Handle(IntPtr wparam, IntPtr lparam);



        protected override sealed void OnDispose()
        {
            UnhookWindowsHookEx(hook);
            AfterDispose();
        }

        protected virtual void AfterDispose()
        {
        }

    }

    public enum GlobalHookTypes
    {
        BeforeWindow = 4, //WH_CALLWNDPROC 
        AfterWindow = 12, //WH_CALLWNDPROCRET 
        KeyBoard = 2, //WH_KEYBOARD
        KeyBoard_Global = 13,  //WH_KEYBOARD_LL
        Mouse = 7, //WH_MOUSE
        Mouse_Global = 14, //WH_MOUSE_LL
        JournalRecord = 0, //WH_JOURNALRECORD
        JournalPlayback = 1, //WH_JOURNALPLAYBACK
        ForeGroundIdle = 11, //WH_FOREGROUNDIDLE
        SystemMessages = 6, //WH_SYSMSGFILTER
        MessageQueue = 3, //WH_GETMESSAGE
        ComputerBasedTraining = 5, //WH_CBT 
        Hardware = 8, //WH_HARDWARE 
        Debug = 9, //WH_DEBUG 
        Shell = 10, //WH_SHELL
    }

    public abstract class FinalizerBase : IDisposable
    {
        protected readonly AppDomain domain;
        public FinalizerBase()
        {
            System.Windows.Forms.Application.ApplicationExit += new EventHandler(Application_ApplicationExit);
            domain = AppDomain.CurrentDomain;
            domain.ProcessExit += new EventHandler(CurrentDomain_ProcessExit);
            domain.DomainUnload += new EventHandler(domain_DomainUnload);            
        }

        private bool disposed;
        public bool IsDisposed{get{return disposed;}}
        public void Dispose()
        {
            if (!disposed)
            {
                GC.SuppressFinalize(this);
                if (domain != null)
                {
                    domain.ProcessExit -= new EventHandler(CurrentDomain_ProcessExit);
                    domain.DomainUnload -= new EventHandler(domain_DomainUnload);
                    System.Windows.Forms.Application.ApplicationExit -= new EventHandler(Application_ApplicationExit);
                }
                disposed = true;
                OnDispose();
            }
        }

        void Application_ApplicationExit(object sender, EventArgs e)
        {
            Dispose();
        }

        void domain_DomainUnload(object sender, EventArgs e)
        {
            Dispose();
        }

        void CurrentDomain_ProcessExit(object sender, EventArgs e)
        {
            Dispose();
        }

        protected abstract void OnDispose();
                /// Destructor
        ~FinalizerBase()
        {
            Dispose();
        }
    }


}