Question

Is there a way to have a desktop vb application take control of an already open browser window? For example, have it mouse click certain coordinates in the window or check if the window contains certain elements.

I’ve looked at using Microsoft Internet Controls(shdocvw) and MSHTML(IHTMLDocument2) but I am struggling on how to access elements of the browser window (e.g. body.innnerHTML).

Answer 1

Add References to Microsoft HTML Object Library and Microsoft Internet Controls

Option Explicit On

Public Class Form1

   'Functions used to set cursor location and mouse clicks
   Public Declare Auto Function SetCursorPos Lib "User32.dll" (ByVal X As Integer, ByVal Y As Integer) As Long
   Public Declare Sub mouse_event Lib "user32" Alias "mouse_event" (ByVal dwFlags As Long, ByVal dx As Long, ByVal dy As Long, ByVal cButtons As Long, ByVal dwExtraInfo As Long)
   Public Declare Auto Function ShowWindow Lib "user32" (ByVal _
   hwnd As Long, ByVal nCmdShow As Long) As Long

   'Constants used for cursor and mouse functions, and to maximize window
   Public Const SW_MAXIMIZE = 3
   Public Const MOUSEEVENTF_LEFTDOWN = &H2 ' left button down
   Public Const MOUSEEVENTF_LEFTUP = &H4 ' left button up
   Public Const MOUSEEVENTF_RIGHTDOWN = &H8 ' right button down
   Public Const MOUSEEVENTF_RIGHTUP = &H10 ' right button up    

   Private Sub Form1_Load(ByVal sender As System.Object, ByVal e As System.EventArgs) Handles MyBase.Load
      'retrieve all the open instances of IE
      Dim shellWindows = New SHDocVw.ShellWindowsClass()
      Dim htmlInput As mshtml.HTMLInputElement
      'for each instance of IE
      For Each ie As SHDocVw.InternetExplorer In shellWindows
         'If the title of the IE window matches the designated title
         If ie.Document.title = "Page Title" Then
            'retrieve the control with the designated field id
            htmlInput = ie.Document.getElementById("fieldID")
            'if the control's inner html matches the designated text
            If htmlInput.innerHTML = "innerHTML" Then
               'show the IE window maximized and with focus
               ShowWindow(ie.HWND, SW_MAXIMIZE)
               'move the cursor to the designated x,y coordinates
               SetCursorPos(xCoord, yCoord)
               'left mouse click down and up
               mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0)
               mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0)
               'send the designated keyboard command
               My.Computer.Keyboard.SendKeys("keyboardCommand")
            End If
         End If
      Next
   End Sub
End Class