Merge pull request #17 from verisoft-ai/feat/multi-monitor

y-schwab · web-flow · commit 810fc4aaf196 · 2026-03-23T11:13:41.000-03:00
feat(display): add support for multi monitor  testing
diff --git a/README.md b/README.md
@@ -594,6 +594,45 @@ button | string | no | Mouse button: `left` (default), `middle`, `right`, `back`
 
 \* Provide either startElementId or both startX and startY; and either endElementId or both endX and endY.
 
+### windows: getMonitors
+
+Returns information about all connected display monitors, including their screen coordinates in the [virtual screen](https://learn.microsoft.com/en-us/windows/win32/gdi/the-virtual-screen) coordinate space, working area, device name, and which monitor is the primary display.
+
+This command takes no arguments.
+
+#### Returns
+
+An array of monitor objects, one per connected display:
+
+Name | Type | Description
+--- | --- | ---
+index | number | Zero-based index of the monitor in the `AllScreens` array.
+deviceName | string | System device name, e.g. `\\.\DISPLAY1`.
+primary | boolean | `true` if this is the primary display.
+bounds | object | Full monitor rectangle: `{ x, y, width, height }` in virtual screen coordinates.
+workingArea | object | Usable area excluding taskbars and docked toolbars: `{ x, y, width, height }`.
+
+#### Example
+
+```javascript
+// WebdriverIO — move app window to the secondary monitor
+const monitors = await driver.executeScript('windows: getMonitors', []);
+const secondary = monitors.find(m => !m.primary);
+if (secondary) {
+    await driver.setWindowRect(secondary.bounds.x, secondary.bounds.y, null, null);
+}
+```
+
+```python
+# Python — click at the center of the secondary monitor
+monitors = driver.execute_script('windows: getMonitors', {})
+secondary = next((m for m in monitors if not m['primary']), None)
+if secondary:
+    cx = secondary['bounds']['x'] + secondary['bounds']['width'] // 2
+    cy = secondary['bounds']['y'] + secondary['bounds']['height'] // 2
+    driver.execute_script('windows: click', {'x': cx, 'y': cy})
+```
+
 ## Development
 
 it is recommended to use Matt Bierner's [Comment tagged templates](https://marketplace.visualstudio.com/items?itemName=bierner.comment-tagged-templates)
diff --git a/lib/commands/extension.ts b/lib/commands/extension.ts
@@ -67,6 +67,7 @@ const EXTENSION_COMMANDS = Object.freeze({
     clickAndDrag: 'executeClickAndDrag',
     getDeviceTime: 'windowsGetDeviceTime',
     getWindowElement: 'getWindowElement',
+    getMonitors: 'windowsGetMonitors',
 } as const);
 
 const ContentType = Object.freeze({
@@ -892,3 +893,23 @@ export async function getWindowElement(this: NovaWindowsDriver): Promise<Element
     }
     return { [W3C_ELEMENT_KEY]: elementId };
 }
+
+const GET_MONITORS_COMMAND = pwsh /* ps1 */ `
+    Add-Type -AssemblyName System.Windows.Forms
+    $index = 0
+    $monitors = @([System.Windows.Forms.Screen]::AllScreens | ForEach-Object {
+        [PSCustomObject]@{
+            index       = $index++
+            deviceName  = $_.DeviceName
+            primary     = $_.Primary
+            bounds      = @{ x = $_.Bounds.X; y = $_.Bounds.Y; width = $_.Bounds.Width; height = $_.Bounds.Height }
+            workingArea = @{ x = $_.WorkingArea.X; y = $_.WorkingArea.Y; width = $_.WorkingArea.Width; height = $_.WorkingArea.Height }
+        }
+    })
+    ConvertTo-Json -InputObject $monitors -Compress
+`;
+
+export async function windowsGetMonitors(this: NovaWindowsDriver): Promise<object[]> {
+    const result = await this.sendPowerShellCommand(GET_MONITORS_COMMAND);
+    return JSON.parse(result.trim());
+}
diff --git a/lib/mcp/tools/window.ts b/lib/mcp/tools/window.ts
@@ -167,4 +167,21 @@ export function registerWindowTools(server: McpServer, session: AppiumSession):
             }
         }
     );
+
+    server.registerTool(
+        'get_monitors',
+        {
+            description: 'List all connected monitors with their bounds, working area, device name, and whether each is the primary display.',
+            annotations: { readOnlyHint: true },
+        },
+        async () => {
+            try {
+                const driver = session.getDriver();
+                const monitors = await driver.executeScript('windows: getMonitors', []);
+                return { content: [{ type: 'text' as const, text: JSON.stringify(monitors, null, 2) }] };
+            } catch (err) {
+                return { isError: true, content: [{ type: 'text' as const, text: formatError(err) }] };
+            }
+        }
+    );
 }
diff --git a/lib/winapi/user32.ts b/lib/winapi/user32.ts
@@ -429,11 +429,9 @@ function makeMouseMoveEvents(args: {
         wheel: boolean,
         /** Set to true if the event is a mouse move with relative coordinates. This argument is ignored for mouse wheel move. */
         relative?: boolean,
-        /** Set to screen resolution [width, height] when the mouse move is absolute. */
-        screenResolutionAndRefreshRate?: ReturnType<typeof getScreenResolutionAndRefreshRate>;
     }
 ): MouseEvent[] {
-    const { x, y, wheel, relative, screenResolutionAndRefreshRate} = args;
+    const { x, y, wheel, relative } = args;
 
     if (wheel) {
         const mouseEvents: MouseEvent[] = [];
@@ -457,18 +455,18 @@ function makeMouseMoveEvents(args: {
 
     const mouseEvent: MouseEvent = makeEmptyMouseEvent();
 
-    if (!screenResolutionAndRefreshRate) {
-        throw new errors.InvalidArgumentError('screenResolution parameter must be set for absolute mouse move.');
-    }
-
-    const [screenWidth, screenHeight] = screenResolutionAndRefreshRate;
-
-    mouseEvent.u.mi.dx = relative ? Math.trunc(x) : Math.trunc((x * UINT16_MAX) / screenWidth);
-    mouseEvent.u.mi.dy = relative ? Math.trunc(y) : Math.trunc((y * UINT16_MAX) / screenHeight);
-    mouseEvent.u.mi.dwFlags = MouseEventFlags.MOUSEEVENTF_MOVE;
-
-    if (!relative) {
-        mouseEvent.u.mi.dwFlags |= MouseEventFlags.MOUSEEVENTF_ABSOLUTE;
+    if (relative) {
+        mouseEvent.u.mi.dx = Math.trunc(x);
+        mouseEvent.u.mi.dy = Math.trunc(y);
+        mouseEvent.u.mi.dwFlags = MouseEventFlags.MOUSEEVENTF_MOVE;
+    } else {
+        const virt = getVirtualScreenBounds();
+        mouseEvent.u.mi.dx = Math.trunc(((x - virt.left) * UINT16_MAX) / virt.width);
+        mouseEvent.u.mi.dy = Math.trunc(((y - virt.top) * UINT16_MAX) / virt.height);
+        mouseEvent.u.mi.dwFlags =
+            MouseEventFlags.MOUSEEVENTF_MOVE |
+            MouseEventFlags.MOUSEEVENTF_ABSOLUTE |
+            MouseEventFlags.MOUSEEVENTF_VIRTUALDESK;
     }
 
     return [mouseEvent];
@@ -651,8 +649,7 @@ function sendMouseButtonInput(button: number, down: boolean) {
 async function sendMouseMoveInput(args: { x: number, y: number, relative: boolean, duration: number, easingFunction?: string }): Promise<void> {
     const { duration } = args;
     let { x, y, easingFunction, relative } = args;
-    const screenResolutionAndRefreshRate = getScreenResolutionAndRefreshRate();
-    const [, , refreshRate] = screenResolutionAndRefreshRate;
+    const refreshRate = getRefreshRate();
     const updateInterval = 1000 / refreshRate;
     const iterations = Math.max(Math.floor(duration / updateInterval), 1);
 
@@ -695,14 +692,14 @@ async function sendMouseMoveInput(args: { x: number, y: number, relative: boolea
                 const interpolatedX = cursorPosition.x + (x - cursorPosition.x) * easedProgress;
                 const interpolatedY = cursorPosition.y + (y - cursorPosition.y) * easedProgress;
 
-                const events = makeMouseMoveEvents({ x: interpolatedX, y: interpolatedY, wheel: false, screenResolutionAndRefreshRate });
+                const events = makeMouseMoveEvents({ x: interpolatedX, y: interpolatedY, wheel: false });
                 const returnCode = SendInput(events.length, events, sizeof(INPUT));
 
                 assertSuccessSendInputReturnCode(returnCode);
             }, i * updateInterval);
         }
     } else {
-        const events = makeMouseMoveEvents({ x, y, wheel: false, screenResolutionAndRefreshRate });
+        const events = makeMouseMoveEvents({ x, y, wheel: false });
         const returnCode = SendInput(events.length, events, sizeof(INPUT));
 
         assertSuccessSendInputReturnCode(returnCode);
@@ -736,33 +733,56 @@ function getResolutionScalingFactor(): number {
     return scalingFactor;
 }
 
-function getScreenResolutionAndRefreshRate(): [number, number, number] {
-    const width = GetSystemMetrics(SystemMetric.SM_CXSCREEN);
-    const height = GetSystemMetrics(SystemMetric.SM_CYSCREEN);
-    let refreshRate: number | null = null;
-
+function getRefreshRate(): number {
     const buffer = Buffer.alloc(sizeof(DEVMODEA));
     EnumDisplaySettingsA(null, -1, buffer);
-    const deviceMode = { dmDisplayFrequency: buffer.readUInt32LE(120) } as DeviceModeAnsi;
-    refreshRate = deviceMode.dmDisplayFrequency;
+    const refreshRate = (buffer.readUInt32LE(120) as DeviceModeAnsi['dmDisplayFrequency']);
 
-    const resolution = [width, height, refreshRate] satisfies ReturnType<typeof getScreenResolutionAndRefreshRate>;
+    const nonMemoizedMethod = getRefreshRate;
+    const currentTime = new Date().getTime();
 
-    const nonMemoizedMethod = getScreenResolutionAndRefreshRate;
+    // @ts-expect-error memoizing the function to prevent repeated calls that might crash Node.js
+    getRefreshRate = () => {
+        if (new Date().getTime() - currentTime > 1000) {
+            // @ts-expect-error reset memoization after 1 second
+            getRefreshRate = nonMemoizedMethod;
+        }
+        return refreshRate;
+    };
+
+    return refreshRate;
+}
+
+function getScreenResolution(): [number, number] {
+    const width = GetSystemMetrics(SystemMetric.SM_CXSCREEN);
+    const height = GetSystemMetrics(SystemMetric.SM_CYSCREEN);
+
+    const resolution = [width, height] satisfies ReturnType<typeof getScreenResolution>;
+
+    const nonMemoizedMethod = getScreenResolution;
     const currentTime = new Date().getTime();
 
     // @ts-expect-error memoizing the function to prevent repeated calls that might crash Node.js
-    getScreenResolutionAndRefreshRate = () => {
+    getScreenResolution = () => {
         if (new Date().getTime() - currentTime > 1000) {
             // @ts-expect-error reset memoization after 1 second
-            getScreenResolutionAndRefreshRate = nonMemoizedMethod;
+            getScreenResolution = nonMemoizedMethod;
         }
         return resolution;
     };
 
     return resolution;
 }
 
+export function getVirtualScreenBounds(): { left: number; top: number; width: number; height: number } {
+    return {
+        left: GetSystemMetrics(SystemMetric.SM_XVIRTUALSCREEN),
+        top: GetSystemMetrics(SystemMetric.SM_YVIRTUALSCREEN),
+        width: GetSystemMetrics(SystemMetric.SM_CXVIRTUALSCREEN),
+        height: GetSystemMetrics(SystemMetric.SM_CYVIRTUALSCREEN),
+    };
+}
+
 export function keyDown(char: string, forceUnicode: boolean = false): void {
     sendKeyInput(char, true, forceUnicode);
 }
@@ -792,7 +812,7 @@ export function mouseUp(button: number = 0): void {
 }
 
 export function getDisplayOrientation(): Orientation {
-    const resolution = getScreenResolutionAndRefreshRate();
+    const resolution = getScreenResolution();
     return resolution[0] > resolution[1] ? 'LANDSCAPE' : 'PORTRAIT';
 }
 
diff --git a/test/e2e/extension-multimonitor.e2e.ts b/test/e2e/extension-multimonitor.e2e.ts
@@ -0,0 +1,143 @@
+import { describe, it, beforeAll, afterAll, beforeEach, expect } from 'vitest';
+import type { Browser } from 'webdriverio';
+import {
+    createCalculatorSession,
+    quitSession,
+    resetCalculator,
+} from './helpers/session.js';
+
+describe('windows: getMonitors extension command', () => {
+    let calc: Browser;
+
+    beforeAll(async () => {
+        calc = await createCalculatorSession();
+    });
+
+    afterAll(async () => {
+        await quitSession(calc);
+    });
+
+    beforeEach(async () => {
+        await resetCalculator(calc);
+    });
+
+    describe('getMonitors — response shape', () => {
+        it('returns a non-empty array', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            expect(Array.isArray(monitors)).toBe(true);
+            expect(monitors.length).toBeGreaterThanOrEqual(1);
+        });
+
+        it('each monitor has required numeric index and non-empty deviceName', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            for (const monitor of monitors) {
+                expect(typeof monitor.index).toBe('number');
+                expect(typeof monitor.deviceName).toBe('string');
+                expect(monitor.deviceName.length).toBeGreaterThan(0);
+            }
+        });
+
+        it('each monitor has a boolean primary field', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            for (const monitor of monitors) {
+                expect(typeof monitor.primary).toBe('boolean');
+            }
+        });
+
+        it('exactly one monitor is marked as primary', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            const primaries = monitors.filter((m: any) => m.primary);
+            expect(primaries).toHaveLength(1);
+        });
+
+        it('each monitor has bounds with positive width and height', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            for (const monitor of monitors) {
+                expect(typeof monitor.bounds.x).toBe('number');
+                expect(typeof monitor.bounds.y).toBe('number');
+                expect(monitor.bounds.width).toBeGreaterThan(0);
+                expect(monitor.bounds.height).toBeGreaterThan(0);
+            }
+        });
+
+        it('each monitor has workingArea with positive width and height', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            for (const monitor of monitors) {
+                expect(typeof monitor.workingArea.x).toBe('number');
+                expect(typeof monitor.workingArea.y).toBe('number');
+                expect(monitor.workingArea.width).toBeGreaterThan(0);
+                expect(monitor.workingArea.height).toBeGreaterThan(0);
+            }
+        });
+
+        it('workingArea is contained within bounds for each monitor', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            for (const monitor of monitors) {
+                expect(monitor.workingArea.x).toBeGreaterThanOrEqual(monitor.bounds.x);
+                expect(monitor.workingArea.y).toBeGreaterThanOrEqual(monitor.bounds.y);
+                expect(monitor.workingArea.width).toBeLessThanOrEqual(monitor.bounds.width);
+                expect(monitor.workingArea.height).toBeLessThanOrEqual(monitor.bounds.height);
+            }
+        });
+
+        it('monitor indices are sequential starting from 0', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            const indices = monitors.map((m: any) => m.index).sort((a: number, b: number) => a - b);
+            for (let i = 0; i < indices.length; i++) {
+                expect(indices[i]).toBe(i);
+            }
+        });
+
+        it('primary monitor bounds origin is at the Windows virtual origin (0, 0)', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            const primary = monitors.find((m: any) => m.primary);
+            expect(primary.bounds.x).toBe(0);
+            expect(primary.bounds.y).toBe(0);
+        });
+    });
+
+    describe('virtual-screen absolute click regression', () => {
+        it('clicking Calculator "9" button by absolute screen coordinates shows 9 in display', async () => {
+            const btn = await calc.$('~num9Button');
+            const loc = await btn.getLocation();
+            const size = await btn.getSize();
+            const windowRect = await calc.getWindowRect();
+
+            const x = Math.round(windowRect.x + loc.x + size.width / 2);
+            const y = Math.round(windowRect.y + loc.y + size.height / 2);
+
+            await calc.executeScript('windows: click', [{ x, y }]);
+
+            const display = await calc.$('~CalculatorResults');
+            expect(await display.getText()).toContain('9');
+        });
+
+        it('clicking Calculator "5" button by absolute screen coordinates shows 5 in display', async () => {
+            const btn = await calc.$('~num5Button');
+            const loc = await btn.getLocation();
+            const size = await btn.getSize();
+            const windowRect = await calc.getWindowRect();
+
+            const x = Math.round(windowRect.x + loc.x + size.width / 2);
+            const y = Math.round(windowRect.y + loc.y + size.height / 2);
+
+            await calc.executeScript('windows: click', [{ x, y }]);
+
+            const display = await calc.$('~CalculatorResults');
+            expect(await display.getText()).toContain('5');
+        });
+
+        it('absolute coordinates derived from getMonitors primary bounds contain the Calculator window', async () => {
+            const monitors = await calc.executeScript('windows: getMonitors', []) as any[];
+            const primary = monitors.find((m: any) => m.primary);
+            const windowRect = await calc.getWindowRect();
+
+            // Calculator window should fall within primary monitor bounds
+            // (it was launched without any monitor preference, so it opens on primary)
+            expect(windowRect.x).toBeGreaterThanOrEqual(primary.bounds.x);
+            expect(windowRect.y).toBeGreaterThanOrEqual(primary.bounds.y);
+            expect(windowRect.x + windowRect.width).toBeLessThanOrEqual(primary.bounds.x + primary.bounds.width);
+            expect(windowRect.y + windowRect.height).toBeLessThanOrEqual(primary.bounds.y + primary.bounds.height);
+        });
+    });
+});
