Skip to content

Commit ff8645e

Browse files
committed
feat: add cursor overlay pipeline for high-fidelity cursor recording and playback
- Implement native bridge for Windows cursor capture via PowerShell/C# - Add cursor-free capture using getDisplayMedia with setDisplayMediaRequestHandler - Update video player and exporters to support native cursor telemetry - Enable system audio capture on Windows via WASAPI loopback - Add interpolation for smoother cursor movement in playback and export - Improve cursor scaling and visibility handling in editor and playback
1 parent e8e6a95 commit ff8645e

File tree

14 files changed

+679
-310
lines changed

14 files changed

+679
-310
lines changed

electron/ipc/handlers.ts

Lines changed: 42 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import fs from "node:fs/promises";
22
import path from "node:path";
33
import { fileURLToPath, pathToFileURL } from "node:url";
44
import { app, BrowserWindow, desktopCapturer, dialog, ipcMain, screen, shell } from "electron";
5+
import type { DesktopCapturerSource } from "electron";
56
import type {
67
CursorRecordingData,
78
CursorRecordingSample,
@@ -19,11 +20,23 @@ const SHORTCUTS_FILE = path.join(app.getPath("userData"), "shortcuts.json");
1920

2021
type SelectedSource = {
2122
name: string;
23+
id?: string;
24+
display_id?: string;
2225
[key: string]: unknown;
2326
};
2427

2528
let selectedSource: SelectedSource | null = null;
29+
let selectedDesktopSource: DesktopCapturerSource | null = null;
30+
let lastEnumeratedSources = new Map<string, DesktopCapturerSource>();
2631
let currentProjectPath: string | null = null;
32+
33+
/**
34+
* Returns the cached DesktopCapturerSource set when the user picked a source.
35+
* Used by setDisplayMediaRequestHandler in main.ts for cursor-free capture.
36+
*/
37+
export function getSelectedDesktopSource(): DesktopCapturerSource | null {
38+
return selectedDesktopSource;
39+
}
2740
let currentVideoPath: string | null = null;
2841

2942
function normalizePath(filePath: string) {
@@ -59,16 +72,12 @@ function isTrustedProjectPath(filePath?: string | null) {
5972
}
6073

6174
const CURSOR_TELEMETRY_VERSION = 2;
62-
const CURSOR_SAMPLE_INTERVAL_MS = 100;
63-
const MAX_CURSOR_SAMPLES = 60 * 60 * 10; // 1 hour @ 10Hz
75+
const CURSOR_SAMPLE_INTERVAL_MS = 33;
76+
const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz
6477

6578
let cursorRecordingSession: CursorRecordingSession | null = null;
6679
let pendingCursorRecordingData: CursorRecordingData | null = null;
6780

68-
function clamp(value: number, min: number, max: number) {
69-
return Math.min(max, Math.max(min, value));
70-
}
71-
7281
function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
7382
if (!sample || typeof sample !== "object") {
7483
return null;
@@ -80,8 +89,8 @@ function normalizeCursorSample(sample: unknown): CursorRecordingSample | null {
8089
typeof point.timeMs === "number" && Number.isFinite(point.timeMs)
8190
? Math.max(0, point.timeMs)
8291
: 0,
83-
cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? clamp(point.cx, 0, 1) : 0.5,
84-
cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? clamp(point.cy, 0, 1) : 0.5,
92+
cx: typeof point.cx === "number" && Number.isFinite(point.cx) ? point.cx : 0.5,
93+
cy: typeof point.cy === "number" && Number.isFinite(point.cy) ? point.cy : 0.5,
8594
assetId: typeof point.assetId === "string" ? point.assetId : null,
8695
visible: typeof point.visible === "boolean" ? point.visible : true,
8796
};
@@ -216,6 +225,10 @@ function getSelectedSourceBounds() {
216225
return (sourceDisplay ?? screen.getDisplayNearestPoint(cursor)).bounds;
217226
}
218227

228+
function getSelectedSourceId() {
229+
return typeof selectedSource?.id === "string" ? selectedSource.id : null;
230+
}
231+
219232
export function registerIpcHandlers(
220233
createEditorWindow: () => void,
221234
createSourceSelectorWindow: () => BrowserWindow,
@@ -225,6 +238,7 @@ export function registerIpcHandlers(
225238
) {
226239
ipcMain.handle("get-sources", async (_, opts) => {
227240
const sources = await desktopCapturer.getSources(opts);
241+
lastEnumeratedSources = new Map(sources.map((source) => [source.id, source]));
228242
return sources.map((source) => ({
229243
id: source.id,
230244
name: source.name,
@@ -234,8 +248,26 @@ export function registerIpcHandlers(
234248
}));
235249
});
236250

237-
ipcMain.handle("select-source", (_, source: SelectedSource) => {
251+
ipcMain.handle("select-source", async (_, source: SelectedSource) => {
238252
selectedSource = source;
253+
// Reuse the exact source object returned during enumeration to avoid
254+
// Windows window-source id mismatches across separate getSources() calls.
255+
selectedDesktopSource =
256+
typeof source.id === "string" ? lastEnumeratedSources.get(source.id) ?? null : null;
257+
258+
if (!selectedDesktopSource && typeof source.id === "string") {
259+
try {
260+
const sources = await desktopCapturer.getSources({
261+
types: ["screen", "window"],
262+
thumbnailSize: { width: 0, height: 0 },
263+
fetchWindowIcons: true,
264+
});
265+
lastEnumeratedSources = new Map(sources.map((candidate) => [candidate.id, candidate]));
266+
selectedDesktopSource = lastEnumeratedSources.get(source.id) ?? null;
267+
} catch {
268+
selectedDesktopSource = null;
269+
}
270+
}
239271
const sourceSelectorWin = getSourceSelectorWindow();
240272
if (sourceSelectorWin) {
241273
sourceSelectorWin.close();
@@ -327,6 +359,7 @@ export function registerIpcHandlers(
327359
maxSamples: MAX_CURSOR_SAMPLES,
328360
platform: process.platform,
329361
sampleIntervalMs: CURSOR_SAMPLE_INTERVAL_MS,
362+
sourceId: getSelectedSourceId(),
330363
});
331364

332365
try {

electron/main.ts

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {
1212
systemPreferences,
1313
Tray,
1414
} from "electron";
15-
import { registerIpcHandlers } from "./ipc/handlers";
15+
import { registerIpcHandlers, getSelectedDesktopSource } from "./ipc/handlers";
1616
import { createEditorWindow, createHudOverlayWindow, createSourceSelectorWindow } from "./windows";
1717

1818
const __dirname = path.dirname(fileURLToPath(import.meta.url));
@@ -316,6 +316,21 @@ app.on("activate", () => {
316316

317317
// Register all IPC handlers when app is ready
318318
app.whenReady().then(async () => {
319+
// Intercept getDisplayMedia to return the pre-selected source without the cursor.
320+
// The source is cached synchronously at select-source time to avoid async delays here.
321+
session.defaultSession.setDisplayMediaRequestHandler((_request, callback) => {
322+
const source = getSelectedDesktopSource();
323+
if (!source) {
324+
callback({});
325+
return;
326+
}
327+
callback({
328+
video: source,
329+
// WASAPI loopback provides system audio capture on Windows.
330+
...(process.platform === "win32" && { audio: "loopback" as const }),
331+
});
332+
});
333+
319334
// Allow microphone/media permission checks
320335
session.defaultSession.setPermissionCheckHandler((_webContents, permission) => {
321336
const allowed = ["media", "audioCapture", "microphone"];

electron/native-bridge/cursor/recording/factory.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ interface CreateCursorRecordingSessionOptions {
88
maxSamples: number;
99
platform: NodeJS.Platform;
1010
sampleIntervalMs: number;
11+
sourceId?: string | null;
1112
}
1213

1314
export function createCursorRecordingSession(
@@ -18,6 +19,7 @@ export function createCursorRecordingSession(
1819
getDisplayBounds: options.getDisplayBounds,
1920
maxSamples: options.maxSamples,
2021
sampleIntervalMs: options.sampleIntervalMs,
22+
sourceId: options.sourceId,
2123
});
2224
}
2325

Lines changed: 216 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,216 @@
1+
export function parseWindowHandleFromSourceId(sourceId?: string | null) {
2+
if (!sourceId?.startsWith("window:")) {
3+
return null;
4+
}
5+
6+
const handlePart = sourceId.split(":")[1];
7+
if (!handlePart || !/^\d+$/.test(handlePart)) {
8+
return null;
9+
}
10+
11+
return handlePart;
12+
}
13+
14+
export function buildPowerShellCommand(sampleIntervalMs: number, windowHandle?: string | null) {
15+
const script = String.raw`
16+
$ErrorActionPreference = 'Stop'
17+
Add-Type -AssemblyName System.Drawing
18+
19+
$targetWindowHandle = ${windowHandle ? `'${windowHandle}'` : '$null'}
20+
21+
$source = @"
22+
using System;
23+
using System.Runtime.InteropServices;
24+
25+
public static class OpenScreenCursorInterop {
26+
[StructLayout(LayoutKind.Sequential)]
27+
public struct POINT {
28+
public int X;
29+
public int Y;
30+
}
31+
32+
[StructLayout(LayoutKind.Sequential)]
33+
public struct CURSORINFO {
34+
public int cbSize;
35+
public int flags;
36+
public IntPtr hCursor;
37+
public POINT ptScreenPos;
38+
}
39+
40+
[StructLayout(LayoutKind.Sequential)]
41+
public struct ICONINFO {
42+
[MarshalAs(UnmanagedType.Bool)]
43+
public bool fIcon;
44+
public int xHotspot;
45+
public int yHotspot;
46+
public IntPtr hbmMask;
47+
public IntPtr hbmColor;
48+
}
49+
50+
[StructLayout(LayoutKind.Sequential)]
51+
public struct RECT {
52+
public int Left;
53+
public int Top;
54+
public int Right;
55+
public int Bottom;
56+
}
57+
58+
[DllImport("user32.dll", SetLastError = true)]
59+
[return: MarshalAs(UnmanagedType.Bool)]
60+
public static extern bool GetCursorInfo(ref CURSORINFO pci);
61+
62+
[DllImport("user32.dll", SetLastError = true)]
63+
[return: MarshalAs(UnmanagedType.Bool)]
64+
public static extern bool GetWindowRect(IntPtr hWnd, out RECT lpRect);
65+
66+
[DllImport("user32.dll", SetLastError = true)]
67+
[return: MarshalAs(UnmanagedType.Bool)]
68+
public static extern bool IsWindow(IntPtr hWnd);
69+
70+
[DllImport("user32.dll", SetLastError = true)]
71+
public static extern IntPtr CopyIcon(IntPtr hIcon);
72+
73+
[DllImport("user32.dll", SetLastError = true)]
74+
[return: MarshalAs(UnmanagedType.Bool)]
75+
public static extern bool DestroyIcon(IntPtr hIcon);
76+
77+
[DllImport("user32.dll", SetLastError = true)]
78+
[return: MarshalAs(UnmanagedType.Bool)]
79+
public static extern bool GetIconInfo(IntPtr hIcon, out ICONINFO piconinfo);
80+
81+
[DllImport("gdi32.dll", SetLastError = true)]
82+
[return: MarshalAs(UnmanagedType.Bool)]
83+
public static extern bool DeleteObject(IntPtr hObject);
84+
}
85+
"@
86+
87+
Add-Type -TypeDefinition $source
88+
89+
function Write-JsonLine($payload) {
90+
[Console]::Out.WriteLine(($payload | ConvertTo-Json -Compress -Depth 6))
91+
}
92+
93+
function Get-TargetBounds() {
94+
if ([string]::IsNullOrWhiteSpace($targetWindowHandle)) {
95+
return $null
96+
}
97+
98+
try {
99+
$handleValue = [int64]::Parse($targetWindowHandle)
100+
$windowHandle = [IntPtr]::new($handleValue)
101+
if (-not [OpenScreenCursorInterop]::IsWindow($windowHandle)) {
102+
return $null
103+
}
104+
105+
$rect = New-Object OpenScreenCursorInterop+RECT
106+
if (-not [OpenScreenCursorInterop]::GetWindowRect($windowHandle, [ref]$rect)) {
107+
return $null
108+
}
109+
110+
$width = $rect.Right - $rect.Left
111+
$height = $rect.Bottom - $rect.Top
112+
if ($width -le 0 -or $height -le 0) {
113+
return $null
114+
}
115+
116+
return @{
117+
x = $rect.Left
118+
y = $rect.Top
119+
width = $width
120+
height = $height
121+
}
122+
}
123+
catch {
124+
return $null
125+
}
126+
}
127+
128+
function Get-CursorAsset($cursorHandle, $cursorId) {
129+
$copiedHandle = [OpenScreenCursorInterop]::CopyIcon($cursorHandle)
130+
if ($copiedHandle -eq [IntPtr]::Zero) {
131+
return $null
132+
}
133+
134+
$iconInfo = New-Object OpenScreenCursorInterop+ICONINFO
135+
$hasIconInfo = [OpenScreenCursorInterop]::GetIconInfo($copiedHandle, [ref]$iconInfo)
136+
137+
try {
138+
$icon = [System.Drawing.Icon]::FromHandle($copiedHandle)
139+
$bitmap = New-Object System.Drawing.Bitmap $icon.Width, $icon.Height, ([System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
140+
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
141+
$memoryStream = New-Object System.IO.MemoryStream
142+
143+
try {
144+
$graphics.Clear([System.Drawing.Color]::Transparent)
145+
$graphics.DrawIcon($icon, 0, 0)
146+
$bitmap.Save($memoryStream, [System.Drawing.Imaging.ImageFormat]::Png)
147+
$base64 = [System.Convert]::ToBase64String($memoryStream.ToArray())
148+
149+
return @{
150+
id = $cursorId
151+
imageDataUrl = "data:image/png;base64,$base64"
152+
width = $bitmap.Width
153+
height = $bitmap.Height
154+
hotspotX = if ($hasIconInfo) { $iconInfo.xHotspot } else { 0 }
155+
hotspotY = if ($hasIconInfo) { $iconInfo.yHotspot } else { 0 }
156+
}
157+
}
158+
finally {
159+
$memoryStream.Dispose()
160+
$graphics.Dispose()
161+
$bitmap.Dispose()
162+
$icon.Dispose()
163+
}
164+
}
165+
finally {
166+
if ($hasIconInfo) {
167+
if ($iconInfo.hbmMask -ne [IntPtr]::Zero) {
168+
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmMask) | Out-Null
169+
}
170+
if ($iconInfo.hbmColor -ne [IntPtr]::Zero) {
171+
[OpenScreenCursorInterop]::DeleteObject($iconInfo.hbmColor) | Out-Null
172+
}
173+
}
174+
[OpenScreenCursorInterop]::DestroyIcon($copiedHandle) | Out-Null
175+
}
176+
}
177+
178+
Write-JsonLine @{ type = 'ready'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds() }
179+
180+
$lastCursorId = $null
181+
while ($true) {
182+
$cursorInfo = New-Object OpenScreenCursorInterop+CURSORINFO
183+
$cursorInfo.cbSize = [Runtime.InteropServices.Marshal]::SizeOf([type][OpenScreenCursorInterop+CURSORINFO])
184+
185+
if (-not [OpenScreenCursorInterop]::GetCursorInfo([ref]$cursorInfo)) {
186+
Write-JsonLine @{ type = 'error'; timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds(); message = 'GetCursorInfo failed' }
187+
Start-Sleep -Milliseconds ${sampleIntervalMs}
188+
continue
189+
}
190+
191+
$visible = ($cursorInfo.flags -band 1) -ne 0
192+
$cursorId = if ($cursorInfo.hCursor -eq [IntPtr]::Zero) { $null } else { ('0x{0:X}' -f $cursorInfo.hCursor.ToInt64()) }
193+
$asset = $null
194+
195+
if ($visible -and $cursorId -and $cursorId -ne $lastCursorId) {
196+
$asset = Get-CursorAsset -cursorHandle $cursorInfo.hCursor -cursorId $cursorId
197+
$lastCursorId = $cursorId
198+
}
199+
200+
Write-JsonLine @{
201+
type = 'sample'
202+
timestampMs = [DateTimeOffset]::UtcNow.ToUnixTimeMilliseconds()
203+
x = $cursorInfo.ptScreenPos.X
204+
y = $cursorInfo.ptScreenPos.Y
205+
visible = $visible
206+
handle = $cursorId
207+
bounds = Get-TargetBounds
208+
asset = $asset
209+
}
210+
211+
Start-Sleep -Milliseconds ${sampleIntervalMs}
212+
}
213+
`;
214+
215+
return Buffer.from(script, "utf16le").toString("base64");
216+
}

0 commit comments

Comments
 (0)