Detect click interception by overlays (#1434)

ctate · web-flow · commit 688e285b8b7b · 2026-06-10T11:50:32.000-05:00
* Detect click interception by overlays

- Hit-test the click point during element resolution and fail with a description of the covering element (e.g. "covered by &lt;p inside div#consent-banner&gt;") instead of dispatching input that silently lands on a cookie banner, modal, or sticky header
- Applies to CSS-selector, snapshot-ref, and iframe resolve paths; the test descends through same-origin frames and starts from the top document so iframe content is checked in the right coordinate space
- Shadow-DOM-aware relation checks plus label/control association prevent false positives on web components and custom checkboxes styled over hidden inputs

* Add covered click docs and regression test
diff --git a/README.md b/README.md
@@ -90,6 +90,10 @@ agent-browser screenshot page.png
 agent-browser close
 ```
 
+Clicks fail early when another element covers the target's click point,
+for example a consent banner or modal. Dismiss or interact with the reported
+covering element, then take a fresh snapshot before retrying the original ref.
+
 Headless Chromium screenshots hide native scrollbars for consistent image output.
 Pass `--hide-scrollbars false` when launching to keep native scrollbars visible.
 
@@ -883,6 +887,10 @@ agent-browser get text @e1                # Get heading text
 agent-browser hover @e4                   # Hover the link
 ```
 
+When a ref click is blocked by an overlay, the error includes the covering
+element, such as `covered by <div#consent-banner>`. Click the banner or dialog
+control first, then run `snapshot` again before reusing refs.
+
 **Why use refs?**
 
 - **Deterministic**: Ref points to exact element from snapshot
diff --git a/cli/src/native/e2e_tests.rs b/cli/src/native/e2e_tests.rs
@@ -2702,6 +2702,89 @@ async fn e2e_error_handling() {
     assert_success(&resp);
 }
 
+#[tokio::test]
+#[ignore]
+async fn e2e_click_reports_covering_overlay() {
+    let mut state = DaemonState::new();
+
+    let resp = execute_command(
+        &json!({ "id": "1", "action": "launch", "headless": true }),
+        &mut state,
+    )
+    .await;
+    assert_success(&resp);
+
+    let html = r#"
+        <html>
+        <body>
+            <button id="target" onclick="document.getElementById('result').textContent = 'clicked'">
+                Target
+            </button>
+            <div id="consent-banner" style="position:fixed;inset:0;z-index:10;background:rgba(0,0,0,0.1)">
+                <button id="dismiss" style="position:absolute;right:20px;bottom:20px"
+                    onclick="document.getElementById('consent-banner').remove()">
+                    Dismiss
+                </button>
+            </div>
+            <div id="result">idle</div>
+        </body>
+        </html>
+    "#;
+    let url = format!("data:text/html;base64,{}", STANDARD.encode(html));
+    let resp = execute_command(
+        &json!({ "id": "2", "action": "navigate", "url": url }),
+        &mut state,
+    )
+    .await;
+    assert_success(&resp);
+
+    let resp = execute_command(
+        &json!({ "id": "3", "action": "click", "selector": "#target" }),
+        &mut state,
+    )
+    .await;
+    assert_eq!(resp["success"], false, "covered target should fail: {resp}");
+    let error = resp["error"].as_str().unwrap_or_default();
+    assert!(
+        error.contains("covered by <div#consent-banner>"),
+        "unexpected covered-click error: {}",
+        error
+    );
+
+    let resp = execute_command(
+        &json!({ "id": "4", "action": "gettext", "selector": "#result" }),
+        &mut state,
+    )
+    .await;
+    assert_success(&resp);
+    assert_eq!(get_data(&resp)["text"], "idle");
+
+    let resp = execute_command(
+        &json!({ "id": "5", "action": "click", "selector": "#dismiss" }),
+        &mut state,
+    )
+    .await;
+    assert_success(&resp);
+
+    let resp = execute_command(
+        &json!({ "id": "6", "action": "click", "selector": "#target" }),
+        &mut state,
+    )
+    .await;
+    assert_success(&resp);
+
+    let resp = execute_command(
+        &json!({ "id": "7", "action": "gettext", "selector": "#result" }),
+        &mut state,
+    )
+    .await;
+    assert_success(&resp);
+    assert_eq!(get_data(&resp)["text"], "clicked");
+
+    let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
+    assert_success(&resp);
+}
+
 // ---------------------------------------------------------------------------
 // Profile cookie persistence across restarts
 // ---------------------------------------------------------------------------
diff --git a/cli/src/native/element.rs b/cli/src/native/element.rs
@@ -231,7 +231,9 @@ async fn resolve_center_in_same_process_frame(
                 y += frameRect.y + win.frameElement.clientTop;
                 win = win.parent;
             }}
-            return {{ x: x, y: y }};
+            const blockerAt = {BLOCKER_AT_JS};
+            const topDoc = win ? win.document : doc;
+            return {{ x: x, y: y, blocker: blockerAt(topDoc, el, x, y) }};
         }}"#,
     );
     let result = client
@@ -246,6 +248,12 @@ async fn resolve_center_in_same_process_frame(
         )
         .await?;
     let value = result.get("result").and_then(|r| r.get("value"));
+    if let Some(blocker) = value
+        .and_then(|v| v.get("blocker"))
+        .and_then(|v| v.as_str())
+    {
+        return Err(intercepted_error(selector, blocker));
+    }
     let x = value.and_then(|v| v.get("x")).and_then(|v| v.as_f64());
     let y = value.and_then(|v| v.get("y")).and_then(|v| v.as_f64());
     match (x, y) {
@@ -320,6 +328,15 @@ pub async fn resolve_element_center(
 
             if let Ok(r) = result {
                 let (x, y) = box_model_center(&r.model);
+                check_node_interception(
+                    client,
+                    effective_session_id,
+                    backend_node_id,
+                    selector_or_ref,
+                    x,
+                    y,
+                )
+                .await?;
                 return Ok((x, y, effective_session_id.to_string()));
             }
             // backend_node_id is stale; re-query the accessibility tree below
@@ -349,6 +366,15 @@ pub async fn resolve_element_center(
             )
             .await?;
         let (x, y) = box_model_center(&result.model);
+        check_node_interception(
+            client,
+            effective_session_id,
+            fresh_id,
+            selector_or_ref,
+            x,
+            y,
+        )
+        .await?;
         return Ok((x, y, effective_session_id.to_string()));
     }
 
@@ -369,6 +395,73 @@ pub async fn resolve_element_center(
     Ok((x, y, session_id.to_string()))
 }
 
+/// Hit-test a ref-resolved node at its computed click point and error if an
+/// unrelated element (overlay, banner, sticky header) would receive the input
+/// instead. Best effort: resolution failures skip the check rather than block
+/// the interaction.
+async fn check_node_interception(
+    client: &CdpClient,
+    session_id: &str,
+    backend_node_id: i64,
+    target: &str,
+    x: f64,
+    y: f64,
+) -> Result<(), String> {
+    let resolved: Result<DomResolveNodeResult, String> = client
+        .send_command_typed(
+            "DOM.resolveNode",
+            &DomResolveNodeParams {
+                backend_node_id: Some(backend_node_id),
+                node_id: None,
+                object_group: Some("agent-browser".to_string()),
+            },
+            Some(session_id),
+        )
+        .await;
+    let Ok(resolved) = resolved else {
+        return Ok(());
+    };
+    let Some(object_id) = resolved.object.object_id else {
+        return Ok(());
+    };
+    // Box-model coordinates are in the top-level viewport space, so the
+    // hit-test starts from the top document. For an OOPIF node the
+    // frameElement walk stops at the process boundary, where the frame's own
+    // document and session-local coordinates are already consistent.
+    let function = format!(
+        r#"function(x, y) {{
+            let topDoc = this.ownerDocument || document;
+            while (topDoc.defaultView && topDoc.defaultView.frameElement) {{
+                topDoc = topDoc.defaultView.frameElement.ownerDocument;
+            }}
+            const blockerAt = {BLOCKER_AT_JS};
+            return blockerAt(topDoc, this, x, y);
+        }}"#,
+    );
+    let result = client
+        .send_command(
+            "Runtime.callFunctionOn",
+            Some(serde_json::json!({
+                "objectId": object_id,
+                "functionDeclaration": function,
+                "arguments": [{ "value": x }, { "value": y }],
+                "returnByValue": true,
+            })),
+            Some(session_id),
+        )
+        .await;
+    if let Ok(value) = result {
+        if let Some(blocker) = value
+            .get("result")
+            .and_then(|r| r.get("value"))
+            .and_then(|v| v.as_str())
+        {
+            return Err(intercepted_error(target, blocker));
+        }
+    }
+    Ok(())
+}
+
 /// Coordinates from DOM.getBoxModel are viewport-relative, and input events
 /// only land inside the viewport, so make sure the node is visible first.
 /// Best effort: a node that cannot be scrolled (display:none, detached) will
@@ -628,10 +721,51 @@ fn build_count_elements_js(selector: &str) -> String {
     }
 }
 
+/// JS function source for `blockerAt(doc, el, x, y)`: returns a short
+/// description of the element that would actually receive a click at (x, y)
+/// when that element is unrelated to `el`, or null when the click would land
+/// on `el` (or something that activates it). Relations that count as "lands
+/// on el": shadow-including ancestors/descendants in either direction, and
+/// label/control association (custom checkboxes hide the input under a styled
+/// sibling inside the same label).
+const BLOCKER_AT_JS: &str = r#"(doc, el, x, y) => {
+    // Descend from the given document through same-origin iframes so a point
+    // over a frame resolves to the element inside it, in that frame's space.
+    let d = doc, lx = x, ly = y;
+    let hit = d.elementFromPoint(lx, ly);
+    while (hit && (hit.tagName === 'IFRAME' || hit.tagName === 'FRAME') && hit.contentDocument && hit !== el) {
+        const r = hit.getBoundingClientRect();
+        lx -= r.x + hit.clientLeft;
+        ly -= r.y + hit.clientTop;
+        d = hit.contentDocument;
+        hit = d.elementFromPoint(lx, ly);
+    }
+    if (!hit || hit === el) return null;
+    const up = (n) => n.parentNode || n.host || (n.getRootNode && n.getRootNode().host) || null;
+    for (let n = hit; n; n = up(n)) { if (n === el) return null; }
+    for (let n = el; n; n = up(n)) { if (n === hit) return null; }
+    const hitLabel = hit.closest ? hit.closest('label') : null;
+    if (hitLabel && (hitLabel.control === el || hitLabel.contains(el))) return null;
+    const elLabel = el.closest ? el.closest('label') : null;
+    if (elLabel && elLabel.contains(hit)) return null;
+    let desc = hit.tagName.toLowerCase();
+    if (hit.id) desc += '#' + hit.id;
+    else if (typeof hit.className === 'string' && hit.className.trim())
+        desc += '.' + hit.className.trim().split(/\s+/).slice(0, 2).join('.');
+    if (!hit.id && hit.closest) {
+        const anchored = hit.closest('[id]');
+        if (anchored && anchored !== hit)
+            desc += ' inside ' + anchored.tagName.toLowerCase() + '#' + anchored.id;
+    }
+    return desc;
+}"#;
+
 fn build_selector_js(selector: &str) -> String {
     let find_expr = build_find_element_js(selector);
     // Input events dispatch at viewport coordinates, so an element outside the
     // viewport must be scrolled into view first or the click lands on nothing.
+    // The blocker check reports an overlay covering the click point instead of
+    // letting the input land on it and silently doing the wrong thing.
     format!(
         r#"(() => {{
             const el = {find_expr};
@@ -645,7 +779,10 @@ fn build_selector_js(selector: &str) -> String {
                 el.scrollIntoView({{ block: 'center', inline: 'center', behavior: 'instant' }});
                 rect = el.getBoundingClientRect();
             }}
-            return {{ x: rect.x + rect.width / 2, y: rect.y + rect.height / 2 }};
+            const x = rect.x + rect.width / 2;
+            const y = rect.y + rect.height / 2;
+            const blockerAt = {BLOCKER_AT_JS};
+            return {{ x: x, y: y, blocker: blockerAt(document, el, x, y) }};
         }})()"#,
     )
 }
@@ -670,6 +807,9 @@ async fn resolve_by_selector(
         .await?;
 
     let val = result.result.value.unwrap_or(Value::Null);
+    if let Some(blocker) = val.get("blocker").and_then(|v| v.as_str()) {
+        return Err(intercepted_error(selector, blocker));
+    }
     let x = val.get("x").and_then(|v| v.as_f64());
     let y = val.get("y").and_then(|v| v.as_f64());
 
@@ -679,6 +819,13 @@ async fn resolve_by_selector(
     }
 }
 
+fn intercepted_error(target: &str, blocker: &str) -> String {
+    format!(
+        "Element '{}' is covered by <{}> at its click point, so the input would land on that element instead. Dismiss or interact with the covering element first (it is often a dialog, banner, or sticky header).",
+        target, blocker
+    )
+}
+
 fn box_model_center(model: &BoxModel) -> (f64, f64) {
     // content quad: [x1,y1, x2,y2, x3,y3, x4,y4]
     if model.content.len() >= 8 {
diff --git a/cli/src/output.rs b/cli/src/output.rs
@@ -1259,6 +1259,9 @@ Usage: agent-browser click <selector> [--new-tab]
 Clicks on the specified element. The selector can be a CSS selector,
 XPath, or an element reference from snapshot (e.g., @e1).
 
+If another element covers the click point, agent-browser reports the
+covering element instead of dispatching a click to the wrong target.
+
 Options:
   --new-tab            Open link in a new tab instead of navigating current tab
                        (only works on elements with href attribute)
diff --git a/docs/src/app/commands/page.mdx b/docs/src/app/commands/page.mdx
@@ -38,6 +38,11 @@ agent-browser close                   # Close browser (aliases: quit, exit)
 agent-browser close --all             # Close all active sessions
 ```
 
+Clicks fail before dispatch when another element covers the target's click
+point. The error names the covering element, for example
+`covered by <div#consent-banner>`. Dismiss or interact with that element,
+take a fresh snapshot, then retry the original action.
+
 Headless Chromium screenshots hide native scrollbars for consistent image output.
 Pass `--hide-scrollbars false` when launching to keep native scrollbars visible.
 
diff --git a/skill-data/core/SKILL.md b/skill-data/core/SKILL.md
@@ -367,8 +367,9 @@ agent-browser snapshot -i
 ```
 
 **Click does nothing / overlay swallows the click**
-Some modals and cookie banners block other clicks. Snapshot, find the
-dismiss/close button, click it, then re-snapshot.
+Some modals and cookie banners block other clicks. If `click` reports
+`covered by <...>`, interact with that covering element first. Otherwise,
+snapshot, find the dismiss/close button, click it, then re-snapshot.
 
 **Fill / type doesn't work**
 Some custom input components intercept key events. Try:
diff --git a/skill-data/core/references/commands.md b/skill-data/core/references/commands.md
@@ -71,6 +71,11 @@ agent-browser drag @e1 @e2        # Drag and drop
 agent-browser upload @e1 file.pdf # Upload files
 ```
 
+Clicks fail before dispatch when another element covers the target's click
+point. The error names the covering element, for example
+`covered by <div#consent-banner>`. Dismiss or interact with that element, run a
+fresh snapshot, then retry the original action.
+
 ## Get Information
 
 ```bash
