Merge refactor/dashboard-at-idea2paper: unified idea2paper.org/dashboard

Cutover to single-domain architecture with CF Tunnel path routing:
- idea2paper.org/         -> static homepage (new ark-homepage.service)
- idea2paper.org/dashboard/* -> webapp (ark-webapp.service, port 9527)
  protected by CF Access (KAUST + invited allowlist, Google SSO)
- dev.idea2paper.org / webapp.idea2paper.org -> deprecated after 48h

See /home/xinj/.claude/plans/bright-wobbling-thimble.md for the full plan.

Commits included:
- Rename Python module ark.webapp -> ark.dashboard
- Merge webpage submodule into website/homepage
- StripPathPrefixMiddleware + APP_BASE templating
- Systemd env vars (ARK_ROOT_PATH, BASE_URL)

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: JihaoXin

.gitmodules

@@ -1,9 +1,6 @@
 [submodule "materials/references/openclaw"]
 	path = materials/references/openclaw
 	url = https://github.com/openclaw/openclaw.git
-[submodule "submodules/webpage"]
-	path = submodules/webpage
-	url = git@github.com:kaust-ark/kaust-ark.github.io.git
 [submodule "submodules/PaperBanana"]
 	path = submodules/PaperBanana
 	url = https://github.com/dwzhu-pku/PaperBanana.git

ark/cli.py

@@ -816,7 +816,7 @@ def _setup_latex_template(code_dir: str, config: dict):
     print(f"{_c('LaTeX Template Setup', Colors.BOLD)}")
 
     # Try bundled venue_templates/ first (same as webapp)
-    from ark.webapp.templates import has_venue_template, copy_venue_template
+    from ark.dashboard.templates import has_venue_template, copy_venue_template
     downloaded = False
 
     _, required = _resolve_template_urls(venue_format)
@@ -1594,7 +1594,7 @@ def _finalize_project(name: str, project_dir: Path, config: dict,
     # ── Register project in webapp DB ──────────────────────────
     project_id = None
     try:
-        from ark.webapp.db import (resolve_db_path, get_session,
+        from ark.dashboard.db import (resolve_db_path, get_session,
                                    get_or_create_user_by_email, create_project as db_create_project)
         import getpass
         db_path = resolve_db_path()
@@ -1876,13 +1876,13 @@ def cmd_run(args):
     project_id = config.get("_project_id", "")
     if not db_path:
         try:
-            from ark.webapp.db import resolve_db_path
+            from ark.dashboard.db import resolve_db_path
             db_path = resolve_db_path()
         except Exception:
             pass
     if not project_id and db_path and Path(db_path).exists():
         try:
-            from ark.webapp.db import get_session, get_project_by_name
+            from ark.dashboard.db import get_session, get_project_by_name
             with get_session(db_path) as session:
                 p = get_project_by_name(session, name)
                 if p:
@@ -1892,7 +1892,7 @@ def cmd_run(args):
 
     # Launch orchestrator in background, preferring per-project conda env
     try:
-        from ark.webapp.jobs import (
+        from ark.dashboard.jobs import (
             find_conda_binary, project_env_ready, project_env_prefix,
         )
         conda_bin = find_conda_binary()
@@ -3112,14 +3112,26 @@ def _cmd_webapp_install(host: str, port: int, dev: bool = False):
             prod_env_link.symlink_to(main_env)
 
     # Environment variables for systemd service
+    # ARK_ROOT_PATH mounts the webapp under a URL path prefix behind the CF
+    # Tunnel (idea2paper.org/dashboard/*). The strip-prefix middleware makes
+    # this transparent to the app, so BASE_URL stays a bare origin.
     env_vars = {
         "ARK_WEBAPP_DB_PATH": str(db_path),
         "PROJECTS_ROOT": str(data_dir / "projects"),
+        "ARK_ROOT_PATH": "/dashboard",
     }
 
     if dev:
         env_vars["ARK_SESSION_COOKIE"] = "session_dev"
+        # Dev BASE_URL stays as the internal IP — dev is only reachable from
+        # KAUST internal network, and magic-link emails from dev must be
+        # clickable inside KAUST. No CF Tunnel routes dev.
         env_vars["BASE_URL"] = f"http://{get_primary_ip()}:{port}"
+        # Dev does NOT support Google OAuth: Google rejects private IPs as
+        # OAuth redirect URIs. Clear the creds so /auth/google/enabled
+        # returns false and the UI hides the Google button.
+        env_vars["GOOGLE_CLIENT_ID"] = ""
+        env_vars["GOOGLE_CLIENT_SECRET"] = ""
         # Load dev-only env overrides
         dev_env_file = get_config_dir() / "webapp-dev.env"
         if not dev_env_file.exists():
@@ -3135,6 +3147,11 @@ def _cmd_webapp_install(host: str, port: int, dev: bool = False):
                     continue
                 k, _, v = line.partition("=")
                 env_vars[k.strip()] = v.strip()
+    else:
+        # Prod BASE_URL is the public origin; /dashboard prefix is added by
+        # ARK_ROOT_PATH, so anything building a full URL uses BASE_URL +
+        # ARK_ROOT_PATH + /path.
+        env_vars["BASE_URL"] = "https://idea2paper.org"
 
     svc_path = _service_file_path(svc_name)
     svc_path.parent.mkdir(parents=True, exist_ok=True)
@@ -3357,8 +3374,8 @@ def cmd_webapp(args):
 
     try:
         import uvicorn
-        from ark.webapp import create_app
-        from ark.webapp.config import get_settings, _env_file
+        from ark.dashboard import create_app
+        from ark.dashboard.config import get_settings, _env_file
     except ImportError:
         print(f"{_c('Error:', Colors.RED)} Webapp dependencies not installed.")
         print(f"  Install with: {_c('pip install ark-research[webapp]', Colors.BOLD)}")
@@ -3428,8 +3445,8 @@ def _scancel(job_id):
     def _submit(pid, mode, max_iter, user_id, con):
         """Try SLURM submit; fallback to 'local'. Returns new job_id."""
         if _sh.which("sbatch"):
-            from ark.webapp.config import get_settings
-            from ark.webapp.jobs import submit_job
+            from ark.dashboard.config import get_settings
+            from ark.dashboard.jobs import submit_job
             settings = get_settings()
             pdir = settings.projects_root / user_id / pid
             log_dir = pdir / "logs"

ark/webapp/__init__.py ark/dashboard/__init__.pyark/webapp/__init__.py renamed to ark/dashboard/__init__.py

@@ -17,7 +17,7 @@
 from .notify import send_completion_email, send_telegram_notify
 from .routes import router
 
-logger = logging.getLogger("ark.webapp")
+logger = logging.getLogger("ark.dashboard")
 
 _log_mtimes: dict[str, float] = {}   # project_id → last log mtime
 
@@ -78,7 +78,11 @@ async def _poll_jobs(app: FastAPI):
                         continue
 
                     pdir = settings.projects_root / p.user_id / p.id
-                    url = f"{settings.base_url}/#project/{p.id}"
+                    # Background task has no request context, so reconstruct the
+                    # public URL from BASE_URL + ARK_ROOT_PATH. BASE_URL is the
+                    # origin (https://idea2paper.org) and ARK_ROOT_PATH is the
+                    # path prefix ("/dashboard" in prod, "" for bare access).
+                    url = f"{settings.base_url}{os.environ.get('ARK_ROOT_PATH', '')}/#project/{p.id}"
 
                     # ── Local subprocess job ──────────────────────────────
                     if p.slurm_job_id.startswith("local:"):
@@ -445,7 +449,7 @@ async def lifespan(app: FastAPI):
     get_engine(settings.db_path)
 
     # Migrate existing project data: populate new DB columns from YAML state files
-    from ark.webapp.db import migrate_project_data
+    from ark.dashboard.db import migrate_project_data
     try:
         migrate_project_data(settings.db_path, str(settings.projects_root))
         logger.info("Project data migration completed.")
@@ -465,8 +469,58 @@ async def lifespan(app: FastAPI):
     logger.info("ARK Webapp stopped.")
 
 
-def create_app() -> FastAPI:
+class StripPathPrefixMiddleware:
+    """ASGI middleware that strips a path prefix from incoming requests.
+
+    When ARK_ROOT_PATH=/dashboard, requests to /dashboard/api/projects arrive
+    with that full path. This middleware rewrites scope['path'] to
+    /api/projects and sets scope['root_path']=/dashboard so Starlette's
+    request.url_for() builds correct absolute URLs.
+
+    Non-prefixed requests pass through unchanged, which means the same
+    process transparently serves both http://host/ and http://host/dashboard/.
+    This is the 'dual access' property: useful for local dev (direct port)
+    while the same app is served at /dashboard in production via tunnel.
+    """
+
+    def __init__(self, app, prefix: str):
+        self.app = app
+        self.prefix = prefix
+
+    async def __call__(self, scope, receive, send):
+        if scope["type"] in ("http", "websocket"):
+            path = scope.get("path", "")
+            if path == self.prefix or path.startswith(self.prefix + "/"):
+                new_path = path[len(self.prefix):] or "/"
+                scope = dict(scope)
+                scope["path"] = new_path
+                # Stash prefix in a custom key. We intentionally do NOT
+                # set scope["root_path"] because Starlette's StaticFiles
+                # mount misroutes when root_path is non-empty. Handlers
+                # read this via request.scope.get("ark_root_path", "").
+                scope["ark_root_path"] = self.prefix
+                # raw_path is used by Starlette routing; must update it too
+                # or mounts (e.g., /static) won't match after prefix strip.
+                raw_path = scope.get("raw_path")
+                if raw_path is not None:
+                    prefix_bytes = self.prefix.encode("latin-1")
+                    if raw_path == prefix_bytes or raw_path.startswith(prefix_bytes + b"/"):
+                        scope["raw_path"] = raw_path[len(prefix_bytes):] or b"/"
+        await self.app(scope, receive, send)
+
+
+def create_app():
+    """Create the FastAPI app. Returns an ASGI callable.
+
+    When ARK_ROOT_PATH env var is set (e.g. /dashboard), the app is wrapped
+    with StripPathPrefixMiddleware so it serves at both / and /<prefix>.
+
+    Note: root_path is NOT passed to FastAPI() — otherwise FastAPI would
+    set scope["root_path"] on every request, breaking dual-access. The
+    middleware sets it only for requests that actually carry the prefix.
+    """
     settings = get_settings()
+    root_path = os.environ.get("ARK_ROOT_PATH", "")
 
     app = FastAPI(
         title="ARK Research Portal",
@@ -491,4 +545,8 @@ def create_app() -> FastAPI:
     static_dir = Path(__file__).parent / "static"
     app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
 
+    # Wrap as outermost ASGI layer so it sees raw incoming paths before any
+    # FastAPI middleware/routing. Without a prefix, return the app unwrapped.
+    if root_path:
+        return StripPathPrefixMiddleware(app, prefix=root_path)
     return app

ark/webapp/app.py ark/dashboard/app.pyark/webapp/app.py renamed to ark/dashboard/app.py

@@ -9,7 +9,7 @@
 
 def _get_fernet(user_id: str) -> Fernet:
     """Derive a user-specific Fernet key using PBKDF2HMAC."""
-    from ark.webapp.config import get_settings
+    from ark.dashboard.config import get_settings
     settings = get_settings()
 
     # PBKDF2HMAC is a high-security key derivation function.

ark/webapp/auth.py ark/dashboard/auth.pyark/webapp/auth.py renamed to ark/dashboard/auth.py

@@ -367,7 +367,7 @@ def submit_job(
 
     safe_api_keys = {k: shlex.quote(v) for k, v in (api_keys or {}).items()}
 
-    from ark.webapp.db import resolve_db_path
+    from ark.dashboard.db import resolve_db_path
     db_path = resolve_db_path()
 
     template_text = _SLURM_TEMPLATE.read_text()
@@ -489,7 +489,7 @@ def launch_local_job(
         python_prefix = [sys.executable]
 
     # Resolve DB path so orchestrator can sync status
-    from ark.webapp.db import resolve_db_path
+    from ark.dashboard.db import resolve_db_path
     db_path = resolve_db_path()
 
     cmd = python_prefix + [

ark/webapp/config.py ark/dashboard/config.pyark/webapp/config.py renamed to ark/dashboard/config.py

@@ -14,7 +14,7 @@
 from email.mime.text import MIMEText
 from pathlib import Path
 
-logger = logging.getLogger("ark.webapp.notify")
+logger = logging.getLogger("ark.dashboard.notify")
 
 # ── Telegram ──────────────────────────────────────────────────────────────────