Add ensemble monitoring dashboard and member surface field plotter

- ensemble_dashboard.py: live dashboard for bred vector ensemble with
  multi-member STDOUT monitoring (selectable members), convergence table
  showing per-variable RMS across breeding cycles, surface field viewer
  per member with variable selector and time slider
- plot_ensemble_surface_fields.sh: SLURM array job (1-50) to generate
  SST/SSS/SSH/KE plots for each member in parallel
- breed_vectors.py: rescale step now writes convergence.json log with
  per-member per-cycle diagnostics for the dashboard to consume

Dashboard serves on port 8051 (separate from control dashboard on 8050),
deployable on GCP login node via tailscale serve.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: fluidnumericsJoe

simulations/glorysv12-curvilinear/workflows/plot_ensemble_surface_fields.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#SBATCH --array=1-50
+#SBATCH -n1
+#SBATCH -c4
+#SBATCH --time=12:00:00
+#SBATCH --job-name=spectre_ens_plot
+#SBATCH --output=ens_plot_%A_%a.out
+#SBATCH --error=ens_plot_%A_%a.out
+
+# Each array task plots surface fields for one ensemble member.
+# SLURM_ARRAY_TASK_ID = member number (1-50)
+
+MEMBER_ID=$(printf "%03d" $SLURM_ARRAY_TASK_ID)
+
+if [ -n "${SLURM_JOB_ID:-}" ]; then
+    SCRIPT_PATH=$(scontrol show job "$SLURM_JOB_ID" --json | jq -r '.jobs[0].command')
+    SCRIPT_DIR=$(dirname "$(readlink -f "$SCRIPT_PATH")")
+    SIMULATION_DIR=$(dirname $SCRIPT_DIR)
+else
+    SCRIPT_DIR=$(dirname "$(readlink -f "$0")")
+    SIMULATION_DIR=$(dirname $SCRIPT_DIR)
+fi
+
+source $SCRIPT_DIR/env.sh
+
+MEMBER_RUN_DIR="${SIMULATION_DIR}/ensemble/member_${MEMBER_ID}/run"
+
+echo "Plotting surface fields for member ${MEMBER_ID}"
+echo "Run dir: ${MEMBER_RUN_DIR}"
+
+srun --container-image=$SPECTRE_UTILS_IMG \
+     --container-mounts=${HOME}:${HOME},${SIMULATION_DIR}:/workspace,${HOST_DATADIR}:/data \
+     python /opt/spectre_utils/plot_surface_fields.py \
+       /workspace/ensemble/member_${MEMBER_ID}/run \
+       --plots-dir /workspace/ensemble/member_${MEMBER_ID}/run/plots \
+       --poll 120 \
+       --start-date 2002-07-01 \
+       --dt 360.0

spectre_utils/breed_vectors.py

@@ -17,6 +17,7 @@
 import os
 import sys
 import re
+import json
 import argparse
 import yaml
 import numpy as np
@@ -311,7 +312,8 @@ def cmd_rescale(config, config_path, cycle):
     print(f"Control pickup (end of cycle {cycle}): {ctrl_pickup}")
     control_fields, meta = read_pickup(ctrl_pickup, Nx, Ny, Nr)
 
-    # Process each member
+    # Process each member and collect diagnostics for convergence log
+    cycle_diags = []
     for m in range(1, n_members + 1):
         member_dir = os.path.join(ensemble_dir, f"{paths['member_prefix']}_{m:03d}")
 
@@ -336,13 +338,32 @@ def cmd_rescale(config, config_path, cycle):
         with open(os.path.join(member_dir, "nIter0.txt"), "w") as f:
             f.write(str(end_iter))
 
+        diag["member"] = m
+        cycle_diags.append(diag)
+
         print(f"  Member {m:03d}: rescale={diag['rescale_factor']:.3f}, "
               f"T_rms={diag.get('Theta_rms', 0):.4f}°C, "
               f"S_rms={diag.get('Salt_rms', 0):.4f}, "
               f"U_rms={diag.get('Uvel_rms', 0):.4f} m/s, "
               f"Eta_rms={diag.get('EtaN_rms', 0):.4f} m")
 
-    print(f"\nCycle {cycle} rescaling complete")
+    # Write convergence log (append per cycle)
+    convergence_path = os.path.join(ensemble_dir, "convergence.json")
+    if os.path.exists(convergence_path):
+        with open(convergence_path, "r") as f:
+            convergence = json.load(f)
+    else:
+        convergence = {"cycles": []}
+
+    convergence["cycles"].append({
+        "cycle": cycle,
+        "iteration": end_iter,
+        "members": cycle_diags,
+    })
+    with open(convergence_path, "w") as f:
+        json.dump(convergence, f, indent=2)
+
+    print(f"\nCycle {cycle} rescaling complete — convergence log: {convergence_path}")
 
 
 def cmd_status(config, config_path, cycle):