Add Apache Superset for xAPI/LRsql analytics

.env/superset.env

@@ -0,0 +1,3 @@
+# Superset Task - Superset enabled (connects to Lrsql for xAPI analytics)
+# Note: Add supporting apps (Lrsql, etc.) to appsettings.Development.json under Launch.Prod or Launch.Dev
+Launch__Superset=true

.vscode/launch.json

@@ -165,6 +165,16 @@
         "group": "a"
       }
     },
+    {
+      "name": "Superset ",
+      "type": "dotnet",
+      "request": "launch",
+      "projectPath": "${workspaceFolder}/Crucible.AppHost/Crucible.AppHost.csproj",
+      "envFile": "${workspaceFolder}/.env/superset.env",
+      "presentation": {
+        "group": "a"
+      }
+    },
 
     // ========== Aspire Configurations ==========
     {
@@ -391,6 +401,20 @@
         "group": "b"
       }
     },
+    {
+      "type": "aspire",
+      "request": "launch",
+      "name": "Superset (aspire)",
+      "program": "",
+      "debuggers": {
+        "apphost": {
+          "envFile": "${workspaceFolder}/.env/superset.env"
+        }
+      },
+      "presentation": {
+        "group": "b"
+      }
+    },
 
     // PHP
     {

Crucible.AppHost/AppHost.cs

@@ -8,6 +8,7 @@
 
 var builder = DistributedApplication.CreateBuilder(args);
 
+
 LaunchOptions launchOptions = builder.Configuration.GetSection("Launch").Get<LaunchOptions>() ?? new();
 
 LogLaunchOptions(launchOptions);
@@ -26,6 +27,7 @@
 builder.AddMoodle(postgres, keycloak, launchOptions);
 builder.AddLrsql(postgres, keycloak, launchOptions);
 builder.AddMisp(postgres, keycloak, launchOptions);
+builder.AddSuperset(postgres, keycloak, launchOptions);
 builder.AddDocs(launchOptions);
 
 builder.Build().Run();
@@ -979,6 +981,72 @@ public static void AddMisp(this IDistributedApplicationBuilder builder, IResourc
         }
     }
 
+    public static void AddSuperset(this IDistributedApplicationBuilder builder, IResourceBuilder<PostgresServerResource> postgres, IResourceBuilder<KeycloakResource> keycloak, LaunchOptions options)
+    {
+        var supersetMode = ResolveMode(options.Superset, "Superset", options);
+
+        if (!options.AddAllApplications && !IsEnabled(supersetMode))
+            return;
+
+        var supersetDb = postgres.AddDatabase("supersetDb", "superset");
+
+        var superset = builder.AddContainer("superset", "superset-custom-image")
+            .WithDockerfile("./resources/superset", "Dockerfile.SupersetCustom")
+            .WithLifetime(ContainerLifetime.Persistent)
+            .WithContainerName("superset")
+            .WaitFor(postgres)
+            .WaitFor(keycloak)
+            .WithHttpEndpoint(port: 8088, targetPort: 8088)
+            .WithHttpHealthCheck(path: "/health", endpointName: "http")
+            .WithBindMount("./resources/superset/superset_config.py", "/app/superset_config.py", isReadOnly: true)
+            .WithBindMount("./resources/superset/init-superset.sh", "/app/init-superset.sh", isReadOnly: true)
+            .WithBindMount("./resources/superset/create-dashboard-orm.py", "/app/create-dashboard-orm.py", isReadOnly: true)
+            .WithEnvironment("SUPERSET_CONFIG_PATH", "/app/superset_config.py")
+            .WithEnvironment("SUPERSET_SECRET_KEY", "crucible-dev-superset-secret-key")
+            .WithEnvironment("KEYCLOAK_EXTERNAL_URL", "http://localhost:8080/realms/crucible")
+            .WithEnvironment(context =>
+            {
+                // Internal Keycloak URL for server-to-server communication
+                var keycloakEndpoint = keycloak.GetEndpoint("http");
+                var keycloakHost = keycloakEndpoint.Property(EndpointProperty.Host);
+                var keycloakPort = keycloakEndpoint.Property(EndpointProperty.Port);
+                context.EnvironmentVariables["KEYCLOAK_INTERNAL_URL"] =
+                    ReferenceExpression.Create($"http://{keycloakHost}:{keycloakPort}/realms/crucible");
+            })
+            .WithEnvironment("KEYCLOAK_CLIENT_ID", "superset")
+            .WithEnvironment("KEYCLOAK_CLIENT_SECRET", "superset-client-secret")
+            .WithEnvironment(context =>
+            {
+                var host = postgres.Resource.PrimaryEndpoint.Property(EndpointProperty.Host);
+                var port = postgres.Resource.PrimaryEndpoint.Property(EndpointProperty.Port);
+                var user = postgres.Resource.UserNameReference;
+                var password = postgres.Resource.PasswordParameter;
+                var dbName = supersetDb.Resource.DatabaseName;
+
+                context.EnvironmentVariables["DATABASE_HOST"] = host;
+                context.EnvironmentVariables["DATABASE_PORT"] = port;
+                context.EnvironmentVariables["DATABASE_USER"] = user;
+                context.EnvironmentVariables["DATABASE_PASSWORD"] = password;
+                context.EnvironmentVariables["DATABASE_DB"] = dbName;
+
+                // Compose the SQLAlchemy URI for Superset metadata DB
+                context.EnvironmentVariables["SUPERSET_SQLALCHEMY_DATABASE_URI"] =
+                    ReferenceExpression.Create($"postgresql+psycopg2://{user}:{password}@{host}:{port}/{dbName}");
+
+                // LRsql database connection for xAPI analytics
+                context.EnvironmentVariables["LRSQL_SQLALCHEMY_URI"] =
+                    ReferenceExpression.Create($"postgresql+psycopg2://{user}:{password}@{host}:{port}/lrsql");
+            })
+            .WithEnvironment("SUPERSET_LOAD_EXAMPLES", "no")
+            .WithEnvironment("CYPRESS_CONFIG", "false")
+            .WithArgs("bash", "/app/init-superset.sh");
+
+        if (!IsEnabled(supersetMode))
+        {
+            superset.WithExplicitStart();
+        }
+    }
+
     private static void ConfigureApiSecrets(
         this IDistributedApplicationBuilder builder,
         string apiProjectPath,

Crucible.AppHost/LaunchOptions.cs

@@ -20,6 +20,7 @@ public class LaunchOptions
     public bool PGAdmin { get; set; }
     public bool Docs { get; set; }
     public bool Misp { get; set; }
+    public bool Superset { get; set; }
 
     // Supporting apps (appsettings.Development.json): arrays grouped by mode
     public string[] Prod { get; set; } = Array.Empty<string>();

Crucible.AppHost/resources/crucible-realm.json

@@ -2668,6 +2668,56 @@
         "offline_access",
         "microprofile-jwt"
       ]
+    },
+    {
+      "id": "d4e5f6a7-b8c9-4d0e-1f2a-3b4c5d6e7f80",
+      "clientId": "superset",
+      "name": "superset",
+      "description": "Apache Superset BI platform",
+      "rootUrl": "http://localhost:8088",
+      "adminUrl": "http://localhost:8088",
+      "baseUrl": "http://localhost:8088",
+      "surrogateAuthRequired": false,
+      "enabled": true,
+      "alwaysDisplayInConsole": false,
+      "clientAuthenticatorType": "client-secret",
+      "secret": "superset-client-secret",
+      "redirectUris": ["http://localhost:8088/*"],
+      "webOrigins": ["http://localhost:8088"],
+      "notBefore": 0,
+      "bearerOnly": false,
+      "consentRequired": false,
+      "standardFlowEnabled": true,
+      "implicitFlowEnabled": false,
+      "directAccessGrantsEnabled": true,
+      "serviceAccountsEnabled": false,
+      "publicClient": false,
+      "frontchannelLogout": true,
+      "protocol": "openid-connect",
+      "attributes": {
+        "oidc.ciba.grant.enabled": "false",
+        "client.secret.creation.time": "1762805282",
+        "display.on.consent.screen": "false",
+        "oauth2.device.authorization.grant.enabled": "false",
+        "backchannel.logout.session.required": "true",
+        "backchannel.logout.revoke.offline.tokens": "false"
+      },
+      "authenticationFlowBindingOverrides": {},
+      "fullScopeAllowed": true,
+      "nodeReRegistrationTimeout": -1,
+      "defaultClientScopes": [
+        "web-origins",
+        "acr",
+        "profile",
+        "roles",
+        "email"
+      ],
+      "optionalClientScopes": [
+        "address",
+        "phone",
+        "offline_access",
+        "microprofile-jwt"
+      ]
     }
   ],
   "clientScopes": [

Crucible.AppHost/resources/superset/Dockerfile.SupersetCustom

@@ -0,0 +1,5 @@
+FROM apache/superset:4.1.2
+
+USER root
+RUN pip install psycopg2-binary authlib
+USER superset

Crucible.AppHost/resources/superset/README.md

@@ -0,0 +1,81 @@
+# Apache Superset Integration
+
+Apache Superset provides business intelligence and data visualization for xAPI learning analytics data stored in LRsql.
+
+## Architecture
+
+- **Superset container** runs on port 8088 with a custom Dockerfile (`Dockerfile.SupersetCustom`) that adds PostgreSQL and OAuth dependencies
+- **PostgreSQL** stores Superset's own metadata in a `superset` database
+- **LRsql database** is auto-registered as a data source on startup
+- **Keycloak** provides OAuth SSO authentication
+
+## Configuration Files
+
+| File | Purpose |
+|------|---------|
+| `Dockerfile.SupersetCustom` | Custom image with psycopg2-binary and authlib |
+| `superset_config.py` | Superset configuration (OAuth, cache, security) |
+| `init-superset.sh` | Startup script: migrations, admin user, LRsql registration, dashboard creation |
+| `create-dashboard-orm.py` | Creates the starter xAPI analytics dashboard using Superset's internal ORM |
+| `create-dashboard.py` | Alternative: REST API-based dashboard creation (Python) |
+| `create-dashboard.js` | Alternative: REST API-based dashboard creation (Node.js) |
+| `create-dashboard.sh` | Alternative: REST API-based dashboard creation (bash) |
+
+## Starter Dashboard
+
+The `xAPI Learning Analytics` dashboard is automatically created on first startup with 7 charts:
+
+1. **Activity by Client App** (pie) - Statement distribution across Crucible apps (Blueprint, CITE, Gallery, Steamfitter, etc.)
+2. **Client App Verb Breakdown** (stacked bar) - Which verbs each app generates
+3. **xAPI Verb Distribution** (pie) - Overall verb frequency
+4. **xAPI Verb Counts** (bar) - Verb counts ranked
+5. **xAPI Activity Over Time** (timeline) - Statement volume over time by verb
+6. **Top Learners by Activity** (table) - Most active learners
+7. **Most Active Learning Objects** (table) - Most referenced activity IRIs
+
+## Authentication
+
+- **Local admin**: `admin` / `admin` (created on startup)
+- **Keycloak SSO**: Click the Keycloak login option on the login page
+
+## Accessing Superset
+
+- Dashboard URL: http://localhost:8088/superset/dashboard/xapi-analytics/
+- SQL Lab: http://localhost:8088/sqllab/ (query LRsql data directly)
+
+## xAPI Data Model
+
+The LRsql database uses these key tables for analytics:
+
+| Table | Purpose |
+|-------|---------|
+| `xapi_statement` | Core xAPI statements with verb_iri, timestamp, JSON payload |
+| `statement_to_actor` | Links statements to actors (Actor, Team, Authority) |
+| `statement_to_activity` | Links statements to activities (Object, context) |
+| `activity` | Activity definitions with IRI and JSON payload |
+| `actor` | Actor definitions |
+
+### Client App Detection
+
+Client applications are identified by port number in activity IRIs:
+
+| Port | Application |
+|------|-------------|
+| 4724 | Blueprint |
+| 4720, 4721 | CITE |
+| 4722, 4723 | Gallery |
+| 4300, 4301 | Player |
+| 4400, 4401 | Steamfitter |
+| 4403 | Alloy |
+| 4310 | Caster |
+| 5000 | TopoMojo |
+| 8081 | Moodle |
+
+### Cross-App Correlation
+
+Activities across multiple Crucible apps can be correlated using:
+
+- **Registration ID** (`registration` column) - UUID linking statements from a single exercise execution
+- **Context Activities** (`context.contextActivities.grouping` in payload) - Shared MSEL/exercise activity IRI
+
+For full cross-app correlation, orchestrators (Alloy, Blueprint) should pass a shared registration UUID to all downstream apps when launching exercises.