Skip to content

Commit a858fc0

Browse files
sei-awellegithub-actions[bot]
sei-awelle
and
github-actions[bot]
authored
Add Apache Superset for xAPI/LRsql analytics (#64)
* Add Apache Superset for xAPI/LRsql analytics - Add Superset container with custom Dockerfile (psycopg2, authlib) - Configure Keycloak OAuth SSO with internal/external URL split - Auto-register LRsql database connection on startup - Add superset client to Keycloak realm - Add launch configurations and env file * Add xAPI analytics dashboard and remove unrelated changes - Add ORM-based dashboard creation script with 7 charts including client app breakdown (Blueprint, CITE, Gallery, Steamfitter) - Auto-create dashboard on Superset startup via init-superset.sh - Add README documenting Superset integration and xAPI data model - Remove unrelated KC_SPI and CSP changes (belong in separate branch) * Add missing document markings --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent 255015e commit a858fc0

File tree

13 files changed

+1352
-0
lines changed

13 files changed

+1352
-0
lines changed

‎.env/superset.env‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Superset Task - Superset enabled (connects to Lrsql for xAPI analytics)
2+
# Note: Add supporting apps (Lrsql, etc.) to appsettings.Development.json under Launch.Prod or Launch.Dev
3+
Launch__Superset=true

‎.vscode/launch.json‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,16 @@
165165
"group": "a"
166166
}
167167
},
168+
{
169+
"name": "Superset ",
170+
"type": "dotnet",
171+
"request": "launch",
172+
"projectPath": "${workspaceFolder}/Crucible.AppHost/Crucible.AppHost.csproj",
173+
"envFile": "${workspaceFolder}/.env/superset.env",
174+
"presentation": {
175+
"group": "a"
176+
}
177+
},
168178

169179
// ========== Aspire Configurations ==========
170180
{
@@ -391,6 +401,20 @@
391401
"group": "b"
392402
}
393403
},
404+
{
405+
"type": "aspire",
406+
"request": "launch",
407+
"name": "Superset (aspire)",
408+
"program": "",
409+
"debuggers": {
410+
"apphost": {
411+
"envFile": "${workspaceFolder}/.env/superset.env"
412+
}
413+
},
414+
"presentation": {
415+
"group": "b"
416+
}
417+
},
394418

395419
// PHP
396420
{

‎Crucible.AppHost/AppHost.cs‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
var builder = DistributedApplication.CreateBuilder(args);
1010

11+
1112
LaunchOptions launchOptions = builder.Configuration.GetSection("Launch").Get<LaunchOptions>() ?? new();
1213

1314
LogLaunchOptions(launchOptions);
@@ -26,6 +27,7 @@
2627
builder.AddMoodle(postgres, keycloak, launchOptions);
2728
builder.AddLrsql(postgres, keycloak, launchOptions);
2829
builder.AddMisp(postgres, keycloak, launchOptions);
30+
builder.AddSuperset(postgres, keycloak, launchOptions);
2931
builder.AddDocs(launchOptions);
3032

3133
builder.Build().Run();
@@ -979,6 +981,72 @@ public static void AddMisp(this IDistributedApplicationBuilder builder, IResourc
979981
}
980982
}
981983

984+
public static void AddSuperset(this IDistributedApplicationBuilder builder, IResourceBuilder<PostgresServerResource> postgres, IResourceBuilder<KeycloakResource> keycloak, LaunchOptions options)
985+
{
986+
var supersetMode = ResolveMode(options.Superset, "Superset", options);
987+
988+
if (!options.AddAllApplications && !IsEnabled(supersetMode))
989+
return;
990+
991+
var supersetDb = postgres.AddDatabase("supersetDb", "superset");
992+
993+
var superset = builder.AddContainer("superset", "superset-custom-image")
994+
.WithDockerfile("./resources/superset", "Dockerfile.SupersetCustom")
995+
.WithLifetime(ContainerLifetime.Persistent)
996+
.WithContainerName("superset")
997+
.WaitFor(postgres)
998+
.WaitFor(keycloak)
999+
.WithHttpEndpoint(port: 8088, targetPort: 8088)
1000+
.WithHttpHealthCheck(path: "/health", endpointName: "http")
1001+
.WithBindMount("./resources/superset/superset_config.py", "/app/superset_config.py", isReadOnly: true)
1002+
.WithBindMount("./resources/superset/init-superset.sh", "/app/init-superset.sh", isReadOnly: true)
1003+
.WithBindMount("./resources/superset/create-dashboard-orm.py", "/app/create-dashboard-orm.py", isReadOnly: true)
1004+
.WithEnvironment("SUPERSET_CONFIG_PATH", "/app/superset_config.py")
1005+
.WithEnvironment("SUPERSET_SECRET_KEY", "crucible-dev-superset-secret-key")
1006+
.WithEnvironment("KEYCLOAK_EXTERNAL_URL", "http://localhost:8080/realms/crucible")
1007+
.WithEnvironment(context =>
1008+
{
1009+
// Internal Keycloak URL for server-to-server communication
1010+
var keycloakEndpoint = keycloak.GetEndpoint("http");
1011+
var keycloakHost = keycloakEndpoint.Property(EndpointProperty.Host);
1012+
var keycloakPort = keycloakEndpoint.Property(EndpointProperty.Port);
1013+
context.EnvironmentVariables["KEYCLOAK_INTERNAL_URL"] =
1014+
ReferenceExpression.Create($"http://{keycloakHost}:{keycloakPort}/realms/crucible");
1015+
})
1016+
.WithEnvironment("KEYCLOAK_CLIENT_ID", "superset")
1017+
.WithEnvironment("KEYCLOAK_CLIENT_SECRET", "superset-client-secret")
1018+
.WithEnvironment(context =>
1019+
{
1020+
var host = postgres.Resource.PrimaryEndpoint.Property(EndpointProperty.Host);
1021+
var port = postgres.Resource.PrimaryEndpoint.Property(EndpointProperty.Port);
1022+
var user = postgres.Resource.UserNameReference;
1023+
var password = postgres.Resource.PasswordParameter;
1024+
var dbName = supersetDb.Resource.DatabaseName;
1025+
1026+
context.EnvironmentVariables["DATABASE_HOST"] = host;
1027+
context.EnvironmentVariables["DATABASE_PORT"] = port;
1028+
context.EnvironmentVariables["DATABASE_USER"] = user;
1029+
context.EnvironmentVariables["DATABASE_PASSWORD"] = password;
1030+
context.EnvironmentVariables["DATABASE_DB"] = dbName;
1031+
1032+
// Compose the SQLAlchemy URI for Superset metadata DB
1033+
context.EnvironmentVariables["SUPERSET_SQLALCHEMY_DATABASE_URI"] =
1034+
ReferenceExpression.Create($"postgresql+psycopg2://{user}:{password}@{host}:{port}/{dbName}");
1035+
1036+
// LRsql database connection for xAPI analytics
1037+
context.EnvironmentVariables["LRSQL_SQLALCHEMY_URI"] =
1038+
ReferenceExpression.Create($"postgresql+psycopg2://{user}:{password}@{host}:{port}/lrsql");
1039+
})
1040+
.WithEnvironment("SUPERSET_LOAD_EXAMPLES", "no")
1041+
.WithEnvironment("CYPRESS_CONFIG", "false")
1042+
.WithArgs("bash", "/app/init-superset.sh");
1043+
1044+
if (!IsEnabled(supersetMode))
1045+
{
1046+
superset.WithExplicitStart();
1047+
}
1048+
}
1049+
9821050
private static void ConfigureApiSecrets(
9831051
this IDistributedApplicationBuilder builder,
9841052
string apiProjectPath,

‎Crucible.AppHost/LaunchOptions.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ public class LaunchOptions
2020
public bool PGAdmin { get; set; }
2121
public bool Docs { get; set; }
2222
public bool Misp { get; set; }
23+
public bool Superset { get; set; }
2324

2425
// Supporting apps (appsettings.Development.json): arrays grouped by mode
2526
public string[] Prod { get; set; } = Array.Empty<string>();

‎Crucible.AppHost/resources/crucible-realm.json‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2668,6 +2668,56 @@
26682668
"offline_access",
26692669
"microprofile-jwt"
26702670
]
2671+
},
2672+
{
2673+
"id": "d4e5f6a7-b8c9-4d0e-1f2a-3b4c5d6e7f80",
2674+
"clientId": "superset",
2675+
"name": "superset",
2676+
"description": "Apache Superset BI platform",
2677+
"rootUrl": "http://localhost:8088",
2678+
"adminUrl": "http://localhost:8088",
2679+
"baseUrl": "http://localhost:8088",
2680+
"surrogateAuthRequired": false,
2681+
"enabled": true,
2682+
"alwaysDisplayInConsole": false,
2683+
"clientAuthenticatorType": "client-secret",
2684+
"secret": "superset-client-secret",
2685+
"redirectUris": ["http://localhost:8088/*"],
2686+
"webOrigins": ["http://localhost:8088"],
2687+
"notBefore": 0,
2688+
"bearerOnly": false,
2689+
"consentRequired": false,
2690+
"standardFlowEnabled": true,
2691+
"implicitFlowEnabled": false,
2692+
"directAccessGrantsEnabled": true,
2693+
"serviceAccountsEnabled": false,
2694+
"publicClient": false,
2695+
"frontchannelLogout": true,
2696+
"protocol": "openid-connect",
2697+
"attributes": {
2698+
"oidc.ciba.grant.enabled": "false",
2699+
"client.secret.creation.time": "1762805282",
2700+
"display.on.consent.screen": "false",
2701+
"oauth2.device.authorization.grant.enabled": "false",
2702+
"backchannel.logout.session.required": "true",
2703+
"backchannel.logout.revoke.offline.tokens": "false"
2704+
},
2705+
"authenticationFlowBindingOverrides": {},
2706+
"fullScopeAllowed": true,
2707+
"nodeReRegistrationTimeout": -1,
2708+
"defaultClientScopes": [
2709+
"web-origins",
2710+
"acr",
2711+
"profile",
2712+
"roles",
2713+
"email"
2714+
],
2715+
"optionalClientScopes": [
2716+
"address",
2717+
"phone",
2718+
"offline_access",
2719+
"microprofile-jwt"
2720+
]
26712721
}
26722722
],
26732723
"clientScopes": [

‎Crucible.AppHost/resources/superset/Dockerfile.SupersetCustom‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
FROM apache/superset:4.1.2
2+
3+
USER root
4+
RUN pip install psycopg2-binary authlib
5+
USER superset

‎Crucible.AppHost/resources/superset/README.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Apache Superset Integration
2+
3+
Apache Superset provides business intelligence and data visualization for xAPI learning analytics data stored in LRsql.
4+
5+
## Architecture
6+
7+
- **Superset container** runs on port 8088 with a custom Dockerfile (`Dockerfile.SupersetCustom`) that adds PostgreSQL and OAuth dependencies
8+
- **PostgreSQL** stores Superset's own metadata in a `superset` database
9+
- **LRsql database** is auto-registered as a data source on startup
10+
- **Keycloak** provides OAuth SSO authentication
11+
12+
## Configuration Files
13+
14+
| File | Purpose |
15+
|------|---------|
16+
| `Dockerfile.SupersetCustom` | Custom image with psycopg2-binary and authlib |
17+
| `superset_config.py` | Superset configuration (OAuth, cache, security) |
18+
| `init-superset.sh` | Startup script: migrations, admin user, LRsql registration, dashboard creation |
19+
| `create-dashboard-orm.py` | Creates the starter xAPI analytics dashboard using Superset's internal ORM |
20+
| `create-dashboard.py` | Alternative: REST API-based dashboard creation (Python) |
21+
| `create-dashboard.js` | Alternative: REST API-based dashboard creation (Node.js) |
22+
| `create-dashboard.sh` | Alternative: REST API-based dashboard creation (bash) |
23+
24+
## Starter Dashboard
25+
26+
The `xAPI Learning Analytics` dashboard is automatically created on first startup with 7 charts:
27+
28+
1. **Activity by Client App** (pie) - Statement distribution across Crucible apps (Blueprint, CITE, Gallery, Steamfitter, etc.)
29+
2. **Client App Verb Breakdown** (stacked bar) - Which verbs each app generates
30+
3. **xAPI Verb Distribution** (pie) - Overall verb frequency
31+
4. **xAPI Verb Counts** (bar) - Verb counts ranked
32+
5. **xAPI Activity Over Time** (timeline) - Statement volume over time by verb
33+
6. **Top Learners by Activity** (table) - Most active learners
34+
7. **Most Active Learning Objects** (table) - Most referenced activity IRIs
35+
36+
## Authentication
37+
38+
- **Local admin**: `admin` / `admin` (created on startup)
39+
- **Keycloak SSO**: Click the Keycloak login option on the login page
40+
41+
## Accessing Superset
42+
43+
- Dashboard URL: http://localhost:8088/superset/dashboard/xapi-analytics/
44+
- SQL Lab: http://localhost:8088/sqllab/ (query LRsql data directly)
45+
46+
## xAPI Data Model
47+
48+
The LRsql database uses these key tables for analytics:
49+
50+
| Table | Purpose |
51+
|-------|---------|
52+
| `xapi_statement` | Core xAPI statements with verb_iri, timestamp, JSON payload |
53+
| `statement_to_actor` | Links statements to actors (Actor, Team, Authority) |
54+
| `statement_to_activity` | Links statements to activities (Object, context) |
55+
| `activity` | Activity definitions with IRI and JSON payload |
56+
| `actor` | Actor definitions |
57+
58+
### Client App Detection
59+
60+
Client applications are identified by port number in activity IRIs:
61+
62+
| Port | Application |
63+
|------|-------------|
64+
| 4724 | Blueprint |
65+
| 4720, 4721 | CITE |
66+
| 4722, 4723 | Gallery |
67+
| 4300, 4301 | Player |
68+
| 4400, 4401 | Steamfitter |
69+
| 4403 | Alloy |
70+
| 4310 | Caster |
71+
| 5000 | TopoMojo |
72+
| 8081 | Moodle |
73+
74+
### Cross-App Correlation
75+
76+
Activities across multiple Crucible apps can be correlated using:
77+
78+
- **Registration ID** (`registration` column) - UUID linking statements from a single exercise execution
79+
- **Context Activities** (`context.contextActivities.grouping` in payload) - Shared MSEL/exercise activity IRI
80+
81+
For full cross-app correlation, orchestrators (Alloy, Blueprint) should pass a shared registration UUID to all downstream apps when launching exercises.

0 commit comments

Comments
 (0)
⚡