refactor(trends): extract buildDerivedConfigs helper + perf fixes #107163
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Tasks Sandbox Container Image CD | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| workflow_dispatch: | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| if: github.repository == 'PostHog/posthog' | |
| name: Determine if sandbox image needs to be built | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| sandbox_image: ${{ steps.filter.outputs.sandbox_image }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| id: app-token | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }} | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| with: | |
| token: ${{ steps.app-token.outputs.token || github.token }} | |
| filters: | | |
| sandbox_image: | |
| - '.github/workflows/cd-sandbox-base-image.yml' | |
| - 'products/tasks/backend/sandbox/images/**' | |
| - 'products/*/skills/**' | |
| - 'products/posthog_ai/scripts/**' | |
| build_skills: | |
| needs: changes | |
| name: Build agent skills from source | |
| if: | | |
| github.repository == 'PostHog/posthog' && ( | |
| needs.changes.outputs.sandbox_image == 'true' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'build-tasks-sandbox-image') | |
| ) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| permissions: | |
| contents: read | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Start Docker services | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| run: bin/ci-wait-for-docker launch | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 | |
| with: | |
| version: '0.10.2' | |
| enable-cache: true | |
| save-cache: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install python dependencies | |
| run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Install Rust | |
| uses: dtolnay/rust-toolchain@0b1efabc08b657293548b77fb76cc02d26091c7e | |
| with: | |
| toolchain: stable | |
| components: cargo | |
| - name: Cache Rust dependencies | |
| uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 | |
| with: | |
| shared-key: 'v2-rust-skills' | |
| workspaces: rust | |
| save-if: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install sqlx-cli | |
| run: cargo install sqlx-cli --version 0.8.0 --features postgres --no-default-features --locked | |
| - name: Wait for Docker services | |
| env: | |
| COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml | |
| run: bin/ci-wait-for-docker wait | |
| - name: Run migrations | |
| env: | |
| SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| REDIS_URL: 'redis://localhost' | |
| OBJECT_STORAGE_ENABLED: 'False' | |
| TEST: 1 | |
| DEBUG: 1 | |
| run: | | |
| DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ | |
| sqlx database create | |
| DATABASE_URL="postgres://posthog:posthog@localhost:5432/posthog_persons" \ | |
| sqlx migrate run --source rust/persons_migrations/ | |
| python manage.py migrate --noinput | |
| python manage.py setup_dev --no-data | |
| - name: Build skills | |
| env: | |
| SECRET_KEY: '6b01eee4f945ca25045b5aab440b953461faf08693a9abbf1166dc7c6b9772da' | |
| DATABASE_URL: 'postgres://posthog:posthog@localhost:5432/posthog' | |
| REDIS_URL: 'redis://localhost' | |
| OBJECT_STORAGE_ENABLED: 'False' | |
| TEST: 1 | |
| DEBUG: 1 | |
| run: ./bin/hogli build:skills | |
| - name: Download context-mill skills | |
| run: | | |
| CONTEXT_MILL_ZIP_URL="https://github.com/PostHog/context-mill/releases/latest/download/skills-mcp-resources.zip" | |
| TARGET_DIR="products/posthog_ai/dist/skills" | |
| tmp_dir=$(mktemp -d) | |
| trap 'rm -rf "$tmp_dir"' EXIT | |
| echo "Downloading context-mill skills from ${CONTEXT_MILL_ZIP_URL}..." | |
| curl -fsSL --retry 2 --retry-delay 3 -o "$tmp_dir/cm.zip" "$CONTEXT_MILL_ZIP_URL" | |
| unzip -q -o "$tmp_dir/cm.zip" -d "$tmp_dir/outer" | |
| mkdir -p "$TARGET_DIR" | |
| count=0 | |
| while read -r inner_zip; do | |
| skill_name="$(basename "$inner_zip" .zip)" | |
| skill_name="${skill_name#omnibus-}" | |
| skill_dir="$TARGET_DIR/$skill_name" | |
| mkdir -p "$skill_dir" | |
| unzip -q -o "$inner_zip" -d "$skill_dir" | |
| find "$skill_dir" -name 'SKILL.md' -type f -exec sed -i 's/^\(name: *\)omnibus-/\1/' {} + | |
| count=$((count + 1)) | |
| done < <(find "$tmp_dir/outer" -name 'omnibus-*.zip' -type f) | |
| echo "Installed ${count} context-mill skills into ${TARGET_DIR}" | |
| - name: Upload built skills | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| with: | |
| name: built-skills | |
| path: products/posthog_ai/dist/skills/ | |
| retention-days: 1 | |
| sandbox_base_build: | |
| needs: [changes, build_skills] | |
| name: Build and push Tasks Sandbox container image | |
| if: | | |
| github.repository == 'PostHog/posthog' && ( | |
| needs.changes.outputs.sandbox_image == 'true' || | |
| github.event_name == 'workflow_dispatch' || | |
| contains(github.event.pull_request.labels.*.name, 'build-tasks-sandbox-image') | |
| ) | |
| runs-on: depot-ubuntu-latest | |
| timeout-minutes: 10 | |
| permissions: | |
| id-token: write # allow issuing OIDC tokens for this workflow run | |
| contents: read # allow at least reading the repo contents, add other permissions if necessary | |
| packages: write # allow push to ghcr.io | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 2 | |
| - name: Download built skills | |
| uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 | |
| with: | |
| name: built-skills | |
| path: products/posthog_ai/dist/skills/ | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 | |
| - name: Set up Depot CLI | |
| uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1 | |
| - name: Login to ghcr.io | |
| uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| logout: false | |
| - name: Build and push container image (base) | |
| id: build | |
| uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.17.0 | |
| with: | |
| context: . # use local checkout (includes downloaded artifact) | |
| file: ./products/tasks/backend/sandbox/images/Dockerfile.sandbox-base | |
| buildx-fallback: false # the fallback is so slow it's better to just fail | |
| push: true | |
| tags: ${{ github.ref == 'refs/heads/master' && format('ghcr.io/posthog/posthog-sandbox-base:master,ghcr.io/posthog/posthog-sandbox-base:{0}', github.sha) || format('ghcr.io/posthog/posthog-sandbox-base:{0}', github.sha) }} | |
| platforms: linux/arm64,linux/amd64 | |
| build-args: COMMIT_HASH=${{ github.sha }} | |
| - name: Build and push container image (notebook) | |
| id: build-notebook | |
| uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.17.0 | |
| with: | |
| context: . # use local checkout (includes downloaded artifact) | |
| file: ./products/tasks/backend/sandbox/images/Dockerfile.sandbox-notebook | |
| buildx-fallback: false # the fallback is so slow it's better to just fail | |
| push: true | |
| tags: ${{ github.ref == 'refs/heads/master' && format('ghcr.io/posthog/posthog-sandbox-notebook:master,ghcr.io/posthog/posthog-sandbox-notebook:{0}', github.sha) || format('ghcr.io/posthog/posthog-sandbox-notebook:{0}', github.sha) }} | |
| platforms: linux/arm64,linux/amd64 | |
| build-args: COMMIT_HASH=${{ github.sha }} | |
| - name: Build and push container image (pi) | |
| id: build-pi | |
| uses: depot/build-push-action@5f3b3c2e5a00f0093de47f657aeaefcedff27d18 # v1.17.0 | |
| with: | |
| context: . # use local checkout (includes downloaded artifact) | |
| file: ./products/tasks/backend/sandbox/images/Dockerfile.sandbox-pi | |
| buildx-fallback: false # the fallback is so slow it's better to just fail | |
| push: true | |
| tags: ${{ github.ref == 'refs/heads/master' && format('ghcr.io/posthog/posthog-sandbox-pi:master,ghcr.io/posthog/posthog-sandbox-pi:{0}', github.sha) || format('ghcr.io/posthog/posthog-sandbox-pi:{0}', github.sha) }} | |
| platforms: linux/arm64,linux/amd64 | |
| build-args: | | |
| BASE_IMAGE=ghcr.io/posthog/posthog-sandbox-base:${{ github.sha }} | |
| # Single required status check for branch protection. Add future sandbox | |
| # image jobs to `needs` here rather than reconfiguring GitHub. | |
| sandbox_base_image_check: | |
| needs: [changes, build_skills, sandbox_base_build] | |
| name: Sandbox Base Image Pass | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| permissions: {} | |
| steps: | |
| - name: Check all sandbox image jobs | |
| run: | | |
| # Fail if change detection itself failed | |
| if [[ "${{ needs.changes.result }}" == "failure" ]]; then | |
| echo "Change detection job failed." | |
| exit 1 | |
| fi | |
| # Pass if build wasn't required (no relevant changes, no dispatch, no label) | |
| if [[ "${{ needs.build_skills.result }}" == "skipped" && "${{ needs.sandbox_base_build.result }}" == "skipped" ]]; then | |
| echo "Sandbox image build was skipped (no relevant changes)." | |
| exit 0 | |
| fi | |
| if [[ "${{ needs.build_skills.result }}" != "success" && "${{ needs.build_skills.result }}" != "skipped" ]]; then | |
| echo "Agent skills build failed." | |
| exit 1 | |
| fi | |
| if [[ "${{ needs.sandbox_base_build.result }}" != "success" && "${{ needs.sandbox_base_build.result }}" != "skipped" ]]; then | |
| echo "Sandbox base image build failed." | |
| exit 1 | |
| fi | |
| echo "All sandbox image checks passed." |