refactor(trends): extract buildDerivedConfigs helper + perf fixes #137624
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs generic Python tests global to Dagster and PostHog. | |
| name: Python CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| merge_group: | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| # Job to decide if we should run python ci | |
| # See https://github.com/dorny/paths-filter#conditional-execution for more details | |
| changes: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| name: Determine need to run python checks | |
| # Set job outputs to values from filter step | |
| outputs: | |
| python: ${{ steps.filter.outputs.python || 'true' }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code, but we | |
| # also want this to run on master so we need to checkout | |
| - uses: actions/checkout@v6 | |
| - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| id: app-token | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }} | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| if: github.event_name != 'push' # Run all tests on master push | |
| with: | |
| token: ${{ steps.app-token.outputs.token || github.token }} | |
| filters: | | |
| python: | |
| - 'pyproject.toml' | |
| - 'uv.lock' | |
| - 'ee/**/*.py' | |
| - 'posthog/**/*' | |
| - 'products/**/*.py' | |
| # Make sure we run if someone is explicitly change the workflows | |
| - .github/workflows/ci-python.yml | |
| - .github/workflows/ci-dagster.yml | |
| - .github/workflows/ci-backend.yml | |
| - .flox/env/manifest.toml | |
| - bin/check_uv_python_compatibility.py | |
| # Schema changes need to trigger Python CI to validate schema.py is regenerated | |
| - 'frontend/src/queries/schema.json' | |
| # Products changes need to trigger Python CI to validate the json usage is still valid | |
| - 'frontend/src/products.json' | |
| # hogli CLI changes need validation | |
| - 'tools/hogli/**' | |
| - 'tools/hogli-commands/hogli_commands/**' | |
| - 'hogli.yaml' | |
| - 'bin/**' | |
| # LLM Gateway service | |
| - 'services/llm-gateway/**/*.py' | |
| # GitHub scripts | |
| - '.github/scripts/**/*.py' | |
| # PR approval agent | |
| - 'tools/pr-approval-agent/**' | |
| code-quality: | |
| needs: changes | |
| if: needs.changes.outputs.python == 'true' | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ vars.BLACKSMITH_SHADOW_ENABLED == 'true' && fromJson('[{"runner":"depot-ubuntu-latest","is_shadow":false},{"runner":"blacksmith-2vcpu-ubuntu-2404","is_shadow":true}]') || fromJson('[{"runner":"depot-ubuntu-latest","is_shadow":false}]') }} | |
| name: Python code quality (${{ matrix.runner }}) | |
| runs-on: ${{ matrix.runner }} | |
| continue-on-error: ${{ matrix.is_shadow }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version: 3.12.12 | |
| - name: Install uv | |
| id: setup-uv | |
| uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 | |
| with: | |
| version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| save-cache: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Check uv and Python version compatibility | |
| shell: bash | |
| run: | | |
| python3 bin/check_uv_python_compatibility.py | |
| - name: Install SAML (python3-saml) dependencies | |
| if: steps.setup-uv.outputs.cache-hit != 'true' | |
| shell: bash | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install Python dependencies | |
| shell: bash | |
| run: | | |
| UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Download GeoIP database | |
| shell: bash | |
| run: | | |
| ./bin/download-mmdb | |
| - name: Check for syntax errors, import sort, and code style violations | |
| shell: bash | |
| run: | | |
| ruff check . | |
| - name: Check formatting | |
| shell: bash | |
| run: | | |
| ruff format --check --diff . | |
| - name: Add ty Problem Matcher | |
| shell: bash | |
| run: | | |
| echo "::add-matcher::.github/ty-problem-matcher.json" | |
| - name: Check ty type checking | |
| shell: bash | |
| run: | | |
| echo "📊 Feedback welcome in #team-devex" | |
| uv run ty check | |
| - name: Add mypy Problem Matcher | |
| shell: bash | |
| run: | | |
| echo "::add-matcher::.github/mypy-problem-matcher.json" | |
| - name: Restore mypy cache | |
| uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: .mypy_cache | |
| key: mypy-cache-${{ runner.os }}-3.12-${{ hashFiles('**/pyproject.toml', '**/mypy.ini') }} | |
| restore-keys: | | |
| mypy-cache-${{ runner.os }}-3.12- | |
| - name: Check static typing | |
| shell: bash -e {0} | |
| run: | | |
| mypy --version && mypy --cache-fine-grained . | |
| - name: Save mypy cache | |
| # Shadow restores the saved cache for apples-to-apples timing. | |
| if: github.ref == 'refs/heads/master' && !matrix.is_shadow | |
| uses: actions/cache/save@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5 | |
| with: | |
| path: .mypy_cache | |
| key: mypy-cache-${{ runner.os }}-3.12-${{ hashFiles('**/pyproject.toml', '**/mypy.ini') }} | |
| - name: Check if "schema.py" is up to date | |
| shell: bash | |
| run: | | |
| npm run schema:build:python && git diff --exit-code | |
| - name: Check hogli manifest completeness | |
| shell: bash | |
| run: | | |
| ./bin/hogli meta:check | |
| - name: Run hogli framework tests | |
| shell: bash | |
| run: | | |
| pytest tools/hogli/tests -v --junitxml=junit-hogli.xml | |
| - name: Run hogli extension tests | |
| shell: bash | |
| run: | | |
| pytest tools/hogli-commands/hogli_commands/tests -v --junitxml=junit-hogli-commands.xml | |
| - name: Upload test results | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() && !matrix.is_shadow | |
| with: | |
| name: junit-results-hogli | |
| path: | | |
| junit-hogli.xml | |
| junit-hogli-commands.xml | |
| - name: Run dependency detection script tests | |
| shell: bash | |
| run: | | |
| pytest bin/test/test_find_python_dependencies.py -v --junitxml=junit-dependency-detection.xml | |
| - name: Upload test results for dependency detection | |
| uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 | |
| if: always() && !matrix.is_shadow | |
| with: | |
| name: junit-results-dependency-detection | |
| path: junit-dependency-detection.xml | |
| # - name: Check if "taxonomy.json/taxonomy.tsx" is up to date | |
| # if: needs.changes.outputs.python == 'true' | |
| # shell: bash | |
| # run: | | |
| # npm run taxonomy:build:json && git diff --exit-code | |
| # Collate job - single required status check for branch protection. | |
| # The code-quality job skips when no Python changes, but this job always | |
| # runs and reports success, allowing PRs to merge. | |
| python_tests: | |
| needs: [changes, code-quality] | |
| name: Python code quality checks | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| if: always() | |
| steps: | |
| - name: Check Python CI status | |
| run: | | |
| # Fail if change detection itself failed | |
| if [[ "${{ needs.changes.result }}" == "failure" ]]; then | |
| echo "Change detection job failed." | |
| exit 1 | |
| fi | |
| # Pass if no Python changes detected (jobs were skipped) | |
| if [[ "${{ needs.changes.outputs.python }}" != "true" ]]; then | |
| echo "Python checks were skipped (no relevant changes)" | |
| exit 0 | |
| fi | |
| # Check actual job result | |
| if [[ "${{ needs.code-quality.result }}" != "success" && "${{ needs.code-quality.result }}" != "skipped" ]]; then | |
| echo "Python code quality checks failed." | |
| exit 1 | |
| fi | |
| echo "All Python checks passed." |