refactor(trends): extract buildDerivedConfigs helper + perf fixes #189458
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow runs all of our backend django tests. | |
| # | |
| # If these tests get too slow, look at increasing concurrency and re-timing the tests by manually dispatching | |
| # .github/workflows/ci-backend-update-test-timing.yml action | |
| name: Hog CI | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| paths-ignore: | |
| - rust/** | |
| - livestream/** | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| # Job to decide if we should run backend ci | |
| # See https://github.com/dorny/paths-filter#conditional-execution for more details | |
| changes: | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| name: Determine need to run Hog checks | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| # Set job outputs to values from filter step | |
| outputs: | |
| hog: ${{ steps.filter.outputs.hog || 'true' }} | |
| steps: | |
| # For pull requests it's not necessary to checkout the code, but we | |
| # also want this to run on master so we need to checkout | |
| - uses: actions/checkout@v6 | |
| - uses: actions/create-github-app-token@1b10c78c7865c340bc4f6099eb2f838309f1e8c3 # v3.1.1 | |
| id: app-token | |
| if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository | |
| with: | |
| client-id: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_APP_ID }} | |
| private-key: ${{ secrets.GH_APP_POSTHOG_PATHS_FILTER_PRIVATE_KEY }} | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: filter | |
| if: github.event_name != 'push' # Run all tests on master push | |
| with: | |
| token: ${{ steps.app-token.outputs.token || github.token }} | |
| filters: | | |
| hog: | |
| # Avoid running tests for irrelevant changes | |
| - 'common/hogvm/**/*' | |
| - 'posthog/hogql/**/*' | |
| - 'bin/hog' | |
| - 'bin/hoge' | |
| - 'package.json' | |
| - requirements.txt | |
| - requirements-dev.txt | |
| - .github/workflows/ci-hog.yml | |
| hog-tests: | |
| needs: changes | |
| timeout-minutes: 30 | |
| name: Hog tests | |
| runs-on: ubuntu-24.04 | |
| if: needs.changes.outputs.hog == 'true' | |
| outputs: | |
| committed-version: ${{ steps.check-package-version.outputs.committed-version }} | |
| published-version: ${{ steps.check-package-version.outputs.published-version }} | |
| is-new-version: ${{ steps.check-package-version.outputs.is-new-version }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| - name: Install uv | |
| id: setup-uv | |
| uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 | |
| with: | |
| version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| save-cache: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install SAML (python3-saml) dependencies | |
| if: steps.setup-uv.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install Python dependencies | |
| run: | | |
| UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| - name: Fix node-gyp permissions | |
| run: chmod +x ~/setup-pnpm/node_modules/.pnpm/pnpm@*/node_modules/pnpm/dist/node_modules/node-gyp/gyp/gyp_main.py | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| cache: 'pnpm' | |
| - name: Check if ANTLR definitions are up to date | |
| run: | | |
| cd .. | |
| sudo apt-get update && sudo apt-get install default-jre | |
| mkdir antlr | |
| cd antlr | |
| curl --fail --location https://www.antlr.org/download/antlr-${ANTLR_VERSION}-complete.jar --output antlr.jar || curl --fail --location https://repo.maven.apache.org/maven2/org/antlr/antlr4/${ANTLR_VERSION}/antlr4-${ANTLR_VERSION}-complete.jar --output antlr.jar | |
| export PWD=`pwd` | |
| echo '#!/bin/bash' > antlr | |
| echo "java -jar $PWD/antlr.jar \$*" >> antlr | |
| chmod +x antlr | |
| export CLASSPATH=".:$PWD/antlr.jar:$CLASSPATH" | |
| export PATH="$PWD:$PATH" | |
| cd ../posthog | |
| antlr | grep "Version" | |
| npm run grammar:build && git diff --exit-code | |
| env: | |
| # Installing a version of ANTLR compatible with what's in Homebrew as of August 2024 (version 4.13.2), | |
| # as apt-get is quite out of date. The same version must be set in common/hogql_parser/pyproject.toml | |
| ANTLR_VERSION: '4.13.2' | |
| - name: Check if STL bytecode is up to date | |
| run: | | |
| python -m common.hogvm.stl.compile | |
| git diff --exit-code | |
| env: | |
| TEST: 1 | |
| - name: Run HogVM Python tests | |
| run: | | |
| pytest common/hogvm | |
| - name: Run HogVM TypeScript tests | |
| run: | | |
| pnpm --filter=@posthog/hogvm install --frozen-lockfile | |
| pnpm --filter=@posthog/hogvm test | |
| - name: Run Hog tests | |
| run: | | |
| pnpm --filter=@posthog/hogvm install --frozen-lockfile | |
| pnpm --filter=@posthog/hogvm compile:stl | |
| bin/turbo --filter=@posthog/hogvm build | |
| cd common/hogvm | |
| ./test.sh && git diff --exit-code | |
| env: | |
| TEST: 1 | |
| - name: Check package version and detect an update | |
| id: check-package-version | |
| uses: PostHog/check-package-version@f540540cb12d45ec3c58d42dbf1d60070ed2d268 # v2.1.1 | |
| with: | |
| path: common/hogvm/typescript | |
| release-hogvm: | |
| name: Release new HogVM TypeScript version | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| needs: hog-tests | |
| if: needs.hog-tests.outputs.is-new-version == 'true' | |
| permissions: | |
| contents: read | |
| id-token: write | |
| env: | |
| COMMITTED_VERSION: ${{ needs.hog-tests.outputs.committed-version }} | |
| PUBLISHED_VERSION: ${{ needs.hog-tests.outputs.published-version }} | |
| steps: | |
| - name: Checkout the repository | |
| uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 1 | |
| - name: Set up Python | |
| uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 | |
| with: | |
| python-version-file: 'pyproject.toml' | |
| - name: Install uv | |
| id: setup-uv-hogql-python | |
| uses: astral-sh/setup-uv@eac588ad8def6316056a12d4907a9d4d84ff7a3b # v7.3.0 | |
| with: | |
| version: '0.10.2' # pinned: unpinned setup-uv calls GH API on every job, exhausts rate limit | |
| enable-cache: true | |
| cache-dependency-glob: uv.lock | |
| save-cache: ${{ github.ref == 'refs/heads/master' }} | |
| - name: Install SAML (python3-saml) dependencies | |
| if: steps.setup-uv-hogql-python.outputs.cache-hit != 'true' | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install libxml2-dev libxmlsec1 libxmlsec1-dev libxmlsec1-openssl | |
| - name: Install Python dependencies | |
| run: | | |
| UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v5.0.0 | |
| - name: Fix node-gyp permissions | |
| run: chmod +x ~/setup-pnpm/node_modules/.pnpm/pnpm@*/node_modules/pnpm/dist/node_modules/node-gyp/gyp/gyp_main.py | |
| - name: Set up Node.js | |
| uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0 | |
| with: | |
| node-version-file: .nvmrc | |
| cache: 'pnpm' | |
| registry-url: https://registry.npmjs.org | |
| - name: Install package.json dependencies | |
| run: pnpm --filter=@posthog/hogvm install --frozen-lockfile | |
| - name: Publish the package in the npm registry | |
| run: cd common/hogvm/typescript && npm publish --access public |