[improve][all] Upgraded Jackson to 2.21 LTS and fixed a few gradle assemble warnings #28013
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # | |
| name: Pulsar CI | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - branch-* | |
| - pulsar-* | |
| schedule: | |
| # scheduled job with JDK 21 | |
| - cron: '0 12 * * *' | |
| # scheduled job with JDK 25 | |
| # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job | |
| - cron: '0 6 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| jdk_major_version: | |
| description: 'JDK major version to use for the build' | |
| required: true | |
| type: choice | |
| options: | |
| - '21' | |
| - '25' | |
| default: '21' | |
| trace_test_resource_cleanup: | |
| description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.' | |
| required: true | |
| type: choice | |
| options: | |
| - 'off' | |
| - 'on' | |
| - 'full' | |
| default: 'off' | |
| thread_leak_detector_wait_millis: | |
| description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.' | |
| required: true | |
| type: number | |
| default: 10000 | |
| netty_leak_detection: | |
| description: 'Controls Netty leak detection. When set to "report", Netty leak detection is enabled. When set to "fail_on_leak", Netty leak detection is enabled and a build job will fail if leaks are detected. When set to "off", Netty leak detection is disabled.' | |
| required: true | |
| type: choice | |
| options: | |
| - 'report' | |
| - 'fail_on_leak' | |
| - 'off' | |
| default: 'report' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }} | |
| cancel-in-progress: true | |
| env: | |
| # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job | |
| # it's possible to rerun individual failed jobs when the build artifacts are available | |
| # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR | |
| ARTIFACT_RETENTION_DAYS: 3 | |
| JDK_DISTRIBUTION: corretto | |
| jobs: | |
| preconditions: | |
| name: Preconditions | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| docs_only: ${{ steps.check_changes.outputs.docs_only }} | |
| changed_tests: ${{ steps.changes.outputs.tests_files }} | |
| jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }} | |
| java_non_tests: ${{ steps.changes.outputs.java_non_tests }} | |
| netty_leak_detection: ${{ steps.netty_leak_detection.outputs.netty_leak_detection }} | |
| steps: | |
| - name: Cancel scheduled jobs in forks by default | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }} | |
| uses: actions/github-script@v8 | |
| with: | |
| script: | | |
| await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId}); | |
| process.exit(1); | |
| - name: Select JDK major version | |
| id: jdk_major_version | |
| run: | | |
| # use JDK 25 for the scheduled build with cron expression '0 6 * * *' | |
| if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then | |
| echo "jdk_major_version=25" >> $GITHUB_OUTPUT | |
| exit 0 | |
| fi | |
| # use JDK 21 for build unless overridden with workflow_dispatch input | |
| echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '21'}}" >> $GITHUB_OUTPUT | |
| - name: checkout | |
| if: ${{ github.event_name == 'pull_request' }} | |
| uses: actions/checkout@v6 | |
| - name: Detect changed files | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: changes | |
| uses: apache/pulsar-test-infra/paths-filter@master | |
| with: | |
| filters: .github/changes-filter.yaml | |
| list-files: csv | |
| - name: Check changed files | |
| if: ${{ github.event_name == 'pull_request' }} | |
| id: check_changes | |
| run: | | |
| if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then | |
| echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT | |
| else | |
| echo docs_only=false >> $GITHUB_OUTPUT | |
| fi | |
| - name: Set Netty leak detection mode | |
| id: netty_leak_detection | |
| run: | | |
| echo "netty_leak_detection=${{ | |
| github.event_name == 'workflow_dispatch' && github.event.inputs.netty_leak_detection || 'report' | |
| }}" >> $GITHUB_OUTPUT | |
| build-and-license-check: | |
| needs: preconditions | |
| name: Build and License check | |
| env: | |
| JOB_NAME: Build and License check | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| cache-read-only: false | |
| - name: Build, check licenses and code style | |
| run: >- | |
| ./gradlew assemble rat spotlessCheck checkstyleMain checkstyleTest | |
| --no-configuration-cache | |
| - name: Check binary licenses | |
| run: | | |
| src/check-binary-license.sh ./distribution/server/build/distributions/apache-pulsar-*-bin.tar.gz | |
| - name: Upload Gradle reports | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ !success() }} | |
| with: | |
| name: gradle-reports-build-and-license-check | |
| path: | | |
| **/build/reports/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Save build outputs for test jobs | |
| run: | | |
| # Tar up compiled classes, resources, and distribution archives so test jobs | |
| # don't need to recompile. Distribution tarballs are needed for Docker image builds. | |
| find . \( -path '*/build/classes/*' -o -path '*/build/generated/*' -o -path '*/build/resources/*' \ | |
| -o -path '*/build/libs/*' -o -path '*/build/tmp/jar/*' -o -path '*/build/distributions/*' \) -print \ | |
| | tar cf /tmp/gradle-build-outputs.tar --files-from=- | |
| - name: Upload build outputs | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: gradle-build-outputs | |
| path: /tmp/gradle-build-outputs.tar | |
| retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} | |
| compression-level: 1 | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| unit-tests: | |
| name: CI - Unit - ${{ matrix.name }} | |
| env: | |
| JOB_NAME: CI - Unit - ${{ matrix.name }} | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }} | |
| TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/build/trace-test-resource-cleanup | |
| THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }} | |
| THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/build/thread-leak-dumps | |
| NETTY_LEAK_DETECTION: "${{ needs.preconditions.outputs.netty_leak_detection }}" | |
| NETTY_LEAK_DUMP_DIR: ${{ github.workspace }}/build/netty-leak-dumps | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: ${{ matrix.timeout || 60 }} | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Other | |
| group: OTHER | |
| timeout: 75 | |
| - name: Brokers - Broker Group 1 | |
| group: BROKER_GROUP_1 | |
| - name: Brokers - Broker Group 2 | |
| group: BROKER_GROUP_2 | |
| - name: Brokers - Broker Group 3 | |
| group: BROKER_GROUP_3 | |
| timeout: 75 | |
| - name: Brokers - Broker Group 4 | |
| group: BROKER_GROUP_4 | |
| - name: Brokers - Broker Group 5 | |
| group: BROKER_GROUP_5 | |
| - name: Brokers - Client Api | |
| group: BROKER_CLIENT_API | |
| - name: Brokers - Client Impl | |
| group: BROKER_CLIENT_IMPL | |
| - name: Proxy | |
| group: PROXY | |
| - name: Pulsar IO | |
| group: PULSAR_IO | |
| - name: Pulsar Client | |
| group: CLIENT | |
| - name: Pulsar Metadata | |
| group: METADATA | |
| - name: Protobuf v4 | |
| group: PROTOBUFV4 | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Clean Disk when tracing test resource cleanup | |
| if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} | |
| uses: ./.github/actions/clean-disk | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Restore build outputs from build job | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: gradle-build-outputs | |
| - name: Extract build outputs | |
| run: tar xf gradle-build-outputs.tar | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run unit test group '${{ matrix.group }}' | |
| run: | | |
| ./pulsar-build/run_unit_group_gradle.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/pulsar-build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregate test reports to ./test-reports directory | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Report detected thread leaks | |
| if: ${{ always() }} | |
| run: | | |
| if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then | |
| cd "$THREAD_LEAK_DETECTOR_DIR" | |
| cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}' | |
| fi | |
| - name: Report detected Netty leaks | |
| if: ${{ always() && env.NETTY_LEAK_DETECTION != 'off' }} | |
| run: $GITHUB_WORKSPACE/pulsar-build/pulsar_ci_tool.sh report_netty_leaks | |
| - name: Upload test reports | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }} | |
| with: | |
| name: Unit-${{ matrix.group }}-test-reports | |
| path: | | |
| test-reports/ | |
| **/build/reports/tests/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload possible heap dump, core dump or crash files | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ always() }} | |
| with: | |
| name: Unit-${{ matrix.group }}-dumps | |
| path: | | |
| /tmp/*.hprof | |
| **/hs_err_*.log | |
| **/core.* | |
| build/threaddumps/ | |
| ${{ env.NETTY_LEAK_DUMP_DIR }}/* | |
| ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/* | |
| ${{ env.THREAD_LEAK_DETECTOR_DIR }}/* | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Wait for ssh connection when build fails | |
| # ssh access is enabled for builds in own forks | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| macos-build: | |
| name: Build Pulsar on MacOS | |
| runs-on: macos-latest | |
| timeout-minutes: 120 | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Build with Gradle | |
| run: ./gradlew assemble | |
| pulsar-java-test-image: | |
| name: Build java-test-image Docker image | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Restore build outputs from build job | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: gradle-build-outputs | |
| - name: Extract build outputs | |
| run: tar xf gradle-build-outputs.tar | |
| - name: Build java-test-image Docker image | |
| run: ./gradlew :tests:java-test-image:dockerBuild${{ env.CI_JDK_MAJOR_VERSION != '21' && format(' -PdockerJavaVersion={0}', env.CI_JDK_MAJOR_VERSION) || '' }} | |
| - name: Save docker image to file | |
| run: | | |
| docker save apachepulsar/java-test-image:latest | gzip > /tmp/java-test-image.tar.gz | |
| - name: Upload docker image | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: java-test-image | |
| path: /tmp/java-test-image.tar.gz | |
| retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} | |
| compression-level: 0 | |
| - name: Wait for ssh connection when build fails | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| integration-tests: | |
| name: CI - Integration - ${{ matrix.name }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: ${{ matrix.timeout || 60 }} | |
| needs: ['preconditions', 'pulsar-java-test-image'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| JOB_NAME: CI - Integration - ${{ matrix.name }} | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Backwards Compatibility | |
| group: BACKWARDS_COMPAT | |
| - name: Cli | |
| group: CLI | |
| - name: Messaging | |
| group: MESSAGING | |
| - name: LoadBalance | |
| group: LOADBALANCE | |
| - name: Shade on Java 17 | |
| group: SHADE_RUN | |
| upload_name: SHADE_RUN_17 | |
| runtime_jdk: 17 | |
| - name: Shade on Java 21 | |
| group: SHADE_RUN | |
| upload_name: SHADE_RUN_21 | |
| runtime_jdk: 21 | |
| - name: Shade on Java 25 | |
| group: SHADE_RUN | |
| upload_name: SHADE_RUN_25 | |
| runtime_jdk: 25 | |
| - name: Standalone | |
| group: STANDALONE | |
| - name: Transaction | |
| group: TRANSACTION | |
| - name: Metrics | |
| group: METRICS | |
| - name: Upgrade | |
| group: UPGRADE | |
| - name: Kubernetes | |
| group: PULSAR_K8S | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}${{ matrix.runtime_jdk && matrix.runtime_jdk != env.CI_JDK_MAJOR_VERSION && format(' and {0}', matrix.runtime_jdk) || '' }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: | | |
| ${{ matrix.runtime_jdk != env.CI_JDK_MAJOR_VERSION && matrix.runtime_jdk || '' }} | |
| ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Restore build outputs from build job | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: gradle-build-outputs | |
| - name: Extract build outputs | |
| run: tar xf gradle-build-outputs.tar | |
| - name: Download docker image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: java-test-image | |
| path: /tmp | |
| - name: Load docker image | |
| run: docker load -i /tmp/java-test-image.tar.gz | |
| - name: Run setup commands | |
| if: ${{ matrix.setup }} | |
| run: | | |
| ${{ matrix.setup }} | |
| - name: Run integration test group '${{ matrix.group }}' | |
| run: | | |
| ./pulsar-build/run_integration_group_gradle.sh ${{ matrix.group }}${{ matrix.runtime_jdk && matrix.runtime_jdk != env.CI_JDK_MAJOR_VERSION && format(' -PtestJavaVersion={0}', matrix.runtime_jdk) || '' }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/pulsar-build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregate test reports to ./test-reports directory | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Upload test reports | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ !success() }} | |
| with: | |
| name: Integration-${{ matrix.upload_name || matrix.group }}-test-reports | |
| path: | | |
| test-reports/ | |
| **/build/reports/tests/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload possible heap dump, core dump or crash files | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ always() }} | |
| with: | |
| name: Integration-${{ matrix.upload_name || matrix.group }}-dumps | |
| path: | | |
| /tmp/*.hprof | |
| **/hs_err_*.log | |
| **/core.* | |
| build/threaddumps/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: Integration-${{ matrix.upload_name || matrix.group }}-container-logs | |
| path: tests/integration/build/container-logs | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| pulsar-test-latest-version-image: | |
| name: Build pulsar-test-latest-version Docker image | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'build-and-license-check'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Restore build outputs from build job | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: gradle-build-outputs | |
| - name: Extract build outputs | |
| run: tar xf gradle-build-outputs.tar | |
| - name: Build pulsar-test-latest-version Docker image | |
| run: ./gradlew :tests:latest-version-image:dockerBuild${{ env.CI_JDK_MAJOR_VERSION != '21' && format(' -PdockerJavaVersion={0}', env.CI_JDK_MAJOR_VERSION) || '' }} | |
| - name: Check binary licenses | |
| run: | | |
| src/check-binary-license.sh ./distribution/server/build/distributions/apache-pulsar-*-bin.tar.gz | |
| src/check-binary-license.sh ./distribution/shell/build/distributions/apache-pulsar-shell-*-bin.tar.gz | |
| - name: Run Trivy container scan | |
| id: trivy_scan | |
| uses: lhotari/sandboxed-trivy-action@555963036b2012b44c1071508a236e569db28ebb | |
| if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| scan-type: 'image' | |
| scan-ref: "apachepulsar/pulsar:latest" | |
| scanners: vuln | |
| severity: CRITICAL,HIGH,MEDIUM,LOW | |
| limit-severities-for-sarif: true | |
| format: 'sarif' | |
| output: 'trivy-results.sarif' | |
| - name: Upload Trivy scan results to GitHub Security tab | |
| uses: github/codeql-action/upload-sarif@v4 | |
| if: ${{ steps.trivy_scan.outcome == 'success' && github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| - name: Save docker image to file | |
| run: | | |
| docker save apachepulsar/pulsar-test-latest-version:latest | gzip > /tmp/pulsar-test-latest-version.tar.gz | |
| - name: Upload docker image | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: pulsar-test-latest-version-image | |
| path: /tmp/pulsar-test-latest-version.tar.gz | |
| retention-days: ${{ env.ARTIFACT_RETENTION_DAYS }} | |
| compression-level: 0 | |
| - name: Wait for ssh connection when build fails | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| system-tests: | |
| name: CI - System - ${{ matrix.name }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'pulsar-test-latest-version-image'] | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| JOB_NAME: CI - System - ${{ matrix.name }} | |
| PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: Tiered FileSystem | |
| group: TIERED_FILESYSTEM | |
| - name: Tiered JCloud | |
| group: TIERED_JCLOUD | |
| - name: Function | |
| group: FUNCTION | |
| - name: Schema | |
| group: SCHEMA | |
| - name: Pulsar Connectors - Thread | |
| group: PULSAR_CONNECTORS_THREAD | |
| - name: Pulsar Connectors - Process | |
| group: PULSAR_CONNECTORS_PROCESS | |
| - name: Plugin | |
| group: PLUGIN | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Restore build outputs from build job | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: gradle-build-outputs | |
| - name: Extract build outputs | |
| run: tar xf gradle-build-outputs.tar | |
| - name: Download docker image | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: pulsar-test-latest-version-image | |
| path: /tmp | |
| - name: Load docker image | |
| run: docker load -i /tmp/pulsar-test-latest-version.tar.gz | |
| - name: Run system test group '${{ matrix.group }}' | |
| run: | | |
| ./pulsar-build/run_integration_group_gradle.sh ${{ matrix.group }} | |
| - name: print JVM thread dumps when cancelled | |
| if: cancelled() | |
| run: $GITHUB_WORKSPACE/pulsar-build/pulsar_ci_tool.sh print_thread_dumps | |
| - name: Aggregate test reports to ./test-reports directory | |
| if: ${{ always() }} | |
| uses: ./.github/actions/copy-test-reports | |
| - name: Upload test reports | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ !success() }} | |
| with: | |
| name: System-${{ matrix.group }}-test-reports | |
| path: | | |
| test-reports/ | |
| **/build/reports/tests/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload possible heap dump, core dump or crash files | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ always() }} | |
| with: | |
| name: System-${{ matrix.group }}-dumps | |
| path: | | |
| /tmp/*.hprof | |
| **/hs_err_*.log | |
| **/core.* | |
| build/threaddumps/ | |
| retention-days: 7 | |
| if-no-files-found: ignore | |
| - name: Upload container logs | |
| uses: actions/upload-artifact@v7 | |
| if: ${{ !success() }} | |
| continue-on-error: true | |
| with: | |
| name: System-${{ matrix.group }}-container-logs | |
| path: tests/integration/build/container-logs | |
| retention-days: 7 | |
| - name: Wait for ssh connection when build fails | |
| uses: ./.github/actions/ssh-access | |
| if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| continue-on-error: true | |
| with: | |
| action: wait | |
| codeql: | |
| name: Run CodeQL Analysis | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 60 | |
| needs: ['preconditions', 'unit-tests'] | |
| if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }} | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| env: | |
| CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }} | |
| CODEQL_LANGUAGE: java-kotlin | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v6 | |
| - name: Tune Runner VM | |
| uses: ./.github/actions/tune-runner-vm | |
| - name: Setup ssh access to build runner VM | |
| # ssh access is enabled for builds in own forks | |
| if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }} | |
| uses: ./.github/actions/ssh-access | |
| continue-on-error: true | |
| with: | |
| limit-access-to-actor: true | |
| - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }} | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: ${{ env.JDK_DISTRIBUTION }} | |
| java-version: ${{ env.CI_JDK_MAJOR_VERSION }} | |
| - name: Setup Gradle | |
| uses: ./.github/actions/setup-gradle | |
| with: | |
| develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }} | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v4 | |
| with: | |
| languages: ${{ env.CODEQL_LANGUAGE }} | |
| - name: Build Java code | |
| run: ./gradlew assemble | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v4 | |
| with: | |
| category: "/language:${{ env.CODEQL_LANGUAGE }}" | |
| # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the | |
| # protected_branches section for master branch required_status_checks. | |
| # It depends on all other jobs in this workflow. | |
| pulsar-ci-checks-completed: | |
| name: "Pulsar CI checks completed" | |
| # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled | |
| # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled | |
| # jobs are not enabled for other than apache/pulsar repository. | |
| if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }} | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| needs: [ | |
| 'preconditions', | |
| 'unit-tests', | |
| 'integration-tests', | |
| 'system-tests', | |
| 'macos-build', | |
| 'codeql' | |
| ] | |
| steps: | |
| - name: Check that all required jobs were completed successfully | |
| if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true' }} | |
| run: | | |
| if [[ ! ( \ | |
| "${{ needs.preconditions.result }}" == "success" \ | |
| && "${{ needs.unit-tests.result }}" == "success" \ | |
| && "${{ needs.integration-tests.result }}" == "success" \ | |
| && "${{ needs.system-tests.result }}" == "success" \ | |
| && "${{ needs.macos-build.result }}" == "success" \ | |
| && ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \ | |
| ) ]]; then | |
| echo "Required jobs haven't been completed successfully." | |
| exit 1 | |
| fi | |
| - name: Delete build artifacts | |
| if: ${{ needs.preconditions.outputs.docs_only != 'true' }} | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| gh api repos/${{ github.repository }}/actions/runs/${{ github.run_id }}/artifacts \ | |
| --jq '.artifacts[] | select(.name == "gradle-build-outputs" or .name == "java-test-image" or .name == "pulsar-test-latest-version-image") | .id' \ | |
| | xargs -I{} gh api -X DELETE repos/${{ github.repository }}/actions/artifacts/{} || true |