Integration Tests #208
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Integration Tests | |
| on: | |
| schedule: | |
| # Run at 7pm PST (3am UTC next day) Monday-Friday | |
| # This translates to Tuesday-Saturday 3am UTC | |
| - cron: '0 3 * * 2-6' | |
| release: | |
| types: [published, edited] | |
| # published → latest-release binary test (full releases only) | |
| # edited + prerelease → nightly binary test (nightly pre-release gets edited with new artifacts) | |
| workflow_dispatch: # Allow manual triggering | |
| # NOTE: This workflow can only be manually triggered from develop or main branches | |
| # The other branch will not pass the OIDC permission check | |
| inputs: | |
| install_mode: | |
| description: '"code" to install from source (make init), "nightly-release" to install SAM CLI nightly binary, "latest-release" to install latest stable release binary' | |
| required: false | |
| default: 'code' | |
| type: choice | |
| options: | |
| - code | |
| - nightly-release | |
| - latest-release | |
| permissions: | |
| id-token: write # Required for OIDC | |
| contents: read | |
| env: | |
| AWS_DEFAULT_REGION: us-east-1 | |
| SAM_CLI_DEV: 1 | |
| SAM_CLI_TELEMETRY: 0 | |
| SAM_CLI_CONTAINER_CONNECTION_TIMEOUT: 60 | |
| NODE_VERSION: "22.21.1" | |
| AWS_S3: "AWS_S3_TESTING" | |
| AWS_ECR: "AWS_ECR_TESTING" | |
| CARGO_LAMBDA_VERSION: "v0.17.1" | |
| NOSE_PARAMETERIZED_NO_WARN: 1 | |
| BY_CANARY: true | |
| UV_PYTHON: python3.11 | |
| CREDENTIAL_DISTRIBUTION_LAMBDA_ARN: ${{ secrets.CREDENTIAL_DISTRIBUTION_LAMBDA_ARN }} | |
| ACCOUNT_RESET_LAMBDA_ARN: ${{ secrets.ACCOUNT_RESET_LAMBDA_ARN }} | |
| jobs: | |
| integration-tests: | |
| # Only run scheduled jobs on the main aws/aws-sam-cli repository, not on forks | |
| if: github.event_name != 'schedule' || github.repository == 'aws/aws-sam-cli' | |
| name: ${{ matrix.test_suite }} | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test_suite: | |
| # Build jobs (no-container mode — requires language toolchains) | |
| - build-x86-1 | |
| - build-x86-2 | |
| - build-arm64 | |
| # Build jobs (docker container mode) | |
| - build-x86-container-1 | |
| - build-x86-container-2 | |
| - build-arm64-container-1 | |
| - build-arm64-container-2 | |
| # Docker-only test suites | |
| - terraform-build | |
| - terraform-start-api | |
| - terraform-invoke-start-lambda | |
| - package | |
| - deploy | |
| - sync-code | |
| - sync-watch | |
| - local-invoke | |
| - local-start-api | |
| - local-start-lambda | |
| - other-and-e2e | |
| # Runs only credential-requiring tests from build and local | |
| - cloud-based-tests | |
| # Tier 1: Cross-platform smoke tests with Finch runtime | |
| - tier1-finch | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ github.event_name == 'schedule' && 'develop' || github.ref }} | |
| - name: Free up disk space | |
| run: bash tests/free_disk_space.sh | |
| - name: Configure AWS credentials via OIDC | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| - name: Set up Python | |
| uses: actions/setup-python@v6 | |
| with: | |
| python-version: | | |
| 3.11 | |
| 3.9 | |
| 3.10 | |
| 3.12 | |
| 3.13 | |
| 3.14 | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v7 | |
| - name: Set up Node.js | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: ${{ env.NODE_VERSION }} | |
| - name: Set up Java 25 | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "sync-code", "sync-watch", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: actions/setup-java@v5 | |
| with: | |
| distribution: 'corretto' | |
| java-version: '25' | |
| - name: Set up Go | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: actions/setup-go@v6 | |
| with: | |
| go-version: '1.25' | |
| - name: Install Maven and Gradle | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "sync-code", "sync-watch", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| run: | | |
| sudo apt-get remove -y maven || true | |
| wget https://dlcdn.apache.org/maven/maven-3/3.9.12/binaries/apache-maven-3.9.12-bin.zip -P /tmp | |
| sudo unzip -d /opt/mvn /tmp/apache-maven-*.zip | |
| wget https://services.gradle.org/distributions/gradle-9.2.0-bin.zip -P /tmp | |
| sudo unzip -d /opt/gradle /tmp/gradle-*.zip | |
| sudo ln -sf /opt/mvn/apache-maven-3.9.12/bin/mvn /usr/local/bin/mvn | |
| sudo ln -sf /opt/gradle/gradle-9.2.0/bin/gradle /usr/local/bin/gradle | |
| echo "/opt/mvn/apache-maven-3.9.12/bin" >> $GITHUB_PATH | |
| echo "/opt/gradle/gradle-9.2.0/bin" >> $GITHUB_PATH | |
| echo "MAVEN_HOME=/opt/mvn/apache-maven-3.9.12" >> $GITHUB_ENV | |
| export PATH="/opt/mvn/apache-maven-3.9.12/bin:/opt/gradle/gradle-9.2.0/bin:$PATH" | |
| mvn --version | |
| gradle --version | |
| - name: Install .NET 10 SDK | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "other-and-e2e", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: actions/setup-dotnet@v5 | |
| with: | |
| dotnet-version: '10.0.x' | |
| - name: Set up Ruby 3.3.7 | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "other-and-e2e", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.3.7' | |
| - name: Set up Ruby 3.2.7 | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "other-and-e2e", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.2.7' | |
| - name: Set up Ruby 3.4.7 | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "sync-code", "sync-watch", "other-and-e2e", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| uses: ruby/setup-ruby@v1 | |
| with: | |
| ruby-version: '3.4.7' | |
| - name: Install Rust toolchain and cargo-lambda | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| run: bash tests/install-rust.sh | |
| - name: Install Terraform | |
| if: contains(fromJSON('["terraform-build", "terraform-start-api", "terraform-invoke-start-lambda", "cloud-based-tests", "tier1-finch"]'), matrix.test_suite) | |
| run: | | |
| for i in {1..3}; do | |
| TER_VER=$(curl -s https://api.github.com/repos/hashicorp/terraform/releases/latest | grep tag_name | cut -d: -f2 | tr -d \"\,\v | awk '{$1=$1};1') | |
| if [ -n "$TER_VER" ]; then | |
| if wget https://releases.hashicorp.com/terraform/${TER_VER}/terraform_${TER_VER}_linux_amd64.zip -P /tmp; then | |
| sudo unzip -d /opt/terraform /tmp/terraform_${TER_VER}_linux_amd64.zip | |
| sudo mv /opt/terraform/terraform /usr/local/bin/ | |
| terraform -version | |
| break | |
| fi | |
| fi | |
| echo "Terraform installation attempt $i failed, retrying..." | |
| sleep 5 | |
| done | |
| - name: Setup Finch runtime | |
| if: matrix.test_suite == 'tier1-finch' | |
| run: bash tests/setup_finch.sh | |
| - name: Setup QEMU for ARM64 emulation | |
| if: "!contains(fromJSON('[\"build-x86-1\", \"build-x86-2\", \"build-x86-container-1\", \"build-x86-container-2\", \"tier1-finch\"]'), matrix.test_suite)" | |
| run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes | |
| - name: Determine test type | |
| run: | | |
| echo "CONTAINER_RUNTIME=${{ matrix.test_suite == 'tier1-finch' && 'finch' || 'docker' }}" >> $GITHUB_ENV | |
| if [[ "${{ inputs.install_mode }}" == "nightly-release" ]] || \ | |
| [[ "${{ github.event.action }}" == "edited" && "${{ github.event.release.prerelease }}" == "true" ]]; then | |
| echo "TEST_TYPE=nightly-release" >> $GITHUB_ENV | |
| elif [[ "${{ inputs.install_mode }}" == "latest-release" ]] || \ | |
| [[ "${{ github.event.action }}" == "published" && "${{ github.event.release.prerelease }}" != "true" ]]; then | |
| echo "TEST_TYPE=latest-release" >> $GITHUB_ENV | |
| elif [[ "${{ github.ref }}" == "refs/heads/master" ]]; then | |
| echo "TEST_TYPE=master" >> $GITHUB_ENV | |
| elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ github.ref }}" == "refs/heads/develop" ]]; then | |
| echo "TEST_TYPE=develop" >> $GITHUB_ENV | |
| else | |
| echo "TEST_TYPE=other" >> $GITHUB_ENV | |
| fi | |
| - name: Initialize integration test | |
| if: env.TEST_TYPE == 'develop' || env.TEST_TYPE == 'master' || env.TEST_TYPE == 'other' | |
| run: | | |
| make init | |
| echo "SCRIPT_PY=python3.11" >> $GITHUB_ENV | |
| - name: Initialize nightly release test | |
| if: env.TEST_TYPE == 'nightly-release' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| make setup-pytest | |
| make init-nightly | |
| echo "SAM_CLI_DEV=" >> $GITHUB_ENV | |
| echo "SCRIPT_PY=$HOME/pytest/bin/python3" >> $GITHUB_ENV | |
| - name: Initialize latest release test | |
| if: env.TEST_TYPE == 'latest-release' | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| make setup-pytest | |
| make init-latest-release | |
| echo "SAM_CLI_DEV=" >> $GITHUB_ENV | |
| echo "SCRIPT_PY=$HOME/pytest/bin/python3" >> $GITHUB_ENV | |
| - name: Get testing resources and credentials | |
| if: "!contains(fromJSON('[\"build-x86-1\", \"build-x86-2\", \"build-arm64\", \"build-x86-container-1\", \"build-x86-container-2\", \"build-arm64-container-1\", \"build-arm64-container-2\", \"local-invoke\", \"local-start-api\", \"local-start-lambda\"]'), matrix.test_suite)" | |
| run: $SCRIPT_PY tests/setup_testing_resources.py | |
| - name: Login to Public ECR | |
| run: | | |
| if [[ "${{ matrix.test_suite }}" == "tier1-finch" ]]; then | |
| aws ecr-public get-login-password --region us-east-1 | sudo finch login --username AWS --password-stdin public.ecr.aws || { echo "FATAL: Finch Public ECR login failed"; exit 1; } | |
| else | |
| aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws || { echo "FATAL: Docker Public ECR login failed"; exit 1; } | |
| fi | |
| - name: Clear credentials and skip account reset for non-credential jobs | |
| if: contains(fromJSON('["build-x86-1", "build-x86-2", "build-arm64", "build-x86-container-1", "build-x86-container-2", "build-arm64-container-1", "build-arm64-container-2", "local-invoke", "local-start-api", "local-start-lambda"]'), matrix.test_suite) | |
| run: | | |
| echo "SKIP_ACCOUNT_RESET=true" >> $GITHUB_ENV | |
| echo "AWS_ACCESS_KEY_ID=" >> $GITHUB_ENV | |
| echo "AWS_SECRET_ACCESS_KEY=" >> $GITHUB_ENV | |
| echo "AWS_SESSION_TOKEN=" >> $GITHUB_ENV | |
| - name: Run tests | |
| run: | | |
| case "${{ matrix.test_suite }}" in | |
| "build-x86-1") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/buildcmd/test_build_cmd.py \ | |
| tests/integration/buildcmd/test_build_cmd_python.py \ | |
| tests/integration/buildcmd/test_build_in_source.py \ | |
| tests/integration/buildcmd/test_build_samconfig.py \ | |
| -k "not container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "build-x86-2") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/buildcmd \ | |
| --ignore=tests/integration/buildcmd/test_build_cmd.py \ | |
| --ignore=tests/integration/buildcmd/test_build_cmd_python.py \ | |
| --ignore=tests/integration/buildcmd/test_build_in_source.py \ | |
| --ignore=tests/integration/buildcmd/test_build_samconfig.py \ | |
| --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py \ | |
| --ignore=tests/integration/buildcmd/test_build_terraform_applications.py \ | |
| --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py \ | |
| -k "not container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "build-arm64") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/buildcmd/test_build_cmd_arm64.py \ | |
| -k "not container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "build-x86-container-1") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/buildcmd/test_build_cmd.py \ | |
| tests/integration/buildcmd/test_build_in_source.py \ | |
| tests/integration/buildcmd/test_build_samconfig.py \ | |
| -k "container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "build-x86-container-2") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/buildcmd \ | |
| --ignore=tests/integration/buildcmd/test_build_cmd.py \ | |
| --ignore=tests/integration/buildcmd/test_build_in_source.py \ | |
| --ignore=tests/integration/buildcmd/test_build_samconfig.py \ | |
| --ignore=tests/integration/buildcmd/test_build_cmd_arm64.py \ | |
| --ignore=tests/integration/buildcmd/test_build_terraform_applications.py \ | |
| --ignore=tests/integration/buildcmd/test_build_terraform_applications_other_cases.py \ | |
| -k "container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "build-arm64-container-1") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra and not java" \ | |
| tests/integration/buildcmd/test_build_cmd_arm64.py \ | |
| -k "container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "build-arm64-container-2") | |
| pytest -vv -n 2 --reruns 3 -m "not requires_credential and not tier1_extra and java" \ | |
| tests/integration/buildcmd/test_build_cmd_arm64.py \ | |
| -k "container" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "terraform-build") | |
| pytest -vv -n 4 --reruns 3 -m "not tier1_extra" \ | |
| tests/integration/buildcmd/test_build_terraform_applications.py \ | |
| tests/integration/buildcmd/test_build_terraform_applications_other_cases.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "terraform-start-api") | |
| pytest -vv --reruns 3 -m "not tier1_extra" \ | |
| tests/integration/local/start_api/test_start_api_with_terraform_application.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "terraform-invoke-start-lambda") | |
| pytest -vv --reruns 3 -m "not tier1_extra" \ | |
| tests/integration/local/invoke/test_invoke_terraform_applications.py \ | |
| tests/integration/local/start_lambda/test_start_lambda_terraform_applications.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "package") | |
| pytest -vv tests/integration/package tests/integration/delete \ | |
| --dist=loadgroup -n 4 --reruns 3 -m "not tier1_extra" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "deploy") | |
| pytest -vv tests/integration/deploy -m "not tier1_extra" \ | |
| --dist=loadgroup -n 4 --reruns 4 \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "sync-watch") | |
| pytest -vv tests/integration/sync -n 4 --reruns 3 -m "not tier1_extra" \ | |
| --ignore=tests/integration/sync/test_sync_code.py \ | |
| --ignore=tests/integration/sync/test_sync_infra.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "sync-code") | |
| pytest -vv -n 5 --reruns 3 --dist loadscope -m "not tier1_extra" \ | |
| tests/integration/sync/test_sync_code.py \ | |
| tests/integration/sync/test_sync_infra.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "local-invoke") | |
| pytest -vv --reruns 3 --dist loadgroup -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/local/invoke \ | |
| tests/integration/local/generate_event \ | |
| tests/integration/local/callback \ | |
| tests/integration/local/execution \ | |
| --ignore=tests/integration/local/invoke/test_invoke_terraform_applications.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "local-start-api") | |
| pytest -vv --reruns 3 --dist loadgroup -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/local/start_api \ | |
| --ignore=tests/integration/local/start_api/test_start_api_with_terraform_application.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "local-start-lambda") | |
| pytest -vv --reruns 3 --dist loadgroup -m "not requires_credential and not tier1_extra" \ | |
| tests/integration/local/start_lambda \ | |
| --ignore=tests/integration/local/start_lambda/test_start_lambda_terraform_applications.py \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "other-and-e2e") | |
| pytest -vv -n 5 --reruns 3 --dist loadgroup -m "not tier1_extra" \ | |
| tests/integration tests/end_to_end tests/regression \ | |
| --ignore=tests/integration/buildcmd \ | |
| --ignore=tests/integration/delete \ | |
| --ignore=tests/integration/deploy \ | |
| --ignore=tests/integration/package \ | |
| --ignore=tests/integration/sync \ | |
| --ignore=tests/integration/local \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "cloud-based-tests") | |
| pytest -vv --reruns 3 \ | |
| tests/integration/buildcmd \ | |
| tests/integration/local \ | |
| -m "requires_credential and not tier1_extra" \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| "tier1-finch") | |
| export CONTAINER_RUNTIME=finch | |
| pytest -vv --reruns 3 -m "tier1 or tier1_extra" -n 2\ | |
| tests/integration \ | |
| tests/regression \ | |
| --json-report --json-report-file=TEST_REPORT-${{ matrix.test_suite }}.json | |
| ;; | |
| esac | |
| - name: Upload test results | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: test-results-${{ matrix.test_suite }} | |
| path: TEST_REPORT-*.json | |
| - name: Re-authenticate with OIDC for cleanup | |
| if: always() | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: ${{ secrets.OIDC_ROLE_ARN }} | |
| aws-region: us-east-1 | |
| - name: Assume test reporting role | |
| if: always() | |
| uses: aws-actions/configure-aws-credentials@v6 | |
| with: | |
| role-to-assume: ${{ secrets.TESTREPORTING_ARN }} | |
| aws-region: us-east-1 | |
| role-chaining: true | |
| - name: Reset test account and upload reports | |
| if: always() | |
| env: | |
| TESTREPORTING_S3: ${{ secrets.TESTREPORTING_S3 }} | |
| run: $SCRIPT_PY tests/reset_testing_resources.py ${{ matrix.test_suite }} |