Add multi-architecture canary support (ARM64 Lambda) (#4606)

vcjana · web-flow · commit 592237ddfd0b · 2026-04-20T14:41:35.000-07:00
## Motivation Addresses #4380. Supersedes #4428. The smithy-rs canary only deploys x86_64 Lambda functions. This meant the [crc-fast 1.4 SIGILL incident](#4264) was not caught by smithy-rs CI it was discovered later in the aws-sdk-rust canary, after release. This PR expands the canary to also deploy **ARM64 (aarch64) Lambda functions** on AWS Graviton, catching architecture-specific runtime bugs before merge. ## Approach The existing x86_64 canary runs inside a Docker container via the `docker-build` action. The Docker image is x86_64-only in ECR, so instead of fixing Docker for ARM, this PR adds a **separate `canary-arm` job** that runs directly on `ubuntu-24.04-arm` without Docker. The ARM runner compiles the canary natively, deploys an ARM64 Lambda on Graviton, and exercises real S3/checksum code paths — the same paths where crc-fast SIGILL would manifest. ## Changes **Canary runner (`run.rs`):** - Add `--architecture` CLI flag to `RunArgs`/`Options` - Wire architecture through to `build_bundle()` (replaces hardcoded x86_64) - Set `.architectures(lambda_arch)` on Lambda creation (ARM64 → `Arm64`) - Use `provided.al2023` runtime for ARM64 (al2 is deprecated) - Architecture-suffixed Lambda function names to prevent parallel collisions **Build bundle (`build_bundle.rs`):** - Skip `cross` tool when building natively on ARM (`std::env::consts::ARCH` check) - Reduce hash truncation from 24→16 chars to leave headroom for arch suffix in 64-char Lambda name limit **Run script (`run-canary`):** - Accept optional 5th argument for architecture (defaults to `x86_64`) - Pass `--architecture` to canary runner **CI workflows (`ci.yml`, `manual-canary.yml`):** - Add new `canary-arm` job on `ubuntu-24.04-arm` (no Docker) - Existing x86_64 canary is completely unchanged ## Testing - Canary runner: `cargo check` ✓, `cargo clippy` ✓, `cargo test` 11/11 ✓ - YAML validation: both workflow files pass syntax check - Full CI verification pending (this PR) ## What is NOT changed - Existing x86_64 canary behavior (zero risk) - Docker infrastructure (`Dockerfile`, `acquire-build-image`, `docker-build` action) - Any runtime crates, codegen, or Smithy models - `arch.rs` (already supports `Aarch64`)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -371,7 +371,55 @@ jobs:
       uses: ./smithy-rs/.github/actions/docker-build
       with:
         action: run-canary
-        action-arguments: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} ${{ steps.creds.outputs.aws-access-key-id }} ${{ steps.creds.outputs.aws-secret-access-key }} ${{ steps.creds.outputs.aws-session-token }}
+        action-arguments: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} ${{ steps.creds.outputs.aws-access-key-id }} ${{ steps.creds.outputs.aws-secret-access-key }} ${{ steps.creds.outputs.aws-session-token }} x86_64
+
+  canary-arm:
+    name: Canary (aarch64)
+    if: ${{ inputs.run_canary }}
+    needs: generate
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 30
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: smithy-rs
+        ref: ${{ inputs.git_ref }}
+    - name: Download artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: artifacts-generate-aws-sdk
+        path: artifacts-generate-aws-sdk
+    - name: Extract artifacts
+      run: tar xfz artifacts-generate-aws-sdk/artifacts-generate-aws-sdk.tar.gz
+    - name: Install Rust
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        targets: aarch64-unknown-linux-musl
+    - name: Install build dependencies
+      run: sudo apt-get update && sudo apt-get install -y musl-tools cmake perl clang pkg-config
+    - name: Configure credentials
+      id: creds
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+        output-credentials: true
+    - name: Run canary
+      run: |
+        export RUST_STABLE_VERSION="$(rustc --version | cut -d' ' -f2)"
+        # Install the pinned toolchain from rust-toolchain.toml and add musl target
+        rustup toolchain install 1.91.1
+        rustup target add aarch64-unknown-linux-musl wasm32-wasip2 --toolchain 1.91.1
+        ./smithy-rs/tools/ci-scripts/run-canary \
+          ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} \
+          ${{ steps.creds.outputs.aws-access-key-id }} \
+          ${{ steps.creds.outputs.aws-secret-access-key }} \
+          ${{ steps.creds.outputs.aws-session-token }} \
+          aarch64
 
   # This is always a failing job since forked repositories do not have necessary repository secrets
   # to run the PR bot workflow or the canary workflow
diff --git a/.github/workflows/manual-canary.yml b/.github/workflows/manual-canary.yml
@@ -117,4 +117,53 @@ jobs:
       uses: ./smithy-rs/.github/actions/docker-build
       with:
         action: run-canary
-        action-arguments: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} ${{ steps.creds.outputs.aws-access-key-id }} ${{ steps.creds.outputs.aws-secret-access-key }} ${{ steps.creds.outputs.aws-session-token }}
+        action-arguments: ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} ${{ steps.creds.outputs.aws-access-key-id }} ${{ steps.creds.outputs.aws-secret-access-key }} ${{ steps.creds.outputs.aws-session-token }} x86_64
+
+  canary-arm:
+    name: Canary (aarch64)
+    needs:
+    - generate
+    - get-pr-info
+    runs-on: ubuntu-24.04-arm
+    timeout-minutes: 30
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        path: smithy-rs
+        ref: ${{ inputs.commit_sha }}
+    - name: Download artifacts
+      uses: actions/download-artifact@v4
+      with:
+        name: artifacts-generate-aws-sdk-for-canary
+        path: artifacts-generate-aws-sdk-for-canary
+    - name: Extract artifacts
+      run: tar xfz artifacts-generate-aws-sdk-for-canary/artifacts-generate-aws-sdk-for-canary.tar.gz
+    - name: Install Rust
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        targets: aarch64-unknown-linux-musl
+    - name: Install build dependencies
+      run: sudo apt-get update && sudo apt-get install -y musl-tools cmake perl clang pkg-config
+    - name: Configure credentials
+      id: creds
+      uses: aws-actions/configure-aws-credentials@v4
+      with:
+        role-to-assume: ${{ secrets.CANARY_GITHUB_ACTIONS_ROLE_ARN }}
+        role-session-name: GitHubActions
+        aws-region: us-west-2
+        output-credentials: true
+    - name: Run canary
+      run: |
+        export RUST_STABLE_VERSION="$(rustc --version | cut -d' ' -f2)"
+        # Install the pinned toolchain from rust-toolchain.toml and add musl target
+        rustup toolchain install 1.91.1
+        rustup target add aarch64-unknown-linux-musl wasm32-wasip2 --toolchain 1.91.1
+        ./smithy-rs/tools/ci-scripts/run-canary \
+          ${{ secrets.CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME }} \
+          ${{ steps.creds.outputs.aws-access-key-id }} \
+          ${{ steps.creds.outputs.aws-secret-access-key }} \
+          ${{ steps.creds.outputs.aws-session-token }} \
+          aarch64
diff --git a/tools/ci-cdk/canary-runner/src/arch.rs b/tools/ci-cdk/canary-runner/src/arch.rs
@@ -22,3 +22,21 @@ impl std::str::FromStr for Arch {
         }
     }
 }
+
+impl From<Arch> for aws_sdk_lambda::types::Architecture {
+    fn from(arch: Arch) -> Self {
+        match arch {
+            Arch::X86_64 => aws_sdk_lambda::types::Architecture::X8664,
+            Arch::Aarch64 => aws_sdk_lambda::types::Architecture::Arm64,
+        }
+    }
+}
+
+impl From<Arch> for aws_sdk_lambda::types::Runtime {
+    fn from(arch: Arch) -> Self {
+        match arch {
+            Arch::X86_64 => aws_sdk_lambda::types::Runtime::Providedal2,
+            Arch::Aarch64 => aws_sdk_lambda::types::Runtime::Providedal2023,
+        }
+    }
+}
diff --git a/tools/ci-cdk/canary-runner/src/bench.rs b/tools/ci-cdk/canary-runner/src/bench.rs
@@ -257,15 +257,6 @@ async fn create_lambda_fn(
     Ok(())
 }
 
-impl From<Arch> for lambda::types::Architecture {
-    fn from(arch: Arch) -> Self {
-        match arch {
-            Arch::X86_64 => lambda::types::Architecture::X8664,
-            Arch::Aarch64 => lambda::types::Architecture::Arm64,
-        }
-    }
-}
-
 async fn run_benchmark(
     lambda_client: &lambda::Client,
     bundle_name: &str,
diff --git a/tools/ci-cdk/canary-runner/src/build_bundle.rs b/tools/ci-cdk/canary-runner/src/build_bundle.rs
@@ -333,8 +333,9 @@ fn name_hashed_bundle(
     rust_version: Option<&str>,
     sdk_release_tag: Option<&ReleaseTag>,
 ) -> Result<String> {
-    // The Lambda name must be less than 64 characters, so truncate the hash a bit
-    let bin_hash = &bin_hash[..24];
+    // The Lambda name must be less than 64 characters, so truncate the hash a bit.
+    // Using 16 chars leaves headroom for the architecture suffix added in run.rs.
+    let bin_hash = &bin_hash[..16];
     // Lambda function names can't have periods in them
     let rust_version = rust_version.map(|s| s.replace('.', ""));
     let rust_version = rust_version.as_deref().unwrap_or("unknown");
@@ -390,8 +391,8 @@ pub async fn build_bundle(opt: BuildBundleArgs) -> Result<Option<PathBuf>> {
     };
 
     if !opt.manifest_only {
-        // Check if cross is needed and available
-        let use_cross = opt.architecture == Arch::Aarch64;
+        // Only use cross when cross-compiling (host arch != target arch)
+        let use_cross = opt.architecture == Arch::Aarch64 && std::env::consts::ARCH != "aarch64";
         if use_cross {
             let cross_check = Command::new("cross").arg("--version").output();
             if cross_check.is_err() || !cross_check.unwrap().status.success() {
@@ -883,7 +884,7 @@ aws-smithy-wasm = { version = "0.1.0" }
             assert_eq!(expected, actual);
         }
         check(
-            "canary-release20221216-1621-7ae6085d2105d5d1e13b10f8.zip",
+            "canary-release20221216-1621-7ae6085d2105d5d1.zip",
             &name_hashed_bundle(
                 "7ae6085d2105d5d1e13b10f882c6cb072ff5bbf8",
                 Some("1.62.1"),
@@ -892,7 +893,7 @@ aws-smithy-wasm = { version = "0.1.0" }
             .unwrap(),
         );
         check(
-            "canary-release202212162-1621-7ae6085d2105d5d1e13b10f8.zip",
+            "canary-release202212162-1621-7ae6085d2105d5d1.zip",
             &name_hashed_bundle(
                 "7ae6085d2105d5d1e13b10f882c6cb072ff5bbf8",
                 Some("1.62.1"),
@@ -901,7 +902,7 @@ aws-smithy-wasm = { version = "0.1.0" }
             .unwrap(),
         );
         check(
-            "canary-untagged-1621-7ae6085d2105d5d1e13b10f8.zip",
+            "canary-untagged-1621-7ae6085d2105d5d1.zip",
             &name_hashed_bundle(
                 "7ae6085d2105d5d1e13b10f882c6cb072ff5bbf8",
                 Some("1.62.1"),
@@ -910,7 +911,7 @@ aws-smithy-wasm = { version = "0.1.0" }
             .unwrap(),
         );
         check(
-            "canary-release20221216-unknown-7ae6085d2105d5d1e13b10f8.zip",
+            "canary-release20221216-unknown-7ae6085d2105d5d1.zip",
             &name_hashed_bundle(
                 "7ae6085d2105d5d1e13b10f882c6cb072ff5bbf8",
                 None,
diff --git a/tools/ci-cdk/canary-runner/src/run.rs b/tools/ci-cdk/canary-runner/src/run.rs
@@ -119,6 +119,10 @@ pub struct RunArgs {
     /// The ARN of the role that the Lambda will execute as
     #[clap(long, required_unless_present = "cdk-output")]
     lambda_execution_role_arn: Option<String>,
+
+    /// Lambda architecture
+    #[clap(long, default_value = "x86_64")]
+    pub(crate) architecture: crate::arch::Arch,
 }
 
 #[derive(Debug, Eq, PartialEq)]
@@ -134,6 +138,7 @@ struct Options {
     lambda_test_s3_mrap_bucket_arn: String,
     lambda_test_s3_express_bucket_name: String,
     lambda_execution_role_arn: String,
+    architecture: crate::arch::Arch,
 }
 
 impl Options {
@@ -203,6 +208,7 @@ impl Options {
                 lambda_test_s3_mrap_bucket_arn,
                 lambda_test_s3_express_bucket_name,
                 lambda_execution_role_arn,
+                architecture: run_opt.architecture,
             })
         } else {
             Ok(Options {
@@ -223,6 +229,7 @@ impl Options {
                     .lambda_test_s3_express_bucket_name
                     .expect("required"),
                 lambda_execution_role_arn: run_opt.lambda_execution_role_arn.expect("required"),
+                architecture: run_opt.architecture,
             })
         }
     }
@@ -319,9 +326,17 @@ async fn run_canary(options: &Options, config: &aws_config::SdkConfig) -> Result
         "Creating the canary Lambda function named {}...",
         bundle_name
     );
+    let function_name = format!(
+        "{}-{}",
+        bundle_name,
+        match options.architecture {
+            crate::arch::Arch::X86_64 => "x86_64",
+            crate::arch::Arch::Aarch64 => "aarch64",
+        }
+    );
     create_lambda_fn(
         lambda_client.clone(),
-        bundle_name,
+        &function_name,
         bundle_file_name,
         options,
     )
@@ -330,11 +345,11 @@ async fn run_canary(options: &Options, config: &aws_config::SdkConfig) -> Result
 
     info!("Invoking the canary Lambda...");
     let invoke_start_time = SystemTime::now();
-    let invoke_result = invoke_lambda(lambda_client.clone(), bundle_name).await;
+    let invoke_result = invoke_lambda(lambda_client.clone(), &function_name).await;
     let invoke_time = invoke_start_time.elapsed().expect("time in range");
 
     info!("Deleting the canary Lambda...");
-    delete_lambda_fn(lambda_client, bundle_name)
+    delete_lambda_fn(lambda_client, &function_name)
         .await
         .context(here!())?;
 
@@ -362,7 +377,7 @@ async fn build_bundle(options: &Options) -> Result<PathBuf> {
         sdk_release_tag: options.sdk_release_tag.clone(),
         sdk_path: options.sdk_path.clone(),
         musl: options.musl,
-        architecture: crate::arch::Arch::X86_64,
+        architecture: options.architecture,
         manifest_only: false,
         disable_jitter_entropy: true,
         feature_override: None,
@@ -433,10 +448,13 @@ async fn create_lambda_fn(
             ),
     };
 
+    let lambda_arch: Architecture = options.architecture.into();
+    let lambda_runtime: Runtime = options.architecture.into();
+
     lambda_client
         .create_function()
         .function_name(bundle_name)
-        .runtime(Runtime::Providedal2)
+        .runtime(lambda_runtime)
         .role(&options.lambda_execution_role_arn)
         .handler("aws-sdk-rust-lambda-canary")
         .code(
@@ -449,6 +467,7 @@ async fn create_lambda_fn(
         .environment(env_builder.build())
         .timeout(180)
         .memory_size(options.lambda_function_memory_size_in_mb)
+        .architectures(lambda_arch)
         .send()
         .await
         .context(here!("failed to create canary Lambda function"))?;
@@ -565,7 +584,8 @@ mod tests {
                 lambda_test_s3_bucket_name: None,
                 lambda_execution_role_arn: None,
                 lambda_test_s3_mrap_bucket_arn: None,
-                lambda_test_s3_express_bucket_name: None
+                lambda_test_s3_express_bucket_name: None,
+                architecture: crate::arch::Arch::X86_64,
             },
             RunArgs::try_parse_from([
                 "run",
@@ -616,6 +636,7 @@ mod tests {
                 lambda_test_s3_mrap_bucket_arn: "arn:aws:s3::000000000000:accesspoint/example.mrap"
                     .to_owned(),
                 lambda_test_s3_express_bucket_name: "test--usw2-az1--x-s3".to_owned(),
+                architecture: crate::arch::Arch::X86_64,
             },
             Options::load_from(run_args).unwrap(),
         );
diff --git a/tools/ci-scripts/run-canary b/tools/ci-scripts/run-canary
@@ -10,6 +10,7 @@ CANARY_STACK_CDK_OUTPUTS_BUCKET_NAME=$1
 export AWS_ACCESS_KEY_ID=$2
 export AWS_SECRET_ACCESS_KEY=$3
 export AWS_SESSION_TOKEN=$4
+ARCHITECTURE="${5:-x86_64}"
 export AWS_REGION=us-west-2
 export RUST_LOG=debug
 
@@ -21,4 +22,5 @@ cargo run -- \
   run --rust-version "${RUST_STABLE_VERSION}" \
       --sdk-path "${SDK_PATH}" \
       --cdk-output cdk-outputs.json \
-      --musl
+      --musl \
+      --architecture "${ARCHITECTURE}"
