feat: add cli client support for image builds to enable buildkit (#8772)

* feat: add support for using CLI container clients (#8661)

* feat: add build_client

* feat: add --use-buildkit option

* feat: use build client when building with images

* lint and format

* feat: add command to cli

* fix: use absolute path for dockerfile

* test: add unit test for build_client

* test: add unit test for build_context

* fix: additional test fixes

* fix: schema update

* fix: remove hardcoded unix pathseps

* fix: remove provenance and sbom from docker buildx

* fix: refactor code to use SDKBuildClient

This change also refactors a lot of docker specific things to be "container_client" rather than "docker_client".
In addition, moving from instantiating the image build client in build context to lazily doing it in the app builder.

* test: add additional unit tests

* fix: fix error handling in CLIBuildClient

* fix: reorder error handling in CLIImageBuilder

* test: add integration tests for CLI client (#8689)

* test: add basic integration test

* test: more comprehensive integ tests

* test: add arm64 test and mark as tier1

* fix: update dockerfile error messages

* test: remove arm64 test

* fix: allow finch without sudo

Author: reedham-aws

samcli/commands/_utils/options.py

@@ -862,6 +862,20 @@ def use_container_build_option(f):
     return use_container_build_click_option()(f)
 
 
+def use_buildkit_click_option():
+    return click.option(
+        "--use-buildkit/--no-use-buildkit",
+        required=False,
+        default=False,
+        is_flag=True,
+        help="Enable buildkit for container image builds. Requires Docker with buildx plugin or Finch CLI.",
+    )
+
+
+def use_buildkit_option(f):
+    return use_buildkit_click_option()(f)
+
+
 def mount_symlinks_click_option():
     return click.option(
         "--mount-symlinks/--no-mount-symlinks",

samcli/commands/build/build_context.py

@@ -83,6 +83,7 @@ def __init__(
         build_in_source: Optional[bool] = None,
         mount_with: str = MountMode.READ.value,
         mount_symlinks: Optional[bool] = False,
+        use_buildkit: Optional[bool] = False,
     ) -> None:
         """
         Initialize the class
@@ -142,6 +143,8 @@ def __init__(
             Mount mode of source code directory when building inside container, READ ONLY by default
         mount_symlinks Optional[bool]:
             Indicates if symlinks should be mounted inside the container
+        use_buildkit Optional[bool]:
+            Enable buildkit for container image builds
         """
 
         self._resource_identifier = resource_identifier
@@ -184,6 +187,7 @@ def __init__(
         self._build_result: Optional[ApplicationBuildResult] = None
         self._mount_with = MountMode(mount_with)
         self._mount_symlinks = mount_symlinks
+        self._use_buildkit = use_buildkit
 
     def __enter__(self) -> "BuildContext":
         self.set_up()
@@ -278,6 +282,7 @@ def run(self) -> None:
                 build_in_source=self._build_in_source,
                 mount_with_write=mount_with_write,
                 mount_symlinks=self._mount_symlinks,
+                use_buildkit=self._use_buildkit,
             )
 
             self._check_exclude_warning()

samcli/commands/build/command.py

@@ -29,6 +29,7 @@
     skip_prepare_infra_option,
     template_option_without_build,
     terraform_project_root_path_option,
+    use_buildkit_option,
     use_container_build_option,
 )
 from samcli.commands.build.click_container import ContainerOptions
@@ -82,6 +83,7 @@
 )
 @skip_prepare_infra_option
 @use_container_build_option
+@use_buildkit_option
 @build_in_source_option
 @click.option(
     "--container-env-var",
@@ -161,6 +163,7 @@ def cli(
     terraform_project_root_path: Optional[str],
     build_in_source: Optional[bool],
     mount_symlinks: Optional[bool],
+    use_buildkit: Optional[bool],
 ) -> None:
     """
     `sam build` command entry point
@@ -193,6 +196,7 @@ def cli(
         build_in_source,
         mount_with,
         mount_symlinks,
+        use_buildkit,
     )  # pragma: no cover
 
 
@@ -220,6 +224,7 @@ def do_cli(  # pylint: disable=too-many-locals, too-many-statements
     build_in_source: Optional[bool],
     mount_with: str,
     mount_symlinks: Optional[bool],
+    use_buildkit: Optional[bool],
 ) -> None:
     """
     Implementation of the ``cli`` method
@@ -260,6 +265,7 @@ def do_cli(  # pylint: disable=too-many-locals, too-many-statements
         build_in_source=build_in_source,
         mount_with=mount_with,
         mount_symlinks=mount_symlinks,
+        use_buildkit=use_buildkit,
     ) as ctx:
         ctx.run()
 

samcli/commands/build/core/options.py

@@ -16,6 +16,7 @@
 
 CONTAINER_OPTION_NAMES: List[str] = [
     "use_container",
+    "use_buildkit",
     "container_env_var",
     "container_env_var_file",
     "build_image",

samcli/lib/build/app_builder.py

@@ -60,7 +60,9 @@
 )
 from samcli.lib.utils.stream_writer import StreamWriter
 from samcli.local.docker.container import ContainerContext
-from samcli.local.docker.exceptions import ContainerArchiveImageLoadFailedException
+from samcli.local.docker.container_client import ContainerClient
+from samcli.local.docker.exceptions import BuildkitNotAvailableException, ContainerArchiveImageLoadFailedException
+from samcli.local.docker.image_build_client import CLIBuildClient, ImageBuildClient, SDKBuildClient
 from samcli.local.docker.lambda_build_container import LambdaBuildContainer
 from samcli.local.docker.manager import ContainerManager, DockerImagePullFailedException
 from samcli.local.docker.utils import (
@@ -106,14 +108,15 @@ def __init__(
         parallel: bool = False,
         mode: Optional[str] = None,
         stream_writer: Optional[StreamWriter] = None,
-        docker_client: Optional[docker.DockerClient] = None,
+        container_client: Optional[ContainerClient] = None,
         container_env_var: Optional[Dict] = None,
         container_env_var_file: Optional[str] = None,
         build_images: Optional[Dict] = None,
         combine_dependencies: bool = True,
         build_in_source: Optional[bool] = None,
         mount_with_write: bool = False,
         mount_symlinks: Optional[bool] = False,
+        use_buildkit: Optional[bool] = False,
     ) -> None:
         """
         Initialize the class
@@ -144,8 +147,8 @@ def __init__(
             Optional, name of the build mode to use ex: 'debug'
         stream_writer : Optional[StreamWriter]
             An optional stream writer to accept stderr output
-        docker_client : Optional[docker.DockerClient]
-            An optional Docker client object to replace the default one loaded from env
+        container_client : Optional[ContainerClient]
+            An optional container client object to replace the default one loaded from env
         container_env_var : Optional[Dict]
             An optional dictionary of environment variables to pass to the container
         container_env_var_file : Optional[str]
@@ -161,6 +164,8 @@ def __init__(
             Mount source code directory with write permissions when building inside container.
         mount_symlinks: Optional[bool]
             True if symlinks should be mounted in the container.
+        use_buildkit: Optional[bool]
+            Optional flag for building Image functions with buildkit support.
         """
         self._resources_to_build = resources_to_build
         self._build_dir = build_dir
@@ -175,11 +180,12 @@ def __init__(
         self._mode = mode
         self._stream_writer = stream_writer if stream_writer else StreamWriter(stream=osutils.stderr(), auto_flush=True)
 
-        # Store docker_client parameter for lazy initialization
-        # Only validate container runtime when Docker client is actually accessed
+        # Store container_client parameter for lazy initialization
+        # Only validate container runtime when container client is actually accessed
         # This prevents unnecessary validation for builds that don't require containers
-        self._docker_client_param = docker_client
-        self._validated_docker_client: Optional[docker.DockerClient] = None
+        # NOTE: It seems like at this point container_client is only ever passed in the tests for mocking.
+        self._container_client_param = container_client
+        self._validated_container_client: Optional[ContainerClient] = None
 
         self._deprecated_runtimes = DEPRECATED_RUNTIMES
         self._colored = Colored()
@@ -190,16 +196,18 @@ def __init__(
         self._build_in_source = build_in_source
         self._mount_with_write = mount_with_write
         self._mount_symlinks = mount_symlinks
+        self._use_buildkit = use_buildkit
+        self._image_build_client: Optional[ImageBuildClient] = None
 
     @property
-    def _docker_client(self) -> docker.DockerClient:
+    def _container_client(self) -> ContainerClient:
         """
-        Lazy initialization of Docker client. Only validates container runtime when actually accessed.
+        Lazy initialization of container client. Only validates container runtime when actually accessed.
         This prevents unnecessary container runtime validation for builds that don't require containers.
         """
-        if self._validated_docker_client is None:
-            self._validated_docker_client = self._docker_client_param or get_validated_container_client()
-        return self._validated_docker_client
+        if self._validated_container_client is None:
+            self._validated_container_client = self._container_client_param or get_validated_container_client()
+        return self._validated_container_client
 
     def build(self) -> ApplicationBuildResult:
         """
@@ -453,14 +461,28 @@ def _build_lambda_image(self, function_name: str, metadata: Dict, architecture:
             build_args["target"] = cast(str, docker_build_target)
 
         try:
-            build_image, build_logs = self._docker_client.images.build(**build_args)
-            LOG.debug("%s image is built for %s function", build_image, function_name)
+            if not self._image_build_client:
+                if self._use_buildkit:
+                    container_client = self._container_client
+                    engine_type = container_client.get_runtime_type()
+
+                    is_available, error_msg = CLIBuildClient.is_available(engine_type)
+                    if not is_available:
+                        raise BuildkitNotAvailableException(error_msg)
+
+                    self._image_build_client = CLIBuildClient(engine_type=engine_type)
+                    LOG.debug(f"Using CLIBuildClient with engine_type {engine_type}")
+                else:
+                    self._image_build_client = SDKBuildClient(self._container_client)
+                    LOG.debug("Using SDKBuildClient")
+            build_logs = self._image_build_client.build_image(**build_args)  # type: ignore[arg-type]
+            LOG.debug(f"Image built for {function_name} function")
         except docker.errors.BuildError as ex:
             LOG.error("Failed building function %s", function_name)
             self._stream_lambda_image_build_logs(ex.build_log, function_name, False)
             raise DockerBuildFailed(str(ex)) from ex
         except docker.errors.APIError as e:
-            if self._docker_client.is_dockerfile_error(e):
+            if self._container_client.is_dockerfile_error(e):
                 raise DockerfileOutSideOfContext(e.explanation) from e
 
             # Re-raise other API errors
@@ -469,9 +491,9 @@ def _build_lambda_image(self, function_name: str, metadata: Dict, architecture:
         # The Docker-py low level api will stream logs back but if an exception is raised by the api
         # this is raised when accessing the generator. So we need to wrap accessing build_logs in a try: except.
         try:
-            self._stream_lambda_image_build_logs(build_logs, function_name)
+            self._stream_lambda_image_build_logs(build_logs, function_name)  # type: ignore[arg-type]
         except docker.errors.APIError as e:
-            if self._docker_client.is_dockerfile_error(e):
+            if self._container_client.is_dockerfile_error(e):
                 raise DockerfileOutSideOfContext(e.explanation) from e
 
             # Not sure what else can be raise that we should be catching but re-raising for now
@@ -501,7 +523,7 @@ def _stream_lambda_image_build_logs(
     def _load_lambda_image(self, image_archive_path: str) -> str:
         try:
             with open(image_archive_path, mode="rb") as image_archive:
-                image = self._docker_client.load_image_from_archive(image_archive)
+                image = self._container_client.load_image_from_archive(image_archive)
                 return f"{image.id}"
         except (docker.errors.APIError, OSError, ContainerArchiveImageLoadFailedException) as ex:
             raise DockerBuildFailed(msg=str(ex)) from ex

samcli/local/docker/container_client.py

@@ -382,22 +382,23 @@ def is_dockerfile_error(self, error: Union[Exception, str]) -> bool:
         Check if error is a dockerfile-related error for Docker.
 
         Docker-specific error patterns for dockerfile-related issues typically
-        contain "Cannot locate specified Dockerfile" in the error message.
+        contain "Cannot locate specified Dockerfile" or "failed to read dockerfile" in the error message.
 
         Args:
             error: Exception or error message to check
 
         Returns:
             bool: True if the error indicates a dockerfile-related issue
         """
+        patterns = ["Cannot locate specified Dockerfile", "failed to read dockerfile"]
         if isinstance(error, docker.errors.APIError):
             if not error.is_server_error:
                 return False
             if not hasattr(error, "explanation") or error.explanation is None:
                 return False
-            return "Cannot locate specified Dockerfile" in str(error.explanation)
+            return any(pattern in str(error.explanation) for pattern in patterns)
         elif isinstance(error, str):
-            return "Cannot locate specified Dockerfile" in error
+            return any(pattern in error for pattern in patterns)
         return False
 
     def list_containers_by_image(self, image_name: str, all_containers: bool = True) -> List[Any]:

samcli/local/docker/exceptions.py

@@ -68,3 +68,9 @@ class ContainerInvalidSocketPathException(UserException):
     """
     Failed to load Docker/Finch container image from archive file
     """
+
+
+class BuildkitNotAvailableException(UserException):
+    """
+    Raised when --with-buildkit is specified but buildkit is not available
+    """