feat: Add experimental flag to boost 'package' command in conjunction with similar flag for 'build' command (#7923)

Co-authored-by: Roger Zhang <ruojiazh@amazon.com>
Co-authored-by: vicheey <181402101+vicheey@users.noreply.github.com>

Author: 3 people

samcli/commands/_utils/experimental.py

@@ -46,6 +46,9 @@ class ExperimentalFlag:
     BuildPerformance = ExperimentalEntry(
         "experimentalBuildPerformance", EXPERIMENTAL_ENV_VAR_PREFIX + "BUILD_PERFORMANCE"
     )
+    PackagePerformance = ExperimentalEntry(
+        "experimentalPackagePerformance", EXPERIMENTAL_ENV_VAR_PREFIX + "PACKAGE_PERFORMANCE"
+    )
     IaCsSupport = {
         "terraform": ExperimentalEntry(
             "experimentalTerraformSupport", EXPERIMENTAL_ENV_VAR_PREFIX + "TERRAFORM_SUPPORT"

samcli/lib/package/artifact_exporter.py

@@ -19,6 +19,7 @@
 
 from botocore.utils import set_value_from_jmespath
 
+from samcli.commands._utils.experimental import ExperimentalFlag, is_experimental_enabled
 from samcli.commands.package import exceptions
 from samcli.lib.package.code_signer import CodeSigner
 from samcli.lib.package.local_files_utils import get_uploaded_s3_object_name, mktempfile
@@ -275,6 +276,10 @@ def export(self) -> Dict:
         self._apply_global_values()
         self.template_dict = self._export_global_artifacts(self.template_dict)
 
+        cache: Optional[Dict] = None
+        if is_experimental_enabled(ExperimentalFlag.PackagePerformance):
+            cache = {}
+
         for resource_logical_id, resource in self.template_dict["Resources"].items():
             resource_type = resource.get("Type", None)
             resource_dict = resource.get("Properties", {})
@@ -287,7 +292,7 @@ def export(self) -> Dict:
                 if resource_dict.get("PackageType", ZIP) != exporter_class.ARTIFACT_TYPE:
                     continue
                 # Export code resources
-                exporter = exporter_class(self.uploaders, self.code_signer)
+                exporter = exporter_class(self.uploaders, self.code_signer, cache)
                 exporter.export(full_path, resource_dict, self.template_dir)
 
         return self.template_dict

samcli/lib/package/packageable_resources.py

@@ -67,9 +67,10 @@ class Resource:
     EXPORT_DESTINATION: Destination
     ARTIFACT_TYPE: Optional[str] = None
 
-    def __init__(self, uploaders: Uploaders, code_signer):
+    def __init__(self, uploaders: Uploaders, code_signer, cache: Optional[Dict] = None):
         self.uploaders = uploaders
         self.code_signer = code_signer
+        self.cache = cache
 
     @property
     def uploader(self) -> Union[S3Uploader, ECRUploader]:
@@ -167,6 +168,7 @@ def do_export(
             uploader,
             artifact_extension,
             local_path,
+            self.cache,
         )
         if should_sign_package:
             uploaded_url = self.code_signer.sign_package(

samcli/lib/package/utils.py

@@ -141,6 +141,7 @@ def upload_local_artifacts(
     uploader: S3Uploader,
     extension: Optional[str] = None,
     local_path: Optional[str] = None,
+    previously_uploaded: Optional[Dict] = None,
 ) -> str:
     """
     Upload local artifacts referenced by the property at given resource and
@@ -154,18 +155,20 @@ def upload_local_artifacts(
 
     If path is already a path to S3 object, this method does nothing.
 
-    :param resource_type:   Type of the CloudFormation resource
-    :param resource_id:     Id of the CloudFormation resource
-    :param resource_dict:   Dictionary containing resource definition
-    :param property_path:   Json path to the property of SAM or CloudFormation resource where the
-                            local path is present
-    :param parent_dir:      Resolve all relative paths with respect to this
-                            directory
-    :param uploader:        Method to upload files to S3
-    :param extension:       Extension of the uploaded artifact
-    :param local_path:      Local path for the cases when search return more than single result
-    :return:                S3 URL of the uploaded object
-    :raise:                 ValueError if path is not a S3 URL or a local path
+    :param resource_type:         Type of the CloudFormation resource
+    :param resource_id:           Id of the CloudFormation resource
+    :param resource_dict:         Dictionary containing resource definition
+    :param property_path:         Json path to the property of SAM or CloudFormation resource
+                                  where the local path is present
+    :param parent_dir:            Resolve all relative paths with respect to this
+                                  directory
+    :param uploader:              Method to upload files to S3
+    :param extension:             Extension of the uploaded artifact
+    :param local_path:            Local path for the cases when search return more than single result
+    :param previously_uploaded:   Cache mapping from (evaluated) local path to previous result of
+                                  this function
+    :return:                      S3 URL of the uploaded object
+    :raise:                       ValueError if path is not a S3 URL or a local path
     """
 
     if local_path is None:
@@ -183,18 +186,29 @@ def upload_local_artifacts(
 
     local_path = make_abs_path(parent_dir, local_path)
 
+    if previously_uploaded and local_path in previously_uploaded:
+        result = previously_uploaded[local_path]
+        LOG.debug("Skipping upload of %s since is already uploaded to %s", local_path, result)
+        return cast(str, result)
+
     # Or, pointing to a folder. Zip the folder and upload (zip_method is changed based on resource type)
     if is_local_folder(local_path):
-        return zip_and_upload(
+        result = zip_and_upload(
             local_path,
             uploader,
             extension,
             zip_method=make_zip_with_lambda_permissions if resource_type in LAMBDA_LOCAL_RESOURCES else make_zip,
         )
+        if previously_uploaded is not None:
+            previously_uploaded[local_path] = result
+        return cast(str, result)
 
     # Path could be pointing to a file. Upload the file
     if is_local_file(local_path):
-        return uploader.upload_with_dedup(local_path)
+        result = uploader.upload_with_dedup(local_path)
+        if previously_uploaded is not None:
+            previously_uploaded[local_path] = result
+        return cast(str, result)
 
     raise InvalidLocalPathError(resource_id=resource_id, property_name=property_path, local_path=local_path)
 

tests/unit/commands/_utils/test_experimental.py

@@ -57,16 +57,16 @@ def test_set_experimental(self):
         self.gc_mock.return_value.set_value.assert_called_once_with(config_entry, False, is_flag=True, flush=False)
 
     def test_get_all_experimental(self):
-        self.assertEqual(len(get_all_experimental()), 4)
+        self.assertEqual(len(get_all_experimental()), 5)
 
     def test_get_all_experimental_statues(self):
-        self.assertEqual(len(get_all_experimental_statues()), 4)
+        self.assertEqual(len(get_all_experimental_statues()), 5)
 
     def test_get_all_experimental_env_vars(self):
-        self.assertEqual(len(get_all_experimental_env_vars()), 4)
+        self.assertEqual(len(get_all_experimental_env_vars()), 5)
 
     def test_get_enabled_experimental_flags(self):
-        self.assertEqual(len(get_enabled_experimental_flags()), 4)
+        self.assertEqual(len(get_enabled_experimental_flags()), 5)
 
     @patch("samcli.commands._utils.experimental.set_experimental")
     @patch("samcli.commands._utils.experimental.get_all_experimental")