diff --git a/.vscode/cspell.json b/.vscode/cspell.json index a81d9c73375b..84fb693a847d 100644 --- a/.vscode/cspell.json +++ b/.vscode/cspell.json @@ -27,7 +27,7 @@ "sdk/**/_generated/**", "**/*requirement*.txt", "**/assets.json", - + "**/pyproject.toml", "sdk/anomalydetector/**", "sdk/applicationinsights/azure-applicationinsights/**", "sdk/batch/azure-batch/**", @@ -95,9 +95,13 @@ ".gitignore", "tools/azure-sdk-tools/devtools_testutils/fake_credentials.py", "tools/azure-sdk-tools/packaging_tools/**", - "tools/azure-sdk-tools/setup.py" + "tools/azure-sdk-tools/setup.py", + "tools/azure-sdk-tools/tests/test_servicemetadata.py" ], "words": [ + "pyprojecttoml", + "tomli", + "sdkrel", "qnamaker", "mindependency", "automl", diff --git a/doc/eng_sys_checks.md b/doc/eng_sys_checks.md index ca5394b6ed30..e8bfe6d3228d 100644 --- a/doc/eng_sys_checks.md +++ b/doc/eng_sys_checks.md @@ -4,7 +4,9 @@ - [Targeting a specific package at build queue time](#targeting-a-specific-package-at-build-queue-time) - [Skipping a tox test environment at build queue time](#skipping-a-tox-test-environment-at-build-queue-time) - [Skipping entire sections of builds](#skipping-entire-sections-of-builds) + - [The pyproject.toml](#the-pyprojecttoml) - [Environment variables important to CI](#environment-variables-important-to-ci) + - [Atomic Overrides](#atomic-overrides) - [Analyze Checks](#analyze-checks) - [MyPy](#mypy) - [Pylint](#pylint) @@ -102,6 +104,28 @@ This is the most useful skip, but the following skip variables are also supporte - `Skip.VerifyDependencies` - Omit checking that a package's dependencies are on PyPI before releasing. +## The pyproject.toml + +Starting with [this pr](https://github.com/Azure/azure-sdk-for-python/pull/28345), which checks apply to which packages are now **established** in a `pyproject.toml`, right next to each package's `setup.py`. This not only allows devs to fine-tune which checks that are applied at a package-level, but also seriously reduces confusion as to which checks apply when. + +We default to **enabling** most of our checks like `pylint`, `mypy`, etc. Due to that, most `pyproject.toml` settings will likely be **disabling** checks. + +Here's an example: + +```toml +# from sdk/core/azure-servicemanagement-legacy/pyproject.toml, which is a legacy package +# as a result, all of these checks are disabled +[tool.azure-sdk-build] +mypy = false +type_check_samples = false +verifytypes = false +pyright = false +pylint = false +black = false +``` + +If a package does not yet have a `pyproject.toml`, creating one with just the section `[tool.azure-sdk-build]` will do no harm to the release of the package in question. + ## Environment variables important to CI There are a few differences from a standard local invocation of `tox `. Primarily, these differences adjust the checks to be friendly to parallel invocation. These adjustments are necessary to prevent random CI crashes. @@ -114,6 +138,22 @@ There are a few differences from a standard local invocation of `tox `. Pri The various tooling abstracted by the environments within `eng/tox/tox.ini` take the above variables into account automatically. +### Atomic Overrides + +Packages with classifier `Development Status :: 7 - Inactive`, are **not** built by default and as such normal `checks` like `mypy` and `pylint` are also not run against them. Older "core" packages like `azure-common` and `azure-servicemanagement-legacy` are present, but excluded from the build due to this restriction. + +To temporarily **override** this restriction, a dev need only set the queue time variable: `ENABLE_PACKAGE_NAME`. The `-` in package names should be replaced by an `_`, as that is how the environment variable will be set on the actual CI machine anyway. + +- `ENABLE_AZURE_COMMON=true` +- `ENABLE_AZURE_SERVICEMANAGEMENT_LEGACY=true` + +This same methodology also applies to _individual checks_ that run during various phases of CI. Developers can use a queue time variable of format `PACKAGE_NAME_CHECK=true/false`. + +The name that you should use is visible based on what the `tox environment` that the check refers to! Here are a few examples of enabling/disabling checks: + +- `AZURE_SERVICEBUS_PYRIGHT=true` <-- enable a check that normally is disabled in `pyproject.toml` +- `AZURE_CORE_PYLINT=false` <-- disable a check that normally runs + ## Analyze Checks Analyze job in both nightly CI and pull request validation pipeline runs a set of static analysis using external and internal tools. Following are the list of these static analysis. @@ -122,15 +162,8 @@ Analyze job in both nightly CI and pull request validation pipeline runs a set o [`MyPy`](https://pypi.org/project/mypy/) is a static analysis tool that runs type checking of python package. MyPy is an opt-in check for packages. Following are the steps to run `MyPy` locally for a specific package: -1. Add the package name to the end of the [`mypy_hard_failure_packages.py`](https://github.com/Azure/azure-sdk-for-python/blob/main/eng/tox/mypy_hard_failure_packages.py) file: - ```python - MYPY_HARD_FAILURE_OPTED = [ - ..., - "azure-my-package", - ] - ``` -2. Go to root of the package -3. Execute following command: `tox -e mypy -c ../../../eng/tox/tox.ini` +1. Go to root of the package +2. Execute following command: `tox -e mypy -c ../../../eng/tox/tox.ini` ### Pylint diff --git a/eng/pipelines/templates/steps/build-artifacts.yml b/eng/pipelines/templates/steps/build-artifacts.yml index d98fcf9f0ff5..07f835526406 100644 --- a/eng/pipelines/templates/steps/build-artifacts.yml +++ b/eng/pipelines/templates/steps/build-artifacts.yml @@ -63,7 +63,7 @@ steps: condition: and(succeeded(),eq(variables['SetDevVersion'],'true')) - pwsh: | - sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} + sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive displayName: 'Generate Packages' condition: succeededOrFailed() diff --git a/eng/pipelines/templates/steps/run_black.yml b/eng/pipelines/templates/steps/run_black.yml index 91c322ba6c7b..03ed7e8e9880 100644 --- a/eng/pipelines/templates/steps/run_black.yml +++ b/eng/pipelines/templates/steps/run_black.yml @@ -11,7 +11,7 @@ steps: condition: succeededOrFailed() - script: | - pip install black==21.6b0 + pip install black==21.6b0 tools/azure-sdk-tools["build"] displayName: 'Prep Environment' condition: succeededOrFailed() diff --git a/eng/scripts/get_package_properties.py b/eng/scripts/get_package_properties.py index cfc149643d2b..c0bea2a83592 100644 --- a/eng/scripts/get_package_properties.py +++ b/eng/scripts/get_package_properties.py @@ -17,6 +17,7 @@ if "setup.py" in files: try: parsed = ParsedSetup.from_path(root) + print( "{0} {1} {2} {3}".format( parsed.name, parsed.version, parsed.is_new_sdk, os.path.dirname(parsed.setup_filename) diff --git a/eng/tox/run_mypy.py b/eng/tox/run_mypy.py index e79a12c152b6..78e4bdef9d8a 100644 --- a/eng/tox/run_mypy.py +++ b/eng/tox/run_mypy.py @@ -15,8 +15,7 @@ from ci_tools.environment_exclusions import ( is_ignored_package, - MYPY_OPT_OUT, - TYPE_CHECK_SAMPLES_OPT_OUT, + is_check_enabled ) logging.getLogger().setLevel(logging.INFO) @@ -36,7 +35,7 @@ args = parser.parse_args() package_name = os.path.basename(os.path.abspath(args.target_package)) - if package_name in MYPY_OPT_OUT or is_ignored_package(package_name): + if not is_check_enabled(args.target_package, "mypy", True) or is_ignored_package(package_name): logging.info( f"Package {package_name} opts-out of mypy check. See https://aka.ms/python/typing-guide for information." ) @@ -62,7 +61,7 @@ except CalledProcessError as src_err: src_code_error = src_err - if package_name in TYPE_CHECK_SAMPLES_OPT_OUT: + if not is_check_enabled(args.target_package, "type_check_samples", True): logging.info( f"Package {package_name} opts-out of mypy check on samples." ) diff --git a/eng/tox/run_pylint.py b/eng/tox/run_pylint.py index 826766305b56..fc1cacdf4550 100644 --- a/eng/tox/run_pylint.py +++ b/eng/tox/run_pylint.py @@ -14,7 +14,7 @@ import logging import sys -from ci_tools.environment_exclusions import PYLINT_OPT_OUT +from ci_tools.environment_exclusions import is_check_enabled from ci_tools.parsing import ParsedSetup logging.getLogger().setLevel(logging.INFO) @@ -43,7 +43,7 @@ top_level_module = pkg_details.namespace.split('.')[0] - if pkg_details.name not in PYLINT_OPT_OUT: + if is_check_enabled(args.target_package, "pylint"): try: check_call( [ diff --git a/eng/tox/run_pyright.py b/eng/tox/run_pyright.py index 80d5acbf59fe..02d3324075d2 100644 --- a/eng/tox/run_pyright.py +++ b/eng/tox/run_pyright.py @@ -15,8 +15,7 @@ from ci_tools.environment_exclusions import ( is_ignored_package, - PYRIGHT_OPT_OUT, - TYPE_CHECK_SAMPLES_OPT_OUT, + is_check_enabled, ) logging.getLogger().setLevel(logging.INFO) @@ -36,7 +35,7 @@ args = parser.parse_args() package_name = os.path.basename(os.path.abspath(args.target_package)) - if package_name in PYRIGHT_OPT_OUT or is_ignored_package(package_name): + if not is_check_enabled(args.target_package, "pyright") or is_ignored_package(package_name): logging.info( f"Package {package_name} opts-out of pyright check. See https://aka.ms/python/typing-guide for information." ) @@ -46,7 +45,7 @@ os.path.join(args.target_package, "azure"), os.path.join(args.target_package, "samples"), ] - if package_name in TYPE_CHECK_SAMPLES_OPT_OUT: + if not is_check_enabled(args.target_package, "type_check_samples"): logging.info( f"Package {package_name} opts-out of pyright check on samples." ) diff --git a/eng/tox/run_verifytypes.py b/eng/tox/run_verifytypes.py index 3f11856975b3..ef2444b243e0 100644 --- a/eng/tox/run_verifytypes.py +++ b/eng/tox/run_verifytypes.py @@ -16,7 +16,7 @@ import logging import sys -from ci_tools.environment_exclusions import is_ignored_package, VERIFYTYPES_OPT_OUT +from ci_tools.environment_exclusions import is_ignored_package, is_check_enabled logging.getLogger().setLevel(logging.INFO) @@ -98,7 +98,7 @@ def get_type_complete_score(commands, check_pytyped=False): module = package_name.replace("-", ".") setup_path = os.path.abspath(args.target_package) - if package_name in VERIFYTYPES_OPT_OUT or is_ignored_package(package_name): + if not is_check_enabled(args.target_package, "type_check_samples") or is_ignored_package(package_name): logging.info( f"{package_name} opts-out of verifytypes check. See https://aka.ms/python/typing-guide for information." ) diff --git a/eng/tox/tox.ini b/eng/tox/tox.ini index 66922cd243d3..cff76a4cebb9 100644 --- a/eng/tox/tox.ini +++ b/eng/tox/tox.ini @@ -39,6 +39,7 @@ pkgs = wheel==0.37.0 packaging==20.4 urllib3==1.26.12 + tomli==2.0.1 [testenv] diff --git a/scripts/devops_tasks/common_tasks.py b/scripts/devops_tasks/common_tasks.py index 2becb325a420..9d0c82a56374 100644 --- a/scripts/devops_tasks/common_tasks.py +++ b/scripts/devops_tasks/common_tasks.py @@ -31,7 +31,7 @@ from packaging.version import Version from packaging.version import parse -from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, lambda_filter_azure_pkg +from ci_tools.functions import MANAGEMENT_PACKAGE_IDENTIFIERS, lambda_filter_azure_pkg, str_to_bool from ci_tools.parsing import parse_require, ParsedSetup DEV_REQ_FILE = "dev_requirements.txt" @@ -41,8 +41,6 @@ logging.getLogger().setLevel(logging.INFO) - - def log_file(file_location, is_error=False): with open(file_location, "r") as file: for line in file: @@ -81,17 +79,6 @@ def clean_coverage(coverage_dir): raise -def str_to_bool(input_string): - if isinstance(input_string, bool): - return input_string - elif input_string.lower() in ("true", "t", "1"): - return True - elif input_string.lower() in ("false", "f", "0"): - return False - else: - return False - - def run_check_call( command_array, working_directory, diff --git a/scripts/devops_tasks/test_regression.py b/scripts/devops_tasks/test_regression.py index cddfd35afdbe..9d8847bb2ffc 100644 --- a/scripts/devops_tasks/test_regression.py +++ b/scripts/devops_tasks/test_regression.py @@ -23,8 +23,7 @@ find_whl, find_tools_packages, get_installed_packages, - extend_dev_requirements, - str_to_bool, + extend_dev_requirements ) from git_helper import ( @@ -34,7 +33,7 @@ clone_repo, ) -from ci_tools.functions import discover_targeted_packages +from ci_tools.functions import discover_targeted_packages, str_to_bool from ci_tools.parsing import ParsedSetup AZURE_GLOB_STRING = "azure*" diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index d11700cc1778..1a456e2b46a2 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -323,12 +323,12 @@ def prep_and_run_tox(targeted_packages: List[str], parsed_args: Namespace, optio inject_custom_reqs(destination_dev_req, parsed_args.injected_packages, package_dir) if parsed_args.tox_env: - filtered_tox_environment_set = filter_tox_environment_string(parsed_args.tox_env, package_name) + filtered_tox_environment_set = filter_tox_environment_string(parsed_args.tox_env, package_dir) if not filtered_tox_environment_set: logging.info( f"All requested tox environments for package {package_name} have been excluded by the environment exclusion list." - + " Check file /tools/azure-sdk-tools/ci_tools/environment_exclusions.py" + + " Check file /tools/azure-sdk-tools/ci_tools/environment_exclusions.py and the pyproject.toml." ) continue diff --git a/scripts/devops_tasks/validate_formatting.py b/scripts/devops_tasks/validate_formatting.py index ffdf138afb59..aca9f3475b83 100644 --- a/scripts/devops_tasks/validate_formatting.py +++ b/scripts/devops_tasks/validate_formatting.py @@ -12,29 +12,40 @@ import subprocess logging.getLogger().setLevel(logging.INFO) +from ci_tools.functions import discover_targeted_packages +from ci_tools.environment_exclusions import is_check_enabled root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", "..")) sdk_dir = os.path.join(root_dir, "sdk") def run_black(service_dir): + results = [] logging.info("Running black for {}".format(service_dir)) - out = subprocess.Popen([sys.executable, "-m", "black", "-l", "120", "sdk/{}".format(service_dir)], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd = root_dir - ) + discovered_packages = discover_targeted_packages("azure*", os.path.join(root_dir, "sdk", service_dir)) + + for package in discovered_packages: + package_name = os.path.basename(package) + + if is_check_enabled(package, "black", True): + out = subprocess.Popen([sys.executable, "-m", "black", "-l", "120", "sdk/{}/{}".format(service_dir, package_name)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + cwd = root_dir + ) - stdout,stderr = out.communicate() + stdout,stderr = out.communicate() - if stderr: - raise RuntimeError("black ran into some trouble during its invocation: " + stderr) + if stderr: + results.append((package_name, stderr)) - if stdout: - if "reformatted" in stdout.decode('utf-8'): - return False + if stdout: + if "reformatted" in stdout.decode('utf-8'): + results.append((package_name, False)) + else: + print(f"black succeeded against {package_name}") - return True + return results if __name__ == "__main__": @@ -51,7 +62,13 @@ def run_black(service_dir): args = parser.parse_args() if args.validate != "False": - if not run_black(args.service_directory): + results = run_black(args.service_directory) + + if len(results) > 0: + for result in results: + error = "Code needs reformat." if result[1] == False else error + logging.error(f"Black run for {result[0]} ran into an issue: {error}") + raise ValueError("Found difference between formatted code and current commit. Please re-generate with the latest autorest.") else: diff --git a/sdk/core/azure-common/pyproject.toml b/sdk/core/azure-common/pyproject.toml new file mode 100644 index 000000000000..482c53ffc174 --- /dev/null +++ b/sdk/core/azure-common/pyproject.toml @@ -0,0 +1,8 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false +pylint = false +regression = false +black = false \ No newline at end of file diff --git a/sdk/core/azure-core-experimental/pyproject.toml b/sdk/core/azure-core-experimental/pyproject.toml new file mode 100644 index 000000000000..4010cf355786 --- /dev/null +++ b/sdk/core/azure-core-experimental/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false \ No newline at end of file diff --git a/sdk/core/azure-core-tracing-opencensus/pyproject.toml b/sdk/core/azure-core-tracing-opencensus/pyproject.toml new file mode 100644 index 000000000000..4010cf355786 --- /dev/null +++ b/sdk/core/azure-core-tracing-opencensus/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false \ No newline at end of file diff --git a/sdk/core/azure-core-tracing-opentelemetry/pyproject.toml b/sdk/core/azure-core-tracing-opentelemetry/pyproject.toml new file mode 100644 index 000000000000..4010cf355786 --- /dev/null +++ b/sdk/core/azure-core-tracing-opentelemetry/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false \ No newline at end of file diff --git a/sdk/core/azure-core/pyproject.toml b/sdk/core/azure-core/pyproject.toml new file mode 100644 index 000000000000..c98953fca712 --- /dev/null +++ b/sdk/core/azure-core/pyproject.toml @@ -0,0 +1,8 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +# For test environments or static checks where a check should be run by default, not explicitly disabling will enable the check. +# pylint is enabled by default, so there is no reason for a pylint = true in every pyproject.toml. +# +# For newly added checks that are not enabled by default, packages should opt IN by " = true". \ No newline at end of file diff --git a/sdk/core/azure-mgmt-core/pyproject.toml b/sdk/core/azure-mgmt-core/pyproject.toml new file mode 100644 index 000000000000..a9a644ab469a --- /dev/null +++ b/sdk/core/azure-mgmt-core/pyproject.toml @@ -0,0 +1,5 @@ +[tool.azure-sdk-build] +mypy = false +verifytypes = false +pyright = false +type_check_samples = false diff --git a/sdk/core/azure-mgmt/pyproject.toml b/sdk/core/azure-mgmt/pyproject.toml new file mode 100644 index 000000000000..f09a5b522bde --- /dev/null +++ b/sdk/core/azure-mgmt/pyproject.toml @@ -0,0 +1,9 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false +pylint = false +regression = false +sphinx = false +black = false \ No newline at end of file diff --git a/sdk/core/azure-mgmt/sdk_packaging.toml b/sdk/core/azure-mgmt/sdk_packaging.toml index e10a1aae3a66..e7687fdae93b 100644 --- a/sdk/core/azure-mgmt/sdk_packaging.toml +++ b/sdk/core/azure-mgmt/sdk_packaging.toml @@ -1,2 +1,2 @@ [packaging] -auto_update = falsetitle = \ No newline at end of file +auto_update = false \ No newline at end of file diff --git a/sdk/core/azure-servicemanagement-legacy/pyproject.toml b/sdk/core/azure-servicemanagement-legacy/pyproject.toml new file mode 100644 index 000000000000..9dcba289df80 --- /dev/null +++ b/sdk/core/azure-servicemanagement-legacy/pyproject.toml @@ -0,0 +1,7 @@ +[tool.azure-sdk-build] +mypy = false +type_check_samples = false +verifytypes = false +pyright = false +pylint = false +black = false \ No newline at end of file diff --git a/sdk/core/azure/pyproject.toml b/sdk/core/azure/pyproject.toml new file mode 100644 index 000000000000..f09a5b522bde --- /dev/null +++ b/sdk/core/azure/pyproject.toml @@ -0,0 +1,9 @@ +[tool.azure-sdk-build] +type_check_samples = false +verifytypes = false +pyright = false +mypy = false +pylint = false +regression = false +sphinx = false +black = false \ No newline at end of file diff --git a/sdk/core/ci.yml b/sdk/core/ci.yml index 6603617297b0..76a666cf0ad8 100644 --- a/sdk/core/ci.yml +++ b/sdk/core/ci.yml @@ -35,6 +35,7 @@ extends: template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml parameters: ServiceDirectory: core + ValidateFormatting: true Artifacts: - name: azure-core safeName: azurecore diff --git a/sdk/cosmos/azure-mgmt-documentdb/setup.py b/sdk/cosmos/azure-mgmt-documentdb/setup.py index 6921e0e1eb11..e94bb9de1106 100644 --- a/sdk/cosmos/azure-mgmt-documentdb/setup.py +++ b/sdk/cosmos/azure-mgmt-documentdb/setup.py @@ -51,7 +51,7 @@ author_email='ptvshelp@microsoft.com', url='https://github.com/Azure/azure-sdk-for-python', classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', diff --git a/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml b/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml index 92335d051e16..58e60f17bd62 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml +++ b/sdk/regionmove/azure-mgmt-regionmove/sdk_packaging.toml @@ -6,3 +6,4 @@ package_doc_id = "" is_stable = false is_arm = true title = "RegionMoveServiceAPI" +auto_update = false \ No newline at end of file diff --git a/sdk/regionmove/azure-mgmt-regionmove/setup.py b/sdk/regionmove/azure-mgmt-regionmove/setup.py index c80db06cec50..d63fb5669455 100644 --- a/sdk/regionmove/azure-mgmt-regionmove/setup.py +++ b/sdk/regionmove/azure-mgmt-regionmove/setup.py @@ -47,7 +47,7 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 4 - Beta', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', diff --git a/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml b/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml index 0f1a6417619a..af79b133f248 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml +++ b/sdk/scheduler/azure-mgmt-scheduler/sdk_packaging.toml @@ -5,3 +5,4 @@ package_doc_id = "scheduler" is_stable = true sample_link = "" title = "SchedulerManagementClient" +auto_update = false \ No newline at end of file diff --git a/sdk/scheduler/azure-mgmt-scheduler/setup.py b/sdk/scheduler/azure-mgmt-scheduler/setup.py index 4b0e8374318e..5913934e2c36 100644 --- a/sdk/scheduler/azure-mgmt-scheduler/setup.py +++ b/sdk/scheduler/azure-mgmt-scheduler/setup.py @@ -47,7 +47,7 @@ url='https://github.com/Azure/azure-sdk-for-python', keywords="azure, azure sdk", # update with search keywords relevant to the azure service / product classifiers=[ - 'Development Status :: 5 - Production/Stable', + 'Development Status :: 7 - Inactive', 'Programming Language :: Python', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3', diff --git a/tools/azure-sdk-tools/ci_tools/build.py b/tools/azure-sdk-tools/ci_tools/build.py index f7f9de9c1987..b0097acdd5ec 100644 --- a/tools/azure-sdk-tools/ci_tools/build.py +++ b/tools/azure-sdk-tools/ci_tools/build.py @@ -11,6 +11,7 @@ from ci_tools.versioning.version_set_dev import get_dev_version, format_build_id from ci_tools.logging import initialize_logger, run_logged + def build() -> None: parser = argparse.ArgumentParser( description="""This is the primary entrypoint for the "build" action. This command is used to build any package within the azure-sdk-for-python repository.""", @@ -57,6 +58,17 @@ def build() -> None: ), ) + parser.add_argument( + "--inactive", + default=False, + dest="inactive", + action="store_true", + help=( + "Include inactive packages when assembling artifacts. CI builds will include inactive packages as a way to ensure that the yml" + + " controlled artifacts can be associated with a wheel/sdist." + ), + ) + parser.add_argument( "--produce_apiview_artifact", default=False, @@ -93,7 +105,9 @@ def build() -> None: else: target_dir = repo_root - targeted_packages = discover_targeted_packages(args.glob_string, target_dir, args.package_filter_string) + targeted_packages = discover_targeted_packages( + args.glob_string, target_dir, args.package_filter_string, filter_type="Build", compatibility_filter=True, include_inactive=args.inactive + ) artifact_directory = get_artifact_directory(args.distribution_directory) build_id = format_build_id(args.build_id or DEFAULT_BUILD_ID) @@ -129,7 +143,7 @@ def build_packages( new_version = get_dev_version(setup_parsed.version, build_id) logger.log(level=logging.DEBUG, msg=f"{setup_parsed.name}: {setup_parsed.version} -> {new_version}") - + set_version_py(setup_parsed.setup_filename, new_version) set_dev_classifier(setup_parsed.setup_filename, new_version) @@ -150,6 +164,12 @@ def create_package( setup_directory_or_file = os.path.dirname(setup_directory_or_file) if enable_wheel: - run_logged([sys.executable, "setup.py", "bdist_wheel", "-d", dist], prefix="create_wheel", cwd=setup_directory_or_file) + run_logged( + [sys.executable, "setup.py", "bdist_wheel", "-d", dist], prefix="create_wheel", cwd=setup_directory_or_file + ) if enable_sdist: - run_logged([sys.executable, "setup.py", "sdist", "--format", "zip", "-d", dist], prefix="create_sdist", cwd=setup_directory_or_file) + run_logged( + [sys.executable, "setup.py", "sdist", "--format", "zip", "-d", dist], + prefix="create_sdist", + cwd=setup_directory_or_file, + ) diff --git a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py index 5b5579a52488..b7692245d0ac 100644 --- a/tools/azure-sdk-tools/ci_tools/environment_exclusions.py +++ b/tools/azure-sdk-tools/ci_tools/environment_exclusions.py @@ -4,7 +4,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. See License.txt in the project root for license information. # -------------------------------------------------------------------------------------------- -import logging +from ci_tools.functions import get_config_setting +import os PYLINT_OPT_OUT = [ "azure-applicationinsights", @@ -83,9 +84,6 @@ "azure-confidentialledger", "azure-containerregistry", "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-iot-deviceupdate", "azure-digitaltwins-core", "azure-eventhub-checkpointstoreblob", @@ -138,11 +136,6 @@ "azure-communication-sms", "azure-confidentialledger", "azure-containerregistry", - "azure-core", - "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-cosmos", "azure-developer-devcenter", "azure-iot-deviceupdate", @@ -287,11 +280,6 @@ "azure-communication-sms", "azure-confidentialledger", "azure-containerregistry", - "azure-core", - "azure-mgmt-core", - "azure-core-experimental", - "azure-core-tracing-opencensus", - "azure-core-tracing-opentelemetry", "azure-cosmos", "azure-developer-devcenter", "azure-iot-deviceupdate", @@ -337,7 +325,6 @@ "azure-messaging-webpubsubservice", ] - # -------------------------------------------------------------------------------------------------------------------- # DO NOT add packages to the below lists. They are used to omit packages that will never run type checking. IGNORE_FILTER = ["nspkg", "mgmt", "cognitiveservices"] @@ -361,30 +348,52 @@ ] -def filter_tox_environment_string(namespace_argument: str, package_name: str) -> str: +def is_check_enabled(package_path: str, check: str, default: bool = True) -> bool: + if package_path.endswith("setup.py"): + package_path = os.path.dirname(package_path) + + if package_path == ".": + package_path = os.getcwd() + + enabled = default + package_name = os.path.basename(package_path) + + # check the original exclusion lists for the package name + exclusions_for_env = [] + try: + exclusions_for_env = globals()[f"{check.strip().upper()}_OPT_OUT"] + except Exception as e: + pass + + if package_name in exclusions_for_env: + enabled = False + + # now pull the new pyproject.toml configuration + config = get_config_setting(package_path, check.strip().lower(), True) + + return (config and enabled) + + +def filter_tox_environment_string(namespace_argument: str, package_path: str) -> str: """ Takes an incoming comma separated list of tox environments and package name. Resolves whether or not each given tox environment should run, given comparison to single unified exclusion file in `environment_exclusions`. - :param namespace_argument: A namespace argument. - :param package_name: The name of the package. This takes the form of a comma separated list: "whl,sdist,mindependency". "whl". "lint,pyright,sphinx". + :param namespace_argument: A namespace argument. This takes the form of a comma separated list: "whl,sdist,mindependency". "whl". "lint,pyright,sphinx". + :param package_path: The path to the package. """ + if package_path.endswith("setup.py"): + package_path = os.path.dirname(package_path) + + package_name = os.path.basename(package_path) + if namespace_argument: tox_envs = namespace_argument.strip().split(",") filtered_set = [] for tox_env in tox_envs: - exclusions_for_env = [] - try: - exclusions_for_env = globals()[f"{tox_env.strip().upper()}_OPT_OUT"] - except Exception as e: - pass - - if exclusions_for_env: - if package_name in exclusions_for_env: - continue - - filtered_set.append(tox_env) + if is_check_enabled(package_path, tox_env, True): + filtered_set.append(tox_env) return ",".join(filtered_set) return namespace_argument diff --git a/tools/azure-sdk-tools/ci_tools/functions.py b/tools/azure-sdk-tools/ci_tools/functions.py index 6b76b4b68afb..58e9cbeee489 100644 --- a/tools/azure-sdk-tools/ci_tools/functions.py +++ b/tools/azure-sdk-tools/ci_tools/functions.py @@ -6,34 +6,27 @@ from ci_tools.variables import discover_repo_root, get_artifact_directory, DEV_BUILD_IDENTIFIER import os, sys, platform, glob, re -from ci_tools.parsing import ParsedSetup +from ci_tools.parsing import ParsedSetup, get_build_config from pypi_tools.pypi import PyPIClient -from typing import List +from typing import List, Any import logging +INACTIVE_CLASSIFIER = "Development Status :: 7 - Inactive" -OMITTED_CI_PACKAGES = [ - "azure-mgmt-documentdb", - "azure-servicemanagement-legacy", - "azure-mgmt-scheduler", - "azure", - "azure-mgmt", - "azure-storage", - "azure-monitor", - "azure-mgmt-regionmove", -] MANAGEMENT_PACKAGE_IDENTIFIERS = [ "mgmt", + "nspkg", "azure-cognitiveservices", "azure-servicefabric", - "nspkg", "azure-keyvault", "azure-synapse", "azure-ai-anomalydetector", ] + META_PACKAGES = ["azure", "azure-mgmt", "azure-keyvault"] + REGRESSION_EXCLUDED_PACKAGES = [ "azure-common", ] @@ -49,8 +42,9 @@ and "mgmt" not in x and os.path.basename(x) not in MANAGEMENT_PACKAGE_IDENTIFIERS and os.path.basename(x) not in META_PACKAGES - and os.path.basename(x) not in REGRESSION_EXCLUDED_PACKAGES + and str_to_bool(get_config_setting(x, "regression", True)) ) + omit_docs = lambda x: "nspkg" not in x and os.path.basename(x) not in META_PACKAGES omit_build = lambda x: x # Dummy lambda to match omit type lambda_filter_azure_pkg = lambda x: x.startswith("azure") and "-nspkg" not in x @@ -66,7 +60,7 @@ } -def filter_for_compatibility(package_set: List[str]) -> List[str]: +def apply_compatibility_filter(package_set: List[str]) -> List[str]: """ This function takes in a set of paths to python packages. It returns the set filtered by compatibility with the currently running python executable. If a package is unsupported by the executable, it will be omitted from the returned list. @@ -80,7 +74,12 @@ def filter_for_compatibility(package_set: List[str]) -> List[str]: running_major_version = Version(".".join([str(v[0]), str(v[1]), str(v[2])])) for pkg in package_set: - spec_set = SpecifierSet(ParsedSetup.from_path(pkg).python_requires) + try: + spec_set = SpecifierSet(ParsedSetup.from_path(pkg).python_requires) + except RuntimeError as e: + logging.error(f"Unable to parse metadata for package {pkg}, omitting from build.") + continue + pkg_specs_override = TEST_COMPATIBILITY_MAP.get(os.path.basename(pkg), None) if pkg_specs_override: @@ -89,6 +88,9 @@ def filter_for_compatibility(package_set: List[str]) -> List[str]: if running_major_version in spec_set: collected_packages.append(pkg) + logging.debug("Target packages after applying compatibility filter: {}".format(collected_packages)) + logging.debug("Package(s) omitted by compatibility filter: {}".format(generate_difference(package_set, collected_packages))) + return collected_packages @@ -112,6 +114,34 @@ def str_to_bool(input_string: str) -> bool: else: return False +def generate_difference(original_packages: List[str], filtered_packages: List[str]): + return list(set(original_packages) - set(filtered_packages)) + +def glob_packages(glob_string: str, target_root_dir: str) -> List[str]: + if glob_string: + individual_globs = glob_string.split(",") + else: + individual_globs = "azure-*" + collected_top_level_directories = [] + + for glob_string in individual_globs: + globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py")) + glob.glob( + os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py") + ) + collected_top_level_directories.extend([os.path.dirname(p) for p in globbed]) + + # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" + return list(set(collected_top_level_directories)) + + +def apply_business_filter(collected_packages: List[str], filter_type: str) -> List[str]: + pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), collected_packages)) + + logging.debug("Target packages after applying business filter: {}".format(pkg_set_ci_filtered)) + logging.debug("Package(s) omitted by business filter: {}".format(generate_difference(collected_packages, pkg_set_ci_filtered))) + + return pkg_set_ci_filtered + def discover_targeted_packages( glob_string: str, @@ -119,6 +149,7 @@ def discover_targeted_packages( additional_contains_filter: str = "", filter_type: str = "Build", compatibility_filter: bool = True, + include_inactive: bool = False ) -> List[str]: """ During build and test, the set of targeted packages may expand or contract depending on the needs of the invocation. @@ -130,48 +161,60 @@ def discover_targeted_packages( :param str filter_type: One a string representing a filter function as a set of options. Options [ "Build", "Docs", "Regression", "Omit_management" ] Defaults to "Build". :param bool compatibility_filter: Enables or disables compatibility filtering of found packages. If the invoking python executable does not match a found package's specifiers, the package will be omitted. Defaults to True. """ - if glob_string: - individual_globs = glob_string.split(",") - else: - individual_globs = "azure-*" - collected_top_level_directories = [] - for glob_string in individual_globs: - globbed = glob.glob(os.path.join(target_root_dir, glob_string, "setup.py")) + glob.glob( - os.path.join(target_root_dir, "sdk/*/", glob_string, "setup.py") - ) - collected_top_level_directories.extend([os.path.dirname(p) for p in globbed]) + # glob the starting package set + collected_packages = glob_packages(glob_string, target_root_dir) - # deduplicate, in case we have double coverage from the glob strings. Example: "azure-mgmt-keyvault,azure-mgmt-*" - collected_directories = list(set([p for p in collected_top_level_directories if additional_contains_filter in p])) - pkg_set_ci_filtered = collected_directories + # apply the additional contains filter + collected_packages = [pkg for pkg in collected_packages if additional_contains_filter in pkg] - # if we have individually queued this specific package, it's obvious that we want to build it specifically - # in this case, do not honor the omission list - if len(collected_directories) == 1: - if compatibility_filter: - pkg_set_ci_filtered = filter_for_compatibility(collected_directories) + # filter for compatibility, this means excluding a package that doesn't support py36 when we are running a py36 executable + if compatibility_filter: + collected_packages = apply_compatibility_filter(collected_packages) - # however, if there are multiple packages being built, we should honor the omission list and NOT build the omitted - # packages - else: - allowed_package_set = remove_omitted_packages(collected_directories) - if compatibility_filter: - pkg_set_ci_filtered = filter_for_compatibility(allowed_package_set) + # apply package-specific exclusions only if we have gotten more than one + if len(collected_packages) > 1: + if not include_inactive: + collected_packages = apply_inactive_filter(collected_packages) # Apply filter based on filter type. for e.g. Docs, Regression, Management - pkg_set_ci_filtered = list(filter(omit_function_dict.get(filter_type, omit_build), pkg_set_ci_filtered)) - logging.info("Target packages after filtering by CI Type: {}".format(pkg_set_ci_filtered)) - logging.info( - "Package(s) omitted by CI filter: {}".format(list(set(collected_directories) - set(pkg_set_ci_filtered))) - ) - return sorted(pkg_set_ci_filtered) + collected_packages = apply_business_filter(collected_packages, filter_type) + + return sorted(collected_packages) + + +def get_config_setting(package_path: str, setting: str, default: Any = True) -> Any: + # we should always take the override if one is present + override_value = os.getenv(f"{os.path.basename(package_path).upper()}_{setting.upper()}", None) + if override_value: + return override_value + + # if no override, check for the config setting in the pyproject.toml + config = get_build_config(package_path) + + if config: + if setting.lower() in config: + return config[setting.lower()] + + return default + + +def is_package_active(package_path: str): + disabled = INACTIVE_CLASSIFIER in ParsedSetup.from_path(package_path).classifiers + + override_value = os.getenv(f"ENABLE_{os.path.basename(package_path).upper()}", None) + + if override_value: + return str_to_bool(override_value) + else: + return not disabled + +def apply_inactive_filter(collected_packages: List[str]) -> List[str]: + packages = [pkg for pkg in collected_packages if is_package_active(pkg)] -def remove_omitted_packages(collected_directories): - packages = [ - package_dir for package_dir in collected_directories if os.path.basename(package_dir) not in OMITTED_CI_PACKAGES - ] + logging.debug("Target packages after applying inactive filter: {}".format(collected_packages)) + logging.debug("Package(s) omitted by inactive filter: {}".format(generate_difference(collected_packages, packages))) return packages @@ -217,7 +260,7 @@ def get_package_from_repo(pkg_name: str, repo_root: str = None) -> ParsedSetup: return None -def get_version_from_repo(pkg_name: str, repo_root: str = None): +def get_version_from_repo(pkg_name: str, repo_root: str = None) -> str: pkg_info = get_package_from_repo(pkg_name, repo_root) if pkg_info: # Remove dev build part if version for this package is already updated to dev build @@ -235,7 +278,7 @@ def get_version_from_repo(pkg_name: str, repo_root: str = None): exit(1) -def get_base_version(pkg_name): +def get_base_version(pkg_name: str) -> str: root_dir = discover_repo_root() # find version for the package from source. This logic should be revisited to find version from devops feed glob_path = os.path.join(root_dir, "sdk", "*", pkg_name, "setup.py") diff --git a/tools/azure-sdk-tools/ci_tools/parsing/__init__.py b/tools/azure-sdk-tools/ci_tools/parsing/__init__.py index 12b5f7641fac..4d48f7f720d4 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/__init__.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/__init__.py @@ -5,6 +5,7 @@ get_name_from_specifier, ParsedSetup, read_setup_py_content, + get_build_config ) __all__ = [ @@ -14,4 +15,5 @@ "get_name_from_specifier", "ParsedSetup", "read_setup_py_content", + "get_build_config" ] diff --git a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 7ed9d36924e1..ca54909c4f2d 100644 --- a/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -2,7 +2,15 @@ import ast import textwrap import re -from typing import Dict, List, Tuple + +try: + # py 311 adds this library natively + import tomllib as toml +except: + # otherwise fall back to pypi package tomli + import tomli as toml + +from typing import Dict, List, Tuple, Any # Assumes the presence of setuptools from pkg_resources import ( @@ -29,23 +37,26 @@ def __init__( self, name: str, version: str, - python_requires: List[str], + python_requires: str, requires: List[str], is_new_sdk: bool, setup_filename: str, name_space: str, - package_data: Dict, + package_data: Dict[str, Any], include_package_data: bool, + classifiers: List[str], ): self.name: str = name self.version: str = version - self.python_requires: List[str] = python_requires + self.python_requires: str = python_requires self.requires: List[str] = requires self.is_new_sdk: bool = is_new_sdk self.setup_filename: str = setup_filename - self.namespace = name_space - self.package_data = package_data - self.include_package_data = include_package_data + self.namespace: str = name_space + self.package_data: Dict[str, Any] = package_data + self.include_package_data: bool = include_package_data + self.classifiers: List[str] = classifiers + self.folder = os.path.dirname(self.setup_filename) @classmethod @@ -60,6 +71,7 @@ def from_path(cls, parse_directory_or_file: str): name_space, package_data, include_package_data, + classifiers, ) = parse_setup(parse_directory_or_file) return cls( @@ -72,8 +84,29 @@ def from_path(cls, parse_directory_or_file: str): name_space, package_data, include_package_data, + classifiers, ) + def get_build_config(self) -> Dict[str, Any]: + return get_build_config(self.folder) + +def get_build_config(package_path: str) -> Dict[str, Any]: + if package_path.lower().endswith("setup.py"): + package_path = os.path.dirname(package_path) + + toml_file = os.path.join(package_path, "pyproject.toml") + + if os.path.exists(toml_file): + try: + with open(toml_file, "rb") as f: + toml_dict = toml.load(f) + if "tool" in toml_dict: + tool_configs = toml_dict["tool"] + if "azure-sdk-build" in tool_configs: + return tool_configs["azure-sdk-build"] + except: + return {} + def read_setup_py_content(setup_filename: str) -> str: """ @@ -84,7 +117,7 @@ def read_setup_py_content(setup_filename: str) -> str: return content -def parse_setup(setup_filename: str) -> Tuple[str, str, List[str], List[str], bool, str]: +def parse_setup(setup_filename: str) -> Tuple[str, str, str, List[str], bool, str, str, Dict[str, Any], bool, List[str]]: """ Used to evaluate a setup.py (or a directory containing a setup.py) and return a tuple containing: ( @@ -96,7 +129,8 @@ def parse_setup(setup_filename: str) -> Tuple[str, str, List[str], List[str], bo , , , - + , + ) """ if not setup_filename.endswith("setup.py"): @@ -165,6 +199,10 @@ def setup(*args, **kwargs): if "include_package_data" in kwargs: include_package_data = kwargs["include_package_data"] + classifiers = [] + if "classifiers" in kwargs: + classifiers = kwargs["classifiers"] + is_new_sdk = name in NEW_REQ_PACKAGES or any(map(lambda x: (parse_require(x)[0] in NEW_REQ_PACKAGES), requires)) return ( @@ -177,6 +215,7 @@ def setup(*args, **kwargs): name_space, package_data, include_package_data, + classifiers ) diff --git a/tools/azure-sdk-tools/dev_requirements.txt b/tools/azure-sdk-tools/dev_requirements.txt index 252afe5ae37a..fc7e3b122c21 100644 --- a/tools/azure-sdk-tools/dev_requirements.txt +++ b/tools/azure-sdk-tools/dev_requirements.txt @@ -1 +1 @@ --e ../azure-devtools \ No newline at end of file +-e ../azure-devtools["ci_tools"] \ No newline at end of file diff --git a/tools/azure-sdk-tools/setup.py b/tools/azure-sdk-tools/setup.py index d8781864f91c..96863b58e3e6 100644 --- a/tools/azure-sdk-tools/setup.py +++ b/tools/azure-sdk-tools/setup.py @@ -20,7 +20,8 @@ "pyopenssl", "python-dotenv", "PyYAML", - "urllib3" + "urllib3", + "tomli" ] setup( diff --git a/tools/azure-sdk-tools/tests/integration/test_package_discovery.py b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py new file mode 100644 index 000000000000..1bbc7762033f --- /dev/null +++ b/tools/azure-sdk-tools/tests/integration/test_package_discovery.py @@ -0,0 +1,111 @@ +import os + +from ci_tools.parsing import ParsedSetup +from ci_tools.functions import discover_targeted_packages + + +repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +core_service_root = os.path.join(repo_root, "sdk", "core") +storage_service_root = os.path.join(repo_root, "sdk", "storage") + + +def test_toml_result(): + package_with_toml = os.path.join(core_service_root, "azure-core") + + parsed_setup = ParsedSetup.from_path(package_with_toml) + result = parsed_setup.get_build_config() + + expected = { + "type_check_samples": False, + "verifytypes": False, + "pyright": False, + } + + assert expected == result + + +def test_discovery(): + results = discover_targeted_packages("azure*", core_service_root) + + # if in a set, this should be empty + non_empty_results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root) + + assert len(results) > 1 + assert len(non_empty_results) == 1 + + +def test_discovery_omit_mgmt(): + results = discover_targeted_packages("azure*", storage_service_root, filter_type="Omit_management") + + assert [os.path.basename(result) for result in results] == [ + "azure-storage-blob", + "azure-storage-blob-changefeed", + "azure-storage-file-datalake", + "azure-storage-file-share", + "azure-storage-queue" + ] + +def test_discovery_omit_build(): + results = discover_targeted_packages("azure*", core_service_root, filter_type="Build") + + assert [os.path.basename(result) for result in results] == [ + "azure-core", + "azure-core-experimental", + "azure-core-tracing-opencensus", + "azure-core-tracing-opentelemetry", + "azure-mgmt-core", + ] + +def test_discovery_single_package(): + results = discover_targeted_packages("azure-servicemanagement-legacy", core_service_root, filter_type="Build") + + assert [os.path.basename(result) for result in results] == [ + "azure-servicemanagement-legacy", + ] + +def test_discovery_omit_regression(): + results = discover_targeted_packages("azure*", core_service_root, filter_type="Regression") + + assert [os.path.basename(result) for result in results] == [ + "azure-core", + "azure-core-experimental", + "azure-core-tracing-opencensus", + "azure-core-tracing-opentelemetry" + ] + + storage_results = discover_targeted_packages("azure*", storage_service_root, filter_type="Regression") + + assert [os.path.basename(result) for result in storage_results] == [ + "azure-storage-blob", + "azure-storage-blob-changefeed", + "azure-storage-file-datalake", + "azure-storage-file-share", + "azure-storage-queue" + ] + + +def test_discovery_honors_contains_filter(): + + storage_results = discover_targeted_packages("azure*", storage_service_root, "file", filter_type="Regression") + + assert [os.path.basename(result) for result in storage_results] == [ + "azure-storage-file-datalake", + "azure-storage-file-share", + ] + + + +def test_discovery_honors_override(): + os.environ["ENABLE_AZURE_COMMON"] = "true" + os.environ["ENABLE_AZURE_SERVICEMANAGEMENT_LEGACY"] = "false" + + results = discover_targeted_packages("azure*", core_service_root) + + assert [os.path.basename(result) for result in results] == [ + "azure-common", + "azure-core", + "azure-core-experimental", + "azure-core-tracing-opencensus", + "azure-core-tracing-opentelemetry", + "azure-mgmt-core", + ] diff --git a/tools/azure-sdk-tools/tests/test_parse_functionality.py b/tools/azure-sdk-tools/tests/test_parse_functionality.py index 28127df91a9a..cc62eb58d9a5 100644 --- a/tools/azure-sdk-tools/tests/test_parse_functionality.py +++ b/tools/azure-sdk-tools/tests/test_parse_functionality.py @@ -6,7 +6,6 @@ package_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) - def test_parse_require(): test_scenarios = [ ("ConfigArgParse>=0.12.0", "configargparse", ">=0.12.0"), @@ -114,3 +113,6 @@ def test_sdk_sample_setup(test_patch): assert "pytyped" in result.package_data assert result.include_package_data == True assert result.folder == package_root + assert len(result.classifiers) > 0 + assert result.classifiers[0] == "Development Status :: 5 - Production/Stable" + assert result.classifiers[5] == "Programming Language :: Python :: 3.8" \ No newline at end of file diff --git a/tools/azure-sdk-tools/tests/test_pypi_client.py b/tools/azure-sdk-tools/tests/test_pypi_client.py index cde13d5b1eb0..cb2de3dfa154 100644 --- a/tools/azure-sdk-tools/tests/test_pypi_client.py +++ b/tools/azure-sdk-tools/tests/test_pypi_client.py @@ -31,10 +31,6 @@ def test_package_version_retrieve(self): assert result["info"]["name"] == "azure-core" assert result["info"]["release_url"] == "https://pypi.org/project/azure-core/1.8.0/" - @pytest.mark.skipif( - os.environ.get("TF_BUILD", "None") == True, - reason=f"This test isn't worth recording and could be flaky. Skipping in CI.", - ) @patch("pypi_tools.pypi.sys") def test_package_filter_for_compatibility(self, mock_sys): mock_sys.version_info = (2, 7, 0) diff --git a/tools/azure-sdk-tools/tests/test_servicemetadata.py b/tools/azure-sdk-tools/tests/test_servicemetadata.py index f8a701b11264..cd62466c0763 100644 --- a/tools/azure-sdk-tools/tests/test_servicemetadata.py +++ b/tools/azure-sdk-tools/tests/test_servicemetadata.py @@ -5,7 +5,7 @@ from pathlib import Path import pytest -from packaging_tools.auto_codegen import update_servicemetadata +from packaging_tools.generate_utils import update_servicemetadata """ @@ -13,9 +13,9 @@ Update metadata file -Update MANIFETS.IN +Update MANIFEST.IN -No need to update MANIFETS.IN +No need to update MANIFEST.IN """ MANIFEST_TEMP = """recursive-include tests *.py *.yaml