Refactor: Rename avdutils to pyavd-utils

.github/workflows/pull-request-management.yml

@@ -105,7 +105,7 @@ jobs:
             rust_src_or_ci_changed:
               - '.github/**/*'
               - 'rust/**/*'
-              - 'python/avdutils/**/*'
+              - 'pyavd-utils/**/*'
               - 'Cargo.toml'
 
   # -------------------------------------- #
@@ -682,6 +682,10 @@ jobs:
         # This is needed for arm machines since they don't ship with 2024 edition.
       - name: Upgrade rust and cargo
         run: rustup update stable
+        # This is needed because we have pyo3 tests.
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.10"
       - name: Check code formatting with cargo fmt
         run: cargo fmt --verbose
         if: |
@@ -749,7 +753,7 @@ jobs:
         run: |
           pip install tox tox-gh-actions --upgrade
       - name: Run pytest via tox
-        working-directory: python/avdutils
+        working-directory: pyavd-utils
         # Running tox  which will build the package (including rust).
         # For most runs without an environment set here. The environment is mapped with the tox gh-action plugin.
         run: |
@@ -772,11 +776,11 @@ jobs:
         run: |
           pip install tox tox-gh-actions --upgrade
       - name: Run pytest via tox
-        working-directory: python/avdutils
+        working-directory: pyavd-utils
         run: |
           tox -e coverage,report
       - name: Upload coverage from pytest
         uses: actions/upload-artifact@v4
         with:
-          name: pytest-avdutils-coverage
-          path: python/avdutils/coverage.xml
+          name: pytest-pyavd-utils-coverage
+          path: pyavd-utils/coverage.xml

.github/workflows/sonar.yml

@@ -46,14 +46,14 @@ jobs:
           path: python-avd/
           merge-multiple: true
 
-      - name: Download coverage from pytest avdutils (bindings for rust)
+      - name: Download coverage from pytest pyavd-utils (bindings for rust)
         continue-on-error: true
         uses: actions/download-artifact@v5
         with:
-          name: pytest-avdutils-coverage
+          name: pytest-pyavd-utils-coverage
           github-token: ${{ secrets.GITHUB_TOKEN }}
           run-id: ${{ github.event.workflow_run.id }}
-          path: python/avdutils/
+          path: pyavd-utils/
           merge-multiple: true
 
       - name: Download eos_designs compiled templates from pytest

.gitignore

@@ -11,18 +11,6 @@ __pycache__/
 coverage.xml
 *.so
 
-# Python source
-/python/**/build/
-/python/**/dist/
-/python/**/*.egg-info/
-/python/**/.tox/
-
-# pyavd ignores
-/python-avd/build/
-/python-avd/dist/
-/python-avd/pyavd.egg-info/
-/python-avd/.tox/
-
 # Ignore cloned cloudvision-python repo created during generation of _cv/api
 /python-avd/pyavd/_cv/cloudvision-apis/
 
@@ -40,6 +28,11 @@ coverage.xml
 ansible_collections/arista/avd/.ansible
 
 # Development
+build/
+dist/
+*.egg-info/
+.tox/
+.pytest_cache/
 ## pyenv
 .python-version
 ## .vscode/*

Cargo.toml

@@ -8,9 +8,23 @@ members = [
     "rust/avdschema_macros",
     "rust/validation",
     "rust/passwords",
+    "pyavd-utils/rust/pypasswords",
+    "pyavd-utils/rust/pyvalidation",
     ]
 resolver = "2"
 
+[workspace.dependencies]
+derive_more = { version = "2.0.1" }
+log = { version = "0.4.26" }
+ordermap = { version = "0.5.4" }
+pyo3 = { version = "0.26.0" }
+pyo3-build-config = { version = "0.26.0"}
+pyo3-log = { version = "0.13.1" }
+regex = { version = "1.11.1" }
+serde = { version = "1.0.217" }
+serde_json = { version = "1.0.135" }
+serde_yaml = { version = "0.9.33" }
+
 [profile.release-lto]
 inherits = "release"
 lto = true

mkdocs.yml

@@ -99,7 +99,7 @@ plugins:
         - development/*
         - galaxy-importer/*
         - python-avd/*
-        - python/*
+        - pyavd-utils/*
         - rust/*
       regex:
         # Exclude examples common md snippets

pyavd-utils/Makefile

@@ -0,0 +1,36 @@
+CURRENT_DIR = $(shell pwd)
+ANSIBLE_AVD_DIR ?= ..
+SCRIPTS_DIR = $(CURRENT_DIR)/scripts
+
+# export PYTHONPATH=$(CURRENT_DIR) # Uncomment to test from source
+
+.PHONY: help
+help: ## Display help message
+	@grep -E '^[0-9a-zA-Z_-]+\.*[0-9a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+.PHONY: build
+build: ## Build pyavd package
+	pip3 install build
+	rm -rf $(CURRENT_DIR)/build/ $(CURRENT_DIR)/dist/ $(CURRENT_DIR)/pyavd.egg-info/
+	python3 -m build --wheel
+
+.PHONY: uv-build
+uv-build: ## Build pyavd package
+	uv pip install build
+	rm -rf $(CURRENT_DIR)/build/ $(CURRENT_DIR)/dist/ $(CURRENT_DIR)/pyavd.egg-info/
+	python3 -m build --wheel
+
+
+.PHONY: publish
+publish: ## Publish pyavd package to PyPI (build first)
+	pip3 install twine && \
+	twine check dist/* && \
+	twine upload -r testpypi dist/* && \
+	twine upload dist/*
+
+.PHONY: uv-publish
+uv-publish: ## Publish pyavd package to PyPI (build first)
+	uv pip install twine && \
+	twine check dist/* && \
+	twine upload -r testpypi dist/* && \
+	twine upload dist/*

python/avdutils/pyproject.toml → pyavd-utils/pyproject.toml

@@ -1,9 +1,9 @@
 [build-system]
-requires = ["setuptools", "setuptools-rust"]
+requires = ["setuptools", "setuptools-rust>=1.12.0"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "avdutils"
+name = "pyavd-utils"
 version = "0.0.1"
 
 [dependency-groups]
@@ -14,20 +14,19 @@ coverage = ["coverage[toml]"]
 py-limited-api = "cp310"
 
 [tool.setuptools.packages.find]
-include = ["avdutils"]
+include = ["pyavd_utils"]
 
 [tool.setuptools.package-data]
-"avdutils" = ["*.pyi"]
+"pyavd_utils" = ["*.pyi"]
 
 [[tool.setuptools-rust.ext-modules]]
-target = "avdutils.validation"
-path = "../../rust/validation/Cargo.toml"
+target = "pyavd_utils.passwords"
+path = "rust/pypasswords/Cargo.toml"
 binding = "PyO3"
-features = ["python_bindings"]
 
 [[tool.setuptools-rust.ext-modules]]
-target = "avdutils.passwords"
-path = "../../rust/passwords/Cargo.toml"
+target = "pyavd_utils.validation"
+path = "rust/pyvalidation/Cargo.toml"
 binding = "PyO3"
 
 [tool.pytest.ini_options]

pyavd-utils/rust/pypasswords/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "pypasswords"
+version.workspace = true
+edition = "2024"
+license.workspace = true
+
+[build-dependencies]
+pyo3-build-config = { workspace = true }
+
+[dependencies]
+pyo3 = {workspace = true, "features" = ["abi3-py310"] }
+passwords = { path = "../../../rust/passwords" }
+
+[lib]
+crate-type = ["cdylib", "lib"]

rust/passwords/build.rs → pyavd-utils/rust/pypasswords/build.rs

@@ -8,11 +8,10 @@ fn main() {
     println!("cargo:rerun-if-changed=build.rs");
     println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS");
 
-    if let Ok(current_target_os) = env::var("CARGO_CFG_TARGET_OS") {
-        println!("cargo:warning=Compiling on 'macos'");
-        if current_target_os == "macos" {
+    if let Ok(current_target_os) = env::var("CARGO_CFG_TARGET_OS")
+        && current_target_os == "macos" {
+            println!("cargo:warning=Compiling on 'macos'");
             // Needed for MacOS when using pyo3 extension-module
             pyo3_build_config::add_extension_module_link_args();
         }
-    }
 }

pyavd-utils/rust/pypasswords/src/lib.rs

@@ -0,0 +1,28 @@
+// Copyright (c) 2025 Arista Networks, Inc.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the LICENSE file.
+
+use pyo3::pymodule;
+
+#[pymodule]
+#[pyo3(name = "passwords")]
+mod passwords {
+
+    use pyo3::{PyResult, pyfunction};
+
+    #[pyfunction]
+    /// Computes the SHA512 crypt value for the password given the salt
+    pub fn sha512_crypt(password: String, salt: String) -> PyResult<String> {
+        passwords::sha512_crypt(&password, &salt).map_err(|err| {
+            // Mapping our crates error to Python errors.
+            match err {
+                passwords::Sha512CryptError::InvalidSalt(_) => {
+                    pyo3::exceptions::PyValueError::new_err(format!("{err}"))
+                }
+                passwords::Sha512CryptError::ShaCrypt(_) => {
+                    pyo3::exceptions::PyRuntimeError::new_err(format!("{err}"))
+                }
+            }
+        })
+    }
+}

pyavd-utils/rust/pyvalidation/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "pyvalidation"
+version.workspace = true
+edition = "2024"
+license.workspace = true
+
+[build-dependencies]
+pyo3-build-config = { workspace = true }
+
+[dependencies]
+avdschema = { path = "../../../rust/avdschema", features = ["dump_load_files"] }
+included_store = { path = "../../../rust/included_store" }
+validation = { path = "../../../rust/validation", features = ["python_bindings"] }
+
+derive_more = { workspace = true, features = ["display", "from"] }
+log = { workspace = true }
+pyo3 = { workspace = true, "features" = ["abi3-py310"] }
+pyo3-log = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+serde_json = { workspace = true, features = [
+    "arbitrary_precision", # To support very big numbers
+    "preserve_order",
+] }
+serde_yaml = { workspace = true }
+
+[lib]
+crate-type = ["cdylib", "lib"]

pyavd-utils/rust/pyvalidation/build.rs

@@ -0,0 +1,17 @@
+// Copyright (c) 2025 Arista Networks, Inc.
+// Use of this source code is governed by the Apache License 2.0
+// that can be found in the LICENSE file.
+
+use std::env;
+
+fn main() {
+    println!("cargo:rerun-if-changed=build.rs");
+    println!("cargo:rerun-if-env-changed=CARGO_CFG_TARGET_OS");
+
+    if let Ok(current_target_os) = env::var("CARGO_CFG_TARGET_OS")
+        && current_target_os == "macos" {
+            println!("cargo:warning=Compiling on 'macos'");
+            // Needed for MacOS when using pyo3 extension-module
+            pyo3_build_config::add_extension_module_link_args();
+        }
+}

rust/validation/src/python_bindings.rs → pyavd-utils/rust/pyvalidation/src/lib.rs

@@ -28,9 +28,8 @@ mod validation {
 
     use pyo3::{Bound, PyResult, exceptions::PyRuntimeError, pyfunction, types::PyModule};
 
-    use crate::{
-        StoreValidate as _, coercion::Coercion, context::Context, validation::Validation,
-        validation_result::ValidationResult,
+    use validation::{
+        Coercion as _, Context, StoreValidate as _, Validation as _, ValidationResult,
     };
 
     use super::{STORE, get_store};
@@ -42,7 +41,7 @@ mod validation {
     }
 
     #[pymodule_export]
-    use crate::feedback::{
+    use ::validation::feedback::{
         CoercionNote, Feedback, Issue, Type, Value, Violation, ViolationValidValues,
     };
 
@@ -112,3 +111,64 @@ mod validation {
         Ok(ctx.into())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use pyo3::{types::PyAnyMethods as _};
+    use super::validation;
+
+    #[test]
+    fn validate_json_py_ok() {
+        // Partial implementation of the pytest but here using pyo3 wrappers in Rust, to ensure we get coverage data
+        // and that we can catch issues in Rust without building the Python first.
+        pyo3::append_to_inittab!(validation);
+        pyo3::Python::initialize();
+        pyo3::Python::attach(|py| {
+            let crate_dir = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"));
+            let eos_cli_config_gen_fragment_dir = crate_dir
+                .join("../../../python-avd/pyavd/_eos_cli_config_gen/schema/schema_fragments");
+            let eos_designs_fragment_dir =
+                crate_dir.join("../../../python-avd/pyavd/_eos_designs/schema/schema_fragments");
+
+            let module = py.import("validation").unwrap();
+            {
+                let args = ();
+                let kwargs = pyo3::types::PyDict::new(py);
+                kwargs.set_item("eos_cli_config_gen", eos_cli_config_gen_fragment_dir).unwrap();
+                kwargs.set_item("eos_designs", eos_designs_fragment_dir).unwrap();
+                module.call_method("init_store_from_fragments", args, Some(&kwargs)).unwrap();
+            };
+
+            let data_as_json_str = serde_json::json!({"ethernet_interfaces": [{"name": "Ethernet1", "description": 12345}, {"name": "Ethernet1"}, {}]}).to_string();
+            let validation_result = {
+                let args = ();
+                let kwargs= pyo3::types::PyDict::new(py);
+                kwargs.set_item("data_as_json", data_as_json_str).unwrap();
+                kwargs.set_item("schema_name", "eos_cli_config_gen").unwrap();
+                module.call_method("validate_json", args, Some(&kwargs)).unwrap()
+            };
+            assert!(validation_result.hasattr("violations").unwrap());
+            let violations = validation_result.getattr("violations").unwrap();
+            assert!(violations.is_instance_of::<pyo3::types::PyList>());
+            assert_eq!(violations.len().unwrap(), 3);
+
+            let issue_enum = module.getattr("Issue").unwrap();
+            let violation_enum = module.getattr("Violation").unwrap();
+
+            // Checking the first violation only. The rest are checked in the pytest implementation.
+            let feedback = violations.get_item(0).unwrap();
+            let path = feedback.getattr("path").unwrap().cast_into_exact::<pyo3::types::PyList>().unwrap();
+            let expected_path = pyo3::types::PyList::new(py, ["ethernet_interfaces", "2"]).unwrap();
+            assert!(path.eq(expected_path).unwrap());
+            let issue = feedback.getattr("issue").unwrap();
+            let expected_issue = issue_enum.getattr("Validation").unwrap();
+            assert!(issue.get_type().eq(expected_issue).unwrap());
+            let violation = issue.getattr("_0").unwrap();
+            let expected_violation = violation_enum.getattr("MissingRequiredKey").unwrap();
+            assert!(violation.get_type().eq(expected_violation).unwrap());
+            let key = violation.getattr("key").unwrap();
+            let expected_key = pyo3::types::PyString::new(py, "name");
+            assert!(key.eq(expected_key).unwrap());
+        });
+    }
+}