merged with develop

Author: nheeb

docs/adr/0013-plugin-init.md

@@ -0,0 +1,45 @@
+<!--
+SPDX-FileCopyrightText: 2022 German Aerospace Center (DLR), Forschungszentrum Jülich, Helmholtz-Zentrum Dresden-Rossendorf
+
+SPDX-License-Identifier: CC-BY-SA-4.0
+-->
+# Installing plugins from the marketplace using hermes init
+
+* Status: accepted
+* Deciders: nheeb
+* Date: 2025-03-04
+
+## Context and Problem Statement
+
+For a smooth user experience common plugins (those which are on the marketplace) should be installable via `hermes init`. That however means we need to decide on a way for the plugins to ask question during that same init process.
+
+## Considered Options
+
+* **Full install**: Plugins are installed locally during `hermes init`, can hook into the init process, and ask questions themselves to adjust the `hermes.toml` or similar.
+* **No install**: Plugins are only added to the CI pipeline as `pip install ...` lines. Additional questions for the init process could be added in the marketplace input mask along with a property name under which the answer is stored in the `hermes.toml`.
+* **Partial install**: Plugins are only added to the CI pipeline as `pip install ...` lines. Additional questions for the init process are handled in a "detached" script that is downloaded, executed locally, and then deleted again (this script may only have dependencies of Hermes).
+
+## Decision Outcome
+
+Chosen option: "**Partial install**", because it is a good trade-off between giving plugins more controll over their init process and not being too invasive with local installs.
+
+## Pros and Cons of the Options
+
+### Full install
+- (+) Plugins have a lot of control over how they design their own init process.
+- (+) The plugin can be used locally on the device directly if the user intends to do so.
+- (-) The plugin might be installed unnecessarily, as locally only the init part may be executed.
+- (-) The plugin may introduce many dependencies that the user doesn’t necessarily need.
+- (-) There could be environment complications (e.g., if Hermes was installed with `pipx`).
+
+### No install
+- (+) Nothing is installed locally via `hermes init`.
+- (+) Overall, slightly less effort for the init command and plugin developers.
+- (-) The marketplace would need input masks for an arbitrary number of such questions.
+- (-) Plugins have little control over how they design their own init process.
+- (-) Plugins intended for local use must be installed manually.
+
+### Partial install
+- (+) Nothing is installed locally via `hermes init`.
+- (+) Plugins have relatively high control over how they design their own init process.
+- (-) Plugins intended for local use must be installed manually.

pyproject.toml

@@ -9,7 +9,7 @@
 [tool.poetry]
 # Reference at https://python-poetry.org/docs/pyproject/
 name = "hermes"
-version = "0.9.0"
+version = "0.10.0.dev0"
 description = "Workflow to publish research software with rich metadata"
 homepage = "https://software-metadata.pub"
 license = "Apache-2.0"

src/hermes/commands/marketplace.py

@@ -4,6 +4,8 @@
 # SPDX-FileContributor: Stephan Druskat
 
 """Basic CLI to list plugins from the Hermes marketplace."""
+
+from functools import cache
 from html.parser import HTMLParser
 from typing import List, Optional
 
@@ -12,7 +14,7 @@
 from pydantic.alias_generators import to_camel
 
 from hermes.commands.init.util import slim_click
-from hermes.utils import hermes_doi, hermes_user_agent
+from hermes.utils import hermes_doi, hermes_concept_doi, hermes_user_agent
 
 MARKETPLACE_URL = "https://hermes.software-metadata.pub/marketplace"
 
@@ -64,7 +66,14 @@ class SchemaOrgSoftwareApplication(SchemaOrgModel):
     keywords: List["str"] = None
 
 
-schema_org_hermes = SchemaOrgSoftwareApplication(id_=hermes_doi, name="hermes")
+schema_org_hermes = SchemaOrgSoftwareApplication(
+    id_=(
+        hermes_doi
+        if hermes_doi.startswith("https://doi.org/")
+        else f"https://doi.org/{hermes_doi}"
+    ),
+    name="hermes",
+)
 
 
 class PluginMarketPlaceParser(HTMLParser):
@@ -88,6 +97,49 @@ def handle_data(self, data):
             self.plugins.append(plugin)
 
 
+@cache
+def _doi_is_version_of_concept_doi(doi: str, concept_doi: str) -> bool:
+    """Check whether ``doi`` is a version of ``concept_doi``.
+
+    The check is performed by requesting ``doi`` from the DataCite API and checking
+    whether its related identifier of type ``IsVersionOf`` points to ``concept_doi``.
+    This is the case if ``conecpt_doi`` is the concept DOI of ``doi``.
+    """
+
+    doi = doi.removeprefix("https://doi.org/")
+    concept_doi = concept_doi.removeprefix("https://doi.org/")
+
+    response = requests.get(
+        f"https://api.datacite.org/dois/{doi}",
+        headers={"User-Agent": hermes_user_agent},
+    )
+    response.raise_for_status()
+
+    for identifier in response.json()["data"]["attributes"]["relatedIdentifiers"]:
+        if (
+            identifier["relationType"] == "IsVersionOf"
+            and identifier["relatedIdentifier"] == concept_doi
+        ):
+            return True
+
+    return False
+
+
+def _is_hermes_reference(reference: Optional[SchemaOrgModel]):
+    """Figure out whether ``reference`` refers to HERMES."""
+    if reference is None:
+        return False
+
+    if reference.id_ in [
+        schema_org_hermes.id_,
+        hermes_concept_doi,
+        f"https://doi.org/{hermes_concept_doi}",
+    ]:
+        return True
+
+    return _doi_is_version_of_concept_doi(reference.id_, hermes_concept_doi)
+
+
 def _sort_plugins_by_step(plugins: list[SchemaOrgSoftwareApplication]) -> dict[str, list[SchemaOrgSoftwareApplication]]:
     sorted_plugins = {k: [] for k in ["harvest", "process", "curate", "deposit", "postprocess"]}
     for p in plugins:
@@ -101,44 +153,6 @@ def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str:
     return "builtin" if _plugin.is_part_of == schema_org_hermes else (_plugin.url or "")
 
 
-class PluginInfo:
-    """
-    This class contains all the information about a plugin which are needed for the init-Command.
-    """
-    def __init__(self):
-        self.name: str = ""
-        self.location: str = ""
-        self.step: str = ""
-        self.builtin: bool = True
-        self.install_url: str = ""
-        self.abstract: str = ""
-
-    def __str__(self):
-        step_text = f"[{self.step}]"
-        return f"{step_text} {slim_click.Formats.BOLD.wrap_around(self.name)} ({self.location})"
-
-    def get_pip_install_command(self) -> str:
-        """
-        Returns the pip install command which can be used to install the plugin.
-        Tries to extract the project name from the install_url (PyPI-URL) if possible.
-        Otherwise, it tries to use the location (Git-Project-URL) for the pip install command.
-        """
-        if self.install_url and self.install_url.startswith("https://pypi.org/project/"):
-            project_name = self.install_url.rstrip("/").removeprefix("https://pypi.org/project/")
-            return f"pip install {project_name}"
-        if self.location and self.location.startswith(("https://", "git@", "ssh://")):
-            git_url = self.location.rstrip("/")
-            return f"pip install git+{git_url}"
-        return ""
-
-    def is_valid(self) -> bool:
-        """
-        Returns True if the plugin can be installed. Maybe we'll check the actual repository here later
-        to make sure that other things are valid too.
-        """
-        return self.get_pip_install_command() != ""
-
-
 def get_plugin_infos() -> list[PluginInfo]:
     response = requests.get(MARKETPLACE_URL, headers={"User-Agent": hermes_user_agent})
     response.raise_for_status()
@@ -172,6 +186,13 @@ def main():
         MARKETPLACE_URL + "."
     )
 
+    def _plugin_loc(_plugin: SchemaOrgSoftwareApplication) -> str:
+        return (
+            "builtin"
+            if _is_hermes_reference(_plugin.is_part_of)
+            else (_plugin.url or "")
+        )
+
     if parser.plugins:
         print()
         max_name_len = max(map(lambda plugin: len(plugin.name), parser.plugins))
@@ -189,3 +210,41 @@ def main():
 
 if __name__ == "__main__":
     main()
+
+
+class PluginInfo:
+    """
+    This class contains all the information about a plugin which are needed for the init-Command.
+    """
+    def __init__(self):
+        self.name: str = ""
+        self.location: str = ""
+        self.step: str = ""
+        self.builtin: bool = True
+        self.install_url: str = ""
+        self.abstract: str = ""
+
+    def __str__(self):
+        step_text = f"[{self.step}]"
+        return f"{step_text} {slim_click.Formats.BOLD.wrap_around(self.name)} ({self.location})"
+
+    def get_pip_install_command(self) -> str:
+        """
+        Returns the pip install command which can be used to install the plugin.
+        Tries to extract the project name from the install_url (PyPI-URL) if possible.
+        Otherwise, it tries to use the location (Git-Project-URL) for the pip install command.
+        """
+        if self.install_url and self.install_url.startswith("https://pypi.org/project/"):
+            project_name = self.install_url.rstrip("/").removeprefix("https://pypi.org/project/")
+            return f"pip install {project_name}"
+        if self.location and self.location.startswith(("https://", "git@", "ssh://")):
+            git_url = self.location.rstrip("/")
+            return f"pip install git+{git_url}"
+        return ""
+
+    def is_valid(self) -> bool:
+        """
+        Returns True if the plugin can be installed. Maybe we'll check the actual repository here later
+        to make sure that other things are valid too.
+        """
+        return self.get_pip_install_command() != ""

src/hermes/utils.py

@@ -16,4 +16,7 @@
 # TODO: Fetch this from somewhere
 hermes_doi = "10.5281/zenodo.13311079"  # hermes v0.8.1
 
+hermes_concept_doi = "10.5281/zenodo.13221383"
+"""Fix "concept" DOI that always refers to the newest version."""
+
 hermes_user_agent = f"{hermes_name}/{hermes_version} ({hermes_homepage})"

test/hermes_test/test_marketplace.py

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: 2025 Helmholtz-Zentrum Dresden-Rossendorf
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileContributor: David Pape
+
+import requests_mock
+
+from hermes.commands.marketplace import (
+    schema_org_hermes,
+    SchemaOrgModel,
+    _is_hermes_reference,
+)
+
+
+def test_schema_org_hermes_doi_is_absolute():
+    """The HERMES DOI rendered to the plugins list must always be a full URL."""
+    assert isinstance(schema_org_hermes, SchemaOrgModel)
+    assert schema_org_hermes.id_ is not None
+    assert schema_org_hermes.id_.startswith("https://doi.org/")
+
+
+def test_concept_doi_is_hermes_reference():
+    """The HERMES concept DOI is a reference to HERMES.
+
+    We must be able to figure this out without asking DataCite.
+    """
+    with requests_mock.Mocker() as m:
+        assert _is_hermes_reference(
+            SchemaOrgModel(
+                type_="SoftwareSourceCode",
+                id_="https://doi.org/10.5281/zenodo.13221383",
+            )
+        )
+        # DataCite API was not called
+        assert m.call_count == 0
+
+
+def test_is_hermes_reference_if_datacite_api_returns_concept_doi_as_rel_id():
+    with requests_mock.Mocker() as m:
+        m.get(
+            "https://api.datacite.org/dois/10.9999/fake.1000",
+            text="""
+{
+  "data": {
+    "id": "10.9999/fake.1000",
+    "type": "dois",
+    "attributes": {
+      "doi": "10.9999/fake.1000",
+      "prefix": "10.9999",
+      "suffix": "fake.1000",
+      "relatedIdentifiers": [
+        {
+          "relationType": "IsVersionOf",
+          "relatedIdentifier": "10.5281/zenodo.13221383",
+          "relatedIdentifierType": "DOI"
+        }
+      ]
+    }
+  }
+}
+""".strip(),
+        )
+        # 10.5281/zenodo.13221383 retured from DataCite is HERMES concept DOI
+        assert _is_hermes_reference(
+            SchemaOrgModel(
+                type_="SoftwareSourceCode", id_="https://doi.org/10.9999/fake.1000"
+            )
+        )
+        # DataCite API was called once
+        assert m.call_count == 1
+
+
+def test_not_is_hermes_reference_if_datacite_api_returns_wrong_rel_id():
+    with requests_mock.Mocker() as m:
+        m.get(
+            "https://api.datacite.org/dois/10.9999/fake.2000",
+            text="""
+{
+  "data": {
+    "id": "10.9999/fake.2000",
+    "type": "dois",
+    "attributes": {
+      "doi": "10.9999/fake.2000",
+      "prefix": "10.9999",
+      "suffix": "fake.2000",
+      "relatedIdentifiers": [
+        {
+          "relationType": "IsVersionOf",
+          "relatedIdentifier": "10.9999/fake.1999",
+          "relatedIdentifierType": "DOI"
+        }
+      ]
+    }
+  }
+}
+""".strip(),
+        )
+        # 10.9999/fake.1999 returned from DataCite is not HERMES concept DOI
+        assert not _is_hermes_reference(
+            SchemaOrgModel(
+                type_="SoftwareSourceCode", id_="https://doi.org/10.9999/fake.2000"
+            )
+        )
+        # DataCite API was called once
+        assert m.call_count == 1