updated documentation for plugin development, made curate pluginizable and changed the way commands handle errors in plugin runs

notactuallyfinn · notactuallyfinn · commit 630dd00d4a1c · 2026-03-19T17:43:51.000+01:00
diff --git a/docs/source/_static/custom.css b/docs/source/_static/custom.css
@@ -45,9 +45,14 @@ img {
 }
 
 .bd-sidebar-primary.bd-sidebar {
-    max-width: 340px;
+    max-width: min-content;
+}
+
+.bd-docs-nav {
+    min-width: max-content;
 }
 
 .bd-sidebar-secondary{
-    max-width: min-content;
+    max-width: 15%;
+    width: max-content;
 }
diff --git a/docs/source/tutorials/automated-publication-with-ci.md b/docs/source/tutorials/automated-publication-with-ci.md
@@ -110,7 +110,7 @@ Each step in the publication workflow has its own section.
 
 Configure HERMES to:
 
-- harvest metadata from Git and `CITATION.cff`
+- harvest metadata from `CITATION.cff`
 - deposit on Zenodo Sandbox (which is built on the InvenioRDM)
 - use Zenodo Sandbox as the target publication repository
 
diff --git a/docs/source/tutorials/writing-a-plugin-for-hermes.md b/docs/source/tutorials/writing-a-plugin-for-hermes.md
diff --git a/hermes.toml b/hermes.toml
@@ -5,6 +5,9 @@
 [harvest]
 sources = [ "cff", "toml" ] # ordered priority (first one is most important)
 
+[curate]
+plugin = "pass_curate"
+
 [deposit]
 target = "invenio_rdm"
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -74,6 +74,9 @@ cff_doi = "hermes.commands.postprocess.invenio:cff_doi"
 [project.entry-points."hermes.process"]
 codemeta = "hermes.commands.process.standard_merge:CodemetaProcessPlugin"
 
+[project.entry-points."hermes.curate"]
+pass_curate = "hermes.commands.curate.pass_curate:DoNothingCuratePlugin"
+
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.1.1"
 pytest-cov = "^3.0.0"
diff --git a/src/hermes/commands/cli.py b/src/hermes/commands/cli.py
@@ -21,6 +21,7 @@
     HermesProcessCommand, HermesVersionCommand
 )
 from hermes.commands.base import HermesCommand
+from hermes.error import HermesPluginRunError
 
 
 def main() -> None:
@@ -79,16 +80,15 @@ def main() -> None:
 
         log.info("Run subcommand %s", args.command.command_name)
         args.command(args)
+    except HermesPluginRunError as e:
+        log.error("An error occurred during the execution of a plugin %s (Find details in './hermes.log')",
+                  args.command.command_name)
+        log.debug("Original exception was: %s", e)
+        sys.exit(2)
     except Exception as e:
         log.error("An error occurred during execution of %s (Find details in './hermes.log')",
                   args.command.command_name)
         log.debug("Original exception was: %s", e)
-
-        sys.exit(2)
-
-    if args.command.errors:
-        for e in args.command.errors:
-            log.error(e)
         sys.exit(1)
 
     sys.exit(0)
diff --git a/src/hermes/commands/curate/base.py b/src/hermes/commands/curate/base.py
@@ -8,16 +8,24 @@
 
 from pydantic import BaseModel
 
-from hermes.commands.base import HermesCommand
+from hermes.commands.base import HermesCommand, HermesPlugin
+from hermes.error import HermesPluginRunError, MisconfigurationError
 from hermes.model import SoftwareMetadata
 from hermes.model.context_manager import HermesContext
 from hermes.model.error import HermesValidationError
 
 
+class HermesCuratePlugin(HermesPlugin):
+    """ Base plugin for curate plugins. """
+
+    def __call__(self, command: HermesCommand, metadata: SoftwareMetadata) -> SoftwareMetadata:
+        pass
+
+
 class CurateSettings(BaseModel):
     """Generic deposition settings."""
 
-    pass
+    plugin: str = ""
 
 
 class HermesCurateCommand(HermesCommand):
@@ -26,28 +34,41 @@ class HermesCurateCommand(HermesCommand):
     command_name = "curate"
     settings_class = CurateSettings
 
-    def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None:
-        pass
-
     def __call__(self, args: argparse.Namespace) -> None:
         self.log.info("# Metadata curation")
+        plugin_name = self.settings.plugin
 
         ctx = HermesContext()
         ctx.prepare_step("curate")
 
+        self.log.info("## Load processed metadata")
+        # load processed data
         ctx.prepare_step("process")
-        with ctx["result"] as process_ctx:
-            expanded_data = process_ctx["expanded"]
-            context_data = process_ctx["context"]
+        try:
+            metadata = SoftwareMetadata.load_from_cache(ctx, "result")
+        except Exception as e:
+            self.log.error("The data from the process step could not be loaded or is invalid for some reason.")
+            raise HermesValidationError("The results of the process step are invalid.") from e
         ctx.finalize_step("process")
 
+        self.log.info("## Load curation plugin")
+        # load plugin
         try:
-            data = SoftwareMetadata(expanded_data[0], context_data["@context"][1])
+            plugin_func = self.plugins[plugin_name]()
+        except KeyError as e:
+            self.log.error(f"Plugin {plugin_name} not found.")
+            raise MisconfigurationError(f"Curate plugin {plugin_name} not found.")
+
+        self.log.info("## Run curation plugin")
+        # run plugin
+        try:
+            curated_metadata = plugin_func(self, metadata)
         except Exception as e:
-            raise HermesValidationError("The results of the process step are invalid.") from e
+            self.log.error(f"Unknown error while executing the {plugin_name} plugin.")
+            raise HermesPluginRunError(f"Something went wrong while running the curate plugin {plugin_name}") from e
 
-        with ctx["result"] as curate_ctx:
-            curate_ctx["expanded"] = data.ld_value
-            curate_ctx["context"] = {"@context": data.full_context}
+        self.log.info("## Store curated data")
+        # store metadata
+        curated_metadata.write_to_cache(ctx, "result")
 
         ctx.finalize_step("curate")
diff --git a/src/hermes/commands/curate/pass_curate.py b/src/hermes/commands/curate/pass_curate.py
@@ -0,0 +1,15 @@
+from pydantic import BaseModel
+
+from hermes.model import SoftwareMetadata
+from .base import HermesCurateCommand, HermesCuratePlugin
+
+
+class DoNothingCurateSettings(BaseModel):
+    pass
+
+
+class DoNothingCuratePlugin(HermesCuratePlugin):
+    settings_class = DoNothingCurateSettings
+
+    def __call__(self, command: HermesCurateCommand, metadata: SoftwareMetadata) -> SoftwareMetadata:
+        return metadata
diff --git a/src/hermes/commands/deposit/base.py b/src/hermes/commands/deposit/base.py
@@ -11,6 +11,7 @@
 from pydantic import BaseModel
 
 from hermes.commands.base import HermesCommand, HermesPlugin
+from hermes.error import HermesPluginRunError, MisconfigurationError
 from hermes.model.context_manager import HermesContext
 from hermes.model import SoftwareMetadata
 from hermes.model.error import HermesValidationError
@@ -29,25 +30,26 @@ def __call__(self, command: HermesCommand) -> None:
         """
         self.command = command
         self.ctx = HermesContext()
+        self.ctx.prepare_step("deposit")
 
         self.ctx.prepare_step("curate")
-        self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result")
+        try:
+            self.metadata = SoftwareMetadata.load_from_cache(self.ctx, "result")
+        except Exception as e:
+            raise HermesValidationError("The results of the curate step are invalid.") from e
         self.ctx.finalize_step("curate")
 
         self.prepare()
         deposit = self.map_metadata()
-        self.ctx.prepare_step("deposit")
         with self.ctx[command.settings.target] as cache:
             cache["deposit"] = deposit
-        self.ctx.finalize_step("deposit")
 
         if self.is_initial_publication():
             self.create_initial_version()
         else:
             self.create_new_version()
 
         deposit = self.update_metadata()
-        self.ctx.prepare_step("deposit")
         with self.ctx[command.settings.target] as cache:
             cache["result"] = deposit
         self.ctx.finalize_step("deposit")
@@ -133,16 +135,24 @@ def init_command_parser(self, command_parser: argparse.ArgumentParser) -> None:
                                     help="Allow initial deposition (i.e., minting a new PID).")
 
     def __call__(self, args: argparse.Namespace) -> None:
+        self.log.info("# Metadata deposition")
         self.args = args
         plugin_name = self.settings.target
 
+        self.log.info("## Load deposit plugin")
+        # load plugin
         try:
             plugin_func = self.plugins[plugin_name]()
         except KeyError as e:
-            self.log.error("Plugin '%s' not found.", plugin_name)
-            self.errors.append(e)
+            self.log.error(f"Plugin {plugin_name} not found.")
+            raise MisconfigurationError(f"Deposit plugin {self.settings.plugin} not found.")
+
+        self.log.info("## Run deposit plugin")
+        # run plugin
         try:
             plugin_func(self)
         except HermesValidationError as e:
-            self.log.error("Error while executing %s: %s", plugin_name, e)
-            self.errors.append(e)
+            self.log.error(f"Error while executing {plugin_name}: {e}")
+            raise HermesPluginRunError(
+                f"Something went wrong while running the curate plugin {self.settings.plugin}"
+            ) from e
diff --git a/src/hermes/commands/harvest/base.py b/src/hermes/commands/harvest/base.py
@@ -9,6 +9,7 @@
 from pydantic import BaseModel
 
 from hermes.commands.base import HermesCommand, HermesPlugin
+from hermes.error import HermesPluginRunError, MisconfigurationError
 from hermes.model.context_manager import HermesContext
 from hermes.model.error import HermesValidationError
 from hermes.model import SoftwareMetadata
@@ -20,7 +21,7 @@ class HermesHarvestPlugin(HermesPlugin):
     TODO: describe the harvesting process and how this is mapped to this plugin.
     """
 
-    def __call__(self, command: HermesCommand) -> tuple[SoftwareMetadata, dict]:
+    def __call__(self, command: HermesCommand) -> SoftwareMetadata:
         pass
 
 
@@ -37,28 +38,35 @@ class HermesHarvestCommand(HermesCommand):
     settings_class = HarvestSettings
 
     def __call__(self, args: argparse.Namespace) -> None:
+        self.log.info("# Metadata harvesting")
         self.args = args
 
         # Initialize the harvest cache directory here to indicate the step ran
         ctx = HermesContext()
         ctx.prepare_step('harvest')
 
+        self.log.info("## Load and run the plugins")
         for plugin_name in self.settings.sources:
-            plugin_cls = self.plugins[plugin_name]
+            self.log.info(f"### Load {plugin_name} plugin")
+            # load plugin
+            try:
+                plugin_func = self.plugins[plugin_name]()
+            except KeyError as e:
+                self.log.error(f"Plugin {plugin_name} not found.")
+                raise MisconfigurationError(f"Harvest plugin {plugin_name} not found.")
 
+            self.log.info(f"### Run {plugin_name} plugin")
+            # run plugin
             try:
-                # Load plugin and run the harvester
-                plugin_func = plugin_cls()
                 harvested_data = plugin_func(self)
-
-                with ctx[plugin_name] as plugin_ctx:
-                    plugin_ctx["codemeta"] = harvested_data[0].compact()
-                    plugin_ctx["context"] = {"@context": harvested_data[0].full_context}
-
-                    plugin_ctx["expanded"] = harvested_data[0].ld_value
-
-            except HermesValidationError as e:
-                self.log.error("Error while executing %s: %s", plugin_name, e)
-                self.errors.append(e)
+            except Exception as e:
+                self.log.error(f"Unknown error while executing the {plugin_name} plugin.")
+                raise HermesPluginRunError(
+                    f"Something went wrong while running the harvest plugin {plugin_name}"
+                ) from e
+
+            self.log.info(f"### Store metadata harvested by {plugin_name} plugin")
+            # store harvested data
+            harvested_data.write_to_cache(ctx, plugin_name)
 
         ctx.finalize_step('harvest')
diff --git a/src/hermes/commands/harvest/cff.py b/src/hermes/commands/harvest/cff.py
@@ -59,7 +59,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic
         # TODO Replace the following temp patch for #112 once there is a new cffconvert version with cffconvert#309
         codemeta_dict = self._patch_author_emails(cff_dict, codemeta_dict)
         ld_codemeta = SoftwareMetadata(codemeta_dict, extra_vocabs={'legalName': {'@id': "http://schema.org/name"}})
-        return ld_codemeta, {}
+        return ld_codemeta
 
     def _load_cff_from_file(self, cff_data: str) -> Any:
         yaml = YAML(typ='safe')
diff --git a/src/hermes/commands/harvest/codemeta.py b/src/hermes/commands/harvest/codemeta.py
@@ -40,7 +40,7 @@ def __call__(self, command: HermesHarvestCommand) -> tuple[SoftwareMetadata, dic
             raise HermesValidationError(codemeta_file)
 
         codemeta = json.loads(codemeta_str)
-        return SoftwareMetadata(codemeta), {'local_path': str(codemeta_file)}
+        return SoftwareMetadata(codemeta)  # , {'local_path': str(codemeta_file)}
 
     def _validate(self, codemeta_file: pathlib.Path) -> bool:
         with open(codemeta_file, "r") as fi:
diff --git a/src/hermes/commands/postprocess/base.py b/src/hermes/commands/postprocess/base.py
@@ -9,11 +9,15 @@
 
 from pydantic import BaseModel
 
-from ..base import HermesCommand, HermesPlugin
+from hermes.commands.base import HermesCommand, HermesPlugin
+from hermes.error import HermesPluginRunError, MisconfigurationError
 
 
 class HermesPostprocessPlugin(HermesPlugin):
-    pass
+    """ Base plugin for postprocess plugins. """
+
+    def __call__(self, command: HermesCommand) -> None:
+        pass
 
 
 class PostprocessSettings(BaseModel):
@@ -29,13 +33,26 @@ class HermesPostprocessCommand(HermesCommand):
     settings_class = PostprocessSettings
 
     def __call__(self, args: argparse.Namespace) -> None:
+        self.log.info("# Postprocessing")
         self.args = args
         plugin_names = self.settings.run
 
+        self.log.info("## Load and run the plugins")
         for plugin_name in plugin_names:
+            self.log.info(f"### Load {plugin_name} plugin")
+            # load plugin
             try:
                 plugin_func = self.plugins[plugin_name]()
-                plugin_func(self)
             except KeyError as e:
-                self.log.error("Plugin '%s' not found.", plugin_name)
-                self.errors.append(e)
+                self.log.error(f"Plugin {plugin_name} not found.")
+                raise MisconfigurationError(f"Postprocess plugin {plugin_name} not found.")
+
+            self.log.info(f"### Run {plugin_name} plugin")
+            # run plugin
+            try:
+                plugin_func(self)
+            except Exception as e:
+                self.log.error(f"Unknown error while executing the {plugin_name} plugin.")
+                raise HermesPluginRunError(
+                    f"Something went wrong while running the postprocess plugin {plugin_name}"
+                ) from e
diff --git a/src/hermes/commands/process/base.py b/src/hermes/commands/process/base.py
diff --git a/src/hermes/error.py b/src/hermes/error.py
diff --git a/src/hermes/model/api.py b/src/hermes/model/api.py
diff --git a/test/hermes_test/commands/curate/test_do_nothing_curate.py b/test/hermes_test/commands/curate/test_do_nothing_curate.py

Original file line number	Diff line number	Diff line change
`@@ -45,9 +45,14 @@ img {`
`45`	`45`	`}`
`46`	`46`
`47`	`47`	`.bd-sidebar-primary.bd-sidebar {`
`48`		`- max-width: 340px;`
	`48`	`+ max-width: min-content;`
	`49`	`+}`
	`50`	`+`
	`51`	`+.bd-docs-nav {`
	`52`	`+ min-width: max-content;`
`49`	`53`	`}`
`50`	`54`
`51`	`55`	`.bd-sidebar-secondary{`
`52`		`- max-width: min-content;`
	`56`	`+ max-width: 15%;`
	`57`	`+ width: max-content;`
`53`	`58`	`}`