✨ NEW: Add nb_execution_raise_on_error config (#404)

Author: chrisjsewell

docs/use/execute.md

@@ -161,6 +161,14 @@ tags: [raises-exception]
 print(thisvariabledoesntexist)
 ```
 
+(execute/raise_on_error)=
+## Error Reporting: Warning vs. Failure
+
+When an error occurs in a context where `nb_execution_allow_errors=False`,
+the default behaviour is for this to be reported as a warning.
+This warning will simply be logged and not cause the build to fail unless `sphinx-build` is run with the [`-W` option](https://www.sphinx-doc.org/en/master/man/sphinx-build.html#cmdoption-sphinx-build-W).
+If you would like unexpected execution errors to cause a build failure rather than a warning regardless of the `-W` option, you can achieve this by setting `nb_execution_raise_on_error=True` in your `conf.py`.
+
 (execute/statistics)=
 ## Execution statistics
 

myst_nb/core/config.py

@@ -202,6 +202,14 @@ def __post_init__(self):
             "legacy_name": "execution_allow_errors",
         },
     )
+    execution_raise_on_error: bool = dc.field(
+        default=False,
+        metadata={
+            "validator": instance_of(bool),
+            "help": "Raise an exception on failed execution, "
+            "rather than emitting a warning",
+        },
+    )
     execution_show_tb: bool = dc.field(  # TODO implement
         default=False,
         metadata={

myst_nb/core/execute.py

@@ -35,6 +35,10 @@ class ExecutionResult(TypedDict):
     """traceback if the notebook failed"""
 
 
+class ExecutionError(Exception):
+    """An exception for failed execution and `execution_raise_on_error` is true."""
+
+
 def execute_notebook(
     notebook: NotebookNode,
     source: str,
@@ -111,6 +115,8 @@ def execute_notebook(
             )
 
         if result.err is not None:
+            if nb_config.execution_raise_on_error:
+                raise ExecutionError(str(source)) from result.err
             msg = f"Executing notebook failed: {result.err.__class__.__name__}"
             if nb_config.execution_show_tb:
                 msg += f"\n{result.exc_string}"
@@ -187,6 +193,8 @@ def execute_notebook(
         # handle success / failure cases
         # TODO do in try/except to be careful (in case of database write errors?
         if result.err is not None:
+            if nb_config.execution_raise_on_error:
+                raise ExecutionError(str(source)) from result.err
             msg = f"Executing notebook failed: {result.err.__class__.__name__}"
             if nb_config.execution_show_tb:
                 msg += f"\n{result.exc_string}"

pyproject.toml

@@ -87,7 +87,7 @@ testing = [
     "coverage<5.0",
     "beautifulsoup4",
     "ipykernel~=5.5",
-    "ipython<8.1.0", # see https://github.com/ipython/ipython/issues/13554
+    "ipython!=8.1.0", # see https://github.com/ipython/ipython/issues/13554
     "ipywidgets",
     "jupytext~=1.11.2",
     # TODO: 3.4.0 has some warnings that need to be fixed in the tests.

tests/test_execute.py

@@ -5,6 +5,7 @@
 from IPython import version_info as ipy_version
 import pytest
 
+from myst_nb.core.execute import ExecutionError
 from myst_nb.sphinx_ import NbMetadataCollector
 
 
@@ -153,6 +154,24 @@ def test_allow_errors_auto(sphinx_run, file_regression, check_nbs):
     regress_nb_doc(file_regression, sphinx_run, check_nbs)
 
 
+@pytest.mark.sphinx_params(
+    "basic_failing.ipynb",
+    conf={"nb_execution_raise_on_error": True, "nb_execution_mode": "force"},
+)
+def test_raise_on_error_force(sphinx_run):
+    with pytest.raises(ExecutionError, match="basic_failing.ipynb"):
+        sphinx_run.build()
+
+
+@pytest.mark.sphinx_params(
+    "basic_failing.ipynb",
+    conf={"nb_execution_raise_on_error": True, "nb_execution_mode": "cache"},
+)
+def test_raise_on_error_cache(sphinx_run):
+    with pytest.raises(ExecutionError, match="basic_failing.ipynb"):
+        sphinx_run.build()
+
+
 @pytest.mark.sphinx_params("basic_unrun.ipynb", conf={"nb_execution_mode": "force"})
 def test_outputs_present(sphinx_run, file_regression, check_nbs):
     sphinx_run.build()
@@ -222,13 +241,13 @@ def test_jupyter_cache_path(sphinx_run, file_regression, check_nbs):
 
 # Testing relative paths within the notebook
 @pytest.mark.sphinx_params("basic_relative.ipynb", conf={"nb_execution_mode": "cache"})
-def test_relative_path_cache(sphinx_run, file_regression, check_nbs):
+def test_relative_path_cache(sphinx_run):
     sphinx_run.build()
     assert "Execution Failed" not in sphinx_run.status(), sphinx_run.status()
 
 
 @pytest.mark.sphinx_params("basic_relative.ipynb", conf={"nb_execution_mode": "force"})
-def test_relative_path_force(sphinx_run, file_regression, check_nbs):
+def test_relative_path_force(sphinx_run):
     sphinx_run.build()
     assert "Execution Failed" not in sphinx_run.status(), sphinx_run.status()
 
@@ -238,7 +257,7 @@ def test_relative_path_force(sphinx_run, file_regression, check_nbs):
     "sleep_10.ipynb",
     conf={"nb_execution_mode": "cache", "nb_execution_timeout": 1},
 )
-def test_execution_timeout(sphinx_run, file_regression, check_nbs):
+def test_execution_timeout(sphinx_run):
     """execution should fail given the low timeout value"""
     sphinx_run.build()
     # print(sphinx_run.warnings())
@@ -249,7 +268,7 @@ def test_execution_timeout(sphinx_run, file_regression, check_nbs):
     "sleep_10_metadata_timeout.ipynb",
     conf={"nb_execution_mode": "cache", "nb_execution_timeout": 60},
 )
-def test_execution_metadata_timeout(sphinx_run, file_regression, check_nbs):
+def test_execution_metadata_timeout(sphinx_run):
     """notebook timeout metadata has higher preference then execution_timeout config"""
     sphinx_run.build()
     # print(sphinx_run.warnings())
@@ -260,7 +279,7 @@ def test_execution_metadata_timeout(sphinx_run, file_regression, check_nbs):
     "nb_exec_table.md",
     conf={"nb_execution_mode": "auto"},
 )
-def test_nb_exec_table(sphinx_run, file_regression, check_nbs):
+def test_nb_exec_table(sphinx_run, file_regression):
     """Test that the table gets output into the HTML,
     including a row for the executed notebook.
     """

tests/test_execute/test_allow_errors_auto.ipynb

@@ -21,7 +21,7 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
-      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moopsie!\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moopsie!\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
       "\u001b[0;31mException\u001b[0m: oopsie!"
      ]
     }
@@ -47,7 +47,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.12"
+   "version": "3.8.13"
   },
   "test_name": "notebook1"
  },

tests/test_execute/test_allow_errors_auto.xml

@@ -12,7 +12,7 @@
                 <literal_block classes="output traceback" language="ipythontb" xml:space="preserve">
                     ---------------------------------------------------------------------------
                     Exception                                 Traceback (most recent call last)
-                    Input In [1], in <module>
+                    Input In [1], in <cell line: 1>()
                     ----> 1 raise Exception('oopsie!')
 
                     Exception: oopsie!

tests/test_execute/test_allow_errors_cache.ipynb

@@ -21,7 +21,7 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
-      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moopsie!\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moopsie!\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
       "\u001b[0;31mException\u001b[0m: oopsie!"
      ]
     }
@@ -47,7 +47,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.12"
+   "version": "3.8.13"
   },
   "test_name": "notebook1"
  },

tests/test_execute/test_allow_errors_cache.xml

@@ -12,7 +12,7 @@
                 <literal_block classes="output traceback" language="ipythontb" xml:space="preserve">
                     ---------------------------------------------------------------------------
                     Exception                                 Traceback (most recent call last)
-                    Input In [1], in <module>
+                    Input In [1], in <cell line: 1>()
                     ----> 1 raise Exception('oopsie!')
 
                     Exception: oopsie!

tests/test_execute/test_basic_failing_auto.ipynb

@@ -21,7 +21,7 @@
      "traceback": [
       "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
       "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
-      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moopsie!\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
+      "Input \u001b[0;32mIn [1]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124moopsie!\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
       "\u001b[0;31mException\u001b[0m: oopsie!"
      ]
     }
@@ -47,7 +47,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.12"
+   "version": "3.8.13"
   },
   "test_name": "notebook1"
  },