Skip to content

Commit 66d26b2

Browse files
chrisjsewellchrisjsewell
authored
✨ NEW: nb_merge_streams configuration (#364)
1 parent c54becd commit 66d26b2

7 files changed

Lines changed: 177 additions & 2 deletions

File tree

‎docs/use/start.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,4 +112,7 @@ Then for parsing and output rendering:
112112
* - `nb_output_stderr`
113113
- `show`
114114
- One of 'show', 'remove', 'warn', 'error' or 'severe', [see here](use/format/stderr) for details.
115+
* - `nb_merge_streams`
116+
- `False`
117+
- If `True`, ensure all stdout / stderr output streams are merged into single outputs. This ensures deterministic outputs.
115118
`````

‎myst_nb/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,7 @@ def visit_element_html(self, node):
133133
app.add_config_value("nb_render_plugin", "default", "env")
134134
app.add_config_value("nb_render_text_lexer", "myst-ansi", "env")
135135
app.add_config_value("nb_output_stderr", "show", "env")
136+
app.add_config_value("nb_merge_streams", False, "env")
136137

137138
# Register our post-transform which will convert output bundles to nodes
138139
app.add_post_transform(PasteNodesToDocutils)

‎myst_nb/render_outputs.py‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"""A Sphinx post-transform, to convert notebook outpus to AST nodes."""
22
import os
3+
import re
34
from abc import ABC, abstractmethod
45
from typing import List, Optional
56
from unittest import mock
@@ -91,6 +92,58 @@ def load_renderer(name: str) -> "CellOutputRendererBase":
9192
raise MystNbEntryPointError(f"No Entry Point found for myst_nb.mime_render:{name}")
9293

9394

95+
RGX_CARRIAGERETURN = re.compile(r".*\r(?=[^\n])")
96+
RGX_BACKSPACE = re.compile(r"[^\n]\b")
97+
98+
99+
def coalesce_streams(outputs: List[NotebookNode]) -> List[NotebookNode]:
100+
"""Merge all stream outputs with shared names into single streams.
101+
102+
This ensure deterministic outputs.
103+
104+
Adapted from:
105+
https://github.com/computationalmodelling/nbval/blob/master/nbval/plugin.py.
106+
"""
107+
if not outputs:
108+
return []
109+
110+
new_outputs = []
111+
streams = {}
112+
for output in outputs:
113+
if output["output_type"] == "stream":
114+
if output["name"] in streams:
115+
streams[output["name"]]["text"] += output["text"]
116+
else:
117+
new_outputs.append(output)
118+
streams[output["name"]] = output
119+
else:
120+
new_outputs.append(output)
121+
122+
# process \r and \b characters
123+
for output in streams.values():
124+
old = output["text"]
125+
while len(output["text"]) < len(old):
126+
old = output["text"]
127+
# Cancel out anything-but-newline followed by backspace
128+
output["text"] = RGX_BACKSPACE.sub("", output["text"])
129+
# Replace all carriage returns not followed by newline
130+
output["text"] = RGX_CARRIAGERETURN.sub("", output["text"])
131+
132+
# We also want to ensure stdout and stderr are always in the same consecutive order,
133+
# because they are asynchronous, so order isn't guaranteed.
134+
for i, output in enumerate(new_outputs):
135+
if output["output_type"] == "stream" and output["name"] == "stderr":
136+
if (
137+
len(new_outputs) >= i + 2
138+
and new_outputs[i + 1]["output_type"] == "stream"
139+
and new_outputs[i + 1]["name"] == "stdout"
140+
):
141+
stdout = new_outputs.pop(i + 1)
142+
new_outputs.insert(i, stdout)
143+
144+
return new_outputs
145+
146+
94147
class CellOutputsToNodes(SphinxPostTransform):
95148
"""Use the builder context to transform a CellOutputNode into Sphinx nodes."""
96149

@@ -108,6 +161,8 @@ def run(self):
108161
renderer_cls = load_renderer(node.renderer)
109162
renderers[node.renderer] = renderer_cls
110163
renderer = renderer_cls(self.document, node, abs_dir)
164+
if self.config.nb_merge_streams:
165+
node._outputs = coalesce_streams(node.outputs)
111166
output_nodes = renderer.cell_output_to_nodes(self.env.nb_render_priority)
112167
node.replace_self(output_nodes)
113168

‎tests/notebooks/merge_streams.ipynb‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "code",
5+
"execution_count": 1,
6+
"source": [
7+
"import sys\n",
8+
"print('stdout1', file=sys.stdout)\n",
9+
"print('stdout2', file=sys.stdout)\n",
10+
"print('stderr1', file=sys.stderr)\n",
11+
"print('stderr2', file=sys.stderr)\n",
12+
"print('stdout3', file=sys.stdout)\n",
13+
"print('stderr3', file=sys.stderr)\n",
14+
"1"
15+
],
16+
"outputs": [
17+
{
18+
"output_type": "stream",
19+
"name": "stdout",
20+
"text": [
21+
"stdout1\n",
22+
"stdout2\n"
23+
]
24+
},
25+
{
26+
"output_type": "stream",
27+
"name": "stderr",
28+
"text": [
29+
"stderr1\n",
30+
"stderr2\n"
31+
]
32+
},
33+
{
34+
"output_type": "stream",
35+
"name": "stdout",
36+
"text": [
37+
"stdout3\n"
38+
]
39+
},
40+
{
41+
"output_type": "stream",
42+
"name": "stderr",
43+
"text": [
44+
"stderr3\n"
45+
]
46+
},
47+
{
48+
"output_type": "execute_result",
49+
"data": {
50+
"text/plain": [
51+
"1"
52+
]
53+
},
54+
"metadata": {},
55+
"execution_count": 1
56+
}
57+
],
58+
"metadata": {}
59+
}
60+
],
61+
"metadata": {
62+
"kernelspec": {
63+
"display_name": "Python 3",
64+
"language": "python",
65+
"name": "python3"
66+
},
67+
"language_info": {
68+
"codemirror_mode": {
69+
"name": "ipython",
70+
"version": 3
71+
},
72+
"file_extension": ".py",
73+
"mimetype": "text/x-python",
74+
"name": "python",
75+
"nbconvert_exporter": "python",
76+
"pygments_lexer": "ipython3",
77+
"version": "3.6.1"
78+
}
79+
},
80+
"nbformat": 4,
81+
"nbformat_minor": 2
82+
}

‎tests/test_render_outputs.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,17 @@ def test_stderr_remove(sphinx_run, file_regression):
7272
file_regression.check(doctree.pformat(), extension=".xml", encoding="utf8")
7373

7474

75+
@pytest.mark.sphinx_params(
76+
"merge_streams.ipynb",
77+
conf={"jupyter_execute_notebooks": "off", "nb_merge_streams": True},
78+
)
79+
def test_merge_streams(sphinx_run, file_regression):
80+
sphinx_run.build()
81+
assert sphinx_run.warnings() == ""
82+
doctree = sphinx_run.get_resolved_doctree("merge_streams")
83+
file_regression.check(doctree.pformat(), extension=".xml", encoding="utf8")
84+
85+
7586
@pytest.mark.sphinx_params(
7687
"metadata_image.ipynb",
7788
conf={"jupyter_execute_notebooks": "off", "nb_render_key": "myst"},

‎tests/test_render_outputs/test_merge_streams.xml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<document source="merge_streams">
2+
<CellNode cell_type="code" classes="cell">
3+
<CellInputNode classes="cell_input">
4+
<literal_block language="ipython3" linenos="False" xml:space="preserve">
5+
import sys
6+
print('stdout1', file=sys.stdout)
7+
print('stdout2', file=sys.stdout)
8+
print('stderr1', file=sys.stderr)
9+
print('stderr2', file=sys.stderr)
10+
print('stdout3', file=sys.stdout)
11+
print('stderr3', file=sys.stderr)
12+
1
13+
<CellOutputNode classes="cell_output">
14+
<literal_block classes="output stream" language="myst-ansi" linenos="False" xml:space="preserve">
15+
stdout1
16+
stdout2
17+
stdout3
18+
<literal_block classes="output stderr" language="myst-ansi" linenos="False" xml:space="preserve">
19+
stderr1
20+
stderr2
21+
stderr3
22+
<literal_block classes="output text_plain" language="myst-ansi" linenos="False" xml:space="preserve">
23+
1

‎tox.ini‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,9 @@
1111
# then then deleting compiled files has been found to fix it: `find . -name \*.pyc -delete`
1212

1313
[tox]
14-
envlist = py37-sphinx3
14+
envlist = py37-sphinx4
1515

16-
[testenv:py{36,37,38,39}-sphinx{3,4}]
16+
[testenv:py{37,38,39}-sphinx{3,4}]
1717
extras = testing
1818
deps =
1919
sphinx3: sphinx>=3,<4

0 commit comments

Comments
 (0)
⚡