⬆️ UPGRADE: markdown-it-py v1.0 (#320)

Updated via dependencies which support this version;

- myst-parser -> 0.14.0
- jupytext -> 1.11.2

See https://github.com/executablebooks/markdown-it-py/blob/master/CHANGELOG.md
and https://github.com/executablebooks/MyST-Parser/blob/master/CHANGELOG.md

Also added mypy type checking for the parser code.

Author: chrisjsewell

.pre-commit-config.yaml

@@ -42,8 +42,20 @@ repos:
     - id: flake8
       additional_dependencies: [flake8-bugbear==21.3.1]
 
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v0.812
+    hooks:
+    - id: mypy
+      args: [--config-file=setup.cfg]
+      additional_dependencies:
+      - myst-parser~=0.14.0
+      files: >
+        (?x)^(
+            myst_nb/parser.py|
+        )$
+
   # this is not used for now,
-  # since it converts myst-parser to myst_parser and removes comments
+  # since it converts myst-nb to myst_nb and removes comments
   # - repo: https://github.com/asottile/setup-cfg-fmt
   #   rev: v1.17.0
   #   hooks:

docs/examples/custom-formats.Rmd

@@ -71,7 +71,7 @@ nb_custom_formats = {
 ```
 
 :::{important}
-For full compatibility with `myst-nb`, `jupytext>=1.8.0` should be used.
+For full compatibility with `myst-nb`, `jupytext>=1.11.2` should be used.
 :::
 
 For example:

myst_nb/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.12.3"
+__version__ = "0.13.0"
 
 import os
 from collections.abc import Sequence

myst_nb/parser.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import List, Tuple
+from typing import Any, Dict, List, Tuple
 
 import nbformat as nbf
 from docutils import nodes
@@ -8,7 +8,7 @@
 from markdown_it import MarkdownIt
 from markdown_it.rules_core import StateCore
 from markdown_it.token import Token
-from markdown_it.utils import AttrDict
+from markdown_it.tree import SyntaxTreeNode
 from myst_parser.main import MdParserConfig, default_parser
 from myst_parser.sphinx_parser import MystParser
 from myst_parser.sphinx_renderer import SphinxRenderer
@@ -33,7 +33,9 @@ class NotebookParser(MystParser):
     config_section = "myst-nb parser"
     config_section_dependencies = ("parsers",)
 
-    def parse(self, inputstring: str, document: nodes.document):
+    def parse(
+        self, inputstring: str, document: nodes.document, renderer: str = "sphinx"
+    ) -> None:
 
         self.reporter = document.reporter
         self.env = document.settings.env  # type: BuildEnvironment
@@ -70,7 +72,11 @@ def parse(self, inputstring: str, document: nodes.document):
         # containing global data like reference definitions
         md_parser, env, tokens = nb_to_tokens(
             ntbk,
-            self.env.myst_config if converter is None else converter.config,
+            (
+                self.env.myst_config  # type: ignore[attr-defined]
+                if converter is None
+                else converter.config
+            ),
             self.env.config["nb_render_plugin"],
         )
 
@@ -87,7 +93,7 @@ def parse(self, inputstring: str, document: nodes.document):
 
 def nb_to_tokens(
     ntbk: nbf.NotebookNode, config: MdParserConfig, renderer_plugin: str
-) -> Tuple[MarkdownIt, AttrDict, List[Token]]:
+) -> Tuple[MarkdownIt, Dict[str, Any], List[Token]]:
     """Parse the notebook content to a list of syntax tokens and an env,
     containing global data like reference definitions.
     """
@@ -99,7 +105,7 @@ def nb_to_tokens(
     md.renderer = SphinxNBRenderer(md)
     # make a sandbox where all the parsing global data,
     # like reference definitions will be stored
-    env = AttrDict()
+    env: Dict[str, Any] = {}
     rules = md.core.ruler.get_active_rules()
 
     # First only run pre-inline chains
@@ -185,7 +191,7 @@ def parse_block(src, start_line):
             "",
             0,
             map=[0, 0],
-            content=({k: v for k, v in ntbk.metadata.items()}),
+            content=({k: v for k, v in ntbk.metadata.items()}),  # type: ignore[arg-type]
         )
     ] + state.tokens
 
@@ -205,8 +211,8 @@ def parse_block(src, start_line):
 
 
 def tokens_to_docutils(
-    md: MarkdownIt, env: AttrDict, tokens: List[Token], document: nodes.document
-):
+    md: MarkdownIt, env: Dict[str, Any], tokens: List[Token], document: nodes.document
+) -> None:
     """Render the Markdown tokens to docutils AST."""
     md.options["document"] = document
     md.renderer.render(tokens, md.options, env)
@@ -217,14 +223,14 @@ class SphinxNBRenderer(SphinxRenderer):
     which includes special methods for notebook cells.
     """
 
-    def render_jupyter_widget_state(self, token: Token):
+    def render_jupyter_widget_state(self, token: SyntaxTreeNode) -> None:
         if token.meta["state"]:
             self.document.settings.env.nb_contains_widgets = True
         node = JupyterWidgetStateNode(state=token.meta["state"])
         self.add_line_and_source_path(node, token)
         self.document.append(node)
 
-    def render_nb_code_cell(self, token: Token):
+    def render_nb_code_cell(self, token: SyntaxTreeNode) -> None:
         """Render a Jupyter notebook cell."""
         cell = token.meta["cell"]  # type: nbf.NotebookNode
 

setup.cfg

@@ -29,7 +29,7 @@ keywords =
     markdown
     lexer
     parser
-    development
+    jupyter
     docutils
     sphinx
 project_urls =
@@ -44,7 +44,7 @@ install_requires =
     ipywidgets>=7.0.0,<8
     jupyter-cache~=0.4.1
     jupyter_sphinx~=0.3.2
-    myst-parser~=0.13.5
+    myst-parser~=0.14.0
     nbconvert~=5.6
     nbformat~=5.0
     pyyaml
@@ -74,7 +74,7 @@ rtd =
     bokeh
     coconut~=1.4.3
     ipywidgets
-    jupytext~=1.8.0
+    jupytext~=1.11.2
     matplotlib
     numpy
     pandas
@@ -86,7 +86,7 @@ rtd =
     sympy
 testing =
     coverage<5.0
-    jupytext~=1.8.0
+    jupytext~=1.11.2
     # TODO: 3.4.0 has some warnings that need to be fixed in the tests.
     matplotlib~=3.3.0
     numpy
@@ -99,3 +99,20 @@ testing =
 [flake8]
 max-line-length = 100
 extend-ignore = E203
+
+[mypy]
+show_error_codes = true
+check_untyped_defs = true
+strict_equality = true
+no_implicit_optional = true
+warn_unused_ignores = true
+
+[mypy-myst_nb.*]
+; can only follow these imports when more of the code is typed
+follow_imports = skip
+
+[mypy-jupyter_sphinx.*]
+ignore_missing_imports = True
+
+[mypy-nbformat.*]
+ignore_missing_imports = True

tests/test_mystnb_features.py

@@ -17,6 +17,7 @@ def test_codecell_file(sphinx_run, file_regression, check_nbs, get_test_path):
         "author",
         "source_map",
         "language_info",
+        "wordcount",
     }
     assert sphinx_run.app.env.metadata["mystnb_codecell_file"]["author"] == "Matt"
     assert (
@@ -50,6 +51,7 @@ def test_codecell_file_warnings(sphinx_run, file_regression, check_nbs, get_test
         "author",
         "source_map",
         "language_info",
+        "wordcount",
     }
     assert (
         sphinx_run.app.env.metadata["mystnb_codecell_file_warnings"]["author"]

tests/test_parser.py

@@ -10,6 +10,7 @@ def test_basic_run(sphinx_run, file_regression):
         "test_name",
         "kernelspec",
         "language_info",
+        "wordcount",
     }
     assert sphinx_run.app.env.metadata["basic_run"]["test_name"] == "notebook1"
     assert (
@@ -44,6 +45,7 @@ def test_complex_outputs(sphinx_run, file_regression):
         "jupytext",
         "toc",
         "varInspector",
+        "wordcount",
     }
     assert (
         sphinx_run.app.env.metadata["complex_outputs"]["celltoolbar"] == "Edit Metadata"

tests/test_text_based.py

@@ -15,6 +15,7 @@ def test_basic_run(sphinx_run, file_regression, check_nbs):
         "author",
         "source_map",
         "language_info",
+        "wordcount",
     }
     assert sphinx_run.app.env.metadata["basic_unrun"]["author"] == "Chris"
     assert (