Highlighters: Refactor tim.prettify_markup as a highlighter class

This makes a bit more sense semantically, and the implementation is
a lot cleaner & more robust now.

Author: bczsalba

pytermgui/cmd.py

@@ -326,7 +326,7 @@ def _create_table(source: Iterable[tuple[str, str]]) -> ptg.Container:
 
     def on_exit(self) -> None:
         super().on_exit()
-        print(ptg.tim.prettify_markup(self._input.value))
+        ptg.tim.print(ptg.highlight_tim(self._input.value))
         ptg.tim.print(self._input.value)
 
 

pytermgui/colors.py

@@ -16,15 +16,17 @@
 from dataclasses import dataclass, field
 from functools import cached_property, lru_cache
 from math import sqrt  # pylint: disable=no-name-in-module
-from typing import Generator, Literal, Type
+from typing import TYPE_CHECKING, Generator, Literal, Type
 
 from .ansi_interface import reset as reset_style
 from .color_info import COLOR_TABLE, CSS_COLORS
 from .exceptions import ColorSyntaxError
-from .fancy_repr import FancyYield
 from .input import getch
 from .terminal import ColorSystem, terminal
 
+if TYPE_CHECKING:
+    from .fancy_repr import FancyYield
+
 __all__ = [
     "COLOR_TABLE",
     "XTERM_NAMED_COLORS",

pytermgui/highlighters.py

@@ -6,8 +6,10 @@
 import keyword
 import re
 from dataclasses import dataclass, field
+from functools import lru_cache
 from typing import TYPE_CHECKING, Callable, Generator, Match, Pattern, Protocol
 
+from .markup import Token, consume_tag
 from .regex import RE_MARKUP
 
 if TYPE_CHECKING:
@@ -16,6 +18,7 @@
 __all__ = [
     "Highlighter",
     "RegexHighlighter",
+    "highlight_tim",
     "highlight_python",
 ]
 
@@ -137,7 +140,7 @@ def _insert_style(matchobj: Match) -> str:
         return text
 
     def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
-        """Yields some fancy looking repl text."""
+        """Yields some fancy looking repr text."""
 
         preview = self("highlight_python()") + "\x1b[0m"
         pattern = self._pattern.pattern
@@ -151,6 +154,72 @@ def __fancy_repr__(self) -> Generator[FancyYield, None, None]:
         yield ">"
 
 
+def highlight_tim(text: str, cache: bool = True) -> str:
+    """Highlights some TIM code."""
+
+    @lru_cache(1048)
+    def _highlight(txt: str) -> str:
+        output = ""
+        cursor = 0
+        active_tokens: list[Token] = []
+
+        def _get_active_markup() -> str:
+            active_markup = " ".join(tkn.markup for tkn in active_tokens)
+
+            if active_markup == "":
+                return ""
+
+            return f"[{active_markup}]"
+
+        for matchobj in RE_MARKUP.finditer(txt):
+            start, end = matchobj.span()
+
+            if cursor < start:
+                if cursor > 0:
+                    output += "]"
+
+                output += _get_active_markup()
+                output += f"{txt[cursor:start]}[/]"
+
+            *_, tags = matchobj.groups()
+
+            output += "["
+            for tag in tags.split():
+                token = consume_tag(tag)
+                output += f"{token.prettified_markup} "
+
+                if Token.is_clear(token):
+                    active_tokens = [
+                        tkn for tkn in active_tokens if not token.targets(tkn)
+                    ]
+
+                else:
+                    active_tokens.append(token)
+
+            output = output.rstrip()
+            cursor = end
+
+        if cursor < len(txt) - 1:
+            if cursor > 0:
+                output += "]"
+
+            output += _get_active_markup()
+            output += f"{txt[cursor:]}"
+
+            if len(active_tokens) > 0:
+                output += "[/]"
+
+        if output.count("[") != output.count("]"):
+            output += "]"
+
+        return output
+
+    if cache:
+        return _highlight(text)
+
+    return _highlight.__wrapped__(text)
+
+
 _BUILTIN_NAMES = "|".join(f"(?:{item})" for item in dir(builtins))
 _KEYWORD_NAMES = "|".join(
     f"(?:{keyw})" for keyw in list(keyword.kwlist) + ["builtin", "function", "module"]

pytermgui/markup/language.py

@@ -233,24 +233,6 @@ def group_styles(
             text, tokenizer=tokenizer, context=self.context
         )
 
-    def prettify_markup(self, markup: str) -> str:
-        """Syntax-highlight markup."""
-
-        output = ""
-
-        for span in StyledText.group_styles(
-            markup, tokenizer=tokenize_markup, context=self.context
-        ):
-            tags = " ".join(token.prettified_markup for token in span.tokens[:-1])
-            markup = " ".join(tkn.markup for tkn in span.tokens[:-1])
-
-            if len(tags) > 0 and len(markup) > 0:
-                output += f"[{tags}][{markup}]"
-
-            output += f"{span.plain}[/]"
-
-        return self.parse(output, optimize=True)
-
     def print(self, *args, **kwargs) -> None:
         """Parse all arguments and pass them through to print, along with kwargs."""
 

pytermgui/markup/tokens.py

@@ -8,12 +8,14 @@
 
 from dataclasses import dataclass
 from functools import cached_property
-from typing import Any, Generator, Iterator
+from typing import TYPE_CHECKING, Any, Generator, Iterator
 
 from typing_extensions import TypeGuard
 
 from ..colors import Color
-from ..fancy_repr import FancyYield
+
+if TYPE_CHECKING:
+    from ..fancy_repr import FancyYield
 
 __all__ = [
     "Token",

pytermgui/prettifiers.py

@@ -10,7 +10,7 @@
 from typing import Any
 
 from .fancy_repr import build_fancy_repr, supports_fancy_repr
-from .highlighters import highlight_python
+from .highlighters import highlight_python, highlight_tim
 from .markup import tim
 from .regex import RE_MARKUP
 
@@ -61,8 +61,10 @@ def prettify(  # pylint: disable=too-many-branches
 
     if isinstance(target, str):
         if RE_MARKUP.match(target) is not None:
+            target = highlight_tim(target)
+
             if parse:
-                return f'"{tim.prettify_markup(target)}"'
+                return f'"{tim.parse(target)}"'
 
             return target + "[/]"
 

pytermgui/pretty.py

@@ -8,6 +8,8 @@
     ```
 """
 
+# isort: skip_file
+
 from __future__ import annotations
 
 import builtins
@@ -24,7 +26,9 @@
     # IPython runtime, so if running outside of that context a NameError
     # is raised.
     IPYTHON = get_ipython()  # type: ignore
-    from IPython.core.formatters import BaseFormatter  # pylint: disable=import-error
+    from IPython.core.formatters import (  # type: ignore # pylint: disable=import-error
+        BaseFormatter,
+    )
 
 except NameError:
     IPYTHON = None

pytermgui/terminal.py

@@ -14,12 +14,14 @@
 from enum import Enum
 from functools import cached_property
 from shutil import get_terminal_size
-from typing import Any, Callable, Generator, TextIO
+from typing import TYPE_CHECKING, Any, Callable, Generator, TextIO
 
-from .fancy_repr import FancyYield
 from .input import getch_timeout
 from .regex import RE_PIXEL_SIZE, has_open_sequence, real_length, strip_ansi
 
+if TYPE_CHECKING:
+    from .fancy_repr import FancyYield
+
 __all__ = [
     "terminal",
     "set_global_terminal",