WIP

Author: hugovk

Doc/library/tokenize.rst

@@ -227,7 +227,7 @@ Example of tokenizing from the command line.  The script::
 
 will be tokenized to the following output where the first column is the range
 of the line/column coordinates where the token is found, the second column is
-the name of the token, and the final column is the value of the token (if any)
+the name of the token, and the final column is the value of the token (if any):
 
 .. code-block:: shell-session
 

Lib/_colorize.py

@@ -375,6 +375,14 @@ class Timeit(ThemeSection):
     reset: str = ANSIColors.RESET
 
 
+@dataclass(frozen=True, kw_only=True)
+class Tokenize(ThemeSection):
+    whitespace: str = ANSIColors.GREY
+    error: str = ANSIColors.BOLD_RED
+    position: str = ANSIColors.GREY
+    delimiter: str = ANSIColors.RESET
+
+
 @dataclass(frozen=True, kw_only=True)
 class Traceback(ThemeSection):
     type: str = ANSIColors.BOLD_MAGENTA
@@ -411,6 +419,7 @@ class Theme:
     live_profiler: LiveProfiler = field(default_factory=LiveProfiler)
     syntax: Syntax = field(default_factory=Syntax)
     timeit: Timeit = field(default_factory=Timeit)
+    tokenize: Tokenize = field(default_factory=Tokenize)
     traceback: Traceback = field(default_factory=Traceback)
     unittest: Unittest = field(default_factory=Unittest)
 
@@ -424,6 +433,7 @@ def copy_with(
         live_profiler: LiveProfiler | None = None,
         syntax: Syntax | None = None,
         timeit: Timeit | None = None,
+        tokenize: Tokenize | None = None,
         traceback: Traceback | None = None,
         unittest: Unittest | None = None,
     ) -> Self:
@@ -440,6 +450,7 @@ def copy_with(
             live_profiler=live_profiler or self.live_profiler,
             syntax=syntax or self.syntax,
             timeit=timeit or self.timeit,
+            tokenize=tokenize or self.tokenize,
             traceback=traceback or self.traceback,
             unittest=unittest or self.unittest,
         )
@@ -460,6 +471,7 @@ def no_colors(cls) -> Self:
             live_profiler=LiveProfiler.no_colors(),
             syntax=Syntax.no_colors(),
             timeit=Timeit.no_colors(),
+            tokenize=Tokenize.no_colors(),
             traceback=Traceback.no_colors(),
             unittest=Unittest.no_colors(),
         )

Lib/tokenize.py

@@ -505,6 +505,32 @@ def generate_tokens(readline):
     """
     return _generate_tokens_from_c_tokenizer(readline, extra_tokens=True)
 
+
+def _get_token_colors(syntax, tokenize):
+    """Map token type numbers to theme colors."""
+    return frozendict({
+        COMMENT: syntax.comment,
+        DEDENT: tokenize.whitespace,
+        ENCODING: tokenize.whitespace,
+        ENDMARKER: tokenize.whitespace,
+        ERRORTOKEN: tokenize.error,
+        FSTRING_START: syntax.string,
+        FSTRING_MIDDLE: syntax.string,
+        FSTRING_END: syntax.string,
+        INDENT: tokenize.whitespace,
+        NAME: syntax.reset,
+        NEWLINE: tokenize.whitespace,
+        NL: tokenize.whitespace,
+        NUMBER: syntax.number,
+        OP: syntax.op,
+        SOFT_KEYWORD: syntax.soft_keyword,
+        STRING: syntax.string,
+        TSTRING_START: syntax.string,
+        TSTRING_MIDDLE: syntax.string,
+        TSTRING_END: syntax.string,
+    })
+
+
 def _main(args=None):
     import argparse
 
@@ -545,13 +571,32 @@ def error(message, filename=None, location=None):
 
 
         # Output the tokenization
+        import _colorize
+
+        _theme = _colorize.get_theme()
+        s = _theme.syntax
+        t = _theme.tokenize
+        _token_colors = _get_token_colors(s, t)
         for token in tokens:
             token_type = token.type
             if args.exact:
                 token_type = token.exact_type
-            token_range = "%d,%d-%d,%d:" % (token.start + token.end)
-            print("%-20s%-15s%-15r" %
-                  (token_range, tok_name[token_type], token.string))
+            token_range = (
+                f"{t.position}{token.start[0]}"
+                f"{t.delimiter},{t.position}{token.start[1]}"
+                f"{t.delimiter}-"
+                f"{t.position}{token.end[0]}"
+                f"{t.delimiter},{t.position}{token.end[1]}"
+                f"{t.delimiter}:"
+            )
+            color = _token_colors.get(token_type, s.reset)
+            token_name = tok_name[token_type]
+            visible_range = f"{token.start[0]},{token.start[1]}-{token.end[0]},{token.end[1]}:"
+            print(
+                f"{token_range}{' ' * (20 - len(visible_range))}"
+                f"{color}{token_name:<15}"
+                f"{s.reset}{token.string!r:<15}"
+            )
     except IndentationError as err:
         line, column = err.args[1][1:3]
         error(err.args[0], filename, (line, column))