pythongh-148821: Always reject known multi-byte encodings in pyexpat

serhiy-storchaka · serhiy-storchaka · commit cef74d0d051d · 2026-04-23T14:46:35.000+03:00
The XML parser (pyexpat) now raises ValueError for known unsupported
multi-byte encodings such us "ISO-2022-JP", "utf8" (without hyphen) or
"raw-unicode-escape" instead of failing later, when encounter non-ASCII data.
diff --git a/Include/codecs.h b/Include/codecs.h
@@ -170,6 +170,12 @@ PyAPI_FUNC(PyObject *) PyCodec_NameReplaceErrors(PyObject *exc);
 PyAPI_DATA(const char *) Py_hexdigits;
 #endif
 
+#ifndef Py_LIMITED_API
+PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding(
+   const char *encoding,
+   const char *alternate_command);
+#endif
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/Include/internal/pycore_codecs.h b/Include/internal/pycore_codecs.h
@@ -45,7 +45,7 @@ extern int _PyCodec_UnregisterError(const char *name);
    in Python 3.5+?
 
  */
-extern PyObject* _PyCodec_LookupTextEncoding(
+PyAPI_FUNC(PyObject*) _PyCodec_LookupTextEncoding(
    const char *encoding,
    const char *alternate_command);
 
diff --git a/Lib/codecs.py b/Lib/codecs.py
@@ -93,7 +93,8 @@ class CodecInfo(tuple):
 
     def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
         incrementalencoder=None, incrementaldecoder=None, name=None,
-        *, _is_text_encoding=None):
+        *, _is_text_encoding=None,
+        _is_single_byte=None):
         self = tuple.__new__(cls, (encode, decode, streamreader, streamwriter))
         self.name = name
         self.encode = encode
@@ -104,6 +105,8 @@ def __new__(cls, encode, decode, streamreader=None, streamwriter=None,
         self.streamreader = streamreader
         if _is_text_encoding is not None:
             self._is_text_encoding = _is_text_encoding
+        if _is_single_byte is not None:
+            self._is_single_byte = _is_single_byte
         return self
 
     def __repr__(self):
diff --git a/Lib/encodings/big5.py b/Lib/encodings/big5.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/big5hkscs.py b/Lib/encodings/big5hkscs.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/cp932.py b/Lib/encodings/cp932.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/cp949.py b/Lib/encodings/cp949.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/cp950.py b/Lib/encodings/cp950.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/euc_jis_2004.py b/Lib/encodings/euc_jis_2004.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/euc_jisx0213.py b/Lib/encodings/euc_jisx0213.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/euc_jp.py b/Lib/encodings/euc_jp.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/euc_kr.py b/Lib/encodings/euc_kr.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/gb18030.py b/Lib/encodings/gb18030.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/gb2312.py b/Lib/encodings/gb2312.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/gbk.py b/Lib/encodings/gbk.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/hz.py b/Lib/encodings/hz.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py
@@ -385,4 +385,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_jp.py b/Lib/encodings/iso2022_jp.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_jp_1.py b/Lib/encodings/iso2022_jp_1.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_jp_2.py b/Lib/encodings/iso2022_jp_2.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_jp_2004.py b/Lib/encodings/iso2022_jp_2004.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_jp_3.py b/Lib/encodings/iso2022_jp_3.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_jp_ext.py b/Lib/encodings/iso2022_jp_ext.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/iso2022_kr.py b/Lib/encodings/iso2022_kr.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/johab.py b/Lib/encodings/johab.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py
@@ -250,4 +250,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/raw_unicode_escape.py b/Lib/encodings/raw_unicode_escape.py
@@ -43,4 +43,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/shift_jis.py b/Lib/encodings/shift_jis.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/shift_jis_2004.py b/Lib/encodings/shift_jis_2004.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/shift_jisx0213.py b/Lib/encodings/shift_jisx0213.py
@@ -36,4 +36,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/unicode_escape.py b/Lib/encodings/unicode_escape.py
@@ -43,4 +43,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamwriter=StreamWriter,
         streamreader=StreamReader,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py
@@ -152,4 +152,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_16_be.py b/Lib/encodings/utf_16_be.py
@@ -39,4 +39,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_16_le.py b/Lib/encodings/utf_16_le.py
@@ -39,4 +39,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py
@@ -147,4 +147,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_32_be.py b/Lib/encodings/utf_32_be.py
@@ -34,4 +34,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_32_le.py b/Lib/encodings/utf_32_le.py
@@ -34,4 +34,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_7.py b/Lib/encodings/utf_7.py
@@ -35,4 +35,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_8.py b/Lib/encodings/utf_8.py
@@ -39,4 +39,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/encodings/utf_8_sig.py b/Lib/encodings/utf_8_sig.py
@@ -127,4 +127,5 @@ def getregentry():
         incrementaldecoder=IncrementalDecoder,
         streamreader=StreamReader,
         streamwriter=StreamWriter,
+        _is_single_byte=False,
     )
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
@@ -1892,6 +1892,7 @@ def test_copy(self):
         self.assertIsNot(dup, orig)
         self.assertEqual(dup, orig)
         self.assertTrue(orig._is_text_encoding)
+        self.assertFalse(orig._is_single_byte)
         self.assertEqual(dup.encode, orig.encode)
         self.assertEqual(dup.name, orig.name)
         self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
@@ -1912,6 +1913,7 @@ def test_deepcopy(self):
         self.assertIsNot(dup, orig)
         self.assertEqual(dup, orig)
         self.assertTrue(orig._is_text_encoding)
+        self.assertFalse(orig._is_single_byte)
         self.assertEqual(dup.encode, orig.encode)
         self.assertEqual(dup.name, orig.name)
         self.assertEqual(dup.incrementalencoder, orig.incrementalencoder)
@@ -1940,6 +1942,7 @@ def test_pickle(self):
                      unpickled_codec_info.incrementalencoder
                 )
                 self.assertTrue(unpickled_codec_info._is_text_encoding)
+                self.assertFalse(unpickled_codec_info._is_single_byte)
 
         # Test a CodecInfo with _is_text_encoding equal to false.
         codec_info = codecs.lookup('base64')
diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py
@@ -227,7 +227,7 @@ def _verify_parse_output(self, operations):
             "Character data: '\xb5'",
             "End element: 'root'",
         ]
-        for operation, expected_operation in zip(operations, expected_operations):
+        for operation, expected_operation in zip(operations, expected_operations, strict=True):
             self.assertEqual(operation, expected_operation)
 
     def test_parse_bytes(self):
@@ -276,6 +276,51 @@ def test_parse_again(self):
         self.assertEqual(expat.ErrorString(cm.exception.code),
                           expat.errors.XML_ERROR_FINISHED)
 
+    @support.subTests('enc', ['UTF-8', 'utf-8', 'utf-16', 'koi8-u',
+                              'cp1125', 'cp1251', 'iso8859-5',
+                              'mac_cyrillic'])
+    def test_supportes_ecodings(self, enc):
+        out = self.Outputter()
+        parser = expat.ParserCreate()
+        self._hookup_callbacks(parser, out)
+        data = (f'<?xml version="1.0" encoding="{enc}"?>\n'
+                '<корінь атрибут="значення">зміст</корінь>').encode(enc)
+        parser.Parse(data, True)
+        self.assertEqual(out.out, [
+            ('XML declaration', ('1.0', enc, -1)),
+            "Start element: 'корінь' {'атрибут': 'значення'}",
+            "Character data: 'зміст'",
+            "End element: 'корінь'",
+        ])
+
+    @support.subTests('enc', [
+        'UTF8', 'UTF-7',
+        "unicode-escape", "raw-unicode-escape",
+        "Big5-HKSCS", "Big5",
+        "cp932", "cp949", "cp950",
+        "EUC_JIS-2004", "EUC_JISX0213", "EUC-JP", "EUC-KR",
+        "GB18030", "GB2312", "GBK",
+        "HZ-GB-2312",
+        "ISO-2022-JP", "ISO-2022-JP-1", "ISO-2022-JP-2004",
+        "ISO-2022-JP-2", "ISO-2022-JP-3", "ISO-2022-JP-EXT",
+        "ISO-2022-KR",
+        "johab",
+        "Shift_JIS", "Shift_JIS-2004", "Shift_JISX0213",
+    ])
+    def test_unsupportes_ecodings(self, enc):
+        parser = expat.ParserCreate()
+        data = (f'<?xml version="1.0" encoding="{enc}"?>\n'
+                '<root></root>').encode(enc)
+        with self.assertRaises(ValueError):
+            parser.Parse(data, True)
+
+    def test_unknown_ecoding(self):
+        parser = expat.ParserCreate()
+        data = b'<?xml version="1.0" encoding="xyz"?>\n<root></root>'
+        with self.assertRaises(LookupError):
+            parser.Parse(data, True)
+
+
 class NamespaceSeparatorTest(unittest.TestCase):
     def test_legal(self):
         # Tests that make sure we get errors when the namespace_separator value
diff --git a/Misc/NEWS.d/next/Library/2026-04-23-14-46-30.gh-issue-148821.cR4kMa.rst b/Misc/NEWS.d/next/Library/2026-04-23-14-46-30.gh-issue-148821.cR4kMa.rst
@@ -0,0 +1,4 @@
+The :mod:`XML parser <pyexpat>` now raises :exc:`ValueError` for known
+unsupported multi-byte encodings such us "UTF8", "ISO-2022-JP" or
+"raw-unicode-escape" instead of failing later, when encounter non-ASCII
+data.
diff --git a/Modules/pyexpat.c b/Modules/pyexpat.c
@@ -4,6 +4,7 @@
 
 #include "Python.h"
 #include "pycore_ceval.h"         // _Py_EnterRecursiveCall()
+#include "pycore_codecs.h"        // _PyCodec_LookupTextEncoding()
 #include "pycore_import.h"        // _PyImport_SetModule()
 #include "pycore_pyhash.h"        // _Py_HashSecret
 #include "pycore_traceback.h"     // _PyTraceback_Add()
@@ -1465,6 +1466,31 @@ PyUnknownEncodingHandler(void *encodingHandlerData,
     if (PyErr_Occurred())
         return XML_STATUS_ERROR;
 
+    PyObject *codec = _PyCodec_LookupTextEncoding(name, NULL);
+    if (codec == NULL) {
+        return XML_STATUS_ERROR;
+    }
+    // if (!PyTuple_CheckExact(codec)) {
+    //     PyObject *attr;
+    //     if (PyObject_GetOptionalAttrString(codec, "_is_single_byte", &attr) < 0) {
+    //         Py_DECREF(codec);
+    //         return XML_STATUS_ERROR;
+    //     }
+    //     if (attr != NULL) {
+    //         int is_single_byte = PyObject_IsTrue(attr);
+    //         Py_DECREF(attr);
+    //         if (is_single_byte <= 0) {
+    //             Py_DECREF(codec);
+    //             if (is_single_byte == 0) {
+    //                 PyErr_SetString(PyExc_ValueError,
+    //                                 "multi-byte encodings are not supported");
+    //             }
+    //             return XML_STATUS_ERROR;
+    //         }
+    //     }
+    // }
+    Py_DECREF(codec);
+
     u = PyUnicode_Decode((const char*) template_buffer, 256, name, "replace");
     if (u == NULL) {
         Py_XDECREF(u);
diff --git a/Tools/unicode/gencjkcodecs.py b/Tools/unicode/gencjkcodecs.py

Original file line number	Diff line number	Diff line change
`@@ -36,4 +36,5 @@ def getregentry():`
`36`	`36`	`incrementaldecoder=IncrementalDecoder,`
`37`	`37`	`streamreader=StreamReader,`
`38`	`38`	`streamwriter=StreamWriter,`
	`39`	`+ _is_single_byte=False,`
`39`	`40`	`)`