ls: use Unicode quotes for locale/clocale styles in UTF-8 locales

The locale and clocale quoting styles always emitted ASCII quotes and
escaped embedded apostrophes/double quotes, regardless of the locale.
GNU uses Unicode quotation marks U+2018/U+2019 (keyed off LC_CTYPE) for
both styles in a UTF-8 locale, leaving embedded ASCII quotes untouched.

Should make test tests/ls/quoting-utf8.sh pass.

Author: sylvestre

.vscode/cspell.dictionaries/jargon.wordlist.txt

@@ -254,3 +254,6 @@ Hijri
 Nowruz
 charmap
 hijri
+
+ctype
+clocale

src/uu/ls/src/display.rs

@@ -42,6 +42,7 @@ use uucore::{
     format::human::human_readable,
     fs::display_permissions,
     fsext::metadata_get_time,
+    i18n::{UEncoding, get_ctype_encoding},
     os_str_as_bytes_lossy,
     quoting_style::{QuotingStyle, locale_aware_escape_dir_name, locale_aware_escape_name},
     show,
@@ -168,6 +169,33 @@ fn escape_name_with_locale(name: &OsStr, config: &Config) -> OsString {
 
 fn locale_quote(name: &OsStr, style: LocaleQuoting) -> OsString {
     let bytes = os_str_as_bytes_lossy(name);
+
+    // In a UTF-8 locale GNU's locale/clocale quoting uses Unicode quotation
+    // marks U+2018 (LEFT) and U+2019 (RIGHT) as delimiters for both styles,
+    // keyed off LC_CTYPE. Since the delimiters differ from any ASCII quote,
+    // embedded apostrophes and double quotes are left untouched; only control
+    // characters, backslashes and invalid bytes are escaped.
+    if get_ctype_encoding() == UEncoding::Utf8 {
+        let mut quoted = String::with_capacity(name.len() + 6);
+        quoted.push('\u{2018}');
+        for chunk in bytes.utf8_chunks() {
+            for c in chunk.valid().chars() {
+                if c == '\\' {
+                    quoted.push_str("\\\\");
+                } else if c.is_ascii() && c.is_control() {
+                    push_basic_escape(&mut quoted, c as u8);
+                } else {
+                    quoted.push(c);
+                }
+            }
+            for &byte in chunk.invalid() {
+                let _ = write!(quoted, "\\{byte:03o}");
+            }
+        }
+        quoted.push('\u{2019}');
+        return OsString::from(quoted);
+    }
+
     let mut quoted = String::with_capacity(name.len() + 2);
     match style {
         LocaleQuoting::Single => quoted.push('\''),

src/uucore/src/lib/features/i18n/mod.rs

@@ -92,3 +92,10 @@ pub fn get_numeric_locale() -> &'static (Locale, UEncoding) {
 pub fn get_locale_encoding() -> UEncoding {
     get_collating_locale().1
 }
+
+/// Return the character-type encoding (`LC_CTYPE`) deduced from the environment.
+pub fn get_ctype_encoding() -> UEncoding {
+    static CTYPE_ENCODING: OnceLock<UEncoding> = OnceLock::new();
+
+    *CTYPE_ENCODING.get_or_init(|| get_locale_from_env("LC_CTYPE").1)
+}

tests/by-util/test_ls.rs

@@ -2717,6 +2717,8 @@ fn test_ls_recursive_1() {
 #[cfg(unix)]
 mod quoting {
     use super::TestScenario;
+    use uutests::at_and_ucmd;
+    use uutests::util::is_locale_available;
     use uutests::util_name;
 
     /// Create a directory with "dirname", then for each check, assert that the
@@ -2764,6 +2766,72 @@ mod quoting {
         );
     }
 
+    // Regression test for GNU tests/ls/quoting-utf8.sh: in a UTF-8 locale the
+    // locale/clocale quoting styles use Unicode quotation marks U+2018/U+2019
+    // and must not escape embedded apostrophes or double quotes; in the C
+    // locale they fall back to ASCII single/double quotes.
+    #[test]
+    fn test_ls_quoting_locale_utf8() {
+        if !is_locale_available("en_US.UTF-8") {
+            return;
+        }
+
+        let lq = "\u{2018}";
+        let rq = "\u{2019}";
+
+        for style in ["locale", "clocale"] {
+            let (at, mut ucmd) = at_and_ucmd!();
+            at.touch("hello world");
+            at.touch("it's");
+            at.touch("say \"hi\"");
+            at.touch("tab\there");
+
+            let out = ucmd
+                .env("LC_ALL", "en_US.UTF-8")
+                .arg(format!("--quoting-style={style}"))
+                .arg("-1")
+                .succeeds()
+                .stdout_move_str();
+
+            assert!(
+                out.contains(&format!("{lq}hello world{rq}")),
+                "{style}: 'hello world' not quoted with Unicode quotes: {out:?}"
+            );
+            // Embedded apostrophe and double quote must stay unescaped.
+            assert!(
+                out.contains(&format!("{lq}it's{rq}")),
+                "{style}: embedded apostrophe should not be escaped: {out:?}"
+            );
+            assert!(
+                out.contains(&format!("{lq}say \"hi\"{rq}")),
+                "{style}: embedded double quote should not be escaped: {out:?}"
+            );
+            // Control characters are still C-escaped.
+            assert!(
+                out.contains(&format!("{lq}tab\\there{rq}")),
+                "{style}: tab should be escaped as \\t: {out:?}"
+            );
+        }
+
+        // In the C locale, locale uses ASCII single quotes and clocale uses
+        // ASCII double quotes.
+        let (at, mut ucmd) = at_and_ucmd!();
+        at.touch("hello world");
+        ucmd.env("LC_ALL", "C")
+            .arg("--quoting-style=locale")
+            .arg("-1")
+            .succeeds()
+            .stdout_contains("'hello world'");
+
+        let (at, mut ucmd) = at_and_ucmd!();
+        at.touch("hello world");
+        ucmd.env("LC_ALL", "C")
+            .arg("--quoting-style=clocale")
+            .arg("-1")
+            .succeeds()
+            .stdout_contains("\"hello world\"");
+    }
+
     #[test]
     fn test_ls_quoting_space() {
         check_quoting_dirname(