Merge remote-tracking branch 'ArrayBolt3/arraybolt3/trixie'

Author: adrelanos

.github/workflows/lint.yml

@@ -6,13 +6,15 @@ on:
       - usr/lib/python3/dist-packages/stdisplay/**
       - usr/lib/python3/dist-packages/sanitize_string/**
       - usr/lib/python3/dist-packages/strip_markup/**
+      - usr/lib/python3/dist-packages/unicode_show/**
       - .github/workflows/lint.yml
       - run-tests
   pull_request:
     paths:
       - usr/lib/python3/dist-packages/stdisplay/**
       - usr/lib/python3/dist-packages/sanitize_string/**
       - usr/lib/python3/dist-packages/strip_markup/**
+      - usr/lib/python3/dist-packages/unicode_show/**
       - .github/workflows/lint.yml
       - run-tests
 
@@ -27,7 +29,10 @@ jobs:
           - image: debian:stable
           - image: debian:testing
           - image: debian:unstable
-          - image: ubuntu:latest
+          ## Disabled because ubuntu:latest is currently 24.04, which uses a
+          ## version of Python that does not support the `newline` argument of
+          ## Path.read_text()
+          # - image: ubuntu:latest
           - image: ubuntu:rolling
 
     container:

man/unicode-show.1.ronn

@@ -19,7 +19,8 @@ unicode-show(1) -- Scan and annotate suspicious Unicode characters
 
 ### What is considered suspicious:
 - Characters outside the printable ASCII range (`0x20`-`0x7E`)
-- Control characters (excluding `\n`, `\r`, `\t`)
+- Control characters (excluding `\n` and `\t`)
+- Carriage returns (`\r`), even when used in CRLF pairs
 - Any character not in the standard set of ASCII letters, digits, punctuation, and trailing whitespace
 
 ### Output formatting:
@@ -29,6 +30,7 @@ unicode-show(1) -- Scan and annotate suspicious Unicode characters
 
 ## Color output is disabled if:
 - The environment variable `$NOCOLOR` is set
+- The environment variable `$NO_COLOR` is set to `1`
 - `$TERM` is set to `dumb`
 - Output is redirected (non-interactive terminal)
 
@@ -71,6 +73,7 @@ NOCOLOR=1 unicode-show example.txt
 ## ENVIRONMENT
 
 - **NOCOLOR** - disables color output if set
+- **NO_COLOR** - disables color output if set to `1`
 - **TERM** - if set to `dumb`, disables color output
 
 ## AUTHOR

run-tests

@@ -10,7 +10,7 @@ black=(black --config="${pyrc}" --color --diff --check)
 pylint=(pylint --rcfile="${pyrc}")
 mypy=(mypy --config-file="${pyrc}")
 
-py_lib_to_test_list=(stdisplay sanitize_string strip_markup)
+py_lib_to_test_list=(stdisplay sanitize_string strip_markup unicode_show)
 
 for py_lib_to_test in "${py_lib_to_test_list[@]}"; do
   cd "${pythonpath}/${py_lib_to_test}/"
@@ -23,7 +23,7 @@ for py_lib_to_test in "${py_lib_to_test_list[@]}"; do
 done
 
 stdin_file_read_utils=(stcat stcatn)
-stdin_implicit_read_utils=(sttee stsponge strip-markup)
+stdin_implicit_read_utils=(sttee stsponge strip-markup unicode-show)
 stdin_utils=("${stdin_file_read_utils[@]}" "${stdin_implicit_read_utils[@]}")
 utils=(stprint stecho "${stdin_utils[@]}")
 cd "${git_toplevel}/usr/bin"

usr/bin/grep-find-unicode-wrapper

@@ -71,10 +71,10 @@ set -o errexit
 
 output_message="$(
   {
-    [ -n "$one"   ] && printf '%s\n' "$one"
-    [ -n "$two"   ] && printf '%s\n' "$two"
-    [ -n "$three" ] && printf '%s\n' "$three"
-    [ -n "$four"  ] && printf '%s\n' "$four"
+    [ -n "${one:-}"   ] && printf '%s\n' "$one"   || true
+    [ -n "${two:-}"   ] && printf '%s\n' "$two"   || true
+    [ -n "${three:-}" ] && printf '%s\n' "$three" || true
+    [ -n "${four:-}"  ] && printf '%s\n' "$four"  || true
   } | sort --unique
 )"
 

usr/bin/unicode-show

@@ -3,155 +3,10 @@
 ## Copyright (C) 2025 - 2025 ENCRYPTED SUPPORT LLC <adrelanos@whonix.org>
 ## See the file COPYING for copying conditions.
 
-"""
-This script scans input text or files for non-ASCII and suspicious Unicode characters.
-It prints lines with suspicious characters annotated inline (e.g., [U+XXXX]).
-For each such character, it prints the Unicode codepoint, name, and category.
-
-Exit codes:
-  0 - No suspicious Unicode found
-  1 - Suspicious Unicode found
-  2 - Error (e.g., file I/O or decoding error)
-"""
+# pylint: disable=missing-module-docstring,invalid-name
 
 import sys
-import unicodedata
-import string
-import io
-import os
-
-USE_COLOR = (
-    not os.getenv("NOCOLOR") and
-    os.getenv("TERM") != "dumb" and
-    sys.stdout.isatty()
-)
-
-RED = "\033[91m"
-CYAN = "\033[96m"
-RESET = "\033[0m"
-
-VISIBLE_ASCII_RANGE = range(0x20, 0x7F)
-ALLOWED_WHITESPACE = {'\n', '\r', '\t'}
-SAFE_ASCII_SEMANTIC = set(string.ascii_letters + string.digits + string.punctuation + " \n\r\t")
-
-def colorize(text, color):
-    return f"{color}{text}{RESET}" if USE_COLOR else text
-
-def is_suspicious(c):
-    codepoint_allowed = ord(c) in VISIBLE_ASCII_RANGE or c in ALLOWED_WHITESPACE
-    semantically_allowed = c in SAFE_ASCII_SEMANTIC
-    ## Purposeful redundancy for extra safety.
-    return not (codepoint_allowed and semantically_allowed)
-
-def describe_char(c):
-    """Return a description of a Unicode character including codepoint, name, and category."""
-    code = ord(c)
-    name = unicodedata.name(c, "<unnamed>")
-    cat = unicodedata.category(c)
-
-    codepoint_allowed = code in VISIBLE_ASCII_RANGE or c in ALLOWED_WHITESPACE
-    semantically_allowed = c in SAFE_ASCII_SEMANTIC
-
-    ## Purposeful redundancy for extra safety in character display.
-    if codepoint_allowed and semantically_allowed:
-        display = c
-    else:
-        display = repr(c)
-
-    desc = f"{display} (U+{code:04X}, {name}, {cat})"
-    return colorize(desc, CYAN)
-
-def scan_line(line, lineno=None, filename=None):
-    """Scan a single line for suspicious characters, print annotated line and character info."""
-
-    annotated = ""
-    has_suspicious = False
-    prefix = f"{filename or '<stdin>'}:{lineno}: "
-    suspicious_descrs = []
-
-    for c in line:
-        if is_suspicious(c):
-            has_suspicious = True
-            code = f"[U+{ord(c):04X}]"
-            annotated += colorize(code, RED)
-            suspicious_descrs.append(f"   -> {describe_char(c)}")
-        else:
-            annotated += c
-
-    if annotated and annotated[-1] == "\n":
-        annotated = annotated[:-1]
-
-    annotated_stripped = annotated.rstrip()
-    ## Trailing whitespaces are suspicious.
-    ## https://forums.whonix.org/t/detecting-malicious-unicode-in-source-code-and-pull-requests/13754/28
-    if len(annotated) != len(annotated_stripped):
-        annotated_new = annotated_stripped
-        has_suspicious = True
-        for c in annotated[len(annotated_stripped):]:
-            code = f"[U+{ord(c):04X}]"
-            annotated_new += colorize(code, RED)
-            suspicious_descrs.append(f"   -> {describe_char(c)}")
-        annotated = annotated_new
-
-    if not has_suspicious:
-        return False
-
-    print(prefix + annotated)
-    for suspicious_descr in suspicious_descrs:
-        print(suspicious_descr)
-
-    return True
-
-def scan_file(f, filename=None):
-    """Scan an entire file-like object for suspicious characters."""
-    found = False
-    last_lineno = 0
-    ## Empty files should not report "missing newline at end".
-    #last_line = ""
-    last_line = None
-
-    for lineno, line in enumerate(f, 1):
-        last_lineno = lineno
-        last_line = line
-        if scan_line(line, lineno=lineno, filename=filename):
-            found = True
-
-    if last_line is not None and not last_line.endswith('\n'):
-        found = True
-        ## Missing newline at the end is suspicious.
-        msg = f"{filename or '<stdin>'}:{last_lineno}: " + colorize("[missing newline at end]", RED)
-        print(msg)
-
-    return found
+from unicode_show.unicode_show import main
 
 if __name__ == "__main__":
-    clean = True
-    try:
-        if len(sys.argv) > 1:
-            for fname in sys.argv[1:]:
-                try:
-                    ## Must not use errors='replace' because otherwise suspicious unicode might slip.
-                    ## Fail closed for non-UTF-8.
-                    with open(fname, 'r', encoding='utf-8', errors='strict') as f:
-                        if scan_file(f, filename=fname):
-                            clean = False
-                except UnicodeDecodeError as e:
-                    print(f"[ERROR] Unicode decode error [{fname}]: {e}", file=sys.stderr)
-                    clean = False
-                except Exception as e:
-                    print(f"[ERROR] File read error [{fname}]: {e}", file=sys.stderr)
-                    sys.exit(2)
-        else:
-            try:
-                stdin_buffer = io.TextIOWrapper(sys.stdin.buffer, encoding='utf-8', errors='strict')
-                if scan_file(stdin_buffer):
-                    clean = False
-            except UnicodeDecodeError as e:
-                print(f"[ERROR] Unicode decode error [stdin]: {e}", file=sys.stderr)
-                clean = False
-    except Exception as e:
-        print(f"[ERROR] Unexpected error [main]: {e}", file=sys.stderr)
-        sys.exit(2)
-
-    if not clean:
-        sys.exit(1)
+    sys.exit(main())

usr/lib/python3/dist-packages/sanitize_string/py.typed

@@ -1 +0,0 @@
-#!/usr/bin/python3 -su

usr/lib/python3/dist-packages/sanitize_string/sanitize_string.py

@@ -89,7 +89,3 @@ def main() -> int:
     else:
         sys.stdout.write(sanitized_string)
     return 0
-
-
-if __name__ == "__main__":
-    main()

usr/lib/python3/dist-packages/stdisplay/__init__.py

@@ -1 +0,0 @@
-#!/usr/bin/python3 -su

usr/lib/python3/dist-packages/stdisplay/py.typed

@@ -1,22 +1,25 @@
 #!/usr/bin/python3 -su
 
-## SPDX-FileCopyrightText: 2025 Benjamin Grande M. S. <ben.grande.b@gmail.com>
-## SPDX-FileCopyrightText: 2025 ENCRYPTED SUPPORT LLC <adrelanos@whonix.org>
-##
-## SPDX-License-Identifier: AGPL-3.0-or-later
+## Copyright (C) 2025 - 2025 Benjamin Grande M. S. <ben.grande.b@gmail.com>
+## Copyright (C) 2025 - 2025 ENCRYPTED SUPPORT LLC <adrelanos@whonix.org>
+## See the file COPYING for copying conditions.
 
 """Safely print stdin or file to stdout."""
 
 from pathlib import Path
-from sys import argv, stdin, stdout, modules
+from sys import argv, stdin, stdout
 from stdisplay.stdisplay import stdisplay
 
 
 def main() -> None:
     """Safely print stdin or file to stdout."""
-    # https://github.com/pytest-dev/pytest/issues/4843
-    if "pytest" not in modules and stdin is not None:
-        stdin.reconfigure(errors="replace")  # type: ignore
+    stdout.reconfigure(  # type: ignore
+        encoding="ascii", errors="replace", newline="\n"
+    )
+    if stdin is not None:
+        stdin.reconfigure(  # type: ignore
+            encoding="utf-8", errors="replace", newline="\n"
+        )
     if len(argv) == 1:
         if stdin is not None:
             for untrusted_line in stdin:
@@ -30,10 +33,8 @@ def main() -> None:
                     stdout.write(stdisplay(untrusted_line))
         else:
             path = Path(untrusted_arg)
-            untrusted_text = path.read_text(encoding="utf-8", errors="replace")
+            untrusted_text = path.read_text(
+                encoding="utf-8", errors="replace", newline="\n"
+            )
             stdout.write(stdisplay(untrusted_text))
     stdout.flush()
-
-
-if __name__ == "__main__":
-    main()