fixup! [feature] Add option for computing hashes and storing them as xattrs in the index

Author: mxmlnkn

core/ratarmountcore/hashing.py

@@ -1,16 +1,15 @@
 import hashlib
 import io
+import logging
 import zlib
-from collections.abc import Iterable
 from dataclasses import dataclass
-from typing import IO, Any, Optional
+from typing import IO, Any
 
-from ratarmountcore.utils import RatarmountError
+logger = logging.getLogger(__name__)
 
 
 @dataclass(frozen=True)
-class HashAlgorithmSpec:
-    name: str
+class HashAlgorithm:
     sparse: bool
 
 
@@ -25,49 +24,22 @@ def hexdigest(self) -> str:
         return f"{self._value & 0xFFFFFFFF:08x}"
 
 
-def _normalize_algorithm_name(name: str) -> str:
-    return name.strip().lower().replace('-', '_')
-
-
-def _discover_hashlib_algorithms() -> set[str]:
+def _build_hash_registry() -> dict[str, HashAlgorithm]:
     # shake_* require an explicit output length for digest() / hexdigest() and therefore do not fit this interface.
-    discovered: set[str] = set()
-    for name in hashlib.algorithms_available:
-        normalized = _normalize_algorithm_name(name)
-        if not normalized or normalized.startswith('shake_'):
-            continue
-        discovered.add(normalized)
-    return discovered
-
-
-def _build_hash_registry() -> dict[str, HashAlgorithmSpec]:
-    registry: dict[str, HashAlgorithmSpec] = {
-        name: HashAlgorithmSpec(name=name, sparse=False) for name in sorted(_discover_hashlib_algorithms())
+    hashlib_names = {name for name in hashlib.algorithms_available if name and not name.startswith('shake')}
+    registry: dict[str, HashAlgorithm] = {
+        # Replace sha3_512 to sha3-512 for readability in the --hashes CLI option and the user.hash.sha3-512 key.
+        name.replace('_', '-'): HashAlgorithm(sparse=False)
+        for name in sorted(hashlib_names)
     }
-    registry['crc32'] = HashAlgorithmSpec(name='crc32', sparse=False)
-    registry['smplayer'] = HashAlgorithmSpec(name='smplayer', sparse=True)
+    registry['crc32'] = HashAlgorithm(sparse=False)
+    registry['smplayer'] = HashAlgorithm(sparse=True)
     return registry
 
 
 HASH_REGISTRY = _build_hash_registry()
 
 
-def parse_hash_algorithms(algorithms: Optional[Iterable[str]]) -> list[str]:
-    if not algorithms:
-        return []
-
-    normalized = [
-        _normalize_algorithm_name(part)
-        for algorithm in algorithms
-        for part in algorithm.split(',')
-        if part.strip()
-    ]
-    unknown = [algorithm for algorithm in normalized if algorithm not in HASH_REGISTRY]
-    if unknown:
-        raise RatarmountError(f"Unsupported hash algorithm(s): {', '.join(unknown)}")
-    return list(dict.fromkeys(normalized))
-
-
 def _zero_pad(data: bytes, size: int) -> bytes:
     return data[:size] + b'\0' * max(0, size - len(data))
 
@@ -111,21 +83,23 @@ def compute_hashes(fileObject: IO[bytes], fileSize: int, algorithms: list[str])
     hashers: dict[str, Any] = {}
     sparseAlgorithms: list[str] = []
     for algorithm in algorithms:
-        spec = HASH_REGISTRY.get(algorithm, None)
-        if spec is None:
-            raise RatarmountError(f"Unsupported hash algorithm(s): {algorithm}")
+        if not algorithm:
+            continue
+        specification = HASH_REGISTRY.get(algorithm, None)
+        if specification is None:
+            logger.warning("Unsupported hash algorithm: %s", algorithm)
+            continue
 
-        if spec.sparse:
+        if specification.sparse:
             sparseAlgorithms.append(algorithm)
+        elif algorithm == 'crc32':
+            hashers[algorithm] = _CRC32Hasher()
         else:
-            # Create hasher objects for each algorithm.
-            if algorithm == 'crc32':
-                hashers[algorithm] = _CRC32Hasher()
-                continue
             try:
+                # hashlib.new seems to be stable against _ and - substitutions and probably also case.
                 hashers[algorithm] = hashlib.new(algorithm)
-            except ValueError as exception:
-                raise RatarmountError(f"Unsupported hash algorithm(s): {algorithm}") from exception
+            except ValueError:
+                logger.warning("Unsupported hash: %s", algorithm, exc_info=logger.isEnabledFor(logging.DEBUG))
 
     # Read and hash in chunks of 1 MiB.
     if hashers:

core/tests/test_SQLiteIndexedTar.py

@@ -672,27 +672,23 @@ def test_appending_to_large_archive(parallelization, tmpdir):
 class TestSQLiteIndexedTarXattrs:
     @staticmethod
     def test_hash_xattrs_for_regular_file(parallelization):
+        hashes = ['crc32', 'sha256', 'sha3-256', 'smplayer']
         with (
             copy_test_file("single-file.tar") as path,
-            SQLiteIndexedTar(
-                path, writeIndex=False, parallelization=parallelization, hashes=['sha256', 'sha3_256', 'smplayer']
-            ) as indexedTar,
+            SQLiteIndexedTar(path, writeIndex=False, parallelization=parallelization, hashes=hashes) as indexedTar,
         ):
-            fileInfo = indexedTar.lookup('/bar')
-            assert fileInfo
-            keys = indexedTar.list_xattr(fileInfo)
-
-            assert 'user.hash.sha256' in keys
-            assert 'user.hash.sha3_256' in keys
-            assert 'user.hash.smplayer' in keys
+            path = '/bar'
+            fileInfo = indexedTar.lookup(path)
+            assert fileInfo, path
+            assert set(indexedTar.list_xattr(fileInfo)) == {f'user.hash.{name}' for name in hashes}
 
             data = indexedTar.open(fileInfo).read()
-            assert indexedTar.get_xattr(fileInfo, 'user.hash.sha256') == hashlib.sha256(data).hexdigest().encode(
-                'utf-8'
-            )
-            assert indexedTar.get_xattr(fileInfo, 'user.hash.sha3_256') == hashlib.sha3_256(data).hexdigest().encode(
-                'utf-8'
+            assert indexedTar.get_xattr(fileInfo, 'user.hash.crc32') == b'7e3265a8'
+            assert (
+                indexedTar.get_xattr(fileInfo, 'user.hash.sha256')
+                == b'b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c'
             )
+            assert indexedTar.get_xattr(fileInfo, 'user.hash.sha3-256') == hashlib.sha3_256(data).hexdigest().encode()
             assert indexedTar.get_xattr(fileInfo, 'user.hash.smplayer')
 
     @staticmethod
@@ -709,8 +705,9 @@ def test_hashes_persist_in_written_sidecar_index(parallelization, tmpdir):
                 parallelization=parallelization,
                 hashes=['sha256'],
             ) as indexedTar:
-                fileInfo = indexedTar.lookup('/bar')
-                assert fileInfo
+                path = '/bar'
+                fileInfo = indexedTar.lookup(path)
+                assert fileInfo, path
                 assert indexedTar.get_xattr(fileInfo, 'user.hash.sha256')
 
             # Load existing index with stored xattrs.