feat: schema v2 with 10 new columns, add location to --write-metadata, deprecate dates

- Manifest DB schema v2: add gps_speed, gps_timestamp, timezone_offset,
  asset_subtype, hdr_type, burst_flags, burst_flags_ext, burst_id,
  original_orientation, and raw_fields (JSON blob) columns
- Migration support for v0->v2 and v1->v2 upgrades
- Add 'location' category to --write-metadata for GPS lat/lon/alt
  with float tolerance (~1m) for idempotent delta detection
- Deprecate 'dates' category (accepted with warning, no-op)
- Guard orientation=0 (invalid EXIF, treated as None at extract time)
- Populate all new DB columns from iCloud API in _extract_manifest_metadata
- 93 tests covering all changes (22 manifest, 61 metadata, 10 CLI)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: teh-hippo

src/icloudpd/base.py

@@ -49,18 +49,19 @@
 from icloudpd.filename_policies import build_filename_with_policies, create_filename_builder
 from icloudpd.log_level import LogLevel
 from icloudpd.manifest import ManifestDB
+from icloudpd.metadata_writer import (
+    extract_metadata_update,
+)
+from icloudpd.metadata_writer import (
+    write_metadata as write_file_metadata,
+)
 from icloudpd.mfa_provider import MFAProvider
 from icloudpd.password_provider import PasswordProvider
 from icloudpd.paths import local_download_path, remove_unicode_chars
 from icloudpd.server import serve_app
 from icloudpd.status import Status, StatusExchange
 from icloudpd.string_helpers import parse_timestamp_or_timedelta, truncate_middle
 from icloudpd.xmp_sidecar import build_metadata, generate_xmp_file
-from icloudpd.metadata_writer import (
-    MetadataUpdate,
-    extract_metadata_update,
-    write_metadata as write_file_metadata,
-)
 from pyicloud_ipd.asset_version import (
     AssetVersion,
     add_suffix_to_filename,
@@ -253,7 +254,7 @@ def run_with_configs(global_config: GlobalConfig, user_configs: Sequence[UserCon
 
     # Check exiftool availability if any user has --write-metadata
     if any(uc.write_metadata for uc in user_configs):
-        from icloudpd.metadata_writer import check_exiftool, ExiftoolNotFoundError
+        from icloudpd.metadata_writer import ExiftoolNotFoundError, check_exiftool
         try:
             ver = check_exiftool()
             logger.info("exiftool %s found for --write-metadata", ver)
@@ -667,12 +668,60 @@ def _extract_manifest_metadata(photo: PhotoAsset, version: AssetVersion) -> Dict
         gps_latitude = xmp.GPSLatitude
         gps_longitude = xmp.GPSLongitude
         gps_altitude = xmp.GPSAltitude
+        gps_speed = xmp.GPSSpeed
+        gps_timestamp = xmp.GPSTimeStamp.isoformat() if xmp.GPSTimeStamp else None
         orientation = xmp.Orientation
     except Exception:
         title = description = keywords = None
         gps_latitude = gps_longitude = gps_altitude = None
+        gps_speed = None
+        gps_timestamp = None
         orientation = None
 
+    # Timezone offset
+    timezone_offset = fields.get("timeZoneOffset", {}).get("value")
+    if timezone_offset is not None:
+        with contextlib.suppress(ValueError, TypeError):
+            timezone_offset = int(timezone_offset)
+
+    # Asset subtype / HDR / burst metadata
+    asset_subtype = fields.get("assetSubtypeV2", {}).get("value")
+    if asset_subtype is not None:
+        with contextlib.suppress(ValueError, TypeError):
+            asset_subtype = int(asset_subtype)
+
+    hdr_type = fields.get("assetHDRType", {}).get("value")
+    if hdr_type is not None:
+        with contextlib.suppress(ValueError, TypeError):
+            hdr_type = int(hdr_type)
+
+    burst_flags = fields.get("burstFlags", {}).get("value")
+    if burst_flags is not None:
+        with contextlib.suppress(ValueError, TypeError):
+            burst_flags = int(burst_flags)
+
+    burst_flags_ext = fields.get("burstFlagsExt", {}).get("value")
+    if burst_flags_ext is not None:
+        with contextlib.suppress(ValueError, TypeError):
+            burst_flags_ext = int(burst_flags_ext)
+
+    burst_id = fields.get("burstId", {}).get("value")
+    if burst_id is not None:
+        burst_id = str(burst_id)
+
+    # Original orientation from master record
+    original_orientation = None
+    try:
+        master_fields = photo._master_record.get("fields", {})
+        oo_val = master_fields.get("originalOrientation", {}).get("value")
+        if oo_val is not None:
+            original_orientation = int(oo_val)
+    except (KeyError, TypeError, ValueError):
+        pass
+
+    # Full asset record fields as JSON blob
+    raw_fields = _json.dumps(photo._asset_record.get("fields", {}), default=str)
+
     return {
         "version_checksum": version.checksum,
         "change_tag": photo._asset_record.get("recordChangeTag"),
@@ -693,6 +742,16 @@ def _extract_manifest_metadata(photo: PhotoAsset, version: AssetVersion) -> Dict
         "gps_latitude": gps_latitude,
         "gps_longitude": gps_longitude,
         "gps_altitude": gps_altitude,
+        "gps_speed": gps_speed,
+        "gps_timestamp": gps_timestamp,
+        "timezone_offset": timezone_offset,
+        "asset_subtype": asset_subtype,
+        "hdr_type": hdr_type,
+        "burst_flags": burst_flags,
+        "burst_flags_ext": burst_flags_ext,
+        "burst_id": burst_id,
+        "original_orientation": original_orientation,
+        "raw_fields": raw_fields,
     }
 
 

src/icloudpd/cli.py

@@ -34,7 +34,7 @@ def map_align_raw_to_enum(align_raw_str: str) -> RawTreatmentPolicy:
     return mapping[align_raw_str]
 
 
-_VALID_WRITE_METADATA = {"all", "rating", "keywords", "title", "dates", "orientation"}
+_VALID_WRITE_METADATA = {"all", "rating", "keywords", "title", "dates", "orientation", "location"}
 
 
 def _parse_write_metadata(value: str | None) -> frozenset[str]:

src/icloudpd/manifest.py

@@ -18,9 +18,9 @@
 
 logger = logging.getLogger(__name__)
 
-SCHEMA_VERSION = 1
+SCHEMA_VERSION = 2
 
-_SCHEMA_V1 = """\
+_FRESH_SCHEMA = """\
 CREATE TABLE IF NOT EXISTS manifest (
     asset_id TEXT NOT NULL,
     zone_id TEXT NOT NULL DEFAULT '',
@@ -47,6 +47,16 @@
     gps_latitude REAL,
     gps_longitude REAL,
     gps_altitude REAL,
+    gps_speed REAL,
+    gps_timestamp TEXT,
+    timezone_offset INTEGER,
+    asset_subtype INTEGER,
+    hdr_type INTEGER,
+    burst_flags INTEGER,
+    burst_flags_ext INTEGER,
+    burst_id TEXT,
+    original_orientation INTEGER,
+    raw_fields TEXT,
     PRIMARY KEY (asset_id, zone_id, local_path)
 );
 CREATE INDEX IF NOT EXISTS idx_manifest_path ON manifest(local_path);
@@ -55,8 +65,16 @@
 # Columns added between schema versions, for migration from older DBs.
 # Each entry: (version_introduced, ALTER TABLE statement)
 _MIGRATIONS: list[tuple[int, str]] = [
-    # Future migrations go here, e.g.:
-    # (2, "ALTER TABLE manifest ADD COLUMN new_field TEXT DEFAULT NULL"),
+    (2, "ALTER TABLE manifest ADD COLUMN gps_speed REAL"),
+    (2, "ALTER TABLE manifest ADD COLUMN gps_timestamp TEXT"),
+    (2, "ALTER TABLE manifest ADD COLUMN timezone_offset INTEGER"),
+    (2, "ALTER TABLE manifest ADD COLUMN asset_subtype INTEGER"),
+    (2, "ALTER TABLE manifest ADD COLUMN hdr_type INTEGER"),
+    (2, "ALTER TABLE manifest ADD COLUMN burst_flags INTEGER"),
+    (2, "ALTER TABLE manifest ADD COLUMN burst_flags_ext INTEGER"),
+    (2, "ALTER TABLE manifest ADD COLUMN burst_id TEXT"),
+    (2, "ALTER TABLE manifest ADD COLUMN original_orientation INTEGER"),
+    (2, "ALTER TABLE manifest ADD COLUMN raw_fields TEXT"),
 ]
 
 
@@ -89,14 +107,26 @@ class ManifestRow:
     gps_latitude: float | None
     gps_longitude: float | None
     gps_altitude: float | None
+    gps_speed: float | None
+    gps_timestamp: str | None
+    timezone_offset: int | None
+    asset_subtype: int | None
+    hdr_type: int | None
+    burst_flags: int | None
+    burst_flags_ext: int | None
+    burst_id: str | None
+    original_orientation: int | None
+    raw_fields: str | None
 
 
 _ALL_COLUMNS = (
     "asset_id, zone_id, local_path, version_size, version_checksum, "
     "change_tag, downloaded_at, last_updated_at, item_type, filename, "
     "asset_date, added_date, is_favorite, is_hidden, is_deleted, "
     "original_width, original_height, duration, orientation, "
-    "title, description, keywords, gps_latitude, gps_longitude, gps_altitude"
+    "title, description, keywords, gps_latitude, gps_longitude, gps_altitude, "
+    "gps_speed, gps_timestamp, timezone_offset, asset_subtype, hdr_type, "
+    "burst_flags, burst_flags_ext, burst_id, original_orientation, raw_fields"
 )
 
 
@@ -133,7 +163,7 @@ def open(self) -> None:
             ).fetchone()
             if tables is None:
                 # Brand new DB
-                self._conn.executescript(_SCHEMA_V1)
+                self._conn.executescript(_FRESH_SCHEMA)
             else:
                 # Pre-versioned DB (has table but no user_version) — migrate
                 self._migrate_from_v0()
@@ -167,6 +197,16 @@ def _migrate_from_v0(self) -> None:
             ("gps_latitude", "REAL"),
             ("gps_longitude", "REAL"),
             ("gps_altitude", "REAL"),
+            ("gps_speed", "REAL"),
+            ("gps_timestamp", "TEXT"),
+            ("timezone_offset", "INTEGER"),
+            ("asset_subtype", "INTEGER"),
+            ("hdr_type", "INTEGER"),
+            ("burst_flags", "INTEGER"),
+            ("burst_flags_ext", "INTEGER"),
+            ("burst_id", "TEXT"),
+            ("original_orientation", "INTEGER"),
+            ("raw_fields", "TEXT"),
         ]
         for col_name, col_def in new_columns:
             if col_name not in existing:
@@ -256,13 +296,23 @@ def upsert(
         gps_latitude: float | None = None,
         gps_longitude: float | None = None,
         gps_altitude: float | None = None,
+        gps_speed: float | None = None,
+        gps_timestamp: str | None = None,
+        timezone_offset: int | None = None,
+        asset_subtype: int | None = None,
+        hdr_type: int | None = None,
+        burst_flags: int | None = None,
+        burst_flags_ext: int | None = None,
+        burst_id: str | None = None,
+        original_orientation: int | None = None,
+        raw_fields: str | None = None,
     ) -> None:
         """Insert or update a manifest entry. Auto-flushes every 500 writes."""
         try:
             now = datetime.now(tz=timezone.utc).isoformat()
             self._db.execute(
                 f"INSERT INTO manifest ({_ALL_COLUMNS}) "
-                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "
+                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) "
                 "ON CONFLICT(asset_id, zone_id, local_path) DO UPDATE SET "
                 "version_size=excluded.version_size, "
                 "version_checksum=excluded.version_checksum, "
@@ -284,13 +334,25 @@ def upsert(
                 "keywords=excluded.keywords, "
                 "gps_latitude=excluded.gps_latitude, "
                 "gps_longitude=excluded.gps_longitude, "
-                "gps_altitude=excluded.gps_altitude",
+                "gps_altitude=excluded.gps_altitude, "
+                "gps_speed=excluded.gps_speed, "
+                "gps_timestamp=excluded.gps_timestamp, "
+                "timezone_offset=excluded.timezone_offset, "
+                "asset_subtype=excluded.asset_subtype, "
+                "hdr_type=excluded.hdr_type, "
+                "burst_flags=excluded.burst_flags, "
+                "burst_flags_ext=excluded.burst_flags_ext, "
+                "burst_id=excluded.burst_id, "
+                "original_orientation=excluded.original_orientation, "
+                "raw_fields=excluded.raw_fields",
                 (
                     asset_id, zone_id, local_path, version_size, version_checksum,
                     change_tag, now, now, item_type, filename,
                     asset_date, added_date, is_favorite, is_hidden, is_deleted,
                     original_width, original_height, duration, orientation,
                     title, description, keywords, gps_latitude, gps_longitude, gps_altitude,
+                    gps_speed, gps_timestamp, timezone_offset, asset_subtype, hdr_type,
+                    burst_flags, burst_flags_ext, burst_id, original_orientation, raw_fields,
                 ),
             )
             self._dirty = True