Add movie history for site updates

scripts/import_site.py

@@ -13,32 +13,113 @@ Import rules:
 - If the database has no update date but the site does, the movie is updated.
 - Movies without a newer site update date are left unchanged.
 
-When an existing movie is updated, all fields owned by the site are replaced:
+When an existing movie is updated:
-name, duration, web URL, thumbnail URL, publication date, and update date.
+- The complete current movie row is copied to movie_history.
-Manually maintained metadata fields are not modified.
+- change_summary describes which site-owned fields changed.
+- The site-owned fields in movie are then replaced.
+- The history insert and movie update occur in the same transaction.
+
+History is created only by this site import. OCR processing, CSV migration,
+and manual database changes do not trigger history creation.
+
+Fields owned by the site:
+- name
+- duration_seconds
+- web_url
+- thumbnail
+- published
+- updated
+
+Manually maintained metadata fields are not modified by the site import.
 
 Durations from the site are accepted as mm:ss or hh:mm:ss and stored as
 seconds in the duration_seconds column.
 
-Input:
+Default input:
     /storage/disk1/WCX/import/wcx_site_index.json
 
-Database:
+Default database:
     /storage/disk1/WCX/database/wcx.db
 
-The script prints a summary showing the number of inserted, updated, and
+Usage:
-unchanged records.
+    import_site.py
+
+    import_site.py \
+        --input /path/to/wcx_site_index.json \
+        --database /path/to/wcx.db
+
+The script prints a summary showing the number of inserted, updated,
+historized, and unchanged records.
 """
 
-
+import argparse
 import json
 import sqlite3
 from pathlib import Path
 from typing import Any
 
 
-JSON_FILE = Path("/storage/disk1/WCX/import/wcx_site_index.json")
+DEFAULT_JSON_FILE = Path(
-DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db")
+    "/storage/disk1/WCX/import/wcx_site_index.json"
+)
+
+DEFAULT_DATABASE_FILE = Path(
+    "/storage/disk1/WCX/database/wcx.db"
+)
+
+SITE_FIELDS = (
+    "name",
+    "duration_seconds",
+    "web_url",
+    "thumbnail",
+    "published",
+    "updated",
+)
+
+MOVIE_HISTORY_FIELDS = (
+    "movie_id",
+    "name",
+    "aka",
+    "nationality",
+    "age",
+    "shoot_location",
+    "shoot_date",
+    "duration_seconds",
+    "description",
+    "rating",
+    "web_url",
+    "thumbnail",
+    "published",
+    "updated",
+    "created_at",
+    "modified_at",
+    "ocr_status",
+    "ocr_raw_text",
+    "ocr_error",
+    "ocr_processed_at",
+)
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Import WCX site metadata into SQLite."
+    )
+
+    parser.add_argument(
+        "--input",
+        type=Path,
+        default=DEFAULT_JSON_FILE,
+        help=f"Input JSON file; default: {DEFAULT_JSON_FILE}",
+    )
+
+    parser.add_argument(
+        "--database",
+        type=Path,
+        default=DEFAULT_DATABASE_FILE,
+        help=f"SQLite database; default: {DEFAULT_DATABASE_FILE}",
+    )
+
+    return parser.parse_args()
 
 
 def parse_duration(value: str | None) -> int | None:
@@ -59,21 +140,31 @@ def parse_duration(value: str | None) -> int | None:
     elif len(numbers) == 3:
         hours, minutes, seconds = numbers
     else:
-        raise ValueError(f"Ogiltigt duration-format: {value!r}")
+        raise ValueError(
+            f"Ogiltigt duration-format: {value!r}"
+        )
 
     if hours < 0 or minutes < 0 or seconds < 0:
-        raise ValueError(f"Speltiden får inte vara negativ: {value!r}")
+        raise ValueError(
+            f"Speltiden får inte vara negativ: {value!r}"
+        )
 
     if seconds >= 60:
-        raise ValueError(f"Ogiltigt sekundvärde i speltid: {value!r}")
+        raise ValueError(
+            f"Ogiltigt sekundvärde i speltid: {value!r}"
+        )
 
     if len(numbers) == 3 and minutes >= 60:
-        raise ValueError(f"Ogiltigt minutvärde i speltid: {value!r}")
+        raise ValueError(
+            f"Ogiltigt minutvärde i speltid: {value!r}"
+        )
 
     return hours * 3600 + minutes * 60 + seconds
 
 
-def get_site_updated(movie: dict[str, Any]) -> str | None:
+def get_site_updated(
+    movie: dict[str, Any],
+) -> str | None:
     """
     Read the site's update date.
 
@@ -83,25 +174,50 @@ def get_site_updated(movie: dict[str, Any]) -> str | None:
     return movie.get("update") or movie.get("updated")
 
 
-def validate_movie(movie: Any, index: int) -> dict[str, Any]:
+def validate_movie(
+    movie: Any,
+    index: int,
+) -> dict[str, Any]:
     if not isinstance(movie, dict):
-        raise ValueError(f"Post {index} är inte ett JSON-objekt.")
+        raise ValueError(
+            f"Post {index} är inte ett JSON-objekt."
+        )
 
     for required_field in ("id", "titel"):
         value = movie.get(required_field)
 
         if not isinstance(value, str) or not value.strip():
             raise ValueError(
-                f"Post {index} saknar ett giltigt fält: {required_field}"
+                f"Post {index} saknar ett giltigt fält: "
+                f"{required_field}"
             )
 
     return movie
 
 
-def insert_movie(
+def build_site_values(
-    connection: sqlite3.Connection,
     movie: dict[str, Any],
     site_updated: str | None,
+) -> dict[str, Any]:
+    """
+    Convert one JSON movie to the site-owned values stored in SQLite.
+    """
+    return {
+        "name": movie["titel"],
+        "duration_seconds": parse_duration(
+            movie.get("duration")
+        ),
+        "web_url": movie.get("details"),
+        "thumbnail": movie.get("thumb"),
+        "published": movie.get("published"),
+        "updated": site_updated,
+    }
+
+
+def insert_movie(
+    connection: sqlite3.Connection,
+    movie_id: str,
+    site_values: dict[str, Any],
 ) -> None:
     connection.execute(
         """
@@ -117,21 +233,167 @@ def insert_movie(
         VALUES (?, ?, ?, ?, ?, ?, ?)
         """,
         (
-            movie["id"],
+            movie_id,
-            movie["titel"],
+            site_values["name"],
-            parse_duration(movie.get("duration")),
+            site_values["duration_seconds"],
-            movie.get("details"),
+            site_values["web_url"],
-            movie.get("thumb"),
+            site_values["thumbnail"],
-            movie.get("published"),
+            site_values["published"],
-            site_updated,
+            site_values["updated"],
+        ),
+    )
+
+
+def format_value(
+    field_name: str,
+    value: Any,
+) -> str:
+    """
+    Format a database value for the human-readable change summary.
+    """
+    if value is None:
+        return "NULL"
+
+    if field_name == "duration_seconds":
+        total_seconds = int(value)
+        hours, remainder = divmod(total_seconds, 3600)
+        minutes, seconds = divmod(remainder, 60)
+
+        if hours:
+            return f"{hours}:{minutes:02d}:{seconds:02d}"
+
+        return f"{minutes}:{seconds:02d}"
+
+    return str(value)
+
+
+def build_change_summary(
+    existing: sqlite3.Row,
+    site_values: dict[str, Any],
+) -> str:
+    """
+    Describe changes to fields owned by the site.
+    """
+    changes: list[str] = []
+
+    for field_name in SITE_FIELDS:
+        old_value = existing[field_name]
+        new_value = site_values[field_name]
+
+        if old_value == new_value:
+            continue
+
+        old_text = format_value(
+            field_name,
+            old_value,
+        )
+        new_text = format_value(
+            field_name,
+            new_value,
+        )
+
+        summary = (
+            f"{field_name}: {old_text} -> {new_text}"
+        )
+
+        if (
+            field_name == "duration_seconds"
+            and old_value is not None
+            and new_value is not None
+        ):
+            difference = int(new_value) - int(old_value)
+            sign = "+" if difference >= 0 else "-"
+            absolute_difference = abs(difference)
+
+            difference_text = format_value(
+                field_name,
+                absolute_difference,
+            )
+
+            summary += f" ({sign}{difference_text})"
+
+        changes.append(summary)
+
+    if not changes:
+        return "Site update date was newer, but no site fields changed."
+
+    return "; ".join(changes)
+
+
+def archive_movie(
+    connection: sqlite3.Connection,
+    existing: sqlite3.Row,
+    change_summary: str,
+) -> None:
+    """
+    Copy the complete current movie row to movie_history.
+
+    This function is called only from the site import immediately before
+    the current movie row is updated.
+    """
+    connection.execute(
+        """
+        INSERT INTO movie_history (
+            movie_id,
+            name,
+            aka,
+            nationality,
+            age,
+            shoot_location,
+            shoot_date,
+            duration_seconds,
+            description,
+            rating,
+            web_url,
+            thumbnail,
+            published,
+            updated,
+            created_at,
+            modified_at,
+            ocr_status,
+            ocr_raw_text,
+            ocr_error,
+            ocr_processed_at,
+            change_source,
+            change_summary
+        )
+        VALUES (
+            ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+            ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
+            'site_import',
+            ?
+        )
+        """,
+        (
+            existing["id"],
+            existing["name"],
+            existing["aka"],
+            existing["nationality"],
+            existing["age"],
+            existing["shoot_location"],
+            existing["shoot_date"],
+            existing["duration_seconds"],
+            existing["description"],
+            existing["rating"],
+            existing["web_url"],
+            existing["thumbnail"],
+            existing["published"],
+            existing["updated"],
+            existing["created_at"],
+            existing["modified_at"],
+            existing["ocr_status"],
+            existing["ocr_raw_text"],
+            existing["ocr_error"],
+            existing["ocr_processed_at"],
+            change_summary,
         ),
     )
 
 
 def update_movie(
     connection: sqlite3.Connection,
-    movie: dict[str, Any],
+    movie_id: str,
-    site_updated: str,
+    site_values: dict[str, Any],
 ) -> None:
     connection.execute(
         """
@@ -147,43 +409,93 @@ def update_movie(
         WHERE id = ?
         """,
         (
-            movie["titel"],
+            site_values["name"],
-            parse_duration(movie.get("duration")),
+            site_values["duration_seconds"],
-            movie.get("details"),
+            site_values["web_url"],
-            movie.get("thumb"),
+            site_values["thumbnail"],
-            movie.get("published"),
+            site_values["published"],
-            site_updated,
+            site_values["updated"],
-            movie["id"],
+            movie_id,
         ),
     )
 
 
 def main() -> None:
-    if not JSON_FILE.is_file():
+    args = parse_arguments()
-        raise FileNotFoundError(f"JSON-filen finns inte: {JSON_FILE}")
 
-    if not DATABASE_FILE.is_file():
+    json_file = args.input
-        raise FileNotFoundError(f"Databasen finns inte: {DATABASE_FILE}")
+    database_file = args.database
 
-    with JSON_FILE.open("r", encoding="utf-8") as file:
+    if not json_file.is_file():
+        raise FileNotFoundError(
+            f"JSON-filen finns inte: {json_file}"
+        )
+
+    if not database_file.is_file():
+        raise FileNotFoundError(
+            f"Databasen finns inte: {database_file}"
+        )
+
+    with json_file.open(
+        "r",
+        encoding="utf-8",
+    ) as file:
         movies = json.load(file)
 
     if not isinstance(movies, list):
-        raise ValueError("JSON-filen måste innehålla en array.")
+        raise ValueError(
+            "JSON-filen måste innehålla en array."
+        )
 
     inserted = 0
     updated = 0
+    historized = 0
     unchanged = 0
 
-    with sqlite3.connect(DATABASE_FILE) as connection:
+    with sqlite3.connect(database_file) as connection:
-        for index, raw_movie in enumerate(movies, start=1):
+        connection.row_factory = sqlite3.Row
-            movie = validate_movie(raw_movie, index)
+        connection.execute("PRAGMA foreign_keys = ON")
+
+        for index, raw_movie in enumerate(
+            movies,
+            start=1,
+        ):
+            movie = validate_movie(
+                raw_movie,
+                index,
+            )
+
             movie_id = movie["id"]
             site_updated = get_site_updated(movie)
 
+            site_values = build_site_values(
+                movie,
+                site_updated,
+            )
+
             existing = connection.execute(
                 """
-                SELECT updated
+                SELECT
+                    id,
+                    name,
+                    aka,
+                    nationality,
+                    age,
+                    shoot_location,
+                    shoot_date,
+                    duration_seconds,
+                    description,
+                    rating,
+                    web_url,
+                    thumbnail,
+                    published,
+                    updated,
+                    created_at,
+                    modified_at,
+                    ocr_status,
+                    ocr_raw_text,
+                    ocr_error,
+                    ocr_processed_at
                 FROM movie
                 WHERE id = ?
                 """,
@@ -191,11 +503,16 @@ def main() -> None:
             ).fetchone()
 
             if existing is None:
-                insert_movie(connection, movie, site_updated)
+                insert_movie(
+                    connection,
+                    movie_id,
+                    site_values,
+                )
+
                 inserted += 1
                 continue
 
-            database_updated = existing[0]
+            database_updated = existing["updated"]
 
             should_update = (
                 site_updated is not None
@@ -205,18 +522,45 @@ def main() -> None:
                 )
             )
 
-            if should_update:
+            if not should_update:
-                update_movie(connection, movie, site_updated)
-                updated += 1
-            else:
                 unchanged += 1
+                continue
+
+            change_summary = build_change_summary(
+                existing,
+                site_values,
+            )
+
+            archive_movie(
+                connection,
+                existing,
+                change_summary,
+            )
+
+            update_movie(
+                connection,
+                movie_id,
+                site_values,
+            )
+
+            historized += 1
+            updated += 1
 
     print(f"Poster i JSON: {len(movies)}")
     print(f"Tillagda:       {inserted}")
     print(f"Uppdaterade:    {updated}")
+    print(f"Historiserade:  {historized}")
     print(f"Oförändrade:    {unchanged}")
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except (
+        FileNotFoundError,
+        ValueError,
+        json.JSONDecodeError,
+        sqlite3.Error,
+    ) as error:
+        raise SystemExit(f"Fel: {error}") from error
 

scripts/schema.sql

@@ -21,3 +21,43 @@ CREATE TABLE IF NOT EXISTS movie (
     CHECK (rating IS NULL OR rating BETWEEN 0 AND 10),
     CHECK (duration_seconds IS NULL OR duration_seconds >= 0)
 );
+
+CREATE TABLE IF NOT EXISTS movie_history (
+    history_id INTEGER PRIMARY KEY AUTOINCREMENT,
+
+    movie_id TEXT NOT NULL,
+
+    name TEXT NOT NULL,
+    aka TEXT,
+    nationality TEXT,
+    age INTEGER,
+    shoot_location TEXT,
+    shoot_date TEXT,
+    duration_seconds INTEGER,
+    description TEXT,
+    rating REAL,
+    web_url TEXT,
+    thumbnail TEXT,
+    published TEXT,
+    updated TEXT,
+
+    created_at TEXT,
+    modified_at TEXT,
+
+    ocr_status TEXT,
+    ocr_raw_text TEXT,
+    ocr_error TEXT,
+    ocr_processed_at TEXT,
+
+    archived_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    change_source TEXT NOT NULL DEFAULT 'site_import',
+    change_summary TEXT NOT NULL,
+
+    FOREIGN KEY (movie_id) REFERENCES movie(id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_movie_history_movie_id
+    ON movie_history(movie_id);
+
+CREATE INDEX IF NOT EXISTS idx_movie_history_archived_at
+    ON movie_history(archived_at);