diff --git a/scripts/import_site.py b/scripts/import_site.py index 7ad009d..691f208 100755 --- a/scripts/import_site.py +++ b/scripts/import_site.py @@ -13,32 +13,113 @@ Import rules: - If the database has no update date but the site does, the movie is updated. - Movies without a newer site update date are left unchanged. -When an existing movie is updated, all fields owned by the site are replaced: -name, duration, web URL, thumbnail URL, publication date, and update date. -Manually maintained metadata fields are not modified. +When an existing movie is updated: +- The complete current movie row is copied to movie_history. +- change_summary describes which site-owned fields changed. +- The site-owned fields in movie are then replaced. +- The history insert and movie update occur in the same transaction. + +History is created only by this site import. OCR processing, CSV migration, +and manual database changes do not trigger history creation. + +Fields owned by the site: +- name +- duration_seconds +- web_url +- thumbnail +- published +- updated + +Manually maintained metadata fields are not modified by the site import. Durations from the site are accepted as mm:ss or hh:mm:ss and stored as seconds in the duration_seconds column. -Input: +Default input: /storage/disk1/WCX/import/wcx_site_index.json -Database: +Default database: /storage/disk1/WCX/database/wcx.db -The script prints a summary showing the number of inserted, updated, and -unchanged records. +Usage: + import_site.py + + import_site.py \ + --input /path/to/wcx_site_index.json \ + --database /path/to/wcx.db + +The script prints a summary showing the number of inserted, updated, +historized, and unchanged records. """ - +import argparse import json import sqlite3 from pathlib import Path from typing import Any -JSON_FILE = Path("/storage/disk1/WCX/import/wcx_site_index.json") -DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db") +DEFAULT_JSON_FILE = Path( + "/storage/disk1/WCX/import/wcx_site_index.json" +) + +DEFAULT_DATABASE_FILE = Path( + "/storage/disk1/WCX/database/wcx.db" +) + +SITE_FIELDS = ( + "name", + "duration_seconds", + "web_url", + "thumbnail", + "published", + "updated", +) + +MOVIE_HISTORY_FIELDS = ( + "movie_id", + "name", + "aka", + "nationality", + "age", + "shoot_location", + "shoot_date", + "duration_seconds", + "description", + "rating", + "web_url", + "thumbnail", + "published", + "updated", + "created_at", + "modified_at", + "ocr_status", + "ocr_raw_text", + "ocr_error", + "ocr_processed_at", +) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Import WCX site metadata into SQLite." + ) + + parser.add_argument( + "--input", + type=Path, + default=DEFAULT_JSON_FILE, + help=f"Input JSON file; default: {DEFAULT_JSON_FILE}", + ) + + parser.add_argument( + "--database", + type=Path, + default=DEFAULT_DATABASE_FILE, + help=f"SQLite database; default: {DEFAULT_DATABASE_FILE}", + ) + + return parser.parse_args() def parse_duration(value: str | None) -> int | None: @@ -59,21 +140,31 @@ def parse_duration(value: str | None) -> int | None: elif len(numbers) == 3: hours, minutes, seconds = numbers else: - raise ValueError(f"Ogiltigt duration-format: {value!r}") + raise ValueError( + f"Ogiltigt duration-format: {value!r}" + ) if hours < 0 or minutes < 0 or seconds < 0: - raise ValueError(f"Speltiden får inte vara negativ: {value!r}") + raise ValueError( + f"Speltiden får inte vara negativ: {value!r}" + ) if seconds >= 60: - raise ValueError(f"Ogiltigt sekundvärde i speltid: {value!r}") + raise ValueError( + f"Ogiltigt sekundvärde i speltid: {value!r}" + ) if len(numbers) == 3 and minutes >= 60: - raise ValueError(f"Ogiltigt minutvärde i speltid: {value!r}") + raise ValueError( + f"Ogiltigt minutvärde i speltid: {value!r}" + ) return hours * 3600 + minutes * 60 + seconds -def get_site_updated(movie: dict[str, Any]) -> str | None: +def get_site_updated( + movie: dict[str, Any], +) -> str | None: """ Read the site's update date. @@ -83,25 +174,50 @@ def get_site_updated(movie: dict[str, Any]) -> str | None: return movie.get("update") or movie.get("updated") -def validate_movie(movie: Any, index: int) -> dict[str, Any]: +def validate_movie( + movie: Any, + index: int, +) -> dict[str, Any]: if not isinstance(movie, dict): - raise ValueError(f"Post {index} är inte ett JSON-objekt.") + raise ValueError( + f"Post {index} är inte ett JSON-objekt." + ) for required_field in ("id", "titel"): value = movie.get(required_field) if not isinstance(value, str) or not value.strip(): raise ValueError( - f"Post {index} saknar ett giltigt fält: {required_field}" + f"Post {index} saknar ett giltigt fält: " + f"{required_field}" ) return movie -def insert_movie( - connection: sqlite3.Connection, +def build_site_values( movie: dict[str, Any], site_updated: str | None, +) -> dict[str, Any]: + """ + Convert one JSON movie to the site-owned values stored in SQLite. + """ + return { + "name": movie["titel"], + "duration_seconds": parse_duration( + movie.get("duration") + ), + "web_url": movie.get("details"), + "thumbnail": movie.get("thumb"), + "published": movie.get("published"), + "updated": site_updated, + } + + +def insert_movie( + connection: sqlite3.Connection, + movie_id: str, + site_values: dict[str, Any], ) -> None: connection.execute( """ @@ -117,21 +233,167 @@ def insert_movie( VALUES (?, ?, ?, ?, ?, ?, ?) """, ( - movie["id"], - movie["titel"], - parse_duration(movie.get("duration")), - movie.get("details"), - movie.get("thumb"), - movie.get("published"), - site_updated, + movie_id, + site_values["name"], + site_values["duration_seconds"], + site_values["web_url"], + site_values["thumbnail"], + site_values["published"], + site_values["updated"], + ), + ) + + +def format_value( + field_name: str, + value: Any, +) -> str: + """ + Format a database value for the human-readable change summary. + """ + if value is None: + return "NULL" + + if field_name == "duration_seconds": + total_seconds = int(value) + hours, remainder = divmod(total_seconds, 3600) + minutes, seconds = divmod(remainder, 60) + + if hours: + return f"{hours}:{minutes:02d}:{seconds:02d}" + + return f"{minutes}:{seconds:02d}" + + return str(value) + + +def build_change_summary( + existing: sqlite3.Row, + site_values: dict[str, Any], +) -> str: + """ + Describe changes to fields owned by the site. + """ + changes: list[str] = [] + + for field_name in SITE_FIELDS: + old_value = existing[field_name] + new_value = site_values[field_name] + + if old_value == new_value: + continue + + old_text = format_value( + field_name, + old_value, + ) + new_text = format_value( + field_name, + new_value, + ) + + summary = ( + f"{field_name}: {old_text} -> {new_text}" + ) + + if ( + field_name == "duration_seconds" + and old_value is not None + and new_value is not None + ): + difference = int(new_value) - int(old_value) + sign = "+" if difference >= 0 else "-" + absolute_difference = abs(difference) + + difference_text = format_value( + field_name, + absolute_difference, + ) + + summary += f" ({sign}{difference_text})" + + changes.append(summary) + + if not changes: + return "Site update date was newer, but no site fields changed." + + return "; ".join(changes) + + +def archive_movie( + connection: sqlite3.Connection, + existing: sqlite3.Row, + change_summary: str, +) -> None: + """ + Copy the complete current movie row to movie_history. + + This function is called only from the site import immediately before + the current movie row is updated. + """ + connection.execute( + """ + INSERT INTO movie_history ( + movie_id, + name, + aka, + nationality, + age, + shoot_location, + shoot_date, + duration_seconds, + description, + rating, + web_url, + thumbnail, + published, + updated, + created_at, + modified_at, + ocr_status, + ocr_raw_text, + ocr_error, + ocr_processed_at, + change_source, + change_summary + ) + VALUES ( + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, + 'site_import', + ? + ) + """, + ( + existing["id"], + existing["name"], + existing["aka"], + existing["nationality"], + existing["age"], + existing["shoot_location"], + existing["shoot_date"], + existing["duration_seconds"], + existing["description"], + existing["rating"], + existing["web_url"], + existing["thumbnail"], + existing["published"], + existing["updated"], + existing["created_at"], + existing["modified_at"], + existing["ocr_status"], + existing["ocr_raw_text"], + existing["ocr_error"], + existing["ocr_processed_at"], + change_summary, ), ) def update_movie( connection: sqlite3.Connection, - movie: dict[str, Any], - site_updated: str, + movie_id: str, + site_values: dict[str, Any], ) -> None: connection.execute( """ @@ -147,43 +409,93 @@ def update_movie( WHERE id = ? """, ( - movie["titel"], - parse_duration(movie.get("duration")), - movie.get("details"), - movie.get("thumb"), - movie.get("published"), - site_updated, - movie["id"], + site_values["name"], + site_values["duration_seconds"], + site_values["web_url"], + site_values["thumbnail"], + site_values["published"], + site_values["updated"], + movie_id, ), ) def main() -> None: - if not JSON_FILE.is_file(): - raise FileNotFoundError(f"JSON-filen finns inte: {JSON_FILE}") + args = parse_arguments() - if not DATABASE_FILE.is_file(): - raise FileNotFoundError(f"Databasen finns inte: {DATABASE_FILE}") + json_file = args.input + database_file = args.database - with JSON_FILE.open("r", encoding="utf-8") as file: + if not json_file.is_file(): + raise FileNotFoundError( + f"JSON-filen finns inte: {json_file}" + ) + + if not database_file.is_file(): + raise FileNotFoundError( + f"Databasen finns inte: {database_file}" + ) + + with json_file.open( + "r", + encoding="utf-8", + ) as file: movies = json.load(file) if not isinstance(movies, list): - raise ValueError("JSON-filen måste innehålla en array.") + raise ValueError( + "JSON-filen måste innehålla en array." + ) inserted = 0 updated = 0 + historized = 0 unchanged = 0 - with sqlite3.connect(DATABASE_FILE) as connection: - for index, raw_movie in enumerate(movies, start=1): - movie = validate_movie(raw_movie, index) + with sqlite3.connect(database_file) as connection: + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + + for index, raw_movie in enumerate( + movies, + start=1, + ): + movie = validate_movie( + raw_movie, + index, + ) + movie_id = movie["id"] site_updated = get_site_updated(movie) + site_values = build_site_values( + movie, + site_updated, + ) + existing = connection.execute( """ - SELECT updated + SELECT + id, + name, + aka, + nationality, + age, + shoot_location, + shoot_date, + duration_seconds, + description, + rating, + web_url, + thumbnail, + published, + updated, + created_at, + modified_at, + ocr_status, + ocr_raw_text, + ocr_error, + ocr_processed_at FROM movie WHERE id = ? """, @@ -191,11 +503,16 @@ def main() -> None: ).fetchone() if existing is None: - insert_movie(connection, movie, site_updated) + insert_movie( + connection, + movie_id, + site_values, + ) + inserted += 1 continue - database_updated = existing[0] + database_updated = existing["updated"] should_update = ( site_updated is not None @@ -205,18 +522,45 @@ def main() -> None: ) ) - if should_update: - update_movie(connection, movie, site_updated) - updated += 1 - else: + if not should_update: unchanged += 1 + continue + + change_summary = build_change_summary( + existing, + site_values, + ) + + archive_movie( + connection, + existing, + change_summary, + ) + + update_movie( + connection, + movie_id, + site_values, + ) + + historized += 1 + updated += 1 print(f"Poster i JSON: {len(movies)}") print(f"Tillagda: {inserted}") print(f"Uppdaterade: {updated}") + print(f"Historiserade: {historized}") print(f"Oförändrade: {unchanged}") if __name__ == "__main__": - main() + try: + main() + except ( + FileNotFoundError, + ValueError, + json.JSONDecodeError, + sqlite3.Error, + ) as error: + raise SystemExit(f"Fel: {error}") from error diff --git a/scripts/schema.sql b/scripts/schema.sql index 6ac5ceb..9eaf8b6 100644 --- a/scripts/schema.sql +++ b/scripts/schema.sql @@ -21,3 +21,43 @@ CREATE TABLE IF NOT EXISTS movie ( CHECK (rating IS NULL OR rating BETWEEN 0 AND 10), CHECK (duration_seconds IS NULL OR duration_seconds >= 0) ); + +CREATE TABLE IF NOT EXISTS movie_history ( + history_id INTEGER PRIMARY KEY AUTOINCREMENT, + + movie_id TEXT NOT NULL, + + name TEXT NOT NULL, + aka TEXT, + nationality TEXT, + age INTEGER, + shoot_location TEXT, + shoot_date TEXT, + duration_seconds INTEGER, + description TEXT, + rating REAL, + web_url TEXT, + thumbnail TEXT, + published TEXT, + updated TEXT, + + created_at TEXT, + modified_at TEXT, + + ocr_status TEXT, + ocr_raw_text TEXT, + ocr_error TEXT, + ocr_processed_at TEXT, + + archived_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + change_source TEXT NOT NULL DEFAULT 'site_import', + change_summary TEXT NOT NULL, + + FOREIGN KEY (movie_id) REFERENCES movie(id) +); + +CREATE INDEX IF NOT EXISTS idx_movie_history_movie_id + ON movie_history(movie_id); + +CREATE INDEX IF NOT EXISTS idx_movie_history_archived_at + ON movie_history(archived_at);