Add batch processing for pending OCR

scripts/check_ocr.py

@@ -4,7 +4,7 @@
 Run and validate OCR processing for one movie in the SQLite index.
 
 The script accepts a WCX movie ID, retrieves the movie name and thumbnail URL
-from the database, and then coordinates the complete OCR workflow:
+from the selected database, and coordinates the complete OCR workflow:
 
     1. Run ocr.sh for the thumbnail URL.
     2. Pass the raw OCR text to parse_ocr.py.
@@ -30,21 +30,48 @@ updated.
 
 Usage:
     check_ocr.py <movie-id>
+    check_ocr.py <movie-id> --database /path/to/wcx.db
 
 Example:
     check_ocr.py susana-melo_6707
 """
 
-
+import argparse
 import sqlite3
 import subprocess
 import sys
 from pathlib import Path
 
 
-DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db")
+DEFAULT_DATABASE_FILE = Path(
-OCR_SCRIPT = Path("/storage/disk1/WCX/scripts/ocr.sh")
+    "/storage/disk1/WCX/database/wcx.db"
-PARSER_SCRIPT = Path("/storage/disk1/WCX/scripts/parse_ocr.py")
+)
+OCR_SCRIPT = Path(
+    "/storage/disk1/WCX/scripts/ocr.sh"
+)
+PARSER_SCRIPT = Path(
+    "/storage/disk1/WCX/scripts/parse_ocr.py"
+)
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Run OCR processing for one WCX movie."
+    )
+
+    parser.add_argument(
+        "movie_id",
+        help="Movie ID to process.",
+    )
+
+    parser.add_argument(
+        "--database",
+        type=Path,
+        default=DEFAULT_DATABASE_FILE,
+        help=f"SQLite database; default: {DEFAULT_DATABASE_FILE}",
+    )
+
+    return parser.parse_args()
 
 
 def parse_key_value_output(output: str) -> dict[str, str]:
@@ -147,13 +174,26 @@ def set_ocr_completed(
 
 
 def main() -> None:
-    if len(sys.argv) != 2:
+    args = parse_arguments()
-        print(f"Användning: {sys.argv[0]} <movie-id>", file=sys.stderr)
+    movie_id = args.movie_id
-        sys.exit(1)
+    database_file = args.database
 
-    movie_id = sys.argv[1]
+    if not database_file.is_file():
+        raise FileNotFoundError(
+            f"Database file does not exist: {database_file}"
+        )
 
-    with sqlite3.connect(DATABASE_FILE) as connection:
+    if not OCR_SCRIPT.is_file():
+        raise FileNotFoundError(
+            f"OCR script does not exist: {OCR_SCRIPT}"
+        )
+
+    if not PARSER_SCRIPT.is_file():
+        raise FileNotFoundError(
+            f"OCR parser does not exist: {PARSER_SCRIPT}"
+        )
+
+    with sqlite3.connect(database_file) as connection:
         row = connection.execute(
             """
             SELECT name, thumbnail
@@ -165,7 +205,7 @@ def main() -> None:
 
         if row is None:
             print(
-                f"Filmen finns inte i databasen: {movie_id}",
+                f"Movie does not exist in database: {movie_id}",
                 file=sys.stderr,
             )
             sys.exit(1)
@@ -173,7 +213,7 @@ def main() -> None:
         database_name, thumbnail = row
 
         if not thumbnail:
-            error_message = "Filmen saknar thumbnail."
+            error_message = "Movie has no thumbnail."
 
             set_ocr_failed(
                 connection,
@@ -193,7 +233,7 @@ def main() -> None:
         if ocr_result.returncode != 0:
             error_message = (
                 ocr_result.stderr.strip()
-                or "OCR-körningen misslyckades."
+                or "OCR execution failed."
             )
 
             set_ocr_failed(
@@ -203,7 +243,7 @@ def main() -> None:
                 ocr_result.stdout.strip() or None,
             )
 
-            print("OCR-körningen misslyckades:", file=sys.stderr)
+            print("OCR execution failed:", file=sys.stderr)
             print(error_message, file=sys.stderr)
             sys.exit(1)
 
@@ -219,7 +259,7 @@ def main() -> None:
         if parser_result.returncode != 0:
             error_message = (
                 parser_result.stderr.strip()
-                or "Tolkningen av OCR-resultatet misslyckades."
+                or "OCR output parsing failed."
             )
 
             set_manual_review(
@@ -230,7 +270,7 @@ def main() -> None:
             )
 
             print(
-                "Tolkningen av OCR-resultatet misslyckades:",
+                "OCR output parsing failed:",
                 file=sys.stderr,
             )
             print(error_message, file=sys.stderr)
@@ -253,7 +293,7 @@ def main() -> None:
 
         if missing_fields:
             error_message = (
-                "Parsern saknar fält: "
+                "Parser output is missing fields: "
                 + ", ".join(missing_fields)
             )
 
@@ -275,7 +315,7 @@ def main() -> None:
 
         if not names_match:
             error_message = (
-                "OCR-namnet matchar inte databasnamnet: "
+                "OCR name does not match database name: "
                 f"{ocr_name!r} != {database_name!r}"
             )
 
@@ -299,6 +339,7 @@ def main() -> None:
         )
 
     print(f"id={movie_id}")
+    print(f"database={database_file}")
     print(f"database_name={database_name}")
     print(f"ocr_name={ocr_name}")
     print("name_match=yes")
@@ -309,5 +350,11 @@ def main() -> None:
 
 
 if __name__ == "__main__":
+    try:
         main()
-
+    except (
+        FileNotFoundError,
+        sqlite3.Error,
+    ) as error:
+        print(f"Error: {error}", file=sys.stderr)
+        sys.exit(1)

scripts/process_pending_ocr.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+
+"""
+Process all movies waiting for OCR in the WCX SQLite index.
+
+The script queries the movie table for records whose OCR status is "pending"
+and invokes check_ocr.py once for each movie ID.
+
+check_ocr.py performs the actual OCR request, parsing, validation, and database
+update. This batch script only coordinates those individual executions.
+
+Processing continues if one movie fails. A final summary reports how many
+movies completed successfully and how many failed or require manual review.
+
+By default, all pending movies are processed. Use --limit to restrict the
+number of movies during testing.
+
+Usage:
+    process_pending_ocr.py
+    process_pending_ocr.py --limit 1
+    process_pending_ocr.py --database /path/to/wcx-test.db
+
+Requirements:
+    GOOGLE_VISION_API_KEY must be available in the environment.
+"""
+
+import argparse
+import os
+import sqlite3
+import subprocess
+import sys
+from pathlib import Path
+
+
+DEFAULT_DATABASE_FILE = Path(
+    "/storage/disk1/WCX/database/wcx.db"
+)
+CHECK_OCR_SCRIPT = Path(
+    "/storage/disk1/WCX/scripts/check_ocr.py"
+)
+
+
+def parse_arguments() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Process movies whose OCR status is pending."
+    )
+
+    parser.add_argument(
+        "--database",
+        type=Path,
+        default=DEFAULT_DATABASE_FILE,
+        help=f"SQLite database; default: {DEFAULT_DATABASE_FILE}",
+    )
+
+    parser.add_argument(
+        "--limit",
+        type=int,
+        help="Process at most N pending movies.",
+    )
+
+    return parser.parse_args()
+
+
+def get_pending_movies(
+    database_file: Path,
+    limit: int | None,
+) -> list[tuple[str, str]]:
+    sql = """
+        SELECT id, name
+        FROM movie
+        WHERE ocr_status = 'pending'
+        ORDER BY published, id
+    """
+
+    parameters: tuple[int, ...] = ()
+
+    if limit is not None:
+        sql += "\nLIMIT ?"
+        parameters = (limit,)
+
+    with sqlite3.connect(database_file) as connection:
+        rows = connection.execute(sql, parameters).fetchall()
+
+    return [
+        (str(movie_id), str(name))
+        for movie_id, name in rows
+    ]
+
+
+def get_ocr_status(
+    database_file: Path,
+    movie_id: str,
+) -> tuple[str | None, str | None]:
+    with sqlite3.connect(database_file) as connection:
+        row = connection.execute(
+            """
+            SELECT ocr_status, ocr_error
+            FROM movie
+            WHERE id = ?
+            """,
+            (movie_id,),
+        ).fetchone()
+
+    if row is None:
+        return None, "Movie disappeared from the database."
+
+    return row[0], row[1]
+
+
+def main() -> None:
+    args = parse_arguments()
+
+    if args.limit is not None and args.limit <= 0:
+        raise ValueError("--limit must be greater than zero.")
+
+    if not args.database.is_file():
+        raise FileNotFoundError(
+            f"Database file does not exist: {args.database}"
+        )
+
+    if not CHECK_OCR_SCRIPT.is_file():
+        raise FileNotFoundError(
+            f"OCR processing script does not exist: {CHECK_OCR_SCRIPT}"
+        )
+
+    if not os.environ.get("GOOGLE_VISION_API_KEY"):
+        raise ValueError(
+            "GOOGLE_VISION_API_KEY is not set in the environment."
+        )
+
+    pending_movies = get_pending_movies(
+        database_file=args.database,
+        limit=args.limit,
+    )
+
+    if not pending_movies:
+        print("No movies are waiting for OCR.")
+        return
+
+    print(f"Movies waiting for OCR: {len(pending_movies)}")
+    print()
+
+    completed = 0
+    manual_review = 0
+    failed = 0
+    unexpected = 0
+
+    for position, (movie_id, name) in enumerate(
+        pending_movies,
+        start=1,
+    ):
+        print(
+            f"[{position}/{len(pending_movies)}] "
+            f"{movie_id} - {name}"
+        )
+
+        result = subprocess.run(
+            [
+                str(CHECK_OCR_SCRIPT),
+                movie_id,
+                "--database",
+                str(args.database),
+            ],
+            text=True,
+            capture_output=True,
+        )
+
+        if result.stdout.strip():
+            print(result.stdout.strip())
+
+        if result.stderr.strip():
+            print(result.stderr.strip(), file=sys.stderr)
+
+        status, error_message = get_ocr_status(
+            database_file=args.database,
+            movie_id=movie_id,
+        )
+
+        if status == "completed":
+            completed += 1
+            print("Result: completed")
+        elif status == "manual_review":
+            manual_review += 1
+            print("Result: manual_review")
+        elif status == "failed":
+            failed += 1
+            print("Result: failed")
+        else:
+            unexpected += 1
+            print(f"Result: unexpected status {status!r}")
+
+        if error_message:
+            print(f"Error: {error_message}")
+
+        print()
+
+    print("OCR batch summary")
+    print("-----------------")
+    print(f"Selected:       {len(pending_movies)}")
+    print(f"Completed:      {completed}")
+    print(f"Manual review:  {manual_review}")
+    print(f"Failed:         {failed}")
+    print(f"Unexpected:     {unexpected}")
+
+    if failed or unexpected:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except (
+        FileNotFoundError,
+        ValueError,
+        sqlite3.Error,
+    ) as error:
+        print(f"Error: {error}", file=sys.stderr)
+        sys.exit(1)
+