diff --git a/scripts/check_ocr.py b/scripts/check_ocr.py index dec10df..8ed64dd 100755 --- a/scripts/check_ocr.py +++ b/scripts/check_ocr.py @@ -4,7 +4,7 @@ Run and validate OCR processing for one movie in the SQLite index. The script accepts a WCX movie ID, retrieves the movie name and thumbnail URL -from the database, and then coordinates the complete OCR workflow: +from the selected database, and coordinates the complete OCR workflow: 1. Run ocr.sh for the thumbnail URL. 2. Pass the raw OCR text to parse_ocr.py. @@ -30,21 +30,48 @@ updated. Usage: check_ocr.py + check_ocr.py --database /path/to/wcx.db Example: check_ocr.py susana-melo_6707 """ - +import argparse import sqlite3 import subprocess import sys from pathlib import Path -DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db") -OCR_SCRIPT = Path("/storage/disk1/WCX/scripts/ocr.sh") -PARSER_SCRIPT = Path("/storage/disk1/WCX/scripts/parse_ocr.py") +DEFAULT_DATABASE_FILE = Path( + "/storage/disk1/WCX/database/wcx.db" +) +OCR_SCRIPT = Path( + "/storage/disk1/WCX/scripts/ocr.sh" +) +PARSER_SCRIPT = Path( + "/storage/disk1/WCX/scripts/parse_ocr.py" +) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Run OCR processing for one WCX movie." + ) + + parser.add_argument( + "movie_id", + help="Movie ID to process.", + ) + + parser.add_argument( + "--database", + type=Path, + default=DEFAULT_DATABASE_FILE, + help=f"SQLite database; default: {DEFAULT_DATABASE_FILE}", + ) + + return parser.parse_args() def parse_key_value_output(output: str) -> dict[str, str]: @@ -147,13 +174,26 @@ def set_ocr_completed( def main() -> None: - if len(sys.argv) != 2: - print(f"Användning: {sys.argv[0]} ", file=sys.stderr) - sys.exit(1) + args = parse_arguments() + movie_id = args.movie_id + database_file = args.database - movie_id = sys.argv[1] + if not database_file.is_file(): + raise FileNotFoundError( + f"Database file does not exist: {database_file}" + ) - with sqlite3.connect(DATABASE_FILE) as connection: + if not OCR_SCRIPT.is_file(): + raise FileNotFoundError( + f"OCR script does not exist: {OCR_SCRIPT}" + ) + + if not PARSER_SCRIPT.is_file(): + raise FileNotFoundError( + f"OCR parser does not exist: {PARSER_SCRIPT}" + ) + + with sqlite3.connect(database_file) as connection: row = connection.execute( """ SELECT name, thumbnail @@ -165,7 +205,7 @@ def main() -> None: if row is None: print( - f"Filmen finns inte i databasen: {movie_id}", + f"Movie does not exist in database: {movie_id}", file=sys.stderr, ) sys.exit(1) @@ -173,7 +213,7 @@ def main() -> None: database_name, thumbnail = row if not thumbnail: - error_message = "Filmen saknar thumbnail." + error_message = "Movie has no thumbnail." set_ocr_failed( connection, @@ -193,7 +233,7 @@ def main() -> None: if ocr_result.returncode != 0: error_message = ( ocr_result.stderr.strip() - or "OCR-körningen misslyckades." + or "OCR execution failed." ) set_ocr_failed( @@ -203,7 +243,7 @@ def main() -> None: ocr_result.stdout.strip() or None, ) - print("OCR-körningen misslyckades:", file=sys.stderr) + print("OCR execution failed:", file=sys.stderr) print(error_message, file=sys.stderr) sys.exit(1) @@ -219,7 +259,7 @@ def main() -> None: if parser_result.returncode != 0: error_message = ( parser_result.stderr.strip() - or "Tolkningen av OCR-resultatet misslyckades." + or "OCR output parsing failed." ) set_manual_review( @@ -230,7 +270,7 @@ def main() -> None: ) print( - "Tolkningen av OCR-resultatet misslyckades:", + "OCR output parsing failed:", file=sys.stderr, ) print(error_message, file=sys.stderr) @@ -253,7 +293,7 @@ def main() -> None: if missing_fields: error_message = ( - "Parsern saknar fält: " + "Parser output is missing fields: " + ", ".join(missing_fields) ) @@ -275,7 +315,7 @@ def main() -> None: if not names_match: error_message = ( - "OCR-namnet matchar inte databasnamnet: " + "OCR name does not match database name: " f"{ocr_name!r} != {database_name!r}" ) @@ -299,6 +339,7 @@ def main() -> None: ) print(f"id={movie_id}") + print(f"database={database_file}") print(f"database_name={database_name}") print(f"ocr_name={ocr_name}") print("name_match=yes") @@ -309,5 +350,11 @@ def main() -> None: if __name__ == "__main__": - main() - + try: + main() + except ( + FileNotFoundError, + sqlite3.Error, + ) as error: + print(f"Error: {error}", file=sys.stderr) + sys.exit(1) diff --git a/scripts/process_pending_ocr.py b/scripts/process_pending_ocr.py new file mode 100755 index 0000000..89c55e2 --- /dev/null +++ b/scripts/process_pending_ocr.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 + +""" +Process all movies waiting for OCR in the WCX SQLite index. + +The script queries the movie table for records whose OCR status is "pending" +and invokes check_ocr.py once for each movie ID. + +check_ocr.py performs the actual OCR request, parsing, validation, and database +update. This batch script only coordinates those individual executions. + +Processing continues if one movie fails. A final summary reports how many +movies completed successfully and how many failed or require manual review. + +By default, all pending movies are processed. Use --limit to restrict the +number of movies during testing. + +Usage: + process_pending_ocr.py + process_pending_ocr.py --limit 1 + process_pending_ocr.py --database /path/to/wcx-test.db + +Requirements: + GOOGLE_VISION_API_KEY must be available in the environment. +""" + +import argparse +import os +import sqlite3 +import subprocess +import sys +from pathlib import Path + + +DEFAULT_DATABASE_FILE = Path( + "/storage/disk1/WCX/database/wcx.db" +) +CHECK_OCR_SCRIPT = Path( + "/storage/disk1/WCX/scripts/check_ocr.py" +) + + +def parse_arguments() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Process movies whose OCR status is pending." + ) + + parser.add_argument( + "--database", + type=Path, + default=DEFAULT_DATABASE_FILE, + help=f"SQLite database; default: {DEFAULT_DATABASE_FILE}", + ) + + parser.add_argument( + "--limit", + type=int, + help="Process at most N pending movies.", + ) + + return parser.parse_args() + + +def get_pending_movies( + database_file: Path, + limit: int | None, +) -> list[tuple[str, str]]: + sql = """ + SELECT id, name + FROM movie + WHERE ocr_status = 'pending' + ORDER BY published, id + """ + + parameters: tuple[int, ...] = () + + if limit is not None: + sql += "\nLIMIT ?" + parameters = (limit,) + + with sqlite3.connect(database_file) as connection: + rows = connection.execute(sql, parameters).fetchall() + + return [ + (str(movie_id), str(name)) + for movie_id, name in rows + ] + + +def get_ocr_status( + database_file: Path, + movie_id: str, +) -> tuple[str | None, str | None]: + with sqlite3.connect(database_file) as connection: + row = connection.execute( + """ + SELECT ocr_status, ocr_error + FROM movie + WHERE id = ? + """, + (movie_id,), + ).fetchone() + + if row is None: + return None, "Movie disappeared from the database." + + return row[0], row[1] + + +def main() -> None: + args = parse_arguments() + + if args.limit is not None and args.limit <= 0: + raise ValueError("--limit must be greater than zero.") + + if not args.database.is_file(): + raise FileNotFoundError( + f"Database file does not exist: {args.database}" + ) + + if not CHECK_OCR_SCRIPT.is_file(): + raise FileNotFoundError( + f"OCR processing script does not exist: {CHECK_OCR_SCRIPT}" + ) + + if not os.environ.get("GOOGLE_VISION_API_KEY"): + raise ValueError( + "GOOGLE_VISION_API_KEY is not set in the environment." + ) + + pending_movies = get_pending_movies( + database_file=args.database, + limit=args.limit, + ) + + if not pending_movies: + print("No movies are waiting for OCR.") + return + + print(f"Movies waiting for OCR: {len(pending_movies)}") + print() + + completed = 0 + manual_review = 0 + failed = 0 + unexpected = 0 + + for position, (movie_id, name) in enumerate( + pending_movies, + start=1, + ): + print( + f"[{position}/{len(pending_movies)}] " + f"{movie_id} - {name}" + ) + + result = subprocess.run( + [ + str(CHECK_OCR_SCRIPT), + movie_id, + "--database", + str(args.database), + ], + text=True, + capture_output=True, + ) + + if result.stdout.strip(): + print(result.stdout.strip()) + + if result.stderr.strip(): + print(result.stderr.strip(), file=sys.stderr) + + status, error_message = get_ocr_status( + database_file=args.database, + movie_id=movie_id, + ) + + if status == "completed": + completed += 1 + print("Result: completed") + elif status == "manual_review": + manual_review += 1 + print("Result: manual_review") + elif status == "failed": + failed += 1 + print("Result: failed") + else: + unexpected += 1 + print(f"Result: unexpected status {status!r}") + + if error_message: + print(f"Error: {error_message}") + + print() + + print("OCR batch summary") + print("-----------------") + print(f"Selected: {len(pending_movies)}") + print(f"Completed: {completed}") + print(f"Manual review: {manual_review}") + print(f"Failed: {failed}") + print(f"Unexpected: {unexpected}") + + if failed or unexpected: + sys.exit(1) + + +if __name__ == "__main__": + try: + main() + except ( + FileNotFoundError, + ValueError, + sqlite3.Error, + ) as error: + print(f"Error: {error}", file=sys.stderr) + sys.exit(1) +