From 0d56cfe52396b04a49d717c79f9f4d0f9fbf29d2 Mon Sep 17 00:00:00 2001 From: Urban Date: Fri, 17 Jul 2026 17:52:53 +0200 Subject: [PATCH] Document WCX processing scripts --- scripts/check_ocr.py | 313 +++++++++++++++++++++++++++++++++++++++++ scripts/import_site.py | 31 ++++ scripts/ocr.sh | 102 ++++++++++++++ scripts/parse_ocr.py | 109 ++++++++++++++ 4 files changed, 555 insertions(+) create mode 100755 scripts/check_ocr.py create mode 100755 scripts/ocr.sh create mode 100755 scripts/parse_ocr.py diff --git a/scripts/check_ocr.py b/scripts/check_ocr.py new file mode 100755 index 0000000..dec10df --- /dev/null +++ b/scripts/check_ocr.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +""" +Run and validate OCR processing for one movie in the SQLite index. + +The script accepts a WCX movie ID, retrieves the movie name and thumbnail URL +from the database, and then coordinates the complete OCR workflow: + + 1. Run ocr.sh for the thumbnail URL. + 2. Pass the raw OCR text to parse_ocr.py. + 3. Compare the OCR name with the movie name stored in the database. + 4. Store validated metadata and OCR processing information. + +When processing succeeds and the names match, the script updates: + + nationality + shoot_location + shoot_date + ocr_raw_text + ocr_status = completed + ocr_processed_at + modified_at + +If OCR execution fails, the movie is marked as failed. + +If the OCR text cannot be parsed, required fields are missing, or the OCR name +does not match the database name, the raw OCR text is retained and the movie +is marked as manual_review. In these cases, the parsed metadata fields are not +updated. + +Usage: + check_ocr.py + +Example: + check_ocr.py susana-melo_6707 +""" + + +import sqlite3 +import subprocess +import sys +from pathlib import Path + + +DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db") +OCR_SCRIPT = Path("/storage/disk1/WCX/scripts/ocr.sh") +PARSER_SCRIPT = Path("/storage/disk1/WCX/scripts/parse_ocr.py") + + +def parse_key_value_output(output: str) -> dict[str, str]: + result: dict[str, str] = {} + + for line in output.splitlines(): + if "=" not in line: + continue + + key, value = line.split("=", 1) + result[key.strip()] = value.strip() + + return result + + +def normalize_name(value: str) -> str: + return " ".join(value.casefold().split()) + + +def set_ocr_failed( + connection: sqlite3.Connection, + movie_id: str, + error_message: str, + raw_text: str | None = None, +) -> None: + connection.execute( + """ + UPDATE movie + SET + ocr_status = 'failed', + ocr_raw_text = ?, + ocr_error = ?, + ocr_processed_at = CURRENT_TIMESTAMP, + modified_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + ( + raw_text, + error_message, + movie_id, + ), + ) + + +def set_manual_review( + connection: sqlite3.Connection, + movie_id: str, + raw_text: str, + error_message: str, +) -> None: + connection.execute( + """ + UPDATE movie + SET + ocr_status = 'manual_review', + ocr_raw_text = ?, + ocr_error = ?, + ocr_processed_at = CURRENT_TIMESTAMP, + modified_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + ( + raw_text, + error_message, + movie_id, + ), + ) + + +def set_ocr_completed( + connection: sqlite3.Connection, + movie_id: str, + raw_text: str, + nationality: str, + shoot_location: str, + shoot_date: str, +) -> None: + connection.execute( + """ + UPDATE movie + SET + nationality = ?, + shoot_location = ?, + shoot_date = ?, + ocr_status = 'completed', + ocr_raw_text = ?, + ocr_error = NULL, + ocr_processed_at = CURRENT_TIMESTAMP, + modified_at = CURRENT_TIMESTAMP + WHERE id = ? + """, + ( + nationality, + shoot_location, + shoot_date, + raw_text, + movie_id, + ), + ) + + +def main() -> None: + if len(sys.argv) != 2: + print(f"Användning: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + movie_id = sys.argv[1] + + with sqlite3.connect(DATABASE_FILE) as connection: + row = connection.execute( + """ + SELECT name, thumbnail + FROM movie + WHERE id = ? + """, + (movie_id,), + ).fetchone() + + if row is None: + print( + f"Filmen finns inte i databasen: {movie_id}", + file=sys.stderr, + ) + sys.exit(1) + + database_name, thumbnail = row + + if not thumbnail: + error_message = "Filmen saknar thumbnail." + + set_ocr_failed( + connection, + movie_id, + error_message, + ) + + print(error_message, file=sys.stderr) + sys.exit(1) + + ocr_result = subprocess.run( + [str(OCR_SCRIPT), thumbnail], + capture_output=True, + text=True, + ) + + if ocr_result.returncode != 0: + error_message = ( + ocr_result.stderr.strip() + or "OCR-körningen misslyckades." + ) + + set_ocr_failed( + connection, + movie_id, + error_message, + ocr_result.stdout.strip() or None, + ) + + print("OCR-körningen misslyckades:", file=sys.stderr) + print(error_message, file=sys.stderr) + sys.exit(1) + + raw_text = ocr_result.stdout.strip() + + parser_result = subprocess.run( + [str(PARSER_SCRIPT)], + input=raw_text, + capture_output=True, + text=True, + ) + + if parser_result.returncode != 0: + error_message = ( + parser_result.stderr.strip() + or "Tolkningen av OCR-resultatet misslyckades." + ) + + set_manual_review( + connection, + movie_id, + raw_text, + error_message, + ) + + print( + "Tolkningen av OCR-resultatet misslyckades:", + file=sys.stderr, + ) + print(error_message, file=sys.stderr) + sys.exit(1) + + parsed = parse_key_value_output(parser_result.stdout) + + required_fields = ( + "ocr_name", + "nationality", + "shoot_location", + "shoot_date", + ) + + missing_fields = [ + field + for field in required_fields + if not parsed.get(field) + ] + + if missing_fields: + error_message = ( + "Parsern saknar fält: " + + ", ".join(missing_fields) + ) + + set_manual_review( + connection, + movie_id, + raw_text, + error_message, + ) + + print(error_message, file=sys.stderr) + sys.exit(1) + + ocr_name = parsed["ocr_name"] + names_match = ( + normalize_name(database_name) + == normalize_name(ocr_name) + ) + + if not names_match: + error_message = ( + "OCR-namnet matchar inte databasnamnet: " + f"{ocr_name!r} != {database_name!r}" + ) + + set_manual_review( + connection, + movie_id, + raw_text, + error_message, + ) + + print(error_message, file=sys.stderr) + sys.exit(1) + + set_ocr_completed( + connection=connection, + movie_id=movie_id, + raw_text=raw_text, + nationality=parsed["nationality"], + shoot_location=parsed["shoot_location"], + shoot_date=parsed["shoot_date"], + ) + + print(f"id={movie_id}") + print(f"database_name={database_name}") + print(f"ocr_name={ocr_name}") + print("name_match=yes") + print(f"nationality={parsed['nationality']}") + print(f"shoot_location={parsed['shoot_location']}") + print(f"shoot_date={parsed['shoot_date']}") + print("ocr_status=completed") + + +if __name__ == "__main__": + main() + diff --git a/scripts/import_site.py b/scripts/import_site.py index ebf7049..7ad009d 100755 --- a/scripts/import_site.py +++ b/scripts/import_site.py @@ -1,5 +1,36 @@ #!/usr/bin/env python3 +""" +Import WCX site metadata into the local SQLite movie index. + +The script reads the JSON snapshot produced by the WCX site scraper and +synchronizes it with the movie table in the local SQLite database. + +Import rules: +- A movie whose ID does not exist in the database is inserted. +- An existing movie is updated only when the site provides an update date + and that date is newer than the value currently stored in the database. +- If the database has no update date but the site does, the movie is updated. +- Movies without a newer site update date are left unchanged. + +When an existing movie is updated, all fields owned by the site are replaced: +name, duration, web URL, thumbnail URL, publication date, and update date. +Manually maintained metadata fields are not modified. + +Durations from the site are accepted as mm:ss or hh:mm:ss and stored as +seconds in the duration_seconds column. + +Input: + /storage/disk1/WCX/import/wcx_site_index.json + +Database: + /storage/disk1/WCX/database/wcx.db + +The script prints a summary showing the number of inserted, updated, and +unchanged records. +""" + + import json import sqlite3 from pathlib import Path diff --git a/scripts/ocr.sh b/scripts/ocr.sh new file mode 100755 index 0000000..e6109c7 --- /dev/null +++ b/scripts/ocr.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# --- KONFIGURATION --- +# Skriptet förväntar sig nu miljövariabeln: GOOGLE_VISION_API_KEY +# --------------------- + +usage() { + echo "Användning: $0 [-o utdatafil]" + echo "Exempel 1: $0 kvitto.jpg" + echo "Exempel 2: export GOOGLE_VISION_API_KEY=\"din_nyckel\" && $0 https://example.com/image.png -o text.txt" + exit 1 +} + +# 1. Kontrollera att miljövariabeln är satt +if [ -z "$GOOGLE_VISION_API_KEY" ]; then + echo "Fel: Miljövariabeln GOOGLE_VISION_API_KEY är inte satt." >&2 + echo "Kör detta i terminalen först: export GOOGLE_VISION_API_KEY=\"din_faktiska_nyckel\"" >&2 + exit 1 +fi + +# Kontrollera att vi fick minst ett argument +if [ $# -lt 1 ]; then + usage +fi + +INPUT="$1" +OUTPUT_FILE="" + +# Hantera flaggor för utdatafil (-o) +if [ "$2" == "-o" ] && [ -n "$3" ]; then + OUTPUT_FILE="$3" +fi + +# 2. Hantera bilden (lokal fil vs URL) och koda till Base64 +if [[ "$INPUT" =~ ^https?:// ]]; then + # Det är en URL - ladda ner och koda till Base64 i minnet + if ! command -v curl &> /dev/null; then + echo "Fel: curl krävs men är inte installerat." >&2 + exit 1 + fi + BASE64_IMAGE=$(curl -s "$INPUT" | base64) +else + # Det är en lokal fil + if [ ! -f "$INPUT" ]; then + echo "Fel: Filen '$INPUT' hittades inte." >&2 + exit 1 + fi + # Hanterar både Mac (base64) och Linux (base64 -w 0) för att undvika radbrytningar i Base64-strängen + if [[ "$OSTYPE" == "darwin"* ]]; then + BASE64_IMAGE=$(base64 "$INPUT") + else + BASE64_IMAGE=$(base64 -w 0 "$INPUT") + fi +fi + +# 3. Skapa JSON-payloaden +JSON_PAYLOAD=$(cat < /dev/null; then + RESULT=$(curl -s -X POST \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$JSON_PAYLOAD" \ + "https://vision.googleapis.com/v1/images:annotate?key=$GOOGLE_VISION_API_KEY" | jq -r '.responses[0].textAnnotations[0].description') +else + echo "Tips: Installera 'jq' (t.ex. 'brew install jq' eller 'apt install jq') för att få ren text." >&2 + RESULT=$(curl -s -X POST \ + -H "Content-Type: application/json; charset=utf-8" \ + -d "$JSON_PAYLOAD" \ + "https://vision.googleapis.com/v1/images:annotate?key=$GOOGLE_VISION_API_KEY") +fi + +# Kontrollera om vi fick något svar eller om det blev fel +if [ "$RESULT" == "null" ] || [ -z "$RESULT" ]; then + echo "Ingen text hittades eller så uppstod ett API-fel." >&2 + exit 1 +fi + +# 5. Skriv ut resultatet (till fil eller stdout) +if [ -n "$OUTPUT_FILE" ]; then + echo "$RESULT" > "$OUTPUT_FILE" + echo "Klart! Texten har sparats i $OUTPUT_FILE" +else + echo "$RESULT" +fi + diff --git a/scripts/parse_ocr.py b/scripts/parse_ocr.py new file mode 100755 index 0000000..c45c22a --- /dev/null +++ b/scripts/parse_ocr.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 + +""" +Parse raw OCR text extracted from a WCX thumbnail. + +The script reads OCR text from standard input and expects four non-empty lines: + + 1. Performer name + 2. Nationality + 3. WCX branding text, ignored by the parser + 4. Shoot location and date + +The final line is expected to use the following format: + + Budapest (Hungary), March 9, 2014 + +It is split on the first comma so that the comma inside the date is preserved. +The date is normalized to ISO format, for example 2014-03-09. + +On success, the script writes key/value pairs to standard output: + + ocr_name=... + nationality=... + shoot_location=... + shoot_date=... + +Invalid or unexpected OCR output is reported on standard error and causes a +non-zero exit status. +""" + + +import sys +from datetime import datetime + + +def parse_ocr_text(raw_text: str) -> tuple[str, str, str, str]: + lines = [ + line.strip() + for line in raw_text.splitlines() + if line.strip() + ] + + if len(lines) != 4: + raise ValueError( + f"Förväntade 4 icke-tomma rader, fick {len(lines)}." + ) + + ocr_name = lines[0] + nationality = lines[1] + ignored_text = lines[2] + location_and_date = lines[3] + + if not ocr_name: + raise ValueError("Namn saknas.") + + if not nationality: + raise ValueError("Nationalitet saknas.") + + if not ignored_text: + raise ValueError("Förväntad tredje rad saknas.") + + if "," not in location_and_date: + raise ValueError( + "Sista raden saknar kommatecken mellan plats och datum." + ) + + shoot_location, date_text = location_and_date.split(",", 1) + + shoot_location = shoot_location.strip() + date_text = date_text.strip() + + if not shoot_location: + raise ValueError("Inspelningsplats saknas.") + + try: + shoot_date = datetime.strptime( + date_text, + "%B %d, %Y", + ).date().isoformat() + except ValueError as error: + raise ValueError( + f"Kunde inte tolka datumet: {date_text!r}" + ) from error + + return ocr_name, nationality, shoot_location, shoot_date + + +def main() -> None: + raw_text = sys.stdin.read() + + if not raw_text.strip(): + raise ValueError("Ingen OCR-text mottogs på standard input.") + + ocr_name, nationality, shoot_location, shoot_date = parse_ocr_text( + raw_text + ) + + print(f"ocr_name={ocr_name}") + print(f"nationality={nationality}") + print(f"shoot_location={shoot_location}") + print(f"shoot_date={shoot_date}") + + +if __name__ == "__main__": + try: + main() + except ValueError as error: + print(f"Fel: {error}", file=sys.stderr) + sys.exit(1)