Document WCX processing scripts

scripts/check_ocr.py

@@ -0,0 +1,313 @@
+#!/usr/bin/env python3
+
+"""
+Run and validate OCR processing for one movie in the SQLite index.
+
+The script accepts a WCX movie ID, retrieves the movie name and thumbnail URL
+from the database, and then coordinates the complete OCR workflow:
+
+    1. Run ocr.sh for the thumbnail URL.
+    2. Pass the raw OCR text to parse_ocr.py.
+    3. Compare the OCR name with the movie name stored in the database.
+    4. Store validated metadata and OCR processing information.
+
+When processing succeeds and the names match, the script updates:
+
+    nationality
+    shoot_location
+    shoot_date
+    ocr_raw_text
+    ocr_status = completed
+    ocr_processed_at
+    modified_at
+
+If OCR execution fails, the movie is marked as failed.
+
+If the OCR text cannot be parsed, required fields are missing, or the OCR name
+does not match the database name, the raw OCR text is retained and the movie
+is marked as manual_review. In these cases, the parsed metadata fields are not
+updated.
+
+Usage:
+    check_ocr.py <movie-id>
+
+Example:
+    check_ocr.py susana-melo_6707
+"""
+
+
+import sqlite3
+import subprocess
+import sys
+from pathlib import Path
+
+
+DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db")
+OCR_SCRIPT = Path("/storage/disk1/WCX/scripts/ocr.sh")
+PARSER_SCRIPT = Path("/storage/disk1/WCX/scripts/parse_ocr.py")
+
+
+def parse_key_value_output(output: str) -> dict[str, str]:
+    result: dict[str, str] = {}
+
+    for line in output.splitlines():
+        if "=" not in line:
+            continue
+
+        key, value = line.split("=", 1)
+        result[key.strip()] = value.strip()
+
+    return result
+
+
+def normalize_name(value: str) -> str:
+    return " ".join(value.casefold().split())
+
+
+def set_ocr_failed(
+    connection: sqlite3.Connection,
+    movie_id: str,
+    error_message: str,
+    raw_text: str | None = None,
+) -> None:
+    connection.execute(
+        """
+        UPDATE movie
+        SET
+            ocr_status = 'failed',
+            ocr_raw_text = ?,
+            ocr_error = ?,
+            ocr_processed_at = CURRENT_TIMESTAMP,
+            modified_at = CURRENT_TIMESTAMP
+        WHERE id = ?
+        """,
+        (
+            raw_text,
+            error_message,
+            movie_id,
+        ),
+    )
+
+
+def set_manual_review(
+    connection: sqlite3.Connection,
+    movie_id: str,
+    raw_text: str,
+    error_message: str,
+) -> None:
+    connection.execute(
+        """
+        UPDATE movie
+        SET
+            ocr_status = 'manual_review',
+            ocr_raw_text = ?,
+            ocr_error = ?,
+            ocr_processed_at = CURRENT_TIMESTAMP,
+            modified_at = CURRENT_TIMESTAMP
+        WHERE id = ?
+        """,
+        (
+            raw_text,
+            error_message,
+            movie_id,
+        ),
+    )
+
+
+def set_ocr_completed(
+    connection: sqlite3.Connection,
+    movie_id: str,
+    raw_text: str,
+    nationality: str,
+    shoot_location: str,
+    shoot_date: str,
+) -> None:
+    connection.execute(
+        """
+        UPDATE movie
+        SET
+            nationality = ?,
+            shoot_location = ?,
+            shoot_date = ?,
+            ocr_status = 'completed',
+            ocr_raw_text = ?,
+            ocr_error = NULL,
+            ocr_processed_at = CURRENT_TIMESTAMP,
+            modified_at = CURRENT_TIMESTAMP
+        WHERE id = ?
+        """,
+        (
+            nationality,
+            shoot_location,
+            shoot_date,
+            raw_text,
+            movie_id,
+        ),
+    )
+
+
+def main() -> None:
+    if len(sys.argv) != 2:
+        print(f"Användning: {sys.argv[0]} <movie-id>", file=sys.stderr)
+        sys.exit(1)
+
+    movie_id = sys.argv[1]
+
+    with sqlite3.connect(DATABASE_FILE) as connection:
+        row = connection.execute(
+            """
+            SELECT name, thumbnail
+            FROM movie
+            WHERE id = ?
+            """,
+            (movie_id,),
+        ).fetchone()
+
+        if row is None:
+            print(
+                f"Filmen finns inte i databasen: {movie_id}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+
+        database_name, thumbnail = row
+
+        if not thumbnail:
+            error_message = "Filmen saknar thumbnail."
+
+            set_ocr_failed(
+                connection,
+                movie_id,
+                error_message,
+            )
+
+            print(error_message, file=sys.stderr)
+            sys.exit(1)
+
+        ocr_result = subprocess.run(
+            [str(OCR_SCRIPT), thumbnail],
+            capture_output=True,
+            text=True,
+        )
+
+        if ocr_result.returncode != 0:
+            error_message = (
+                ocr_result.stderr.strip()
+                or "OCR-körningen misslyckades."
+            )
+
+            set_ocr_failed(
+                connection,
+                movie_id,
+                error_message,
+                ocr_result.stdout.strip() or None,
+            )
+
+            print("OCR-körningen misslyckades:", file=sys.stderr)
+            print(error_message, file=sys.stderr)
+            sys.exit(1)
+
+        raw_text = ocr_result.stdout.strip()
+
+        parser_result = subprocess.run(
+            [str(PARSER_SCRIPT)],
+            input=raw_text,
+            capture_output=True,
+            text=True,
+        )
+
+        if parser_result.returncode != 0:
+            error_message = (
+                parser_result.stderr.strip()
+                or "Tolkningen av OCR-resultatet misslyckades."
+            )
+
+            set_manual_review(
+                connection,
+                movie_id,
+                raw_text,
+                error_message,
+            )
+
+            print(
+                "Tolkningen av OCR-resultatet misslyckades:",
+                file=sys.stderr,
+            )
+            print(error_message, file=sys.stderr)
+            sys.exit(1)
+
+        parsed = parse_key_value_output(parser_result.stdout)
+
+        required_fields = (
+            "ocr_name",
+            "nationality",
+            "shoot_location",
+            "shoot_date",
+        )
+
+        missing_fields = [
+            field
+            for field in required_fields
+            if not parsed.get(field)
+        ]
+
+        if missing_fields:
+            error_message = (
+                "Parsern saknar fält: "
+                + ", ".join(missing_fields)
+            )
+
+            set_manual_review(
+                connection,
+                movie_id,
+                raw_text,
+                error_message,
+            )
+
+            print(error_message, file=sys.stderr)
+            sys.exit(1)
+
+        ocr_name = parsed["ocr_name"]
+        names_match = (
+            normalize_name(database_name)
+            == normalize_name(ocr_name)
+        )
+
+        if not names_match:
+            error_message = (
+                "OCR-namnet matchar inte databasnamnet: "
+                f"{ocr_name!r} != {database_name!r}"
+            )
+
+            set_manual_review(
+                connection,
+                movie_id,
+                raw_text,
+                error_message,
+            )
+
+            print(error_message, file=sys.stderr)
+            sys.exit(1)
+
+        set_ocr_completed(
+            connection=connection,
+            movie_id=movie_id,
+            raw_text=raw_text,
+            nationality=parsed["nationality"],
+            shoot_location=parsed["shoot_location"],
+            shoot_date=parsed["shoot_date"],
+        )
+
+    print(f"id={movie_id}")
+    print(f"database_name={database_name}")
+    print(f"ocr_name={ocr_name}")
+    print("name_match=yes")
+    print(f"nationality={parsed['nationality']}")
+    print(f"shoot_location={parsed['shoot_location']}")
+    print(f"shoot_date={parsed['shoot_date']}")
+    print("ocr_status=completed")
+
+
+if __name__ == "__main__":
+    main()
+

scripts/import_site.py

@@ -1,5 +1,36 @@
 #!/usr/bin/env python3
 
+"""
+Import WCX site metadata into the local SQLite movie index.
+
+The script reads the JSON snapshot produced by the WCX site scraper and
+synchronizes it with the movie table in the local SQLite database.
+
+Import rules:
+- A movie whose ID does not exist in the database is inserted.
+- An existing movie is updated only when the site provides an update date
+  and that date is newer than the value currently stored in the database.
+- If the database has no update date but the site does, the movie is updated.
+- Movies without a newer site update date are left unchanged.
+
+When an existing movie is updated, all fields owned by the site are replaced:
+name, duration, web URL, thumbnail URL, publication date, and update date.
+Manually maintained metadata fields are not modified.
+
+Durations from the site are accepted as mm:ss or hh:mm:ss and stored as
+seconds in the duration_seconds column.
+
+Input:
+    /storage/disk1/WCX/import/wcx_site_index.json
+
+Database:
+    /storage/disk1/WCX/database/wcx.db
+
+The script prints a summary showing the number of inserted, updated, and
+unchanged records.
+"""
+
+
 import json
 import sqlite3
 from pathlib import Path

scripts/ocr.sh

@@ -0,0 +1,102 @@
+#!/bin/bash
+
+# --- KONFIGURATION ---
+# Skriptet förväntar sig nu miljövariabeln: GOOGLE_VISION_API_KEY
+# ---------------------
+
+usage() {
+    echo "Användning: $0 <bildfil eller URL> [-o utdatafil]"
+    echo "Exempel 1: $0 kvitto.jpg"
+    echo "Exempel 2: export GOOGLE_VISION_API_KEY=\"din_nyckel\" && $0 https://example.com/image.png -o text.txt"
+    exit 1
+}
+
+# 1. Kontrollera att miljövariabeln är satt
+if [ -z "$GOOGLE_VISION_API_KEY" ]; then
+    echo "Fel: Miljövariabeln GOOGLE_VISION_API_KEY är inte satt." >&2
+    echo "Kör detta i terminalen först: export GOOGLE_VISION_API_KEY=\"din_faktiska_nyckel\"" >&2
+    exit 1
+fi
+
+# Kontrollera att vi fick minst ett argument
+if [ $# -lt 1 ]; then
+    usage
+fi
+
+INPUT="$1"
+OUTPUT_FILE=""
+
+# Hantera flaggor för utdatafil (-o)
+if [ "$2" == "-o" ] && [ -n "$3" ]; then
+    OUTPUT_FILE="$3"
+fi
+
+# 2. Hantera bilden (lokal fil vs URL) och koda till Base64
+if [[ "$INPUT" =~ ^https?:// ]]; then
+    # Det är en URL - ladda ner och koda till Base64 i minnet
+    if ! command -v curl &> /dev/null; then
+        echo "Fel: curl krävs men är inte installerat." >&2
+        exit 1
+    fi
+    BASE64_IMAGE=$(curl -s "$INPUT" | base64)
+else
+    # Det är en lokal fil
+    if [ ! -f "$INPUT" ]; then
+        echo "Fel: Filen '$INPUT' hittades inte." >&2
+        exit 1
+    fi
+    # Hanterar både Mac (base64) och Linux (base64 -w 0) för att undvika radbrytningar i Base64-strängen
+    if [[ "$OSTYPE" == "darwin"* ]]; then
+        BASE64_IMAGE=$(base64 "$INPUT")
+    else
+        BASE64_IMAGE=$(base64 -w 0 "$INPUT")
+    fi
+fi
+
+# 3. Skapa JSON-payloaden
+JSON_PAYLOAD=$(cat <<EOF
+{
+  "requests": [
+    {
+      "image": {
+        "content": "$BASE64_IMAGE"
+      },
+      "features": [
+        {
+          "type": "TEXT_DETECTION"
+        }
+      ]
+    }
+  ]
+}
+EOF
+)
+
+# 4. Skicka till Google Cloud Vision API och filtrera ut texten via 'jq'
+if command -v jq &> /dev/null; then
+    RESULT=$(curl -s -X POST \
+      -H "Content-Type: application/json; charset=utf-8" \
+      -d "$JSON_PAYLOAD" \
+      "https://vision.googleapis.com/v1/images:annotate?key=$GOOGLE_VISION_API_KEY" | jq -r '.responses[0].textAnnotations[0].description')
+else
+    echo "Tips: Installera 'jq' (t.ex. 'brew install jq' eller 'apt install jq') för att få ren text." >&2
+    RESULT=$(curl -s -X POST \
+      -H "Content-Type: application/json; charset=utf-8" \
+      -d "$JSON_PAYLOAD" \
+      "https://vision.googleapis.com/v1/images:annotate?key=$GOOGLE_VISION_API_KEY")
+fi
+
+# Kontrollera om vi fick något svar eller om det blev fel
+if [ "$RESULT" == "null" ] || [ -z "$RESULT" ]; then
+    echo "Ingen text hittades eller så uppstod ett API-fel." >&2
+    exit 1
+fi
+
+# 5. Skriv ut resultatet (till fil eller stdout)
+if [ -n "$OUTPUT_FILE" ]; then
+    echo "$RESULT" > "$OUTPUT_FILE"
+    echo "Klart! Texten har sparats i $OUTPUT_FILE"
+else
+    echo "$RESULT"
+fi
+

scripts/parse_ocr.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+
+"""
+Parse raw OCR text extracted from a WCX thumbnail.
+
+The script reads OCR text from standard input and expects four non-empty lines:
+
+    1. Performer name
+    2. Nationality
+    3. WCX branding text, ignored by the parser
+    4. Shoot location and date
+
+The final line is expected to use the following format:
+
+    Budapest (Hungary), March 9, 2014
+
+It is split on the first comma so that the comma inside the date is preserved.
+The date is normalized to ISO format, for example 2014-03-09.
+
+On success, the script writes key/value pairs to standard output:
+
+    ocr_name=...
+    nationality=...
+    shoot_location=...
+    shoot_date=...
+
+Invalid or unexpected OCR output is reported on standard error and causes a
+non-zero exit status.
+"""
+
+
+import sys
+from datetime import datetime
+
+
+def parse_ocr_text(raw_text: str) -> tuple[str, str, str, str]:
+    lines = [
+        line.strip()
+        for line in raw_text.splitlines()
+        if line.strip()
+    ]
+
+    if len(lines) != 4:
+        raise ValueError(
+            f"Förväntade 4 icke-tomma rader, fick {len(lines)}."
+        )
+
+    ocr_name = lines[0]
+    nationality = lines[1]
+    ignored_text = lines[2]
+    location_and_date = lines[3]
+
+    if not ocr_name:
+        raise ValueError("Namn saknas.")
+
+    if not nationality:
+        raise ValueError("Nationalitet saknas.")
+
+    if not ignored_text:
+        raise ValueError("Förväntad tredje rad saknas.")
+
+    if "," not in location_and_date:
+        raise ValueError(
+            "Sista raden saknar kommatecken mellan plats och datum."
+        )
+
+    shoot_location, date_text = location_and_date.split(",", 1)
+
+    shoot_location = shoot_location.strip()
+    date_text = date_text.strip()
+
+    if not shoot_location:
+        raise ValueError("Inspelningsplats saknas.")
+
+    try:
+        shoot_date = datetime.strptime(
+            date_text,
+            "%B %d, %Y",
+        ).date().isoformat()
+    except ValueError as error:
+        raise ValueError(
+            f"Kunde inte tolka datumet: {date_text!r}"
+        ) from error
+
+    return ocr_name, nationality, shoot_location, shoot_date
+
+
+def main() -> None:
+    raw_text = sys.stdin.read()
+
+    if not raw_text.strip():
+        raise ValueError("Ingen OCR-text mottogs på standard input.")
+
+    ocr_name, nationality, shoot_location, shoot_date = parse_ocr_text(
+        raw_text
+    )
+
+    print(f"ocr_name={ocr_name}")
+    print(f"nationality={nationality}")
+    print(f"shoot_location={shoot_location}")
+    print(f"shoot_date={shoot_date}")
+
+
+if __name__ == "__main__":
+    try:
+        main()
+    except ValueError as error:
+        print(f"Fel: {error}", file=sys.stderr)
+        sys.exit(1)