Document WCX processing scripts
This commit is contained in:
313
scripts/check_ocr.py
Executable file
313
scripts/check_ocr.py
Executable file
@ -0,0 +1,313 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Run and validate OCR processing for one movie in the SQLite index.
|
||||
|
||||
The script accepts a WCX movie ID, retrieves the movie name and thumbnail URL
|
||||
from the database, and then coordinates the complete OCR workflow:
|
||||
|
||||
1. Run ocr.sh for the thumbnail URL.
|
||||
2. Pass the raw OCR text to parse_ocr.py.
|
||||
3. Compare the OCR name with the movie name stored in the database.
|
||||
4. Store validated metadata and OCR processing information.
|
||||
|
||||
When processing succeeds and the names match, the script updates:
|
||||
|
||||
nationality
|
||||
shoot_location
|
||||
shoot_date
|
||||
ocr_raw_text
|
||||
ocr_status = completed
|
||||
ocr_processed_at
|
||||
modified_at
|
||||
|
||||
If OCR execution fails, the movie is marked as failed.
|
||||
|
||||
If the OCR text cannot be parsed, required fields are missing, or the OCR name
|
||||
does not match the database name, the raw OCR text is retained and the movie
|
||||
is marked as manual_review. In these cases, the parsed metadata fields are not
|
||||
updated.
|
||||
|
||||
Usage:
|
||||
check_ocr.py <movie-id>
|
||||
|
||||
Example:
|
||||
check_ocr.py susana-melo_6707
|
||||
"""
|
||||
|
||||
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
DATABASE_FILE = Path("/storage/disk1/WCX/database/wcx.db")
|
||||
OCR_SCRIPT = Path("/storage/disk1/WCX/scripts/ocr.sh")
|
||||
PARSER_SCRIPT = Path("/storage/disk1/WCX/scripts/parse_ocr.py")
|
||||
|
||||
|
||||
def parse_key_value_output(output: str) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
|
||||
for line in output.splitlines():
|
||||
if "=" not in line:
|
||||
continue
|
||||
|
||||
key, value = line.split("=", 1)
|
||||
result[key.strip()] = value.strip()
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def normalize_name(value: str) -> str:
|
||||
return " ".join(value.casefold().split())
|
||||
|
||||
|
||||
def set_ocr_failed(
|
||||
connection: sqlite3.Connection,
|
||||
movie_id: str,
|
||||
error_message: str,
|
||||
raw_text: str | None = None,
|
||||
) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE movie
|
||||
SET
|
||||
ocr_status = 'failed',
|
||||
ocr_raw_text = ?,
|
||||
ocr_error = ?,
|
||||
ocr_processed_at = CURRENT_TIMESTAMP,
|
||||
modified_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
raw_text,
|
||||
error_message,
|
||||
movie_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def set_manual_review(
|
||||
connection: sqlite3.Connection,
|
||||
movie_id: str,
|
||||
raw_text: str,
|
||||
error_message: str,
|
||||
) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE movie
|
||||
SET
|
||||
ocr_status = 'manual_review',
|
||||
ocr_raw_text = ?,
|
||||
ocr_error = ?,
|
||||
ocr_processed_at = CURRENT_TIMESTAMP,
|
||||
modified_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
raw_text,
|
||||
error_message,
|
||||
movie_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def set_ocr_completed(
|
||||
connection: sqlite3.Connection,
|
||||
movie_id: str,
|
||||
raw_text: str,
|
||||
nationality: str,
|
||||
shoot_location: str,
|
||||
shoot_date: str,
|
||||
) -> None:
|
||||
connection.execute(
|
||||
"""
|
||||
UPDATE movie
|
||||
SET
|
||||
nationality = ?,
|
||||
shoot_location = ?,
|
||||
shoot_date = ?,
|
||||
ocr_status = 'completed',
|
||||
ocr_raw_text = ?,
|
||||
ocr_error = NULL,
|
||||
ocr_processed_at = CURRENT_TIMESTAMP,
|
||||
modified_at = CURRENT_TIMESTAMP
|
||||
WHERE id = ?
|
||||
""",
|
||||
(
|
||||
nationality,
|
||||
shoot_location,
|
||||
shoot_date,
|
||||
raw_text,
|
||||
movie_id,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if len(sys.argv) != 2:
|
||||
print(f"Användning: {sys.argv[0]} <movie-id>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
movie_id = sys.argv[1]
|
||||
|
||||
with sqlite3.connect(DATABASE_FILE) as connection:
|
||||
row = connection.execute(
|
||||
"""
|
||||
SELECT name, thumbnail
|
||||
FROM movie
|
||||
WHERE id = ?
|
||||
""",
|
||||
(movie_id,),
|
||||
).fetchone()
|
||||
|
||||
if row is None:
|
||||
print(
|
||||
f"Filmen finns inte i databasen: {movie_id}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
database_name, thumbnail = row
|
||||
|
||||
if not thumbnail:
|
||||
error_message = "Filmen saknar thumbnail."
|
||||
|
||||
set_ocr_failed(
|
||||
connection,
|
||||
movie_id,
|
||||
error_message,
|
||||
)
|
||||
|
||||
print(error_message, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
ocr_result = subprocess.run(
|
||||
[str(OCR_SCRIPT), thumbnail],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if ocr_result.returncode != 0:
|
||||
error_message = (
|
||||
ocr_result.stderr.strip()
|
||||
or "OCR-körningen misslyckades."
|
||||
)
|
||||
|
||||
set_ocr_failed(
|
||||
connection,
|
||||
movie_id,
|
||||
error_message,
|
||||
ocr_result.stdout.strip() or None,
|
||||
)
|
||||
|
||||
print("OCR-körningen misslyckades:", file=sys.stderr)
|
||||
print(error_message, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
raw_text = ocr_result.stdout.strip()
|
||||
|
||||
parser_result = subprocess.run(
|
||||
[str(PARSER_SCRIPT)],
|
||||
input=raw_text,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
if parser_result.returncode != 0:
|
||||
error_message = (
|
||||
parser_result.stderr.strip()
|
||||
or "Tolkningen av OCR-resultatet misslyckades."
|
||||
)
|
||||
|
||||
set_manual_review(
|
||||
connection,
|
||||
movie_id,
|
||||
raw_text,
|
||||
error_message,
|
||||
)
|
||||
|
||||
print(
|
||||
"Tolkningen av OCR-resultatet misslyckades:",
|
||||
file=sys.stderr,
|
||||
)
|
||||
print(error_message, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
parsed = parse_key_value_output(parser_result.stdout)
|
||||
|
||||
required_fields = (
|
||||
"ocr_name",
|
||||
"nationality",
|
||||
"shoot_location",
|
||||
"shoot_date",
|
||||
)
|
||||
|
||||
missing_fields = [
|
||||
field
|
||||
for field in required_fields
|
||||
if not parsed.get(field)
|
||||
]
|
||||
|
||||
if missing_fields:
|
||||
error_message = (
|
||||
"Parsern saknar fält: "
|
||||
+ ", ".join(missing_fields)
|
||||
)
|
||||
|
||||
set_manual_review(
|
||||
connection,
|
||||
movie_id,
|
||||
raw_text,
|
||||
error_message,
|
||||
)
|
||||
|
||||
print(error_message, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
ocr_name = parsed["ocr_name"]
|
||||
names_match = (
|
||||
normalize_name(database_name)
|
||||
== normalize_name(ocr_name)
|
||||
)
|
||||
|
||||
if not names_match:
|
||||
error_message = (
|
||||
"OCR-namnet matchar inte databasnamnet: "
|
||||
f"{ocr_name!r} != {database_name!r}"
|
||||
)
|
||||
|
||||
set_manual_review(
|
||||
connection,
|
||||
movie_id,
|
||||
raw_text,
|
||||
error_message,
|
||||
)
|
||||
|
||||
print(error_message, file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
set_ocr_completed(
|
||||
connection=connection,
|
||||
movie_id=movie_id,
|
||||
raw_text=raw_text,
|
||||
nationality=parsed["nationality"],
|
||||
shoot_location=parsed["shoot_location"],
|
||||
shoot_date=parsed["shoot_date"],
|
||||
)
|
||||
|
||||
print(f"id={movie_id}")
|
||||
print(f"database_name={database_name}")
|
||||
print(f"ocr_name={ocr_name}")
|
||||
print("name_match=yes")
|
||||
print(f"nationality={parsed['nationality']}")
|
||||
print(f"shoot_location={parsed['shoot_location']}")
|
||||
print(f"shoot_date={parsed['shoot_date']}")
|
||||
print("ocr_status=completed")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@ -1,5 +1,36 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Import WCX site metadata into the local SQLite movie index.
|
||||
|
||||
The script reads the JSON snapshot produced by the WCX site scraper and
|
||||
synchronizes it with the movie table in the local SQLite database.
|
||||
|
||||
Import rules:
|
||||
- A movie whose ID does not exist in the database is inserted.
|
||||
- An existing movie is updated only when the site provides an update date
|
||||
and that date is newer than the value currently stored in the database.
|
||||
- If the database has no update date but the site does, the movie is updated.
|
||||
- Movies without a newer site update date are left unchanged.
|
||||
|
||||
When an existing movie is updated, all fields owned by the site are replaced:
|
||||
name, duration, web URL, thumbnail URL, publication date, and update date.
|
||||
Manually maintained metadata fields are not modified.
|
||||
|
||||
Durations from the site are accepted as mm:ss or hh:mm:ss and stored as
|
||||
seconds in the duration_seconds column.
|
||||
|
||||
Input:
|
||||
/storage/disk1/WCX/import/wcx_site_index.json
|
||||
|
||||
Database:
|
||||
/storage/disk1/WCX/database/wcx.db
|
||||
|
||||
The script prints a summary showing the number of inserted, updated, and
|
||||
unchanged records.
|
||||
"""
|
||||
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
|
||||
102
scripts/ocr.sh
Executable file
102
scripts/ocr.sh
Executable file
@ -0,0 +1,102 @@
|
||||
#!/bin/bash
|
||||
|
||||
# --- KONFIGURATION ---
|
||||
# Skriptet förväntar sig nu miljövariabeln: GOOGLE_VISION_API_KEY
|
||||
# ---------------------
|
||||
|
||||
usage() {
|
||||
echo "Användning: $0 <bildfil eller URL> [-o utdatafil]"
|
||||
echo "Exempel 1: $0 kvitto.jpg"
|
||||
echo "Exempel 2: export GOOGLE_VISION_API_KEY=\"din_nyckel\" && $0 https://example.com/image.png -o text.txt"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# 1. Kontrollera att miljövariabeln är satt
|
||||
if [ -z "$GOOGLE_VISION_API_KEY" ]; then
|
||||
echo "Fel: Miljövariabeln GOOGLE_VISION_API_KEY är inte satt." >&2
|
||||
echo "Kör detta i terminalen först: export GOOGLE_VISION_API_KEY=\"din_faktiska_nyckel\"" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Kontrollera att vi fick minst ett argument
|
||||
if [ $# -lt 1 ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
INPUT="$1"
|
||||
OUTPUT_FILE=""
|
||||
|
||||
# Hantera flaggor för utdatafil (-o)
|
||||
if [ "$2" == "-o" ] && [ -n "$3" ]; then
|
||||
OUTPUT_FILE="$3"
|
||||
fi
|
||||
|
||||
# 2. Hantera bilden (lokal fil vs URL) och koda till Base64
|
||||
if [[ "$INPUT" =~ ^https?:// ]]; then
|
||||
# Det är en URL - ladda ner och koda till Base64 i minnet
|
||||
if ! command -v curl &> /dev/null; then
|
||||
echo "Fel: curl krävs men är inte installerat." >&2
|
||||
exit 1
|
||||
fi
|
||||
BASE64_IMAGE=$(curl -s "$INPUT" | base64)
|
||||
else
|
||||
# Det är en lokal fil
|
||||
if [ ! -f "$INPUT" ]; then
|
||||
echo "Fel: Filen '$INPUT' hittades inte." >&2
|
||||
exit 1
|
||||
fi
|
||||
# Hanterar både Mac (base64) och Linux (base64 -w 0) för att undvika radbrytningar i Base64-strängen
|
||||
if [[ "$OSTYPE" == "darwin"* ]]; then
|
||||
BASE64_IMAGE=$(base64 "$INPUT")
|
||||
else
|
||||
BASE64_IMAGE=$(base64 -w 0 "$INPUT")
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3. Skapa JSON-payloaden
|
||||
JSON_PAYLOAD=$(cat <<EOF
|
||||
{
|
||||
"requests": [
|
||||
{
|
||||
"image": {
|
||||
"content": "$BASE64_IMAGE"
|
||||
},
|
||||
"features": [
|
||||
{
|
||||
"type": "TEXT_DETECTION"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
# 4. Skicka till Google Cloud Vision API och filtrera ut texten via 'jq'
|
||||
if command -v jq &> /dev/null; then
|
||||
RESULT=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json; charset=utf-8" \
|
||||
-d "$JSON_PAYLOAD" \
|
||||
"https://vision.googleapis.com/v1/images:annotate?key=$GOOGLE_VISION_API_KEY" | jq -r '.responses[0].textAnnotations[0].description')
|
||||
else
|
||||
echo "Tips: Installera 'jq' (t.ex. 'brew install jq' eller 'apt install jq') för att få ren text." >&2
|
||||
RESULT=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json; charset=utf-8" \
|
||||
-d "$JSON_PAYLOAD" \
|
||||
"https://vision.googleapis.com/v1/images:annotate?key=$GOOGLE_VISION_API_KEY")
|
||||
fi
|
||||
|
||||
# Kontrollera om vi fick något svar eller om det blev fel
|
||||
if [ "$RESULT" == "null" ] || [ -z "$RESULT" ]; then
|
||||
echo "Ingen text hittades eller så uppstod ett API-fel." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 5. Skriv ut resultatet (till fil eller stdout)
|
||||
if [ -n "$OUTPUT_FILE" ]; then
|
||||
echo "$RESULT" > "$OUTPUT_FILE"
|
||||
echo "Klart! Texten har sparats i $OUTPUT_FILE"
|
||||
else
|
||||
echo "$RESULT"
|
||||
fi
|
||||
|
||||
109
scripts/parse_ocr.py
Executable file
109
scripts/parse_ocr.py
Executable file
@ -0,0 +1,109 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
"""
|
||||
Parse raw OCR text extracted from a WCX thumbnail.
|
||||
|
||||
The script reads OCR text from standard input and expects four non-empty lines:
|
||||
|
||||
1. Performer name
|
||||
2. Nationality
|
||||
3. WCX branding text, ignored by the parser
|
||||
4. Shoot location and date
|
||||
|
||||
The final line is expected to use the following format:
|
||||
|
||||
Budapest (Hungary), March 9, 2014
|
||||
|
||||
It is split on the first comma so that the comma inside the date is preserved.
|
||||
The date is normalized to ISO format, for example 2014-03-09.
|
||||
|
||||
On success, the script writes key/value pairs to standard output:
|
||||
|
||||
ocr_name=...
|
||||
nationality=...
|
||||
shoot_location=...
|
||||
shoot_date=...
|
||||
|
||||
Invalid or unexpected OCR output is reported on standard error and causes a
|
||||
non-zero exit status.
|
||||
"""
|
||||
|
||||
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
|
||||
def parse_ocr_text(raw_text: str) -> tuple[str, str, str, str]:
|
||||
lines = [
|
||||
line.strip()
|
||||
for line in raw_text.splitlines()
|
||||
if line.strip()
|
||||
]
|
||||
|
||||
if len(lines) != 4:
|
||||
raise ValueError(
|
||||
f"Förväntade 4 icke-tomma rader, fick {len(lines)}."
|
||||
)
|
||||
|
||||
ocr_name = lines[0]
|
||||
nationality = lines[1]
|
||||
ignored_text = lines[2]
|
||||
location_and_date = lines[3]
|
||||
|
||||
if not ocr_name:
|
||||
raise ValueError("Namn saknas.")
|
||||
|
||||
if not nationality:
|
||||
raise ValueError("Nationalitet saknas.")
|
||||
|
||||
if not ignored_text:
|
||||
raise ValueError("Förväntad tredje rad saknas.")
|
||||
|
||||
if "," not in location_and_date:
|
||||
raise ValueError(
|
||||
"Sista raden saknar kommatecken mellan plats och datum."
|
||||
)
|
||||
|
||||
shoot_location, date_text = location_and_date.split(",", 1)
|
||||
|
||||
shoot_location = shoot_location.strip()
|
||||
date_text = date_text.strip()
|
||||
|
||||
if not shoot_location:
|
||||
raise ValueError("Inspelningsplats saknas.")
|
||||
|
||||
try:
|
||||
shoot_date = datetime.strptime(
|
||||
date_text,
|
||||
"%B %d, %Y",
|
||||
).date().isoformat()
|
||||
except ValueError as error:
|
||||
raise ValueError(
|
||||
f"Kunde inte tolka datumet: {date_text!r}"
|
||||
) from error
|
||||
|
||||
return ocr_name, nationality, shoot_location, shoot_date
|
||||
|
||||
|
||||
def main() -> None:
|
||||
raw_text = sys.stdin.read()
|
||||
|
||||
if not raw_text.strip():
|
||||
raise ValueError("Ingen OCR-text mottogs på standard input.")
|
||||
|
||||
ocr_name, nationality, shoot_location, shoot_date = parse_ocr_text(
|
||||
raw_text
|
||||
)
|
||||
|
||||
print(f"ocr_name={ocr_name}")
|
||||
print(f"nationality={nationality}")
|
||||
print(f"shoot_location={shoot_location}")
|
||||
print(f"shoot_date={shoot_date}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except ValueError as error:
|
||||
print(f"Fel: {error}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
Reference in New Issue
Block a user