WCX/scripts/wcx_sync.py

#!/usr/bin/env python3
import argparse
import html
import json
import re
import sys
import time
import urllib.error
import urllib.request
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import parse_qs, urljoin, urlparse

BASE_URL = "https://www.woodmancastingx.com"
LIST_PATH = "/casting-xxx/"
DEFAULT_OUT = "/storage/disk1/X/wcx_index.json"
MAX_PAGES = 200
DEFAULT_SLEEP_MS = 300
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0"


VOID_TAGS = {
    "area", "base", "br", "col", "embed", "hr", "img", "input",
    "link", "meta", "param", "source", "track", "wbr",
}


class Node:
    def __init__(self, name, attrs=None):
        self.name = name
        self.attrs = dict(attrs or [])
        self.children = []
        self.text = []

    def get_text(self):
        return "".join(self.text + [c.get_text() for c in self.children]).strip()


class TreeParser(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.root = Node("root")
        self.stack = [self.root]

    def handle_starttag(self, tag, attrs):
        node = Node(tag, attrs)
        self.stack[-1].children.append(node)
        if tag not in VOID_TAGS:
            self.stack.append(node)

    def handle_endtag(self, tag):
        for i in range(len(self.stack) - 1, 0, -1):
            if self.stack[i].name == tag:
                self.stack = self.stack[:i]
                return

    def handle_data(self, data):
        if data:
            self.stack[-1].text.append(data)


def classes(node):
    return set((node.attrs.get("class") or "").split())


def walk(node):
    yield node
    for child in node.children:
        yield from walk(child)


def first_under(node, tag, cls=None):
    for n in walk(node):
        if n.name == tag and (cls is None or cls in classes(n)):
            return n
    return None


def attr_under(node, tag, cls, attr):
    n = first_under(node, tag, cls)
    if not n:
        return ""
    return (n.attrs.get(attr) or "").strip()


def absolute_url(href):
    return urljoin(BASE_URL, href)


def file_id_from_url(url):
    path = urlparse(url).path
    filename = Path(path).name
    return filename.rsplit(".", 1)[0]


def slug_title_from_href(href):
    stem = file_id_from_url(href)
    stem = stem.rsplit("_", 1)[0]
    return " ".join(w.capitalize() for w in stem.replace("-", " ").split())


def fetch_url_old(url, timeout, retries, debug=False):
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    last_error = None

    for attempt in range(1, retries + 1):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                raw = resp.read()
                return raw.decode("utf-8", errors="ignore")
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            last_error = e
            if debug:
                print(f"DEBUG fetch failed attempt {attempt}/{retries}: {url} ({e})", file=sys.stderr)
            time.sleep(min(1.0 * attempt, 3.0))

    raise RuntimeError(f"Could not fetch {url}: {last_error}")

def fetch_url(url, timeout, retries, debug=False):
    last_error = None

    for attempt in range(1, retries + 1):
        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": USER_AGENT,
                "Connection": "close",
            },
        )

        try:
            if debug or attempt > 1:
                print(
                    f"Fetching {url} "
                    f"(attempt {attempt}/{retries}, timeout={timeout}s)"
                )

            with urllib.request.urlopen(req, timeout=timeout) as response:
                raw = response.read()

            return raw.decode("utf-8", errors="ignore")

        except (
            urllib.error.HTTPError,
            urllib.error.URLError,
            TimeoutError,
            ConnectionError,
            OSError,
        ) as exc:
            last_error = exc

            print(
                f"WARNING: Fetch attempt {attempt}/{retries} failed "
                f"for {url}: {exc}",
                file=sys.stderr,
            )

            if attempt < retries:
                time.sleep(min(attempt, 3))

    raise RuntimeError(
        f"Could not fetch {url} after {retries} attempts: {last_error}"
    )


def parse_list_page(data):
    parser = TreeParser()
    parser.feed(data)

    containers = [
        n for n in walk(parser.root)
        if n.name == "div" and {"items", "container_3"}.issubset(classes(n))
    ] or [parser.root]

    items = []

    for container in containers:
        anchors = [
            n for n in walk(container)
            if n.name == "a" and {"item", "scene"}.issubset(classes(n))
        ]

        for a in anchors:
            href = (a.attrs.get("href") or "").strip()
            if not href:
                continue

            details = absolute_url(href)

            title_node = first_under(a, "span", "title")
            title = title_node.get_text() if title_node else ""
            title = title or (a.attrs.get("title") or "").strip()
            title = title or attr_under(a, "img", "thumb", "alt")
            title = title or slug_title_from_href(href)
            title = html.unescape(title).strip()

            if not title:
                continue

            duration_node = first_under(a, "span", "duration")
            duration = duration_node.get_text().strip() if duration_node else ""

            thumb = attr_under(a, "img", "thumb", "src")

            item = {
                "id": file_id_from_url(details),
                "titel": title,
                "details": details,
                "duration": duration,
            }

            if thumb:
                item["thumb"] = thumb

            items.append(item)

    return items


def parse_last_page(data):
    parser = TreeParser()
    parser.feed(data)

    base = urlparse(BASE_URL)
    page_numbers = []

    for node in walk(parser.root):
        if node.name != "a":
            continue

        href = (node.attrs.get("href") or "").strip()
        if not href:
            continue

        target = urlparse(urljoin(BASE_URL, href))
        if target.netloc != base.netloc or target.path != LIST_PATH:
            continue

        for value in parse_qs(target.query).get("page", []):
            if value.isdigit() and int(value) >= 1:
                page_numbers.append(int(value))

    if not page_numbers:
        raise RuntimeError("Could not detect a numeric last page from page 1 pagination")

    last_page = max(page_numbers)
    if last_page > MAX_PAGES:
        raise RuntimeError(
            f"Detected last page {last_page}, exceeding safety limit {MAX_PAGES}"
        )

    return last_page


class DetailDateParser(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.in_info_line = False
        self.in_label = False
        self.label = ""
        self.text = ""
        self.result = {}

    def _has_class(self, attrs, wanted):
        attrs = dict(attrs or [])
        return wanted in (attrs.get("class") or "").split()

    def handle_starttag(self, tag, attrs):
        if tag == "p" and self._has_class(attrs, "info_line"):
            self.in_info_line = True
            self.in_label = False
            self.label = ""
            self.text = ""
        elif self.in_info_line and tag == "span" and self._has_class(attrs, "label_info"):
            self.in_label = True

    def handle_endtag(self, tag):
        if tag == "span" and self.in_label:
            self.in_label = False
        elif tag == "p" and self.in_info_line:
            label = self.label.strip().lower()
            match = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", self.text)

            if match:
                if label == "published":
                    self.result["published"] = match.group(1)
                elif label == "updated":
                    self.result["updated"] = match.group(1)

            self.in_info_line = False
            self.in_label = False

    def handle_data(self, data):
        if self.in_info_line:
            self.text += data
            if self.in_label:
                self.label += data


def parse_detail_dates(data):
    parser = DetailDateParser()
    parser.feed(data)
    return parser.result


def unique_by_title(items):
    result = {}
    for item in items:
        result[item["titel"]] = item
    return list(result.values())


def load_existing(path):
    if not path.exists():
        return []

    with path.open("r", encoding="utf-8") as f:
        data = json.load(f)

    if not isinstance(data, list):
        raise ValueError(f"{path} must contain a JSON array")

    return data


def merge_items(old_items, new_items):
    merged = {item.get("titel", ""): dict(item) for item in old_items if item.get("titel")}

    for new in new_items:
        title = new.get("titel")
        if not title:
            continue

        old = merged.get(title)

        if old is None:
            merged[title] = new
            continue

        updated = dict(old)

        for key in ("id", "titel", "details", "duration", "thumb", "published"):
            if new.get(key):
                updated[key] = new[key]

        if new.get("updated"):
            updated["updated"] = new["updated"]
        else:
            updated.pop("updated", None)

        merged[title] = updated

    return list(merged.values())


def sort_items(items):
    return sorted(
        items,
        key=lambda x: x.get("updated") or x.get("published") or "",
        reverse=True,
    )


def write_json_one_object_per_line(path, items):
    tmp = path.with_suffix(path.suffix + ".tmp")

    with tmp.open("w", encoding="utf-8") as f:
        f.write("[\n")
        for i, item in enumerate(items):
            line = json.dumps(item, ensure_ascii=False, separators=(",", ":"))
            if i < len(items) - 1:
                line += ","
            f.write(line + "\n")
        f.write("]\n")

    tmp.replace(path)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=DEFAULT_OUT)
    ap.add_argument("--pages", "--end-page", dest="pages", type=int)
    ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS)
    ap.add_argument("--timeout", type=int, default=20)
    ap.add_argument("--retries", type=int, default=3)
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--debug", action="store_true")
    args = ap.parse_args()

    out_path = Path(args.out)

    requested_pages = args.pages if args.pages is not None else "auto"
    print(f"Start. pages={requested_pages}, out={out_path}, dry_run={args.dry_run}")

    if args.pages is not None and not 1 <= args.pages <= MAX_PAGES:
        ap.error(f"--pages must be between 1 and {MAX_PAGES}")

    list_items = []
    pages_fetched = 0

    first_url = f"{BASE_URL}{LIST_PATH}?page=1"
    print(f"Fetching list page 1: {first_url}")
    first_data = fetch_url(first_url, args.timeout, args.retries, args.debug)

    last_page = args.pages if args.pages is not None else parse_last_page(first_data)
    print(f"Using last list page: {last_page}")

    for page in range(1, last_page + 1):
        url = f"{BASE_URL}{LIST_PATH}?page={page}"
        if page == 1:
            data = first_data
        else:
            print(f"Fetching list page {page}: {url}")
            data = fetch_url(url, args.timeout, args.retries, args.debug)

        page_items = parse_list_page(data)
        if not page_items:
            raise RuntimeError(f"List page {page} returned zero results: {url}")

        list_items.extend(page_items)
        pages_fetched += 1

        if args.debug:
            print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}")

        if page < last_page and args.sleep_ms > 0:
            time.sleep(args.sleep_ms / 1000)

    list_items = unique_by_title(list_items)
    print(f"Unique list items: {len(list_items)}")

    enriched_items = []

    for idx, item in enumerate(list_items, start=1):
        details = item["details"]

        print(f"Fetching detail {idx}/{len(list_items)}: {item['titel']}")
        if args.debug:
            print(f"DEBUG URL: {details}")

        enriched = dict(item)

        try:
            detail_html = fetch_url(details, args.timeout, args.retries, args.debug)
            dates = parse_detail_dates(detail_html)

            if not dates.get("published"):
                print(f"WARNING: No Published date found for: {item['titel']} ({details})", file=sys.stderr)

            enriched.pop("published", None)
            enriched.pop("updated", None)
            enriched.update(dates)

        except RuntimeError as e:
            print(f"WARNING: {e}", file=sys.stderr)
            enriched.pop("published", None)
            enriched.pop("updated", None)

        enriched_items.append(enriched)

        if idx < len(list_items) and args.sleep_ms > 0:
            time.sleep(args.sleep_ms / 1000)

    old_items = load_existing(out_path)
    merged = merge_items(old_items, enriched_items)
    merged = sort_items(merged)

    print(f"Pages fetched: {pages_fetched} / {last_page}")
    print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}")

    if args.dry_run:
        print(f"Dry-run: NOT writing {out_path}")
        return

    out_path.parent.mkdir(parents=True, exist_ok=True)
    write_json_one_object_per_line(out_path, merged)
    print(f"Updated {out_path}")


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        print("\nInterrupted by user.", file=sys.stderr)
        sys.exit(130)