#!/usr/bin/env python3
import argparse
import html
import json
import re
import sys
import time
import urllib.error
import urllib.request
from html.parser import HTMLParser
from pathlib import Path
from urllib.parse import urljoin, urlparse

BASE_URL = "https://www.woodmancastingx.com"
LIST_PATH = "/casting-xxx/"
DEFAULT_OUT = "/storage/disk1/X/wcx_index.json"
DEFAULT_PAGES = 48
DEFAULT_SLEEP_MS = 300
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0"


VOID_TAGS = {
    "area", "base", "br", "col", "embed", "hr", "img", "input",
    "link", "meta", "param", "source", "track", "wbr",
}


class Node:
    def __init__(self, name, attrs=None):
        self.name = name
        self.attrs = dict(attrs or [])
        self.children = []
        self.text = []

    def get_text(self):
        return "".join(self.text + [c.get_text() for c in self.children]).strip()


class TreeParser(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.root = Node("root")
        self.stack = [self.root]

    def handle_starttag(self, tag, attrs):
        node = Node(tag, attrs)
        self.stack[-1].children.append(node)
        if tag not in VOID_TAGS:
            self.stack.append(node)

    def handle_endtag(self, tag):
        for i in range(len(self.stack) - 1, 0, -1):
            if self.stack[i].name == tag:
                self.stack = self.stack[:i]
                return

    def handle_data(self, data):
        if data:
            self.stack[-1].text.append(data)


def classes(node):
    return set((node.attrs.get("class") or "").split())


def walk(node):
    yield node
    for child in node.children:
        yield from walk(child)


def first_under(node, tag, cls=None):
    for n in walk(node):
        if n.name == tag and (cls is None or cls in classes(n)):
            return n
    return None


def attr_under(node, tag, cls, attr):
    n = first_under(node, tag, cls)
    if not n:
        return ""
    return (n.attrs.get(attr) or "").strip()


def absolute_url(href):
    return urljoin(BASE_URL, href)


def file_id_from_url(url):
    path = urlparse(url).path
    filename = Path(path).name
    return filename.rsplit(".", 1)[0]


def slug_title_from_href(href):
    stem = file_id_from_url(href)
    stem = stem.rsplit("_", 1)[0]
    return " ".join(w.capitalize() for w in stem.replace("-", " ").split())


def fetch_url_old(url, timeout, retries, debug=False):
    req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
    last_error = None

    for attempt in range(1, retries + 1):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                raw = resp.read()
                return raw.decode("utf-8", errors="ignore")
        except (urllib.error.URLError, TimeoutError, OSError) as e:
            last_error = e
            if debug:
                print(f"DEBUG fetch failed attempt {attempt}/{retries}: {url} ({e})", file=sys.stderr)
            time.sleep(min(1.0 * attempt, 3.0))

    raise RuntimeError(f"Could not fetch {url}: {last_error}")

def fetch_url(url, timeout, retries, debug=False):
    last_error = None

    for attempt in range(1, retries + 1):
        req = urllib.request.Request(
            url,
            headers={
                "User-Agent": USER_AGENT,
                "Connection": "close",
            },
        )

        try:
            if debug or attempt > 1:
                print(
                    f"Fetching {url} "
                    f"(attempt {attempt}/{retries}, timeout={timeout}s)"
                )

            with urllib.request.urlopen(req, timeout=timeout) as response:
                raw = response.read()

            return raw.decode("utf-8", errors="ignore")

        except (
            urllib.error.HTTPError,
            urllib.error.URLError,
            TimeoutError,
            ConnectionError,
            OSError,
        ) as exc:
            last_error = exc

            print(
                f"WARNING: Fetch attempt {attempt}/{retries} failed "
                f"for {url}: {exc}",
                file=sys.stderr,
            )

            if attempt < retries:
                time.sleep(min(attempt, 3))

    raise RuntimeError(
        f"Could not fetch {url} after {retries} attempts: {last_error}"
    )


def parse_list_page(data):
    parser = TreeParser()
    parser.feed(data)

    containers = [
        n for n in walk(parser.root)
        if n.name == "div" and {"items", "container_3"}.issubset(classes(n))
    ] or [parser.root]

    items = []

    for container in containers:
        anchors = [
            n for n in walk(container)
            if n.name == "a" and {"item", "scene"}.issubset(classes(n))
        ]

        for a in anchors:
            href = (a.attrs.get("href") or "").strip()
            if not href:
                continue

            details = absolute_url(href)

            title_node = first_under(a, "span", "title")
            title = title_node.get_text() if title_node else ""
            title = title or (a.attrs.get("title") or "").strip()
            title = title or attr_under(a, "img", "thumb", "alt")
            title = title or slug_title_from_href(href)
            title = html.unescape(title).strip()

            if not title:
                continue

            duration_node = first_under(a, "span", "duration")
            duration = duration_node.get_text().strip() if duration_node else ""

            thumb = attr_under(a, "img", "thumb", "src")

            item = {
                "id": file_id_from_url(details),
                "titel": title,
                "details": details,
                "duration": duration,
            }

            if thumb:
                item["thumb"] = thumb

            items.append(item)

    return items


class DetailDateParser(HTMLParser):
    def __init__(self):
        super().__init__(convert_charrefs=True)
        self.in_info_line = False
        self.in_label = False
        self.label = ""
        self.text = ""
        self.result = {}

    def _has_class(self, attrs, wanted):
        attrs = dict(attrs or [])
        return wanted in (attrs.get("class") or "").split()

    def handle_starttag(self, tag, attrs):
        if tag == "p" and self._has_class(attrs, "info_line"):
            self.in_info_line = True
            self.in_label = False
            self.label = ""
            self.text = ""
        elif self.in_info_line and tag == "span" and self._has_class(attrs, "label_info"):
            self.in_label = True

    def handle_endtag(self, tag):
        if tag == "span" and self.in_label:
            self.in_label = False
        elif tag == "p" and self.in_info_line:
            label = self.label.strip().lower()
            match = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", self.text)

            if match:
                if label == "published":
                    self.result["published"] = match.group(1)
                elif label == "updated":
                    self.result["updated"] = match.group(1)

            self.in_info_line = False
            self.in_label = False

    def handle_data(self, data):
        if self.in_info_line:
            self.text += data
            if self.in_label:
                self.label += data


def parse_detail_dates(data):
    parser = DetailDateParser()
    parser.feed(data)
    return parser.result


def unique_by_title(items):
    result = {}
    for item in items:
        result[item["titel"]] = item
    return list(result.values())


def load_existing(path):
    if not path.exists():
        return []

    with path.open("r", encoding="utf-8") as f:
        data = json.load(f)

    if not isinstance(data, list):
        raise ValueError(f"{path} must contain a JSON array")

    return data


def merge_items(old_items, new_items):
    merged = {item.get("titel", ""): dict(item) for item in old_items if item.get("titel")}

    for new in new_items:
        title = new.get("titel")
        if not title:
            continue

        old = merged.get(title)

        if old is None:
            merged[title] = new
            continue

        updated = dict(old)

        for key in ("id", "titel", "details", "duration", "thumb", "published"):
            if new.get(key):
                updated[key] = new[key]

        if new.get("updated"):
            updated["updated"] = new["updated"]
        else:
            updated.pop("updated", None)

        merged[title] = updated

    return list(merged.values())


def sort_items(items):
    return sorted(
        items,
        key=lambda x: x.get("updated") or x.get("published") or "",
        reverse=True,
    )


def write_json_one_object_per_line(path, items):
    tmp = path.with_suffix(path.suffix + ".tmp")

    with tmp.open("w", encoding="utf-8") as f:
        f.write("[\n")
        for i, item in enumerate(items):
            line = json.dumps(item, ensure_ascii=False, separators=(",", ":"))
            if i < len(items) - 1:
                line += ","
            f.write(line + "\n")
        f.write("]\n")

    tmp.replace(path)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--out", default=DEFAULT_OUT)
    ap.add_argument("--pages", "--end-page", dest="pages", type=int, default=DEFAULT_PAGES)
    ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS)
    ap.add_argument("--timeout", type=int, default=20)
    ap.add_argument("--retries", type=int, default=3)
    ap.add_argument("--dry-run", action="store_true")
    ap.add_argument("--debug", action="store_true")
    args = ap.parse_args()

    out_path = Path(args.out)

    print(f"Start. pages=1..{args.pages}, out={out_path}, dry_run={args.dry_run}")

    list_items = []
    pages_fetched = 0

    for page in range(1, args.pages + 1):
        url = f"{BASE_URL}{LIST_PATH}?page={page}"
        print(f"Fetching list page {page}: {url}")

        try:
            data = fetch_url(url, args.timeout, args.retries, args.debug)
        except RuntimeError as e:
            print(f"WARNING: {e}", file=sys.stderr)
            continue

        page_items = parse_list_page(data)
        list_items.extend(page_items)
        pages_fetched += 1

        if args.debug:
            print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}")

        if page < args.pages and args.sleep_ms > 0:
            time.sleep(args.sleep_ms / 1000)

    list_items = unique_by_title(list_items)
    print(f"Unique list items: {len(list_items)}")

    enriched_items = []

    for idx, item in enumerate(list_items, start=1):
        details = item["details"]

        print(f"Fetching detail {idx}/{len(list_items)}: {item['titel']}")
        if args.debug:
            print(f"DEBUG URL: {details}")

        enriched = dict(item)

        try:
            detail_html = fetch_url(details, args.timeout, args.retries, args.debug)
            dates = parse_detail_dates(detail_html)

            if not dates.get("published"):
                print(f"WARNING: No Published date found for: {item['titel']} ({details})", file=sys.stderr)

            enriched.pop("published", None)
            enriched.pop("updated", None)
            enriched.update(dates)

        except RuntimeError as e:
            print(f"WARNING: {e}", file=sys.stderr)
            enriched.pop("published", None)
            enriched.pop("updated", None)

        enriched_items.append(enriched)

        if idx < len(list_items) and args.sleep_ms > 0:
            time.sleep(args.sleep_ms / 1000)

    old_items = load_existing(out_path)
    merged = merge_items(old_items, enriched_items)
    merged = sort_items(merged)

    print(f"Pages fetched: {pages_fetched} / {args.pages}")
    print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}")

    if args.dry_run:
        print(f"Dry-run: NOT writing {out_path}")
        return

    out_path.parent.mkdir(parents=True, exist_ok=True)
    write_json_one_object_per_line(out_path, merged)
    print(f"Updated {out_path}")


if __name__ == "__main__":
    try:
        main()
    except KeyboardInterrupt:
        print("\nInterrupted by user.", file=sys.stderr)
        sys.exit(130)