486 lines
14 KiB
Python
Executable File
486 lines
14 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import argparse
|
|
import html
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from html.parser import HTMLParser
|
|
from pathlib import Path
|
|
from urllib.parse import parse_qs, urljoin, urlparse
|
|
|
|
BASE_URL = "https://www.woodmancastingx.com"
|
|
LIST_PATH = "/casting-xxx/"
|
|
DEFAULT_OUT = "/storage/disk1/X/wcx_index.json"
|
|
MAX_PAGES = 200
|
|
DEFAULT_SLEEP_MS = 300
|
|
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0"
|
|
|
|
|
|
VOID_TAGS = {
|
|
"area", "base", "br", "col", "embed", "hr", "img", "input",
|
|
"link", "meta", "param", "source", "track", "wbr",
|
|
}
|
|
|
|
|
|
class Node:
|
|
def __init__(self, name, attrs=None):
|
|
self.name = name
|
|
self.attrs = dict(attrs or [])
|
|
self.children = []
|
|
self.text = []
|
|
|
|
def get_text(self):
|
|
return "".join(self.text + [c.get_text() for c in self.children]).strip()
|
|
|
|
|
|
class TreeParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.root = Node("root")
|
|
self.stack = [self.root]
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
node = Node(tag, attrs)
|
|
self.stack[-1].children.append(node)
|
|
if tag not in VOID_TAGS:
|
|
self.stack.append(node)
|
|
|
|
def handle_endtag(self, tag):
|
|
for i in range(len(self.stack) - 1, 0, -1):
|
|
if self.stack[i].name == tag:
|
|
self.stack = self.stack[:i]
|
|
return
|
|
|
|
def handle_data(self, data):
|
|
if data:
|
|
self.stack[-1].text.append(data)
|
|
|
|
|
|
def classes(node):
|
|
return set((node.attrs.get("class") or "").split())
|
|
|
|
|
|
def walk(node):
|
|
yield node
|
|
for child in node.children:
|
|
yield from walk(child)
|
|
|
|
|
|
def first_under(node, tag, cls=None):
|
|
for n in walk(node):
|
|
if n.name == tag and (cls is None or cls in classes(n)):
|
|
return n
|
|
return None
|
|
|
|
|
|
def attr_under(node, tag, cls, attr):
|
|
n = first_under(node, tag, cls)
|
|
if not n:
|
|
return ""
|
|
return (n.attrs.get(attr) or "").strip()
|
|
|
|
|
|
def absolute_url(href):
|
|
return urljoin(BASE_URL, href)
|
|
|
|
|
|
def file_id_from_url(url):
|
|
path = urlparse(url).path
|
|
filename = Path(path).name
|
|
return filename.rsplit(".", 1)[0]
|
|
|
|
|
|
def slug_title_from_href(href):
|
|
stem = file_id_from_url(href)
|
|
stem = stem.rsplit("_", 1)[0]
|
|
return " ".join(w.capitalize() for w in stem.replace("-", " ").split())
|
|
|
|
|
|
def fetch_url_old(url, timeout, retries, debug=False):
|
|
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
|
last_error = None
|
|
|
|
for attempt in range(1, retries + 1):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
|
raw = resp.read()
|
|
return raw.decode("utf-8", errors="ignore")
|
|
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
|
last_error = e
|
|
if debug:
|
|
print(f"DEBUG fetch failed attempt {attempt}/{retries}: {url} ({e})", file=sys.stderr)
|
|
time.sleep(min(1.0 * attempt, 3.0))
|
|
|
|
raise RuntimeError(f"Could not fetch {url}: {last_error}")
|
|
|
|
def fetch_url(url, timeout, retries, debug=False):
|
|
last_error = None
|
|
|
|
for attempt in range(1, retries + 1):
|
|
req = urllib.request.Request(
|
|
url,
|
|
headers={
|
|
"User-Agent": USER_AGENT,
|
|
"Connection": "close",
|
|
},
|
|
)
|
|
|
|
try:
|
|
if debug or attempt > 1:
|
|
print(
|
|
f"Fetching {url} "
|
|
f"(attempt {attempt}/{retries}, timeout={timeout}s)"
|
|
)
|
|
|
|
with urllib.request.urlopen(req, timeout=timeout) as response:
|
|
raw = response.read()
|
|
|
|
return raw.decode("utf-8", errors="ignore")
|
|
|
|
except (
|
|
urllib.error.HTTPError,
|
|
urllib.error.URLError,
|
|
TimeoutError,
|
|
ConnectionError,
|
|
OSError,
|
|
) as exc:
|
|
last_error = exc
|
|
|
|
print(
|
|
f"WARNING: Fetch attempt {attempt}/{retries} failed "
|
|
f"for {url}: {exc}",
|
|
file=sys.stderr,
|
|
)
|
|
|
|
if attempt < retries:
|
|
time.sleep(min(attempt, 3))
|
|
|
|
raise RuntimeError(
|
|
f"Could not fetch {url} after {retries} attempts: {last_error}"
|
|
)
|
|
|
|
|
|
def parse_list_page(data):
|
|
parser = TreeParser()
|
|
parser.feed(data)
|
|
|
|
containers = [
|
|
n for n in walk(parser.root)
|
|
if n.name == "div" and {"items", "container_3"}.issubset(classes(n))
|
|
] or [parser.root]
|
|
|
|
items = []
|
|
|
|
for container in containers:
|
|
anchors = [
|
|
n for n in walk(container)
|
|
if n.name == "a" and {"item", "scene"}.issubset(classes(n))
|
|
]
|
|
|
|
for a in anchors:
|
|
href = (a.attrs.get("href") or "").strip()
|
|
if not href:
|
|
continue
|
|
|
|
details = absolute_url(href)
|
|
|
|
title_node = first_under(a, "span", "title")
|
|
title = title_node.get_text() if title_node else ""
|
|
title = title or (a.attrs.get("title") or "").strip()
|
|
title = title or attr_under(a, "img", "thumb", "alt")
|
|
title = title or slug_title_from_href(href)
|
|
title = html.unescape(title).strip()
|
|
|
|
if not title:
|
|
continue
|
|
|
|
duration_node = first_under(a, "span", "duration")
|
|
duration = duration_node.get_text().strip() if duration_node else ""
|
|
|
|
thumb = attr_under(a, "img", "thumb", "src")
|
|
|
|
item = {
|
|
"id": file_id_from_url(details),
|
|
"titel": title,
|
|
"details": details,
|
|
"duration": duration,
|
|
}
|
|
|
|
if thumb:
|
|
item["thumb"] = thumb
|
|
|
|
items.append(item)
|
|
|
|
return items
|
|
|
|
|
|
def parse_last_page(data):
|
|
parser = TreeParser()
|
|
parser.feed(data)
|
|
|
|
base = urlparse(BASE_URL)
|
|
page_numbers = []
|
|
|
|
for node in walk(parser.root):
|
|
if node.name != "a":
|
|
continue
|
|
|
|
href = (node.attrs.get("href") or "").strip()
|
|
if not href:
|
|
continue
|
|
|
|
target = urlparse(urljoin(BASE_URL, href))
|
|
if target.netloc != base.netloc or target.path != LIST_PATH:
|
|
continue
|
|
|
|
for value in parse_qs(target.query).get("page", []):
|
|
if value.isdigit() and int(value) >= 1:
|
|
page_numbers.append(int(value))
|
|
|
|
if not page_numbers:
|
|
raise RuntimeError("Could not detect a numeric last page from page 1 pagination")
|
|
|
|
last_page = max(page_numbers)
|
|
if last_page > MAX_PAGES:
|
|
raise RuntimeError(
|
|
f"Detected last page {last_page}, exceeding safety limit {MAX_PAGES}"
|
|
)
|
|
|
|
return last_page
|
|
|
|
|
|
class DetailDateParser(HTMLParser):
|
|
def __init__(self):
|
|
super().__init__(convert_charrefs=True)
|
|
self.in_info_line = False
|
|
self.in_label = False
|
|
self.label = ""
|
|
self.text = ""
|
|
self.result = {}
|
|
|
|
def _has_class(self, attrs, wanted):
|
|
attrs = dict(attrs or [])
|
|
return wanted in (attrs.get("class") or "").split()
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
if tag == "p" and self._has_class(attrs, "info_line"):
|
|
self.in_info_line = True
|
|
self.in_label = False
|
|
self.label = ""
|
|
self.text = ""
|
|
elif self.in_info_line and tag == "span" and self._has_class(attrs, "label_info"):
|
|
self.in_label = True
|
|
|
|
def handle_endtag(self, tag):
|
|
if tag == "span" and self.in_label:
|
|
self.in_label = False
|
|
elif tag == "p" and self.in_info_line:
|
|
label = self.label.strip().lower()
|
|
match = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", self.text)
|
|
|
|
if match:
|
|
if label == "published":
|
|
self.result["published"] = match.group(1)
|
|
elif label == "updated":
|
|
self.result["updated"] = match.group(1)
|
|
|
|
self.in_info_line = False
|
|
self.in_label = False
|
|
|
|
def handle_data(self, data):
|
|
if self.in_info_line:
|
|
self.text += data
|
|
if self.in_label:
|
|
self.label += data
|
|
|
|
|
|
def parse_detail_dates(data):
|
|
parser = DetailDateParser()
|
|
parser.feed(data)
|
|
return parser.result
|
|
|
|
|
|
def unique_by_title(items):
|
|
result = {}
|
|
for item in items:
|
|
result[item["titel"]] = item
|
|
return list(result.values())
|
|
|
|
|
|
def load_existing(path):
|
|
if not path.exists():
|
|
return []
|
|
|
|
with path.open("r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
|
|
if not isinstance(data, list):
|
|
raise ValueError(f"{path} must contain a JSON array")
|
|
|
|
return data
|
|
|
|
|
|
def merge_items(old_items, new_items):
|
|
merged = {item.get("titel", ""): dict(item) for item in old_items if item.get("titel")}
|
|
|
|
for new in new_items:
|
|
title = new.get("titel")
|
|
if not title:
|
|
continue
|
|
|
|
old = merged.get(title)
|
|
|
|
if old is None:
|
|
merged[title] = new
|
|
continue
|
|
|
|
updated = dict(old)
|
|
|
|
for key in ("id", "titel", "details", "duration", "thumb", "published"):
|
|
if new.get(key):
|
|
updated[key] = new[key]
|
|
|
|
if new.get("updated"):
|
|
updated["updated"] = new["updated"]
|
|
else:
|
|
updated.pop("updated", None)
|
|
|
|
merged[title] = updated
|
|
|
|
return list(merged.values())
|
|
|
|
|
|
def sort_items(items):
|
|
return sorted(
|
|
items,
|
|
key=lambda x: x.get("updated") or x.get("published") or "",
|
|
reverse=True,
|
|
)
|
|
|
|
|
|
def write_json_one_object_per_line(path, items):
|
|
tmp = path.with_suffix(path.suffix + ".tmp")
|
|
|
|
with tmp.open("w", encoding="utf-8") as f:
|
|
f.write("[\n")
|
|
for i, item in enumerate(items):
|
|
line = json.dumps(item, ensure_ascii=False, separators=(",", ":"))
|
|
if i < len(items) - 1:
|
|
line += ","
|
|
f.write(line + "\n")
|
|
f.write("]\n")
|
|
|
|
tmp.replace(path)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--out", default=DEFAULT_OUT)
|
|
ap.add_argument("--pages", "--end-page", dest="pages", type=int)
|
|
ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS)
|
|
ap.add_argument("--timeout", type=int, default=20)
|
|
ap.add_argument("--retries", type=int, default=3)
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
ap.add_argument("--debug", action="store_true")
|
|
args = ap.parse_args()
|
|
|
|
out_path = Path(args.out)
|
|
|
|
requested_pages = args.pages if args.pages is not None else "auto"
|
|
print(f"Start. pages={requested_pages}, out={out_path}, dry_run={args.dry_run}")
|
|
|
|
if args.pages is not None and not 1 <= args.pages <= MAX_PAGES:
|
|
ap.error(f"--pages must be between 1 and {MAX_PAGES}")
|
|
|
|
list_items = []
|
|
pages_fetched = 0
|
|
|
|
first_url = f"{BASE_URL}{LIST_PATH}?page=1"
|
|
print(f"Fetching list page 1: {first_url}")
|
|
first_data = fetch_url(first_url, args.timeout, args.retries, args.debug)
|
|
|
|
last_page = args.pages if args.pages is not None else parse_last_page(first_data)
|
|
print(f"Using last list page: {last_page}")
|
|
|
|
for page in range(1, last_page + 1):
|
|
url = f"{BASE_URL}{LIST_PATH}?page={page}"
|
|
if page == 1:
|
|
data = first_data
|
|
else:
|
|
print(f"Fetching list page {page}: {url}")
|
|
data = fetch_url(url, args.timeout, args.retries, args.debug)
|
|
|
|
page_items = parse_list_page(data)
|
|
if not page_items:
|
|
raise RuntimeError(f"List page {page} returned zero results: {url}")
|
|
|
|
list_items.extend(page_items)
|
|
pages_fetched += 1
|
|
|
|
if args.debug:
|
|
print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}")
|
|
|
|
if page < last_page and args.sleep_ms > 0:
|
|
time.sleep(args.sleep_ms / 1000)
|
|
|
|
list_items = unique_by_title(list_items)
|
|
print(f"Unique list items: {len(list_items)}")
|
|
|
|
enriched_items = []
|
|
|
|
for idx, item in enumerate(list_items, start=1):
|
|
details = item["details"]
|
|
|
|
print(f"Fetching detail {idx}/{len(list_items)}: {item['titel']}")
|
|
if args.debug:
|
|
print(f"DEBUG URL: {details}")
|
|
|
|
enriched = dict(item)
|
|
|
|
try:
|
|
detail_html = fetch_url(details, args.timeout, args.retries, args.debug)
|
|
dates = parse_detail_dates(detail_html)
|
|
|
|
if not dates.get("published"):
|
|
print(f"WARNING: No Published date found for: {item['titel']} ({details})", file=sys.stderr)
|
|
|
|
enriched.pop("published", None)
|
|
enriched.pop("updated", None)
|
|
enriched.update(dates)
|
|
|
|
except RuntimeError as e:
|
|
print(f"WARNING: {e}", file=sys.stderr)
|
|
enriched.pop("published", None)
|
|
enriched.pop("updated", None)
|
|
|
|
enriched_items.append(enriched)
|
|
|
|
if idx < len(list_items) and args.sleep_ms > 0:
|
|
time.sleep(args.sleep_ms / 1000)
|
|
|
|
old_items = load_existing(out_path)
|
|
merged = merge_items(old_items, enriched_items)
|
|
merged = sort_items(merged)
|
|
|
|
print(f"Pages fetched: {pages_fetched} / {last_page}")
|
|
print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}")
|
|
|
|
if args.dry_run:
|
|
print(f"Dry-run: NOT writing {out_path}")
|
|
return
|
|
|
|
out_path.parent.mkdir(parents=True, exist_ok=True)
|
|
write_json_one_object_per_line(out_path, merged)
|
|
print(f"Updated {out_path}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
main()
|
|
except KeyboardInterrupt:
|
|
print("\nInterrupted by user.", file=sys.stderr)
|
|
sys.exit(130)
|