Organize sync script and clean migration data
This commit is contained in:
438
scripts/wcx_sync.py
Executable file
438
scripts/wcx_sync.py
Executable file
@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
BASE_URL = "https://www.woodmancastingx.com"
|
||||
LIST_PATH = "/casting-xxx/"
|
||||
DEFAULT_OUT = "/storage/disk1/X/wcx_index.json"
|
||||
DEFAULT_PAGES = 48
|
||||
DEFAULT_SLEEP_MS = 300
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0"
|
||||
|
||||
|
||||
VOID_TAGS = {
|
||||
"area", "base", "br", "col", "embed", "hr", "img", "input",
|
||||
"link", "meta", "param", "source", "track", "wbr",
|
||||
}
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, name, attrs=None):
|
||||
self.name = name
|
||||
self.attrs = dict(attrs or [])
|
||||
self.children = []
|
||||
self.text = []
|
||||
|
||||
def get_text(self):
|
||||
return "".join(self.text + [c.get_text() for c in self.children]).strip()
|
||||
|
||||
|
||||
class TreeParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = Node("root")
|
||||
self.stack = [self.root]
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
node = Node(tag, attrs)
|
||||
self.stack[-1].children.append(node)
|
||||
if tag not in VOID_TAGS:
|
||||
self.stack.append(node)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
for i in range(len(self.stack) - 1, 0, -1):
|
||||
if self.stack[i].name == tag:
|
||||
self.stack = self.stack[:i]
|
||||
return
|
||||
|
||||
def handle_data(self, data):
|
||||
if data:
|
||||
self.stack[-1].text.append(data)
|
||||
|
||||
|
||||
def classes(node):
|
||||
return set((node.attrs.get("class") or "").split())
|
||||
|
||||
|
||||
def walk(node):
|
||||
yield node
|
||||
for child in node.children:
|
||||
yield from walk(child)
|
||||
|
||||
|
||||
def first_under(node, tag, cls=None):
|
||||
for n in walk(node):
|
||||
if n.name == tag and (cls is None or cls in classes(n)):
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
def attr_under(node, tag, cls, attr):
|
||||
n = first_under(node, tag, cls)
|
||||
if not n:
|
||||
return ""
|
||||
return (n.attrs.get(attr) or "").strip()
|
||||
|
||||
|
||||
def absolute_url(href):
|
||||
return urljoin(BASE_URL, href)
|
||||
|
||||
|
||||
def file_id_from_url(url):
|
||||
path = urlparse(url).path
|
||||
filename = Path(path).name
|
||||
return filename.rsplit(".", 1)[0]
|
||||
|
||||
|
||||
def slug_title_from_href(href):
|
||||
stem = file_id_from_url(href)
|
||||
stem = stem.rsplit("_", 1)[0]
|
||||
return " ".join(w.capitalize() for w in stem.replace("-", " ").split())
|
||||
|
||||
|
||||
def fetch_url_old(url, timeout, retries, debug=False):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
last_error = None
|
||||
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||
last_error = e
|
||||
if debug:
|
||||
print(f"DEBUG fetch failed attempt {attempt}/{retries}: {url} ({e})", file=sys.stderr)
|
||||
time.sleep(min(1.0 * attempt, 3.0))
|
||||
|
||||
raise RuntimeError(f"Could not fetch {url}: {last_error}")
|
||||
|
||||
def fetch_url(url, timeout, retries, debug=False):
|
||||
last_error = None
|
||||
|
||||
for attempt in range(1, retries + 1):
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
if debug or attempt > 1:
|
||||
print(
|
||||
f"Fetching {url} "
|
||||
f"(attempt {attempt}/{retries}, timeout={timeout}s)"
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
raw = response.read()
|
||||
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
|
||||
except (
|
||||
urllib.error.HTTPError,
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
) as exc:
|
||||
last_error = exc
|
||||
|
||||
print(
|
||||
f"WARNING: Fetch attempt {attempt}/{retries} failed "
|
||||
f"for {url}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if attempt < retries:
|
||||
time.sleep(min(attempt, 3))
|
||||
|
||||
raise RuntimeError(
|
||||
f"Could not fetch {url} after {retries} attempts: {last_error}"
|
||||
)
|
||||
|
||||
|
||||
def parse_list_page(data):
|
||||
parser = TreeParser()
|
||||
parser.feed(data)
|
||||
|
||||
containers = [
|
||||
n for n in walk(parser.root)
|
||||
if n.name == "div" and {"items", "container_3"}.issubset(classes(n))
|
||||
] or [parser.root]
|
||||
|
||||
items = []
|
||||
|
||||
for container in containers:
|
||||
anchors = [
|
||||
n for n in walk(container)
|
||||
if n.name == "a" and {"item", "scene"}.issubset(classes(n))
|
||||
]
|
||||
|
||||
for a in anchors:
|
||||
href = (a.attrs.get("href") or "").strip()
|
||||
if not href:
|
||||
continue
|
||||
|
||||
details = absolute_url(href)
|
||||
|
||||
title_node = first_under(a, "span", "title")
|
||||
title = title_node.get_text() if title_node else ""
|
||||
title = title or (a.attrs.get("title") or "").strip()
|
||||
title = title or attr_under(a, "img", "thumb", "alt")
|
||||
title = title or slug_title_from_href(href)
|
||||
title = html.unescape(title).strip()
|
||||
|
||||
if not title:
|
||||
continue
|
||||
|
||||
duration_node = first_under(a, "span", "duration")
|
||||
duration = duration_node.get_text().strip() if duration_node else ""
|
||||
|
||||
thumb = attr_under(a, "img", "thumb", "src")
|
||||
|
||||
item = {
|
||||
"id": file_id_from_url(details),
|
||||
"titel": title,
|
||||
"details": details,
|
||||
"duration": duration,
|
||||
}
|
||||
|
||||
if thumb:
|
||||
item["thumb"] = thumb
|
||||
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
class DetailDateParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.in_info_line = False
|
||||
self.in_label = False
|
||||
self.label = ""
|
||||
self.text = ""
|
||||
self.result = {}
|
||||
|
||||
def _has_class(self, attrs, wanted):
|
||||
attrs = dict(attrs or [])
|
||||
return wanted in (attrs.get("class") or "").split()
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "p" and self._has_class(attrs, "info_line"):
|
||||
self.in_info_line = True
|
||||
self.in_label = False
|
||||
self.label = ""
|
||||
self.text = ""
|
||||
elif self.in_info_line and tag == "span" and self._has_class(attrs, "label_info"):
|
||||
self.in_label = True
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == "span" and self.in_label:
|
||||
self.in_label = False
|
||||
elif tag == "p" and self.in_info_line:
|
||||
label = self.label.strip().lower()
|
||||
match = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", self.text)
|
||||
|
||||
if match:
|
||||
if label == "published":
|
||||
self.result["published"] = match.group(1)
|
||||
elif label == "updated":
|
||||
self.result["updated"] = match.group(1)
|
||||
|
||||
self.in_info_line = False
|
||||
self.in_label = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_info_line:
|
||||
self.text += data
|
||||
if self.in_label:
|
||||
self.label += data
|
||||
|
||||
|
||||
def parse_detail_dates(data):
|
||||
parser = DetailDateParser()
|
||||
parser.feed(data)
|
||||
return parser.result
|
||||
|
||||
|
||||
def unique_by_title(items):
|
||||
result = {}
|
||||
for item in items:
|
||||
result[item["titel"]] = item
|
||||
return list(result.values())
|
||||
|
||||
|
||||
def load_existing(path):
|
||||
if not path.exists():
|
||||
return []
|
||||
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, list):
|
||||
raise ValueError(f"{path} must contain a JSON array")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_items(old_items, new_items):
|
||||
merged = {item.get("titel", ""): dict(item) for item in old_items if item.get("titel")}
|
||||
|
||||
for new in new_items:
|
||||
title = new.get("titel")
|
||||
if not title:
|
||||
continue
|
||||
|
||||
old = merged.get(title)
|
||||
|
||||
if old is None:
|
||||
merged[title] = new
|
||||
continue
|
||||
|
||||
updated = dict(old)
|
||||
|
||||
for key in ("id", "titel", "details", "duration", "thumb", "published"):
|
||||
if new.get(key):
|
||||
updated[key] = new[key]
|
||||
|
||||
if new.get("updated"):
|
||||
updated["updated"] = new["updated"]
|
||||
else:
|
||||
updated.pop("updated", None)
|
||||
|
||||
merged[title] = updated
|
||||
|
||||
return list(merged.values())
|
||||
|
||||
|
||||
def sort_items(items):
|
||||
return sorted(
|
||||
items,
|
||||
key=lambda x: x.get("updated") or x.get("published") or "",
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def write_json_one_object_per_line(path, items):
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
|
||||
with tmp.open("w", encoding="utf-8") as f:
|
||||
f.write("[\n")
|
||||
for i, item in enumerate(items):
|
||||
line = json.dumps(item, ensure_ascii=False, separators=(",", ":"))
|
||||
if i < len(items) - 1:
|
||||
line += ","
|
||||
f.write(line + "\n")
|
||||
f.write("]\n")
|
||||
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default=DEFAULT_OUT)
|
||||
ap.add_argument("--pages", "--end-page", dest="pages", type=int, default=DEFAULT_PAGES)
|
||||
ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS)
|
||||
ap.add_argument("--timeout", type=int, default=20)
|
||||
ap.add_argument("--retries", type=int, default=3)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--debug", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
out_path = Path(args.out)
|
||||
|
||||
print(f"Start. pages=1..{args.pages}, out={out_path}, dry_run={args.dry_run}")
|
||||
|
||||
list_items = []
|
||||
pages_fetched = 0
|
||||
|
||||
for page in range(1, args.pages + 1):
|
||||
url = f"{BASE_URL}{LIST_PATH}?page={page}"
|
||||
print(f"Fetching list page {page}: {url}")
|
||||
|
||||
try:
|
||||
data = fetch_url(url, args.timeout, args.retries, args.debug)
|
||||
except RuntimeError as e:
|
||||
print(f"WARNING: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
page_items = parse_list_page(data)
|
||||
list_items.extend(page_items)
|
||||
pages_fetched += 1
|
||||
|
||||
if args.debug:
|
||||
print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}")
|
||||
|
||||
if page < args.pages and args.sleep_ms > 0:
|
||||
time.sleep(args.sleep_ms / 1000)
|
||||
|
||||
list_items = unique_by_title(list_items)
|
||||
print(f"Unique list items: {len(list_items)}")
|
||||
|
||||
enriched_items = []
|
||||
|
||||
for idx, item in enumerate(list_items, start=1):
|
||||
details = item["details"]
|
||||
|
||||
print(f"Fetching detail {idx}/{len(list_items)}: {item['titel']}")
|
||||
if args.debug:
|
||||
print(f"DEBUG URL: {details}")
|
||||
|
||||
enriched = dict(item)
|
||||
|
||||
try:
|
||||
detail_html = fetch_url(details, args.timeout, args.retries, args.debug)
|
||||
dates = parse_detail_dates(detail_html)
|
||||
|
||||
if not dates.get("published"):
|
||||
print(f"WARNING: No Published date found for: {item['titel']} ({details})", file=sys.stderr)
|
||||
|
||||
enriched.pop("published", None)
|
||||
enriched.pop("updated", None)
|
||||
enriched.update(dates)
|
||||
|
||||
except RuntimeError as e:
|
||||
print(f"WARNING: {e}", file=sys.stderr)
|
||||
enriched.pop("published", None)
|
||||
enriched.pop("updated", None)
|
||||
|
||||
enriched_items.append(enriched)
|
||||
|
||||
if idx < len(list_items) and args.sleep_ms > 0:
|
||||
time.sleep(args.sleep_ms / 1000)
|
||||
|
||||
old_items = load_existing(out_path)
|
||||
merged = merge_items(old_items, enriched_items)
|
||||
merged = sort_items(merged)
|
||||
|
||||
print(f"Pages fetched: {pages_fetched} / {args.pages}")
|
||||
print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}")
|
||||
|
||||
if args.dry_run:
|
||||
print(f"Dry-run: NOT writing {out_path}")
|
||||
return
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json_one_object_per_line(out_path, merged)
|
||||
print(f"Updated {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted by user.", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
Reference in New Issue
Block a user