Discover all site pages automatically

scripts/wcx_sync.py

@@ -9,12 +9,12 @@ import urllib.error
 import urllib.request
 from html.parser import HTMLParser
 from pathlib import Path
-from urllib.parse import urljoin, urlparse
+from urllib.parse import parse_qs, urljoin, urlparse
 
 BASE_URL = "https://www.woodmancastingx.com"
 LIST_PATH = "/casting-xxx/"
 DEFAULT_OUT = "/storage/disk1/X/wcx_index.json"
-DEFAULT_PAGES = 48
+MAX_PAGES = 200
 DEFAULT_SLEEP_MS = 300
 USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0"
 
@@ -217,6 +217,41 @@ def parse_list_page(data):
     return items
 
 
+def parse_last_page(data):
+    parser = TreeParser()
+    parser.feed(data)
+
+    base = urlparse(BASE_URL)
+    page_numbers = []
+
+    for node in walk(parser.root):
+        if node.name != "a":
+            continue
+
+        href = (node.attrs.get("href") or "").strip()
+        if not href:
+            continue
+
+        target = urlparse(urljoin(BASE_URL, href))
+        if target.netloc != base.netloc or target.path != LIST_PATH:
+            continue
+
+        for value in parse_qs(target.query).get("page", []):
+            if value.isdigit() and int(value) >= 1:
+                page_numbers.append(int(value))
+
+    if not page_numbers:
+        raise RuntimeError("Could not detect a numeric last page from page 1 pagination")
+
+    last_page = max(page_numbers)
+    if last_page > MAX_PAGES:
+        raise RuntimeError(
+            f"Detected last page {last_page}, exceeding safety limit {MAX_PAGES}"
+        )
+
+    return last_page
+
+
 class DetailDateParser(HTMLParser):
     def __init__(self):
         super().__init__(convert_charrefs=True)
@@ -344,7 +379,7 @@ def write_json_one_object_per_line(path, items):
 def main():
     ap = argparse.ArgumentParser()
     ap.add_argument("--out", default=DEFAULT_OUT)
-    ap.add_argument("--pages", "--end-page", dest="pages", type=int, default=DEFAULT_PAGES)
+    ap.add_argument("--pages", "--end-page", dest="pages", type=int)
     ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS)
     ap.add_argument("--timeout", type=int, default=20)
     ap.add_argument("--retries", type=int, default=3)
@@ -354,29 +389,41 @@ def main():
 
     out_path = Path(args.out)
 
-    print(f"Start. pages=1..{args.pages}, out={out_path}, dry_run={args.dry_run}")
+    requested_pages = args.pages if args.pages is not None else "auto"
+    print(f"Start. pages={requested_pages}, out={out_path}, dry_run={args.dry_run}")
+
+    if args.pages is not None and not 1 <= args.pages <= MAX_PAGES:
+        ap.error(f"--pages must be between 1 and {MAX_PAGES}")
 
     list_items = []
     pages_fetched = 0
 
-    for page in range(1, args.pages + 1):
+    first_url = f"{BASE_URL}{LIST_PATH}?page=1"
-        url = f"{BASE_URL}{LIST_PATH}?page={page}"
+    print(f"Fetching list page 1: {first_url}")
-        print(f"Fetching list page {page}: {url}")
+    first_data = fetch_url(first_url, args.timeout, args.retries, args.debug)
 
-        try:
+    last_page = args.pages if args.pages is not None else parse_last_page(first_data)
+    print(f"Using last list page: {last_page}")
+
+    for page in range(1, last_page + 1):
+        url = f"{BASE_URL}{LIST_PATH}?page={page}"
+        if page == 1:
+            data = first_data
+        else:
+            print(f"Fetching list page {page}: {url}")
             data = fetch_url(url, args.timeout, args.retries, args.debug)
-        except RuntimeError as e:
-            print(f"WARNING: {e}", file=sys.stderr)
-            continue
 
         page_items = parse_list_page(data)
+        if not page_items:
+            raise RuntimeError(f"List page {page} returned zero results: {url}")
+
         list_items.extend(page_items)
         pages_fetched += 1
 
         if args.debug:
             print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}")
 
-        if page < args.pages and args.sleep_ms > 0:
+        if page < last_page and args.sleep_ms > 0:
             time.sleep(args.sleep_ms / 1000)
 
     list_items = unique_by_title(list_items)
@@ -418,7 +465,7 @@ def main():
     merged = merge_items(old_items, enriched_items)
     merged = sort_items(merged)
 
-    print(f"Pages fetched: {pages_fetched} / {args.pages}")
+    print(f"Pages fetched: {pages_fetched} / {last_page}")
     print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}")
 
     if args.dry_run: