diff --git a/.gitignore b/.gitignore index e194813..3e1c23c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,9 @@ -# SQLite database files +# Local databases and runtime state database/*.db database/*.db-shm database/*.db-wal +database/*.lock +database/last_scheduled_update # Imported/generated data import/*.json diff --git a/migration/persons.utf8bom.csv b/migration/persons.utf8bom.csv index cfcefa4..1d21567 100644 --- a/migration/persons.utf8bom.csv +++ b/migration/persons.utf8bom.csv @@ -42,7 +42,7 @@ marushka_838,Marushka,,SLOVAKIAN,18,Budapest (Hungary),2000-07-28,00:49:18,"Osku jennifer-stone_750,Jennifer Stone,,CZECH,19,Prague (Czech Republic ),2000-09-16,00:37:26,"Model - fashion, Vill göra soft, 4 killar, AV - rädd vill inte, ",,https://www.woodmancastingx.com/casting-x/jennifer-stone_750.html,https://s49-pub-ct.high-cdn.com/pics/scene/07/50/750_8_scenedefault_8_244_137_fb_d3f17043.jpg,2009-01-23, mia-stone_286,Mia Stone,,HUNGARIAN,,Budapest (Hungary),2000-10-05,00:55:47,,,https://www.woodmancastingx.com/casting-x/mia-stone_286.html,https://s49-pub-ct.high-cdn.com/pics/scene/02/86/286_8_scenedefault_8_244_137_fb_1ce080eb.jpg,2009-01-18, alexa-may_1674,Alexa May,,UKRAINIAN,,Budapest (Hungary),2000-10-27,00:55:13,,,https://www.woodmancastingx.com/casting-x/alexa-may_1674.html,https://s49-pub-ct.high-cdn.com/pics/scene/16/74/1674_8_scenedefault_8_244_137_fb_e3f92988.jpg,2009-04-02, -,Alexandra,,UKRAINIAN,18,Budapest,2000-11-01,,"Riktigt dåligt ljud, Vet inte vad P gör,Tror att det är modecasting, blir arg, går men kommer tillbaka, vädligt edgy, våldtagen - svårt med sex efter",,,,, +alexandra_916,Alexandra,,UKRAINIAN,18,Budapest,2000-11-01,,"Riktigt dåligt ljud, Vet inte vad P gör,Tror att det är modecasting, blir arg, går men kommer tillbaka, vädligt edgy, våldtagen - svårt med sex efter",,,,, erika-fire_988,Erika Fire ,,HUNGARIAN,18, Budapest,2000-11-08,00:36:03,"student gymnasiet, anser sig vara erfaren, ",,https://www.woodmancastingx.com/casting-x/erika-fire_988.html,https://s49-pub-ct.high-cdn.com/pics/scene/09/88/988_8_scenedefault_8_244_137_fb_77326b4b.jpg,2009-02-08, anita-too_1280,Anita Too,,HUNGARIAN,, Budapest,2001-01-27,00:19:52,,,https://www.woodmancastingx.com/casting-x/anita-too_1280.html,https://s49-pub-ct.high-cdn.com/pics/scene/12/80/1280_8_scenedefault_8_244_137_fb_4744ed83.jpg,2009-03-02, yulia_1124,Yulia,,RUSSIAN,, St Petersburg,2001-02-24,00:46:36,,,https://www.woodmancastingx.com/casting-x/yulia_1124.html,https://s51-pub-ct.high-cdn.com/pics/scene/11/24/1124_8_scenedefault_8_244_137_fb_d162a87d.jpg,2009-02-02, @@ -60,7 +60,6 @@ janett_1722,Janett,,ROMANIAN,, Budapest,2001-09-30,00:54:23,,,https://www.woodma alicia_666,Alicia,,POLISH,,Paris,2001-10-07,00:41:19,"Fransk, ",,https://www.woodmancastingx.com/casting-x/alicia_666.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/66/666_8_scenedefault_8_244_137_fb_2463f186.jpg,2009-01-18, black-diamond_520,Black Diamond,,HUNGARIAN,,Paris (France),2001-11-12,00:34:59,,,https://www.woodmancastingx.com/casting-x/black-diamond_520.html,https://s49-pub-ct.high-cdn.com/pics/scene/05/20/520_8_scenedefault_8_244_137_fb_5635a4f2.jpg,2009-01-09, mili-jay_794,Mili Jay,,CZECH,,Prague,2002-01-04,00:36:12,,,https://www.woodmancastingx.com/casting-x/mili-jay_794.html,https://s49-pub-ct.high-cdn.com/pics/scene/07/94/794_8_scenedefault_8_244_137_fb_df21e018.jpg,2009-02-11, -angel-k_656,Angel K,,UKRAINIAN,,Budapest (Hungary),2002-02-06,00:49:43,,,https://www.woodmancastingx.com/casting-x/angel-k_656.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/56/656_8_scenedefault_8_244_137_fb_c19f1556_v2.jpg,2009-01-16, maggie_968,Maggie,,RUSSIAN,, St Petersburg ,2002-02-16,00:45:24,,,https://www.woodmancastingx.com/casting-x/maggie_968.html,https://s49-pub-ct.high-cdn.com/pics/scene/09/68/968_8_scenedefault_8_244_137_fb_d9dcf5fe.jpg,2009-02-06, claudia-adams_1612,Claudia Adams,,CZECH,19,Prague,2002-04-19,00:38:09,"sett på internet, ",,https://www.woodmancastingx.com/casting-x/claudia-adams_1612.html,https://s49-pub-ct.high-cdn.com/pics/scene/16/12/1612_8_scenedefault_8_244_137_fb_37f8990d.jpg,2009-03-27, anetta-keys_552,Anetta Keys,,CZECH,18,Prague,2002-04-26,00:36:25,,,https://www.woodmancastingx.com/casting-x/anetta-keys_552.html,https://s49-pub-ct.high-cdn.com/pics/scene/05/52/552_8_scenedefault_8_244_137_fb_708634c9.jpg,2009-01-09, @@ -80,7 +79,6 @@ nikka_1896,Nikka,,CZECH,,Prague,2003-05-27,00:41:44,,,https://www.woodmancasting anna_816,Anna,,CZECH,18,Prague,2003-08-08,00:42:45,"Student, Model, tror att det är en model-casting, Säger att det kommer bli dyrt, Utsatt för Pedofil från 5y, Gillar att bli dominerad, AV, ",,https://www.woodmancastingx.com/casting-x/anna_816.html,https://s51-pub-ct.high-cdn.com/pics/scene/08/16/816_8_scenedefault_8_244_137_fb_93004b0c.jpg,2009-01-27, agota_658,Agota,,HUNGARIAN,18,Budapest,2003-08-20,00:34:37,"Arg på otrogen pojkvän, gillar flickor, AV, ",,https://www.woodmancastingx.com/casting-x/agota_658.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/58/658_8_scenedefault_8_244_137_fb_37257b94.jpg,2009-01-16, sandra-bina_5121,Sandra Bina,,SLOVAKIAN,,Budapest (Hungary),2003-10-29,00:58:18,,,https://www.woodmancastingx.com/casting-x/sandra-bina_5121.html,https://s51-pub-ct.high-cdn.com/pics/scene/51/21/5121_8_scenedefault_8_244_137_fb_19cd5337.jpg,2013-02-14, -maggie_968,Maggie,,RUSSIAN,,Rio,2003-11-01,00:45:24,"Oskuld, Stränga föräldrar, gillar flickor, vill börja med A, ",,https://www.woodmancastingx.com/casting-x/maggie_968.html,https://s49-pub-ct.high-cdn.com/pics/scene/09/68/968_8_scenedefault_8_244_137_fb_d9dcf5fe.jpg,2009-02-06, cayenne-klein_4645,Cayenne Klein,,HUNGARIAN,,Budapest (Hungary),2004-08-27,2:16:44,,,https://www.woodmancastingx.com/casting-x/cayenne-klein_4645.html,https://s49-pub-ct.high-cdn.com/pics/scene/46/45/4645_8_scenedefault_8_244_137_fb_eeff119a_v1.jpg,2012-03-12,2016-09-27 xenia-lova_2262,Xenia Lova,,RUSSIAN,,Paris,2004-09-10,00:52:10,,,https://www.woodmancastingx.com/casting-x/xenia-lova_2262.html,https://s49-pub-ct.high-cdn.com/pics/scene/22/62/2262_8_scenedefault_8_244_137_fb_13b14dc9.jpg,2009-06-03, candee-licious_8108,Candee Licious,,HUNGARIAN,,Budupest Hungary,2005-06-03,1:37:19,,,https://www.woodmancastingx.com/casting-x/candee-licious_8108.html,https://s51-pub-ct.high-cdn.com/pics/scene/81/08/8108_8_scenedefault_8_244_137_fb_0f86913a_v2.jpg,2015-06-18,2015-07-30 @@ -408,7 +406,6 @@ lisa-velvet_6193,Lisa Velvet,,RUSSIAN,,Budapest (Hungary),2013-10-15,1:31:12,,,h vicky-braun_6433,Vicky Braun,,HUNGARIAN,21,Budapest ,2013-10-18,1:24:45,"Allready in business, pengar, gillar inte jobbet, rädd för stora, ",,https://www.woodmancastingx.com/casting-x/vicky-braun_6433.html,https://s49-pub-ct.high-cdn.com/pics/scene/64/33/6433_8_scenedefault_8_244_137_fb_1c41ff3d_v3.jpg,2014-01-02,2015-04-20 sissy-gold_6673,Sissy Gold,,CZECH,"18,3w",Budapest Hurgery,2013-10-24,2:44:12,"dansare, vill börja, har en vän som håller på, 3 killar, tystlåtem, inga trosor, spagat, inte Pierre, ",,https://www.woodmancastingx.com/casting-x/sissy-gold_6673.html,https://s51-pub-ct.high-cdn.com/pics/scene/66/73/6673_8_scenedefault_8_244_137_fb_2a276115_v1.jpg,2014-03-03,2014-12-12 kiara-lord_9253,Kiara Lord,,HUNGARIAN,21,Budapest ,2013-10-29,4:23:03,"Allready in business 3-4W, Gillar porr - Fantasi, Vill inte göra A, vän med Vanda Lust, pratsam, bra engelska, 2y 11m later, and yet 3y 9m later, 4y later,",,https://www.woodmancastingx.com/casting-x/kiara-lord_9253.html,https://s51-pub-ct.high-cdn.com/pics/scene/92/53/9253_8_scenedefault_8_244_137_fb_a20d1884_v10.jpg,2013-10-30,2025-08-15 -,Candy Belle,,HUNGARIAN,,Budapest ,2013-10-30,,Med BF,,,,, candybelle_6199,Candybelle,Candybelle (And Roland),HUNGARIAN,,Budapest (Hungary),2013-10-30,1:13:36,,,https://www.woodmancastingx.com/casting-x/candybelle_6199.html,https://s49-pub-ct.high-cdn.com/pics/scene/61/99/6199_8_scenedefault_8_244_137_fb_355567cb.jpg,2013-10-31, alyssa-reece_21841,Alyssa Reece,,CANADIAN,28,Budapest,2013-11-12,2:08:06,"Allready in business - 10y, Only GG, 6yl",,https://www.woodmancastingx.com/casting-x/alyssa-reece_21841.html,https://s51-pub-ct.high-cdn.com/pics/scene/18/41/21841_8_scenedefault_8_244_137_fb_85e02ea5_v4.jpg,2013-11-13,2019-06-24 keira-albina_7487,Keira Albina,,CZECH,21, Prague ,2013-12-03,1:19:40,"Frisörska, vill göra soft,skeptisk till att visa sig naken, pojkvän, AV",,https://www.woodmancastingx.com/casting-x/keira-albina_7487.html,https://s51-pub-ct.high-cdn.com/pics/scene/74/87/7487_8_scenedefault_8_244_137_fb_efcb6be5_v4.jpg,2014-11-06,2014-12-06 @@ -565,7 +562,6 @@ marley-brinx_8216,Marley Brinx,,CANADIAN,,Siofok (Hungary),2015-07-24,1:40:53,,, kayla-kayden_8841,Kayla Kayden,,AMERICAN,26,Las V USA. A 10. 15,2015-08-10,1:42:57,"already in, model och webcam, 30-40 guys, AV, 9m later, vill ha samma som systern, ",,https://www.woodmancastingx.com/casting-x/kayla-kayden_8841.html,https://s49-pub-ct.high-cdn.com/pics/scene/88/41/8841_8_scenedefault_8_244_137_fb_1df0c3e9_v6.jpg,2016-08-25, kendall-kayden_8564,Kendall Kayden,,AMERICAN,24,Las Vegas USA/. August 10,2015-08-10,2:04:49,"Yngre syster till Kayla, inspirerad av systern, AV, ",,https://www.woodmancastingx.com/casting-x/kendall-kayden_8564.html,https://s51-pub-ct.high-cdn.com/pics/scene/85/64/8564_8_scenedefault_8_244_137_fb_7821a4fc_v3.jpg,2016-01-22,2016-04-01 kinsley-eden_8250,Kinsley Eden,,AMERICAN,,Las Vegas ,2015-08-11,2:06:21,,,https://www.woodmancastingx.com/casting-x/kinsley-eden_8250.html,https://s51-pub-ct.high-cdn.com/pics/scene/82/50/8250_8_scenedefault_8_244_137_fb_1ab6237f_v3.jpg,2015-08-16,2017-07-21 -,Aidra Fox,,AMERICAN,19,Los Angeles ,2015-08-20,0:30:00,"Allready in business 2y, only intervju",,,,, chloe-amour_8308,Chloe Amour,,AMERICAN,,Las Ames USA . A 21.2015,2015-08-21,4:09:35,,,https://www.woodmancastingx.com/casting-x/chloe-amour_8308.html,https://s51-pub-ct.high-cdn.com/pics/scene/83/08/8308_8_scenedefault_8_244_137_fb_298bf20e_v3.jpg,2015-09-17,2026-01-31 christiana-cinn_8350,Christiana Cinn,,AMERICAN,,Los Angeles ,2015-08-22,3:08:22,,,https://www.woodmancastingx.com/casting-x/christiana-cinn_8350.html,https://s51-pub-ct.high-cdn.com/pics/scene/83/50/8350_8_scenedefault_8_244_137_fb_95133b45_v2.jpg,2015-10-09,2017-04-22 abella-danger_8278,Abella Danger,,AMERICAN,,Los Angeles ,2015-08-25,1:24:40,,,https://www.woodmancastingx.com/casting-x/abella-danger_8278.html,https://s49-pub-ct.high-cdn.com/pics/scene/82/78/8278_8_scenedefault_8_244_137_fb_4a45388f_v2.jpg,2015-09-03,2017-10-13 @@ -575,7 +571,6 @@ mila-jayde_38393,Mila Jayde,,AMERICAN,,Los Angeles (USA),2015-08-27,1:16:17,,,ht michelle-martinez_8917,Michelle Martinez ,,AMERICAN,,Los Angeles ,2015-08-28,1:35:48,,,https://www.woodmancastingx.com/casting-x/michelle-martinez_8917.html,https://s51-pub-ct.high-cdn.com/pics/scene/89/17/8917_8_scenedefault_8_244_137_fb_92d27aea_v2.jpg,2016-07-15, carla-crouz_8322,Carla Crouz,,HUNGARIAN,,Budapest Hungary / September 4,2015-09-04,1:26:00,,,https://www.woodmancastingx.com/casting-x/carla-crouz_8322.html,https://s51-pub-ct.high-cdn.com/pics/scene/83/22/8322_8_scenedefault_8_244_137_fb_0ccbfc2b_v3.jpg,2015-09-24,2015-12-12 monique-woods_8406,Monique Woods,,HUNGARIAN,,Budapest (Hungary),2015-09-04,1:40:13,,,https://www.woodmancastingx.com/casting-x/monique-woods_8406.html,https://s49-pub-ct.high-cdn.com/pics/scene/84/06/8406_8_scenedefault_8_244_137_fb_db1b8fbe_v3.jpg,2015-11-09,2016-08-15 -monique-woods_8406,Monique Woods ,,HUNGARIAN,,Budapest Hungary. Spr . 2013,2015-09-04,1:40:13,,,https://www.woodmancastingx.com/casting-x/monique-woods_8406.html,https://s49-pub-ct.high-cdn.com/pics/scene/84/06/8406_8_scenedefault_8_244_137_fb_db1b8fbe_v3.jpg,2015-11-09,2016-08-15 virginia-velvet_8514,Virginia Velvet,,HUNGARIAN,,Budapest Howary I. Sepwer 6,2015-09-06,1:05:59,,,https://www.woodmancastingx.com/casting-x/virginia-velvet_8514.html,https://s49-pub-ct.high-cdn.com/pics/scene/85/14/8514_8_scenedefault_8_244_137_fb_bf208df1_v2.jpg,2015-12-31,2018-12-21 empera_8739,Empera,,LATVIAN,,Prague (Czech Republic),2015-09-17,1:23:35,,,https://www.woodmancastingx.com/casting-x/empera_8739.html,https://s51-pub-ct.high-cdn.com/pics/scene/87/39/8739_8_scenedefault_8_244_137_fb_b21d99c4_v1.jpg,2016-04-15,2025-02-28 zoe-valami_8708,Zoe Valami,,HUNGARIAN,19.1,Budapesti wwwry 1. September 17,2015-09-17,1:33:25,"Träggårdsstudent, lite blyg, behöver pengar, AV, pain,Screemer, ",,https://www.woodmancastingx.com/casting-x/zoe-valami_8708.html,https://s49-pub-ct.high-cdn.com/pics/scene/87/08/8708_8_scenedefault_8_244_137_fb_04272308_v1.jpg,2016-04-01,2019-03-09 @@ -939,7 +934,7 @@ valentina-lapiedra_32689,Valentina Lapiedra,,SPANISH,,Budapest (Hungary),2023-01 milev-skaya_32749,Milev Skaya,,BELGIAN,,Budapest (Hungary),2023-02-07,1:56:00,,,https://www.woodmancastingx.com/casting-x/milev-skaya_32749.html,https://s51-pub-ct.high-cdn.com/pics/scene/27/49/32749_8_scenedefault_8_244_137_fb_762cc7dc_v1.jpg,2023-02-13,2023-03-11 sara-isabel_35225,Sara Isabel,,COLOMBIAN,23,Budapest (Hungary),2023-03-28,1:58:19,"already in 8m, hade klädaffär - konkade under pandemin,",,https://www.woodmancastingx.com/casting-x/sara-isabel_35225.html,https://s49-pub-ct.high-cdn.com/pics/scene/52/25/35225_8_scenedefault_8_244_137_fb_8c09c83b_v1.jpg,2023-03-29, hete-tina_35313,Hete Tina,,BELGIAN,,Budapest (Hungary),2023-03-31,1:38:18,,,https://www.woodmancastingx.com/casting-x/hete-tina_35313.html,https://s49-pub-ct.high-cdn.com/pics/scene/53/13/35313_8_scenedefault_8_244_137_fb_523f4652_v1.jpg,2023-04-13,2023-07-07 -,Angelique Lapiedra,,SPANISH,,Budapest (Hungary),2023-04-06,1:56:17,,,https://www.woodmancastingx.com/casting-x/angelique-lapiedra_35273.html,https://s51-pub-ct.high-cdn.com/pics/scene/52/73/35273_8_scenedefault_8_244_137_fb_d81475c4_v3.jpg,2025-10-22, +angelique-lapiedra_35273,Angelique Lapiedra,,SPANISH,,Budapest (Hungary),2023-04-06,1:56:17,,,https://www.woodmancastingx.com/casting-x/angelique-lapiedra_35273.html,https://s51-pub-ct.high-cdn.com/pics/scene/52/73/35273_8_scenedefault_8_244_137_fb_d81475c4_v3.jpg,2025-10-22, scarlett-lapiedra_35385,Scarlett Lapiedra,,VENEZUELAN,24,Budapest (Hungary),2023-04-11,1:24:16,"already in 1y, A trauma, ingen casting, ",,https://www.woodmancastingx.com/casting-x/scarlett-lapiedra_35385.html,https://s51-pub-ct.high-cdn.com/pics/scene/53/85/35385_8_scenedefault_8_244_137_fb_5fbfc23c.jpg,2023-04-27, emily-vander_35493,Emily Vander,,HUNGARIAN,,Budapest (Hungary),2023-05-05,1:54:49,,,https://www.woodmancastingx.com/casting-x/emily-vander_35493.html,https://s49-pub-ct.high-cdn.com/pics/scene/54/93/35493_8_scenedefault_8_244_137_fb_c1c08538_v2.jpg,2023-05-06,2023-06-10 ciel-tokyo_35481,Ciel Tokyo,,JAPANESE,,Budapest (Hungary),2023-05-09,1:35:46,,,https://www.woodmancastingx.com/casting-x/ciel-tokyo_35481.html,https://s49-pub-ct.high-cdn.com/pics/scene/54/81/35481_8_scenedefault_8_244_137_fb_812e1aea_v1.jpg,2023-05-09,2023-05-13 @@ -986,7 +981,6 @@ florencia-nevy_40230,Florencia Nevy,,RUSSIAN,,Budapest (Hungary),2025-09-09,1:59 anna-bella_40254,Anha Bellissima,,TURKISH,,Istabul (Turkey),2025-09-15,1:36:00,,,https://www.woodmancastingx.com/casting-x/anna-bella_40254.html,https://s49-pub-ct.high-cdn.com/pics/scene/02/54/40254_8_scenedefault_8_244_137_fb_47016130_v3.jpg,2025-10-01,2025-11-15 aiyori-midnight_40402,Aiyori Midnight,,FRENCH,21,Budapest (Hungary),2025-10-15,1:55:00,,,https://www.woodmancastingx.com/casting-x/aiyori-midnight_40402.html,https://s49-pub-ct.high-cdn.com/pics/scene/04/02/40402_8_scenedefault_8_244_137_fb_11054bb8_v2.jpg,2025-10-15,2025-10-25 sumiko-smile_40506,Sumiko Smile,,RUSSIAN,24,Belgrade ,2025-11-06,1:50:11,"Already in 2y, 200 killqr, ",,https://www.woodmancastingx.com/casting-x/sumiko-smile_40506.html,https://s51-pub-ct.high-cdn.com/pics/scene/05/06/40506_8_scenedefault_8_244_137_fb_592f7260_v3.jpg,2025-11-10,2025-11-26 -angelique-lapiedra_35273,Angelique Rose,,,,,,3:39:33,,,https://www.woodmancastingx.com/casting-x/angelique-lapiedra_35273.html,https://s49-pub-ct.high-cdn.com/pics/scene/52/73/35273_8_scenedefault_8_244_137_fb_d81475c4_v8.jpg,2023-04-08,2025-12-22 bonnie-woods_40638,Bonnie Woods,,,,,,1:42:39,,,https://www.woodmancastingx.com/casting-x/bonnie-woods_40638.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/38/40638_8_scenedefault_8_244_137_fb_1fc34d9f_v4.jpg,2025-12-14,2026-07-02 kiki-minaj_7225,Kiki Minaj,,,,,,1:21:22,,,https://www.woodmancastingx.com/casting-x/kiki-minaj_7225.html,https://s51-pub-ct.high-cdn.com/pics/scene/72/25/7225_8_scenedefault_8_244_137_fb_af95a702_v1.jpg,2014-08-11,2025-12-05 killa-raketa_9385,Killa Raketa,,,,,,1:57:49,,,https://www.woodmancastingx.com/casting-x/killa-raketa_9385.html,https://s51-pub-ct.high-cdn.com/pics/scene/93/85/9385_8_scenedefault_8_244_137_fb_89579240_v2.jpg,2016-12-27,2025-12-06 diff --git a/scripts/update_wcx.sh b/scripts/update_wcx.sh index 3f9ea8c..4aeb1f2 100755 --- a/scripts/update_wcx.sh +++ b/scripts/update_wcx.sh @@ -18,7 +18,7 @@ set -Eeuo pipefail ROOT_DIR="/storage/disk1/WCX" -SYNC_SCRIPT="${ROOT_DIR}/wcx_sync.py" +SYNC_SCRIPT="${ROOT_DIR}/scripts/wcx_sync.py" IMPORT_SCRIPT="${ROOT_DIR}/scripts/import_site.py" OCR_BATCH_SCRIPT="${ROOT_DIR}/scripts/process_pending_ocr.py" diff --git a/scripts/wcx_sync.py b/scripts/wcx_sync.py new file mode 100755 index 0000000..e49403a --- /dev/null +++ b/scripts/wcx_sync.py @@ -0,0 +1,438 @@ +#!/usr/bin/env python3 +import argparse +import html +import json +import re +import sys +import time +import urllib.error +import urllib.request +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import urljoin, urlparse + +BASE_URL = "https://www.woodmancastingx.com" +LIST_PATH = "/casting-xxx/" +DEFAULT_OUT = "/storage/disk1/X/wcx_index.json" +DEFAULT_PAGES = 48 +DEFAULT_SLEEP_MS = 300 +USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0" + + +VOID_TAGS = { + "area", "base", "br", "col", "embed", "hr", "img", "input", + "link", "meta", "param", "source", "track", "wbr", +} + + +class Node: + def __init__(self, name, attrs=None): + self.name = name + self.attrs = dict(attrs or []) + self.children = [] + self.text = [] + + def get_text(self): + return "".join(self.text + [c.get_text() for c in self.children]).strip() + + +class TreeParser(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.root = Node("root") + self.stack = [self.root] + + def handle_starttag(self, tag, attrs): + node = Node(tag, attrs) + self.stack[-1].children.append(node) + if tag not in VOID_TAGS: + self.stack.append(node) + + def handle_endtag(self, tag): + for i in range(len(self.stack) - 1, 0, -1): + if self.stack[i].name == tag: + self.stack = self.stack[:i] + return + + def handle_data(self, data): + if data: + self.stack[-1].text.append(data) + + +def classes(node): + return set((node.attrs.get("class") or "").split()) + + +def walk(node): + yield node + for child in node.children: + yield from walk(child) + + +def first_under(node, tag, cls=None): + for n in walk(node): + if n.name == tag and (cls is None or cls in classes(n)): + return n + return None + + +def attr_under(node, tag, cls, attr): + n = first_under(node, tag, cls) + if not n: + return "" + return (n.attrs.get(attr) or "").strip() + + +def absolute_url(href): + return urljoin(BASE_URL, href) + + +def file_id_from_url(url): + path = urlparse(url).path + filename = Path(path).name + return filename.rsplit(".", 1)[0] + + +def slug_title_from_href(href): + stem = file_id_from_url(href) + stem = stem.rsplit("_", 1)[0] + return " ".join(w.capitalize() for w in stem.replace("-", " ").split()) + + +def fetch_url_old(url, timeout, retries, debug=False): + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + last_error = None + + for attempt in range(1, retries + 1): + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read() + return raw.decode("utf-8", errors="ignore") + except (urllib.error.URLError, TimeoutError, OSError) as e: + last_error = e + if debug: + print(f"DEBUG fetch failed attempt {attempt}/{retries}: {url} ({e})", file=sys.stderr) + time.sleep(min(1.0 * attempt, 3.0)) + + raise RuntimeError(f"Could not fetch {url}: {last_error}") + +def fetch_url(url, timeout, retries, debug=False): + last_error = None + + for attempt in range(1, retries + 1): + req = urllib.request.Request( + url, + headers={ + "User-Agent": USER_AGENT, + "Connection": "close", + }, + ) + + try: + if debug or attempt > 1: + print( + f"Fetching {url} " + f"(attempt {attempt}/{retries}, timeout={timeout}s)" + ) + + with urllib.request.urlopen(req, timeout=timeout) as response: + raw = response.read() + + return raw.decode("utf-8", errors="ignore") + + except ( + urllib.error.HTTPError, + urllib.error.URLError, + TimeoutError, + ConnectionError, + OSError, + ) as exc: + last_error = exc + + print( + f"WARNING: Fetch attempt {attempt}/{retries} failed " + f"for {url}: {exc}", + file=sys.stderr, + ) + + if attempt < retries: + time.sleep(min(attempt, 3)) + + raise RuntimeError( + f"Could not fetch {url} after {retries} attempts: {last_error}" + ) + + +def parse_list_page(data): + parser = TreeParser() + parser.feed(data) + + containers = [ + n for n in walk(parser.root) + if n.name == "div" and {"items", "container_3"}.issubset(classes(n)) + ] or [parser.root] + + items = [] + + for container in containers: + anchors = [ + n for n in walk(container) + if n.name == "a" and {"item", "scene"}.issubset(classes(n)) + ] + + for a in anchors: + href = (a.attrs.get("href") or "").strip() + if not href: + continue + + details = absolute_url(href) + + title_node = first_under(a, "span", "title") + title = title_node.get_text() if title_node else "" + title = title or (a.attrs.get("title") or "").strip() + title = title or attr_under(a, "img", "thumb", "alt") + title = title or slug_title_from_href(href) + title = html.unescape(title).strip() + + if not title: + continue + + duration_node = first_under(a, "span", "duration") + duration = duration_node.get_text().strip() if duration_node else "" + + thumb = attr_under(a, "img", "thumb", "src") + + item = { + "id": file_id_from_url(details), + "titel": title, + "details": details, + "duration": duration, + } + + if thumb: + item["thumb"] = thumb + + items.append(item) + + return items + + +class DetailDateParser(HTMLParser): + def __init__(self): + super().__init__(convert_charrefs=True) + self.in_info_line = False + self.in_label = False + self.label = "" + self.text = "" + self.result = {} + + def _has_class(self, attrs, wanted): + attrs = dict(attrs or []) + return wanted in (attrs.get("class") or "").split() + + def handle_starttag(self, tag, attrs): + if tag == "p" and self._has_class(attrs, "info_line"): + self.in_info_line = True + self.in_label = False + self.label = "" + self.text = "" + elif self.in_info_line and tag == "span" and self._has_class(attrs, "label_info"): + self.in_label = True + + def handle_endtag(self, tag): + if tag == "span" and self.in_label: + self.in_label = False + elif tag == "p" and self.in_info_line: + label = self.label.strip().lower() + match = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", self.text) + + if match: + if label == "published": + self.result["published"] = match.group(1) + elif label == "updated": + self.result["updated"] = match.group(1) + + self.in_info_line = False + self.in_label = False + + def handle_data(self, data): + if self.in_info_line: + self.text += data + if self.in_label: + self.label += data + + +def parse_detail_dates(data): + parser = DetailDateParser() + parser.feed(data) + return parser.result + + +def unique_by_title(items): + result = {} + for item in items: + result[item["titel"]] = item + return list(result.values()) + + +def load_existing(path): + if not path.exists(): + return [] + + with path.open("r", encoding="utf-8") as f: + data = json.load(f) + + if not isinstance(data, list): + raise ValueError(f"{path} must contain a JSON array") + + return data + + +def merge_items(old_items, new_items): + merged = {item.get("titel", ""): dict(item) for item in old_items if item.get("titel")} + + for new in new_items: + title = new.get("titel") + if not title: + continue + + old = merged.get(title) + + if old is None: + merged[title] = new + continue + + updated = dict(old) + + for key in ("id", "titel", "details", "duration", "thumb", "published"): + if new.get(key): + updated[key] = new[key] + + if new.get("updated"): + updated["updated"] = new["updated"] + else: + updated.pop("updated", None) + + merged[title] = updated + + return list(merged.values()) + + +def sort_items(items): + return sorted( + items, + key=lambda x: x.get("updated") or x.get("published") or "", + reverse=True, + ) + + +def write_json_one_object_per_line(path, items): + tmp = path.with_suffix(path.suffix + ".tmp") + + with tmp.open("w", encoding="utf-8") as f: + f.write("[\n") + for i, item in enumerate(items): + line = json.dumps(item, ensure_ascii=False, separators=(",", ":")) + if i < len(items) - 1: + line += "," + f.write(line + "\n") + f.write("]\n") + + tmp.replace(path) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--out", default=DEFAULT_OUT) + ap.add_argument("--pages", "--end-page", dest="pages", type=int, default=DEFAULT_PAGES) + ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS) + ap.add_argument("--timeout", type=int, default=20) + ap.add_argument("--retries", type=int, default=3) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--debug", action="store_true") + args = ap.parse_args() + + out_path = Path(args.out) + + print(f"Start. pages=1..{args.pages}, out={out_path}, dry_run={args.dry_run}") + + list_items = [] + pages_fetched = 0 + + for page in range(1, args.pages + 1): + url = f"{BASE_URL}{LIST_PATH}?page={page}" + print(f"Fetching list page {page}: {url}") + + try: + data = fetch_url(url, args.timeout, args.retries, args.debug) + except RuntimeError as e: + print(f"WARNING: {e}", file=sys.stderr) + continue + + page_items = parse_list_page(data) + list_items.extend(page_items) + pages_fetched += 1 + + if args.debug: + print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}") + + if page < args.pages and args.sleep_ms > 0: + time.sleep(args.sleep_ms / 1000) + + list_items = unique_by_title(list_items) + print(f"Unique list items: {len(list_items)}") + + enriched_items = [] + + for idx, item in enumerate(list_items, start=1): + details = item["details"] + + print(f"Fetching detail {idx}/{len(list_items)}: {item['titel']}") + if args.debug: + print(f"DEBUG URL: {details}") + + enriched = dict(item) + + try: + detail_html = fetch_url(details, args.timeout, args.retries, args.debug) + dates = parse_detail_dates(detail_html) + + if not dates.get("published"): + print(f"WARNING: No Published date found for: {item['titel']} ({details})", file=sys.stderr) + + enriched.pop("published", None) + enriched.pop("updated", None) + enriched.update(dates) + + except RuntimeError as e: + print(f"WARNING: {e}", file=sys.stderr) + enriched.pop("published", None) + enriched.pop("updated", None) + + enriched_items.append(enriched) + + if idx < len(list_items) and args.sleep_ms > 0: + time.sleep(args.sleep_ms / 1000) + + old_items = load_existing(out_path) + merged = merge_items(old_items, enriched_items) + merged = sort_items(merged) + + print(f"Pages fetched: {pages_fetched} / {args.pages}") + print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}") + + if args.dry_run: + print(f"Dry-run: NOT writing {out_path}") + return + + out_path.parent.mkdir(parents=True, exist_ok=True) + write_json_one_object_per_line(out_path, merged) + print(f"Updated {out_path}") + + +if __name__ == "__main__": + try: + main() + except KeyboardInterrupt: + print("\nInterrupted by user.", file=sys.stderr) + sys.exit(130)