Organize sync script and clean migration data
This commit is contained in:
4
.gitignore
vendored
4
.gitignore
vendored
@ -1,7 +1,9 @@
|
||||
# SQLite database files
|
||||
# Local databases and runtime state
|
||||
database/*.db
|
||||
database/*.db-shm
|
||||
database/*.db-wal
|
||||
database/*.lock
|
||||
database/last_scheduled_update
|
||||
|
||||
# Imported/generated data
|
||||
import/*.json
|
||||
|
||||
@ -42,7 +42,7 @@ marushka_838,Marushka,,SLOVAKIAN,18,Budapest (Hungary),2000-07-28,00:49:18,"Osku
|
||||
jennifer-stone_750,Jennifer Stone,,CZECH,19,Prague (Czech Republic ),2000-09-16,00:37:26,"Model - fashion, Vill göra soft, 4 killar, AV - rädd vill inte, ",,https://www.woodmancastingx.com/casting-x/jennifer-stone_750.html,https://s49-pub-ct.high-cdn.com/pics/scene/07/50/750_8_scenedefault_8_244_137_fb_d3f17043.jpg,2009-01-23,
|
||||
mia-stone_286,Mia Stone,,HUNGARIAN,,Budapest (Hungary),2000-10-05,00:55:47,,,https://www.woodmancastingx.com/casting-x/mia-stone_286.html,https://s49-pub-ct.high-cdn.com/pics/scene/02/86/286_8_scenedefault_8_244_137_fb_1ce080eb.jpg,2009-01-18,
|
||||
alexa-may_1674,Alexa May,,UKRAINIAN,,Budapest (Hungary),2000-10-27,00:55:13,,,https://www.woodmancastingx.com/casting-x/alexa-may_1674.html,https://s49-pub-ct.high-cdn.com/pics/scene/16/74/1674_8_scenedefault_8_244_137_fb_e3f92988.jpg,2009-04-02,
|
||||
,Alexandra,,UKRAINIAN,18,Budapest,2000-11-01,,"Riktigt dåligt ljud, Vet inte vad P gör,Tror att det är modecasting, blir arg, går men kommer tillbaka, vädligt edgy, våldtagen - svårt med sex efter",,,,,
|
||||
alexandra_916,Alexandra,,UKRAINIAN,18,Budapest,2000-11-01,,"Riktigt dåligt ljud, Vet inte vad P gör,Tror att det är modecasting, blir arg, går men kommer tillbaka, vädligt edgy, våldtagen - svårt med sex efter",,,,,
|
||||
erika-fire_988,Erika Fire ,,HUNGARIAN,18, Budapest,2000-11-08,00:36:03,"student gymnasiet, anser sig vara erfaren, ",,https://www.woodmancastingx.com/casting-x/erika-fire_988.html,https://s49-pub-ct.high-cdn.com/pics/scene/09/88/988_8_scenedefault_8_244_137_fb_77326b4b.jpg,2009-02-08,
|
||||
anita-too_1280,Anita Too,,HUNGARIAN,, Budapest,2001-01-27,00:19:52,,,https://www.woodmancastingx.com/casting-x/anita-too_1280.html,https://s49-pub-ct.high-cdn.com/pics/scene/12/80/1280_8_scenedefault_8_244_137_fb_4744ed83.jpg,2009-03-02,
|
||||
yulia_1124,Yulia,,RUSSIAN,, St Petersburg,2001-02-24,00:46:36,,,https://www.woodmancastingx.com/casting-x/yulia_1124.html,https://s51-pub-ct.high-cdn.com/pics/scene/11/24/1124_8_scenedefault_8_244_137_fb_d162a87d.jpg,2009-02-02,
|
||||
@ -60,7 +60,6 @@ janett_1722,Janett,,ROMANIAN,, Budapest,2001-09-30,00:54:23,,,https://www.woodma
|
||||
alicia_666,Alicia,,POLISH,,Paris,2001-10-07,00:41:19,"Fransk, ",,https://www.woodmancastingx.com/casting-x/alicia_666.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/66/666_8_scenedefault_8_244_137_fb_2463f186.jpg,2009-01-18,
|
||||
black-diamond_520,Black Diamond,,HUNGARIAN,,Paris (France),2001-11-12,00:34:59,,,https://www.woodmancastingx.com/casting-x/black-diamond_520.html,https://s49-pub-ct.high-cdn.com/pics/scene/05/20/520_8_scenedefault_8_244_137_fb_5635a4f2.jpg,2009-01-09,
|
||||
mili-jay_794,Mili Jay,,CZECH,,Prague,2002-01-04,00:36:12,,,https://www.woodmancastingx.com/casting-x/mili-jay_794.html,https://s49-pub-ct.high-cdn.com/pics/scene/07/94/794_8_scenedefault_8_244_137_fb_df21e018.jpg,2009-02-11,
|
||||
angel-k_656,Angel K,,UKRAINIAN,,Budapest (Hungary),2002-02-06,00:49:43,,,https://www.woodmancastingx.com/casting-x/angel-k_656.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/56/656_8_scenedefault_8_244_137_fb_c19f1556_v2.jpg,2009-01-16,
|
||||
maggie_968,Maggie,,RUSSIAN,, St Petersburg ,2002-02-16,00:45:24,,,https://www.woodmancastingx.com/casting-x/maggie_968.html,https://s49-pub-ct.high-cdn.com/pics/scene/09/68/968_8_scenedefault_8_244_137_fb_d9dcf5fe.jpg,2009-02-06,
|
||||
claudia-adams_1612,Claudia Adams,,CZECH,19,Prague,2002-04-19,00:38:09,"sett på internet, ",,https://www.woodmancastingx.com/casting-x/claudia-adams_1612.html,https://s49-pub-ct.high-cdn.com/pics/scene/16/12/1612_8_scenedefault_8_244_137_fb_37f8990d.jpg,2009-03-27,
|
||||
anetta-keys_552,Anetta Keys,,CZECH,18,Prague,2002-04-26,00:36:25,,,https://www.woodmancastingx.com/casting-x/anetta-keys_552.html,https://s49-pub-ct.high-cdn.com/pics/scene/05/52/552_8_scenedefault_8_244_137_fb_708634c9.jpg,2009-01-09,
|
||||
@ -80,7 +79,6 @@ nikka_1896,Nikka,,CZECH,,Prague,2003-05-27,00:41:44,,,https://www.woodmancasting
|
||||
anna_816,Anna,,CZECH,18,Prague,2003-08-08,00:42:45,"Student, Model, tror att det är en model-casting, Säger att det kommer bli dyrt, Utsatt för Pedofil från 5y, Gillar att bli dominerad, AV, ",,https://www.woodmancastingx.com/casting-x/anna_816.html,https://s51-pub-ct.high-cdn.com/pics/scene/08/16/816_8_scenedefault_8_244_137_fb_93004b0c.jpg,2009-01-27,
|
||||
agota_658,Agota,,HUNGARIAN,18,Budapest,2003-08-20,00:34:37,"Arg på otrogen pojkvän, gillar flickor, AV, ",,https://www.woodmancastingx.com/casting-x/agota_658.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/58/658_8_scenedefault_8_244_137_fb_37257b94.jpg,2009-01-16,
|
||||
sandra-bina_5121,Sandra Bina,,SLOVAKIAN,,Budapest (Hungary),2003-10-29,00:58:18,,,https://www.woodmancastingx.com/casting-x/sandra-bina_5121.html,https://s51-pub-ct.high-cdn.com/pics/scene/51/21/5121_8_scenedefault_8_244_137_fb_19cd5337.jpg,2013-02-14,
|
||||
maggie_968,Maggie,,RUSSIAN,,Rio,2003-11-01,00:45:24,"Oskuld, Stränga föräldrar, gillar flickor, vill börja med A, ",,https://www.woodmancastingx.com/casting-x/maggie_968.html,https://s49-pub-ct.high-cdn.com/pics/scene/09/68/968_8_scenedefault_8_244_137_fb_d9dcf5fe.jpg,2009-02-06,
|
||||
cayenne-klein_4645,Cayenne Klein,,HUNGARIAN,,Budapest (Hungary),2004-08-27,2:16:44,,,https://www.woodmancastingx.com/casting-x/cayenne-klein_4645.html,https://s49-pub-ct.high-cdn.com/pics/scene/46/45/4645_8_scenedefault_8_244_137_fb_eeff119a_v1.jpg,2012-03-12,2016-09-27
|
||||
xenia-lova_2262,Xenia Lova,,RUSSIAN,,Paris,2004-09-10,00:52:10,,,https://www.woodmancastingx.com/casting-x/xenia-lova_2262.html,https://s49-pub-ct.high-cdn.com/pics/scene/22/62/2262_8_scenedefault_8_244_137_fb_13b14dc9.jpg,2009-06-03,
|
||||
candee-licious_8108,Candee Licious,,HUNGARIAN,,Budupest Hungary,2005-06-03,1:37:19,,,https://www.woodmancastingx.com/casting-x/candee-licious_8108.html,https://s51-pub-ct.high-cdn.com/pics/scene/81/08/8108_8_scenedefault_8_244_137_fb_0f86913a_v2.jpg,2015-06-18,2015-07-30
|
||||
@ -408,7 +406,6 @@ lisa-velvet_6193,Lisa Velvet,,RUSSIAN,,Budapest (Hungary),2013-10-15,1:31:12,,,h
|
||||
vicky-braun_6433,Vicky Braun,,HUNGARIAN,21,Budapest ,2013-10-18,1:24:45,"Allready in business, pengar, gillar inte jobbet, rädd för stora, ",,https://www.woodmancastingx.com/casting-x/vicky-braun_6433.html,https://s49-pub-ct.high-cdn.com/pics/scene/64/33/6433_8_scenedefault_8_244_137_fb_1c41ff3d_v3.jpg,2014-01-02,2015-04-20
|
||||
sissy-gold_6673,Sissy Gold,,CZECH,"18,3w",Budapest Hurgery,2013-10-24,2:44:12,"dansare, vill börja, har en vän som håller på, 3 killar, tystlåtem, inga trosor, spagat, inte Pierre, ",,https://www.woodmancastingx.com/casting-x/sissy-gold_6673.html,https://s51-pub-ct.high-cdn.com/pics/scene/66/73/6673_8_scenedefault_8_244_137_fb_2a276115_v1.jpg,2014-03-03,2014-12-12
|
||||
kiara-lord_9253,Kiara Lord,,HUNGARIAN,21,Budapest ,2013-10-29,4:23:03,"Allready in business 3-4W, Gillar porr - Fantasi, Vill inte göra A, vän med Vanda Lust, pratsam, bra engelska, 2y 11m later, and yet 3y 9m later, 4y later,",,https://www.woodmancastingx.com/casting-x/kiara-lord_9253.html,https://s51-pub-ct.high-cdn.com/pics/scene/92/53/9253_8_scenedefault_8_244_137_fb_a20d1884_v10.jpg,2013-10-30,2025-08-15
|
||||
,Candy Belle,,HUNGARIAN,,Budapest ,2013-10-30,,Med BF,,,,,
|
||||
candybelle_6199,Candybelle,Candybelle (And Roland),HUNGARIAN,,Budapest (Hungary),2013-10-30,1:13:36,,,https://www.woodmancastingx.com/casting-x/candybelle_6199.html,https://s49-pub-ct.high-cdn.com/pics/scene/61/99/6199_8_scenedefault_8_244_137_fb_355567cb.jpg,2013-10-31,
|
||||
alyssa-reece_21841,Alyssa Reece,,CANADIAN,28,Budapest,2013-11-12,2:08:06,"Allready in business - 10y, Only GG, 6yl",,https://www.woodmancastingx.com/casting-x/alyssa-reece_21841.html,https://s51-pub-ct.high-cdn.com/pics/scene/18/41/21841_8_scenedefault_8_244_137_fb_85e02ea5_v4.jpg,2013-11-13,2019-06-24
|
||||
keira-albina_7487,Keira Albina,,CZECH,21, Prague ,2013-12-03,1:19:40,"Frisörska, vill göra soft,skeptisk till att visa sig naken, pojkvän, AV",,https://www.woodmancastingx.com/casting-x/keira-albina_7487.html,https://s51-pub-ct.high-cdn.com/pics/scene/74/87/7487_8_scenedefault_8_244_137_fb_efcb6be5_v4.jpg,2014-11-06,2014-12-06
|
||||
@ -565,7 +562,6 @@ marley-brinx_8216,Marley Brinx,,CANADIAN,,Siofok (Hungary),2015-07-24,1:40:53,,,
|
||||
kayla-kayden_8841,Kayla Kayden,,AMERICAN,26,Las V USA. A 10. 15,2015-08-10,1:42:57,"already in, model och webcam, 30-40 guys, AV, 9m later, vill ha samma som systern, ",,https://www.woodmancastingx.com/casting-x/kayla-kayden_8841.html,https://s49-pub-ct.high-cdn.com/pics/scene/88/41/8841_8_scenedefault_8_244_137_fb_1df0c3e9_v6.jpg,2016-08-25,
|
||||
kendall-kayden_8564,Kendall Kayden,,AMERICAN,24,Las Vegas USA/. August 10,2015-08-10,2:04:49,"Yngre syster till Kayla, inspirerad av systern, AV, ",,https://www.woodmancastingx.com/casting-x/kendall-kayden_8564.html,https://s51-pub-ct.high-cdn.com/pics/scene/85/64/8564_8_scenedefault_8_244_137_fb_7821a4fc_v3.jpg,2016-01-22,2016-04-01
|
||||
kinsley-eden_8250,Kinsley Eden,,AMERICAN,,Las Vegas ,2015-08-11,2:06:21,,,https://www.woodmancastingx.com/casting-x/kinsley-eden_8250.html,https://s51-pub-ct.high-cdn.com/pics/scene/82/50/8250_8_scenedefault_8_244_137_fb_1ab6237f_v3.jpg,2015-08-16,2017-07-21
|
||||
,Aidra Fox,,AMERICAN,19,Los Angeles ,2015-08-20,0:30:00,"Allready in business 2y, only intervju",,,,,
|
||||
chloe-amour_8308,Chloe Amour,,AMERICAN,,Las Ames USA . A 21.2015,2015-08-21,4:09:35,,,https://www.woodmancastingx.com/casting-x/chloe-amour_8308.html,https://s51-pub-ct.high-cdn.com/pics/scene/83/08/8308_8_scenedefault_8_244_137_fb_298bf20e_v3.jpg,2015-09-17,2026-01-31
|
||||
christiana-cinn_8350,Christiana Cinn,,AMERICAN,,Los Angeles ,2015-08-22,3:08:22,,,https://www.woodmancastingx.com/casting-x/christiana-cinn_8350.html,https://s51-pub-ct.high-cdn.com/pics/scene/83/50/8350_8_scenedefault_8_244_137_fb_95133b45_v2.jpg,2015-10-09,2017-04-22
|
||||
abella-danger_8278,Abella Danger,,AMERICAN,,Los Angeles ,2015-08-25,1:24:40,,,https://www.woodmancastingx.com/casting-x/abella-danger_8278.html,https://s49-pub-ct.high-cdn.com/pics/scene/82/78/8278_8_scenedefault_8_244_137_fb_4a45388f_v2.jpg,2015-09-03,2017-10-13
|
||||
@ -575,7 +571,6 @@ mila-jayde_38393,Mila Jayde,,AMERICAN,,Los Angeles (USA),2015-08-27,1:16:17,,,ht
|
||||
michelle-martinez_8917,Michelle Martinez ,,AMERICAN,,Los Angeles ,2015-08-28,1:35:48,,,https://www.woodmancastingx.com/casting-x/michelle-martinez_8917.html,https://s51-pub-ct.high-cdn.com/pics/scene/89/17/8917_8_scenedefault_8_244_137_fb_92d27aea_v2.jpg,2016-07-15,
|
||||
carla-crouz_8322,Carla Crouz,,HUNGARIAN,,Budapest Hungary / September 4,2015-09-04,1:26:00,,,https://www.woodmancastingx.com/casting-x/carla-crouz_8322.html,https://s51-pub-ct.high-cdn.com/pics/scene/83/22/8322_8_scenedefault_8_244_137_fb_0ccbfc2b_v3.jpg,2015-09-24,2015-12-12
|
||||
monique-woods_8406,Monique Woods,,HUNGARIAN,,Budapest (Hungary),2015-09-04,1:40:13,,,https://www.woodmancastingx.com/casting-x/monique-woods_8406.html,https://s49-pub-ct.high-cdn.com/pics/scene/84/06/8406_8_scenedefault_8_244_137_fb_db1b8fbe_v3.jpg,2015-11-09,2016-08-15
|
||||
monique-woods_8406,Monique Woods ,,HUNGARIAN,,Budapest Hungary. Spr . 2013,2015-09-04,1:40:13,,,https://www.woodmancastingx.com/casting-x/monique-woods_8406.html,https://s49-pub-ct.high-cdn.com/pics/scene/84/06/8406_8_scenedefault_8_244_137_fb_db1b8fbe_v3.jpg,2015-11-09,2016-08-15
|
||||
virginia-velvet_8514,Virginia Velvet,,HUNGARIAN,,Budapest Howary I. Sepwer 6,2015-09-06,1:05:59,,,https://www.woodmancastingx.com/casting-x/virginia-velvet_8514.html,https://s49-pub-ct.high-cdn.com/pics/scene/85/14/8514_8_scenedefault_8_244_137_fb_bf208df1_v2.jpg,2015-12-31,2018-12-21
|
||||
empera_8739,Empera,,LATVIAN,,Prague (Czech Republic),2015-09-17,1:23:35,,,https://www.woodmancastingx.com/casting-x/empera_8739.html,https://s51-pub-ct.high-cdn.com/pics/scene/87/39/8739_8_scenedefault_8_244_137_fb_b21d99c4_v1.jpg,2016-04-15,2025-02-28
|
||||
zoe-valami_8708,Zoe Valami,,HUNGARIAN,19.1,Budapesti wwwry 1. September 17,2015-09-17,1:33:25,"Träggårdsstudent, lite blyg, behöver pengar, AV, pain,Screemer, ",,https://www.woodmancastingx.com/casting-x/zoe-valami_8708.html,https://s49-pub-ct.high-cdn.com/pics/scene/87/08/8708_8_scenedefault_8_244_137_fb_04272308_v1.jpg,2016-04-01,2019-03-09
|
||||
@ -939,7 +934,7 @@ valentina-lapiedra_32689,Valentina Lapiedra,,SPANISH,,Budapest (Hungary),2023-01
|
||||
milev-skaya_32749,Milev Skaya,,BELGIAN,,Budapest (Hungary),2023-02-07,1:56:00,,,https://www.woodmancastingx.com/casting-x/milev-skaya_32749.html,https://s51-pub-ct.high-cdn.com/pics/scene/27/49/32749_8_scenedefault_8_244_137_fb_762cc7dc_v1.jpg,2023-02-13,2023-03-11
|
||||
sara-isabel_35225,Sara Isabel,,COLOMBIAN,23,Budapest (Hungary),2023-03-28,1:58:19,"already in 8m, hade klädaffär - konkade under pandemin,",,https://www.woodmancastingx.com/casting-x/sara-isabel_35225.html,https://s49-pub-ct.high-cdn.com/pics/scene/52/25/35225_8_scenedefault_8_244_137_fb_8c09c83b_v1.jpg,2023-03-29,
|
||||
hete-tina_35313,Hete Tina,,BELGIAN,,Budapest (Hungary),2023-03-31,1:38:18,,,https://www.woodmancastingx.com/casting-x/hete-tina_35313.html,https://s49-pub-ct.high-cdn.com/pics/scene/53/13/35313_8_scenedefault_8_244_137_fb_523f4652_v1.jpg,2023-04-13,2023-07-07
|
||||
,Angelique Lapiedra,,SPANISH,,Budapest (Hungary),2023-04-06,1:56:17,,,https://www.woodmancastingx.com/casting-x/angelique-lapiedra_35273.html,https://s51-pub-ct.high-cdn.com/pics/scene/52/73/35273_8_scenedefault_8_244_137_fb_d81475c4_v3.jpg,2025-10-22,
|
||||
angelique-lapiedra_35273,Angelique Lapiedra,,SPANISH,,Budapest (Hungary),2023-04-06,1:56:17,,,https://www.woodmancastingx.com/casting-x/angelique-lapiedra_35273.html,https://s51-pub-ct.high-cdn.com/pics/scene/52/73/35273_8_scenedefault_8_244_137_fb_d81475c4_v3.jpg,2025-10-22,
|
||||
scarlett-lapiedra_35385,Scarlett Lapiedra,,VENEZUELAN,24,Budapest (Hungary),2023-04-11,1:24:16,"already in 1y, A trauma, ingen casting, ",,https://www.woodmancastingx.com/casting-x/scarlett-lapiedra_35385.html,https://s51-pub-ct.high-cdn.com/pics/scene/53/85/35385_8_scenedefault_8_244_137_fb_5fbfc23c.jpg,2023-04-27,
|
||||
emily-vander_35493,Emily Vander,,HUNGARIAN,,Budapest (Hungary),2023-05-05,1:54:49,,,https://www.woodmancastingx.com/casting-x/emily-vander_35493.html,https://s49-pub-ct.high-cdn.com/pics/scene/54/93/35493_8_scenedefault_8_244_137_fb_c1c08538_v2.jpg,2023-05-06,2023-06-10
|
||||
ciel-tokyo_35481,Ciel Tokyo,,JAPANESE,,Budapest (Hungary),2023-05-09,1:35:46,,,https://www.woodmancastingx.com/casting-x/ciel-tokyo_35481.html,https://s49-pub-ct.high-cdn.com/pics/scene/54/81/35481_8_scenedefault_8_244_137_fb_812e1aea_v1.jpg,2023-05-09,2023-05-13
|
||||
@ -986,7 +981,6 @@ florencia-nevy_40230,Florencia Nevy,,RUSSIAN,,Budapest (Hungary),2025-09-09,1:59
|
||||
anna-bella_40254,Anha Bellissima,,TURKISH,,Istabul (Turkey),2025-09-15,1:36:00,,,https://www.woodmancastingx.com/casting-x/anna-bella_40254.html,https://s49-pub-ct.high-cdn.com/pics/scene/02/54/40254_8_scenedefault_8_244_137_fb_47016130_v3.jpg,2025-10-01,2025-11-15
|
||||
aiyori-midnight_40402,Aiyori Midnight,,FRENCH,21,Budapest (Hungary),2025-10-15,1:55:00,,,https://www.woodmancastingx.com/casting-x/aiyori-midnight_40402.html,https://s49-pub-ct.high-cdn.com/pics/scene/04/02/40402_8_scenedefault_8_244_137_fb_11054bb8_v2.jpg,2025-10-15,2025-10-25
|
||||
sumiko-smile_40506,Sumiko Smile,,RUSSIAN,24,Belgrade ,2025-11-06,1:50:11,"Already in 2y, 200 killqr, ",,https://www.woodmancastingx.com/casting-x/sumiko-smile_40506.html,https://s51-pub-ct.high-cdn.com/pics/scene/05/06/40506_8_scenedefault_8_244_137_fb_592f7260_v3.jpg,2025-11-10,2025-11-26
|
||||
angelique-lapiedra_35273,Angelique Rose,,,,,,3:39:33,,,https://www.woodmancastingx.com/casting-x/angelique-lapiedra_35273.html,https://s49-pub-ct.high-cdn.com/pics/scene/52/73/35273_8_scenedefault_8_244_137_fb_d81475c4_v8.jpg,2023-04-08,2025-12-22
|
||||
bonnie-woods_40638,Bonnie Woods,,,,,,1:42:39,,,https://www.woodmancastingx.com/casting-x/bonnie-woods_40638.html,https://s51-pub-ct.high-cdn.com/pics/scene/06/38/40638_8_scenedefault_8_244_137_fb_1fc34d9f_v4.jpg,2025-12-14,2026-07-02
|
||||
kiki-minaj_7225,Kiki Minaj,,,,,,1:21:22,,,https://www.woodmancastingx.com/casting-x/kiki-minaj_7225.html,https://s51-pub-ct.high-cdn.com/pics/scene/72/25/7225_8_scenedefault_8_244_137_fb_af95a702_v1.jpg,2014-08-11,2025-12-05
|
||||
killa-raketa_9385,Killa Raketa,,,,,,1:57:49,,,https://www.woodmancastingx.com/casting-x/killa-raketa_9385.html,https://s51-pub-ct.high-cdn.com/pics/scene/93/85/9385_8_scenedefault_8_244_137_fb_89579240_v2.jpg,2016-12-27,2025-12-06
|
||||
|
||||
|
@ -18,7 +18,7 @@ set -Eeuo pipefail
|
||||
|
||||
ROOT_DIR="/storage/disk1/WCX"
|
||||
|
||||
SYNC_SCRIPT="${ROOT_DIR}/wcx_sync.py"
|
||||
SYNC_SCRIPT="${ROOT_DIR}/scripts/wcx_sync.py"
|
||||
IMPORT_SCRIPT="${ROOT_DIR}/scripts/import_site.py"
|
||||
OCR_BATCH_SCRIPT="${ROOT_DIR}/scripts/process_pending_ocr.py"
|
||||
|
||||
|
||||
438
scripts/wcx_sync.py
Executable file
438
scripts/wcx_sync.py
Executable file
@ -0,0 +1,438 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
BASE_URL = "https://www.woodmancastingx.com"
|
||||
LIST_PATH = "/casting-xxx/"
|
||||
DEFAULT_OUT = "/storage/disk1/X/wcx_index.json"
|
||||
DEFAULT_PAGES = 48
|
||||
DEFAULT_SLEEP_MS = 300
|
||||
USER_AGENT = "Mozilla/5.0 (X11; Linux x86_64) wcx_sync.py/1.0"
|
||||
|
||||
|
||||
VOID_TAGS = {
|
||||
"area", "base", "br", "col", "embed", "hr", "img", "input",
|
||||
"link", "meta", "param", "source", "track", "wbr",
|
||||
}
|
||||
|
||||
|
||||
class Node:
|
||||
def __init__(self, name, attrs=None):
|
||||
self.name = name
|
||||
self.attrs = dict(attrs or [])
|
||||
self.children = []
|
||||
self.text = []
|
||||
|
||||
def get_text(self):
|
||||
return "".join(self.text + [c.get_text() for c in self.children]).strip()
|
||||
|
||||
|
||||
class TreeParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.root = Node("root")
|
||||
self.stack = [self.root]
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
node = Node(tag, attrs)
|
||||
self.stack[-1].children.append(node)
|
||||
if tag not in VOID_TAGS:
|
||||
self.stack.append(node)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
for i in range(len(self.stack) - 1, 0, -1):
|
||||
if self.stack[i].name == tag:
|
||||
self.stack = self.stack[:i]
|
||||
return
|
||||
|
||||
def handle_data(self, data):
|
||||
if data:
|
||||
self.stack[-1].text.append(data)
|
||||
|
||||
|
||||
def classes(node):
|
||||
return set((node.attrs.get("class") or "").split())
|
||||
|
||||
|
||||
def walk(node):
|
||||
yield node
|
||||
for child in node.children:
|
||||
yield from walk(child)
|
||||
|
||||
|
||||
def first_under(node, tag, cls=None):
|
||||
for n in walk(node):
|
||||
if n.name == tag and (cls is None or cls in classes(n)):
|
||||
return n
|
||||
return None
|
||||
|
||||
|
||||
def attr_under(node, tag, cls, attr):
|
||||
n = first_under(node, tag, cls)
|
||||
if not n:
|
||||
return ""
|
||||
return (n.attrs.get(attr) or "").strip()
|
||||
|
||||
|
||||
def absolute_url(href):
|
||||
return urljoin(BASE_URL, href)
|
||||
|
||||
|
||||
def file_id_from_url(url):
|
||||
path = urlparse(url).path
|
||||
filename = Path(path).name
|
||||
return filename.rsplit(".", 1)[0]
|
||||
|
||||
|
||||
def slug_title_from_href(href):
|
||||
stem = file_id_from_url(href)
|
||||
stem = stem.rsplit("_", 1)[0]
|
||||
return " ".join(w.capitalize() for w in stem.replace("-", " ").split())
|
||||
|
||||
|
||||
def fetch_url_old(url, timeout, retries, debug=False):
|
||||
req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT})
|
||||
last_error = None
|
||||
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout) as resp:
|
||||
raw = resp.read()
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||
last_error = e
|
||||
if debug:
|
||||
print(f"DEBUG fetch failed attempt {attempt}/{retries}: {url} ({e})", file=sys.stderr)
|
||||
time.sleep(min(1.0 * attempt, 3.0))
|
||||
|
||||
raise RuntimeError(f"Could not fetch {url}: {last_error}")
|
||||
|
||||
def fetch_url(url, timeout, retries, debug=False):
|
||||
last_error = None
|
||||
|
||||
for attempt in range(1, retries + 1):
|
||||
req = urllib.request.Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": USER_AGENT,
|
||||
"Connection": "close",
|
||||
},
|
||||
)
|
||||
|
||||
try:
|
||||
if debug or attempt > 1:
|
||||
print(
|
||||
f"Fetching {url} "
|
||||
f"(attempt {attempt}/{retries}, timeout={timeout}s)"
|
||||
)
|
||||
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
raw = response.read()
|
||||
|
||||
return raw.decode("utf-8", errors="ignore")
|
||||
|
||||
except (
|
||||
urllib.error.HTTPError,
|
||||
urllib.error.URLError,
|
||||
TimeoutError,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
) as exc:
|
||||
last_error = exc
|
||||
|
||||
print(
|
||||
f"WARNING: Fetch attempt {attempt}/{retries} failed "
|
||||
f"for {url}: {exc}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
if attempt < retries:
|
||||
time.sleep(min(attempt, 3))
|
||||
|
||||
raise RuntimeError(
|
||||
f"Could not fetch {url} after {retries} attempts: {last_error}"
|
||||
)
|
||||
|
||||
|
||||
def parse_list_page(data):
|
||||
parser = TreeParser()
|
||||
parser.feed(data)
|
||||
|
||||
containers = [
|
||||
n for n in walk(parser.root)
|
||||
if n.name == "div" and {"items", "container_3"}.issubset(classes(n))
|
||||
] or [parser.root]
|
||||
|
||||
items = []
|
||||
|
||||
for container in containers:
|
||||
anchors = [
|
||||
n for n in walk(container)
|
||||
if n.name == "a" and {"item", "scene"}.issubset(classes(n))
|
||||
]
|
||||
|
||||
for a in anchors:
|
||||
href = (a.attrs.get("href") or "").strip()
|
||||
if not href:
|
||||
continue
|
||||
|
||||
details = absolute_url(href)
|
||||
|
||||
title_node = first_under(a, "span", "title")
|
||||
title = title_node.get_text() if title_node else ""
|
||||
title = title or (a.attrs.get("title") or "").strip()
|
||||
title = title or attr_under(a, "img", "thumb", "alt")
|
||||
title = title or slug_title_from_href(href)
|
||||
title = html.unescape(title).strip()
|
||||
|
||||
if not title:
|
||||
continue
|
||||
|
||||
duration_node = first_under(a, "span", "duration")
|
||||
duration = duration_node.get_text().strip() if duration_node else ""
|
||||
|
||||
thumb = attr_under(a, "img", "thumb", "src")
|
||||
|
||||
item = {
|
||||
"id": file_id_from_url(details),
|
||||
"titel": title,
|
||||
"details": details,
|
||||
"duration": duration,
|
||||
}
|
||||
|
||||
if thumb:
|
||||
item["thumb"] = thumb
|
||||
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
class DetailDateParser(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.in_info_line = False
|
||||
self.in_label = False
|
||||
self.label = ""
|
||||
self.text = ""
|
||||
self.result = {}
|
||||
|
||||
def _has_class(self, attrs, wanted):
|
||||
attrs = dict(attrs or [])
|
||||
return wanted in (attrs.get("class") or "").split()
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == "p" and self._has_class(attrs, "info_line"):
|
||||
self.in_info_line = True
|
||||
self.in_label = False
|
||||
self.label = ""
|
||||
self.text = ""
|
||||
elif self.in_info_line and tag == "span" and self._has_class(attrs, "label_info"):
|
||||
self.in_label = True
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == "span" and self.in_label:
|
||||
self.in_label = False
|
||||
elif tag == "p" and self.in_info_line:
|
||||
label = self.label.strip().lower()
|
||||
match = re.search(r"([0-9]{4}-[0-9]{2}-[0-9]{2})", self.text)
|
||||
|
||||
if match:
|
||||
if label == "published":
|
||||
self.result["published"] = match.group(1)
|
||||
elif label == "updated":
|
||||
self.result["updated"] = match.group(1)
|
||||
|
||||
self.in_info_line = False
|
||||
self.in_label = False
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.in_info_line:
|
||||
self.text += data
|
||||
if self.in_label:
|
||||
self.label += data
|
||||
|
||||
|
||||
def parse_detail_dates(data):
|
||||
parser = DetailDateParser()
|
||||
parser.feed(data)
|
||||
return parser.result
|
||||
|
||||
|
||||
def unique_by_title(items):
|
||||
result = {}
|
||||
for item in items:
|
||||
result[item["titel"]] = item
|
||||
return list(result.values())
|
||||
|
||||
|
||||
def load_existing(path):
|
||||
if not path.exists():
|
||||
return []
|
||||
|
||||
with path.open("r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
if not isinstance(data, list):
|
||||
raise ValueError(f"{path} must contain a JSON array")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def merge_items(old_items, new_items):
|
||||
merged = {item.get("titel", ""): dict(item) for item in old_items if item.get("titel")}
|
||||
|
||||
for new in new_items:
|
||||
title = new.get("titel")
|
||||
if not title:
|
||||
continue
|
||||
|
||||
old = merged.get(title)
|
||||
|
||||
if old is None:
|
||||
merged[title] = new
|
||||
continue
|
||||
|
||||
updated = dict(old)
|
||||
|
||||
for key in ("id", "titel", "details", "duration", "thumb", "published"):
|
||||
if new.get(key):
|
||||
updated[key] = new[key]
|
||||
|
||||
if new.get("updated"):
|
||||
updated["updated"] = new["updated"]
|
||||
else:
|
||||
updated.pop("updated", None)
|
||||
|
||||
merged[title] = updated
|
||||
|
||||
return list(merged.values())
|
||||
|
||||
|
||||
def sort_items(items):
|
||||
return sorted(
|
||||
items,
|
||||
key=lambda x: x.get("updated") or x.get("published") or "",
|
||||
reverse=True,
|
||||
)
|
||||
|
||||
|
||||
def write_json_one_object_per_line(path, items):
|
||||
tmp = path.with_suffix(path.suffix + ".tmp")
|
||||
|
||||
with tmp.open("w", encoding="utf-8") as f:
|
||||
f.write("[\n")
|
||||
for i, item in enumerate(items):
|
||||
line = json.dumps(item, ensure_ascii=False, separators=(",", ":"))
|
||||
if i < len(items) - 1:
|
||||
line += ","
|
||||
f.write(line + "\n")
|
||||
f.write("]\n")
|
||||
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--out", default=DEFAULT_OUT)
|
||||
ap.add_argument("--pages", "--end-page", dest="pages", type=int, default=DEFAULT_PAGES)
|
||||
ap.add_argument("--sleep-ms", type=int, default=DEFAULT_SLEEP_MS)
|
||||
ap.add_argument("--timeout", type=int, default=20)
|
||||
ap.add_argument("--retries", type=int, default=3)
|
||||
ap.add_argument("--dry-run", action="store_true")
|
||||
ap.add_argument("--debug", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
out_path = Path(args.out)
|
||||
|
||||
print(f"Start. pages=1..{args.pages}, out={out_path}, dry_run={args.dry_run}")
|
||||
|
||||
list_items = []
|
||||
pages_fetched = 0
|
||||
|
||||
for page in range(1, args.pages + 1):
|
||||
url = f"{BASE_URL}{LIST_PATH}?page={page}"
|
||||
print(f"Fetching list page {page}: {url}")
|
||||
|
||||
try:
|
||||
data = fetch_url(url, args.timeout, args.retries, args.debug)
|
||||
except RuntimeError as e:
|
||||
print(f"WARNING: {e}", file=sys.stderr)
|
||||
continue
|
||||
|
||||
page_items = parse_list_page(data)
|
||||
list_items.extend(page_items)
|
||||
pages_fetched += 1
|
||||
|
||||
if args.debug:
|
||||
print(f"DEBUG: page {page} items={len(page_items)}, accumulated={len(list_items)}")
|
||||
|
||||
if page < args.pages and args.sleep_ms > 0:
|
||||
time.sleep(args.sleep_ms / 1000)
|
||||
|
||||
list_items = unique_by_title(list_items)
|
||||
print(f"Unique list items: {len(list_items)}")
|
||||
|
||||
enriched_items = []
|
||||
|
||||
for idx, item in enumerate(list_items, start=1):
|
||||
details = item["details"]
|
||||
|
||||
print(f"Fetching detail {idx}/{len(list_items)}: {item['titel']}")
|
||||
if args.debug:
|
||||
print(f"DEBUG URL: {details}")
|
||||
|
||||
enriched = dict(item)
|
||||
|
||||
try:
|
||||
detail_html = fetch_url(details, args.timeout, args.retries, args.debug)
|
||||
dates = parse_detail_dates(detail_html)
|
||||
|
||||
if not dates.get("published"):
|
||||
print(f"WARNING: No Published date found for: {item['titel']} ({details})", file=sys.stderr)
|
||||
|
||||
enriched.pop("published", None)
|
||||
enriched.pop("updated", None)
|
||||
enriched.update(dates)
|
||||
|
||||
except RuntimeError as e:
|
||||
print(f"WARNING: {e}", file=sys.stderr)
|
||||
enriched.pop("published", None)
|
||||
enriched.pop("updated", None)
|
||||
|
||||
enriched_items.append(enriched)
|
||||
|
||||
if idx < len(list_items) and args.sleep_ms > 0:
|
||||
time.sleep(args.sleep_ms / 1000)
|
||||
|
||||
old_items = load_existing(out_path)
|
||||
merged = merge_items(old_items, enriched_items)
|
||||
merged = sort_items(merged)
|
||||
|
||||
print(f"Pages fetched: {pages_fetched} / {args.pages}")
|
||||
print(f"new_items: {len(enriched_items)} | old_items: {len(old_items)} | merged: {len(merged)}")
|
||||
|
||||
if args.dry_run:
|
||||
print(f"Dry-run: NOT writing {out_path}")
|
||||
return
|
||||
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_json_one_object_per_line(out_path, merged)
|
||||
print(f"Updated {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print("\nInterrupted by user.", file=sys.stderr)
|
||||
sys.exit(130)
|
||||
Reference in New Issue
Block a user