jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
jim
2026-03-26 00:35:24 -05:00
parent 15b484bdf8
commit 4e87daf2b2
10 changed files with 242 additions and 546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
20260215.7z
BIN
View File
Binary file not shown.
RERUN.sqbpro
+14 -507
View File
@@ -1,10 +1,10 @@
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Notes" custom_title="0" dock_id="4" table="4,5:mainNotes"/><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000010100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00ffffff000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="70"/></column_widths><filter_values><column index="1" value="&lt;&gt;1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values><column index="4" value="/^((?!reblog).)*$/"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="19" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="2" value="=1"/><column index="4" value="=0"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="5" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="4" value="&lt;&gt;1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
SET HasNotesGathered = 0
WHERE (BlogName, PostID) IN (
SELECT p.BlogName, p.PostID
FROM Posts p
WHERE p.HasNotesGathered = 1
AND P.notesGatheredDatetime &lt; 1771798752
AND P.notesGatheredDatetime &lt; 1774294520
AND EXISTS (
SELECT 1
FROM Notes n
@@ -12,7 +12,7 @@ WHERE (BlogName, PostID) IN (
AND n.RootBlogName = p.BlogName
--AND n.Type NOT IN ('reblog', 'reply')
)
ORDER BY p.NotesGatheredDateTime ASC
ORDER BY P.PostDate ASC
--LIMIT 500
);</sql><sql name="SQL 2">select *
from Blogs
@@ -21,520 +21,27 @@ where HasBeenOutput = 0
AND
blogname in
(
'pleasantcrusadebeard-blog',
'little-ms-spooky',
'skyhawkmkiv',
'setgendertono',
'sumetal',
'bytorsnwdog',
'lastchild-dissapointment',
'meesh33699',
'zspin',
'bagel-bird',
'ivegotaheartonfire',
'keiths-dolls',
'realmeisnotreal-blog',
'crazy-ones-world',
'mistermaf',
'mooncat163',
'keepmeoutofyourthoughts',
'realityisnotwhatsreal',
'thestuffedanimalqueen',
'afaceinside',
'iwalkedthroughthedoorwithyou',
'kcytna',
'witchymama-c',
'prof-ashraf',
'dawson800',
'omarindigo',
'hassanelazizy',
'ruqyia',
'misterflesheater',
'chobei',
'myromance',
'gsm007g',
'gstopngo',
'cloclomonster',
'sizzle-with-drizzle',
'random-glob-person',
'iblufire',
'kirah69',
'ilovescaredysquirrel2',
'destinationhighervibration',
'allpaintingnowaiting',
'iamaplussizedluckylemon',
'shydeaf',
'daddyryankimberly7482-blog',
'tooiconic',
'coldsightsws',
'atreyu',
'thesaddemon666',
'exploringthrupics',
'pepepsicaryo',
'fangornprincess',
'imcal3b',
'xelderxemox',
'adeadlydull',
'delicatecowboi',
'dom-india',
'dootsie',
'ronaldb',
'qvite--vexing',
'maccloud1592',
'mitymo13',
'moocowofdoom',
'native-rt',
'captron101',
'downworlder69',
'prince-of-the-rodeo',
'matisse-things',
'sadcucky2',
'udynamicchroniclesml',
'uinspiringthoughtsqn',
'jleitzs1',
'hopefulmisanthrope',
'kissmewithyourmouth',
'idkawtd',
'mrweasel2017',
'ertzsogood315',
'wet-panda',
'floridaboy995',
'marlborom6969',
'justus1977posts',
'rox-80-blog',
'totalmayhemmmm',
'rottenlittleshit',
'coolj90',
'bowserhightower',
'singleguy43',
'spiceyginger3',
'wunderltd',
'lilwh0r3xo',
'hereigoagainonmyones',
'vitalenuancer',
'potionhero',
'hamburgerelitist-blog',
'0h-my-glob-blog-blog',
'gabiitrompetero',
'buttcheekdiaries',
'suxametonii',
'kristenkittycat',
'tu-jefa',
'slothalgae',
'sufridakahlo',
'mierdabsurda-blog',
'darkamelot',
'frankencum',
'cuchimiaw',
'the-bands-we-love',
'pelandobananas',
'amiquetuputavida',
'nivyii',
'nomedarisa',
'andres-hmz',
'h4r0ld',
'heyniebla',
'cuntraception-blog',
'o11-59',
'zoomscua',
'laanatomiadelodio',
'not---your-princess-anymore',
'visitmehigh-blog',
'worello-blog',
'nextonagin',
'limiare',
'justicespank',
'kx0g',
'adorable-lesbians',
'mauricioriachuelos',
'the-acoustic-sweet-spot',
'hahahollllaaa-blog',
'fruhling-in-paris',
'secre7lyfamous',
'unexhausted-blog',
'lizamaphone-blog',
'chaoswitchquelaag',
'franjmartin',
'assigned-frog-at-birth',
'nocturnalday',
'nosequehueahagoaqui-blog',
'greggskii',
'unhumanrooster',
'yaggotorres',
'bitelero-blog',
'goodfunwithahandgun',
'anarkopiper',
'tonyclarks',
'empireof-dirt',
'stop-breathe-smile-xo-blog',
'jimmy-is-nerd',
'fcukingjokes-blog',
'nancy-pancy',
'pantiesnifer-blog',
'protocrypto',
'cheekygoober',
'chochelius',
'hologrma',
'cassiecupcake-blog',
'k2m206-blog',
'leedeegaga',
'luisfo',
'thekylainkylaisfantastical-blog',
'peoresnada182-blog',
'fofellie',
'katpuss-blog',
'meditacion-etilika',
'martinespiral-blog-blog-blog',
'20591',
'swords-dance',
'cokethekid',
'theinternationalcanary-blog',
'bitchevaporate-blog',
'veamos1je2-blog',
'maid450',
'charquicandenews-blog',
'poopydoo',
'follancianas-blog',
'wordsinmyskin-blog',
'binarianworld',
'wittyoriginalurl',
'heavy-lies-the-cr0wn',
'ajencerrado',
'rufus101289',
'silvermenthol',
'tutube',
'selfish-wishes',
'xslt',
'thesleepypikachu101',
'luminisciente',
'uriskpuck',
'bermont',
'duarted',
'thebasedfive',
'errormacro-blog',
'blueswicks-blog',
'tardeymal-blog',
'brujorojasdeprecated',
'eldetijuana',
'antiguoshimalayas',
'frogitsme',
'jagg19',
'darkfowar',
'higanbanathan',
'giioh',
'underarrowr',
'alguienyatienemiurl',
'ortegita',
'fuckyeahprocrastinator-blog-blog',
'hitokiricl-blog',
'kingofb',
'coffencing',
'xnothingx',
'rock-feel-live',
'international-industries',
'theanatomyofahangover',
'middleschools-blog',
'rizalwithouttheovercoat',
'hdbskhdhak',
'orchid-nebula',
'sibuenono',
'monrre',
'quetedije',
'justonea',
'tunalunatic',
'angusbeat',
'pelusaur',
'obsonivfx',
'lunnalien',
'shablagoo',
'lainteligenciamepersigue',
'lancelottnsfw',
'blueberrynachos',
'50shadesofbrownie',
'texas-wandering',
'ssgtawesome',
'genesis-does',
'misadventures-of-lauren',
'w00ble12-blog',
'tylerzyazul',
'glaciesdraco',
'fullmetalvenom',
'lifeguardinthesharktank',
'godinbirdsandsataninlongwords',
'livelonglovelife',
'letlive-inthedepths',
'funeralbelle1',
'sleepingpeelz',
'cutie-with-a-b00ty',
'chokerjoker',
'stuffofjosh',
'chasescott',
'thecutestlittlebunbunfairy',
'thewalkingscandal',
'valloir',
'akiraratsuraart',
'longlifenerdking',
'bertau28-blog',
'littlemissmissy',
'icecreamandeviscerations',
'emtvacation',
'maemismile',
'coffeestainedcalamity',
'honestlytrulymaybe',
'mrsrayiito',
'violenceinjupiter',
'atttchooooluieieiieiie',
'coimbrafilho',
'jagneta',
'gloamvalley',
'railheaven',
'1974volvo',
'isukatgam1ng',
'matted33',
'becausetrains',
'worldwiderails',
'charlesdiasme',
'searre',
'railway-express',
'polaris395',
'solitude-now',
'nim-z',
'kingofcrete',
'lorde-of-the-rails',
'ptb8751',
'capewolfe',
'tarandur',
'chrunnous',
'elitekulturen',
'shornd',
'garethgraves',
'thehylianinthetardis',
'meubrucar',
'poroussoul',
'thefakeray',
'theninthdoctortrainlord',
'maninkilt',
'northcountryman',
'restrictedapproach',
'hein-eken',
'qstation',
'impureheart',
'edward-snowdenhands',
'streamlinedtrains',
'rychillacases',
'wncrails',
'curious-ever',
'miamiextra',
'soft-but-ready',
'bellarose357',
'gnostix1',
'armoredwalrus',
'greatnorthernofficial',
'jahnwy',
'maschio61',
'boonatoot',
'qsai',
'whitedogblog',
'allaboutwheels',
'theoca',
'exdraghunt',
'take-me-away-quietly',
'people-places-things-etc',
'timbo5404',
'jeremylawson',
'henkara',
'stressedndepressedlol',
'siderods',
'littlewestern',
'identifyingtrainsinposts',
'amisplacedalphabet',
'belle-aesthetics',
'american-railroading',
'firecooking',
'coolduucks',
'pajjorimre',
'stillnotspamdog',
'legendarycashshoekid',
'dungeon-unionism',
'dracobibliobibulus',
'greatwesternway',
'metalpancam',
'espee-southernpacifc',
'itsongecau-blog',
'gimme-choco',
'mr-wizard-2',
'cupcakesinwonderland80',
'kryptakronickudalite',
'darling-nothing-isokay',
'miss-countess-von-fingerbang',
'violets-n-lavender',
'impostergir007',
'ashleyandred',
'bizarresalmiac',
'themightyseaunicorn',
'terminalfish',
'sanguine----queen',
'versaceking',
'become-one-with-noodle',
'afuture-animator',
'munch-the-moon',
'fleshkiller',
'dancing-cockroaches',
'kyuubicore',
'rollsnoice',
'redrum-all-around-me',
'stone425',
'daydreambeliever1994',
'keefers323',
'raspberri-iced-tea',
'dukethebear85',
'thistooisyuri-ttv',
'sadond',
'noodlekid',
'eggmaster6',
'raybuggybug',
'imaginestillback',
'satan-is-hungry',
'maripos-ita',
'myloveisforever56',
'canarysings',
'heikotheimperial',
'somewhereinthefog',
'the-beautifultragedy',
'chronicallydramatic',
'selectedambientwerks',
'thottyvondoggy',
'letsrevince',
'purpleshewolf',
'3p3rsz3l',
'thirstykurapika',
'over-exxposed',
'chubaccas',
'celadonisle',
'lesbiandumbbitch',
'silver-and-gold27',
'teagrounebulous',
'miscreantfromnarnia',
'pajamas24-7',
'edgeprincess30',
'chicxconpelorosa',
'badbitchandco',
'a-hot-cuppa',
'ohyesizzy',
'inhuman-place-human-monsters',
'andthentheywilleatthestars',
'essencse',
'bard-of-time',
'elementale',
'riddleseekers',
'drowningincolorfultears',
'alienk1ng',
'clarified',
'flannel-penguin',
'jestercult',
'tigaren',
'kingoftheclowns',
'yougotofastforme',
'geekylittlej',
'whyinthewild',
'yahenni',
'pepperstrawberry',
'herecomesthasexy-boom',
'jovannidanda',
'svtudio',
'grown-man-cries',
'reganpaige',
'more-silver-than-gold',
'excaliburunleashed',
'a-literal-cat',
'kelp-kelp',
'halcyon79',
'moonodyssey1994',
'flamedanser',
'flaming-ball-of-basil',
'df-is-going-on',
'grandmapattytabs',
'rainyfdayz',
'goodfellowss',
'perchs-place',
'saberomega',
'polarspur',
'unprincessleah',
'robophantom',
'f1uffy-turtle',
'whitetrashgrandpa',
'the-perplexed-spoon',
'litpit9000',
'kreepykate',
'ohthenovelty',
'jaidskywalker',
'not-just-another-juno',
'peaches-008',
'space-transgressor',
'makeshiftdhole',
'imschlamm',
'post-hummus-leigh',
'na-ta-ri',
'violasarenottheanswer',
'geekitygeek',
'colagummy',
'surprise-bisexual',
'the-mad-loki',
'guava-jelly420-blog',
'curiousmsmeg',
'boredwithmyself71',
'petronyx',
'boabeyyy',
'the-taste-of-stardust',
'bbgirla',
'postergirlwithnoposter',
'becckali-blog',
'personspeople',
'inluvwithagirl',
'voodoo-mamajuju',
'hauntedbiscuit',
'aimforthestars15',
'thealphabagel',
'wolfpupwannabe',
'fairy-faun',
'cheetah-reblogs-shit',
'tropicalwave',
'populusnix',
'the-glitter-acetylcholine',
'levis-pajamas',
'strawberrycreamcheese',
'exterminate-ak',
'amoderntragedy',
'rat-in-impossible-location',
'incdrop',
'roguegoblin',
'official-yimyams',
'foodscarf',
'databent',
'doctorcorbys-junkdrawer',
'angiric',
'silkw0rm',
'themarionettequintet',
'sayitwithbullets',
'unfortunatemizeria',
'smallbabywerewolf',
'mystical-space-witch',
'snepdragon',
'rainbow-smite'
'teaberrybee',
'reddevilgoddesstoo',
'waywardog13',
'wzjustbrowsing-blog',
'lewerta',
'nudenymph',
'caylachief'
)</sql><sql name="SQL 3*">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
from Notes
where --type like 'r%' and
DatetimeCrawled &lt;&gt; '2/12/26 12am'
order by DatetimeCrawled desc, TimeStamp desc</sql><sql name="SQL 5">select *
--
from notes where RootBlogName = 'vaguelydoomed'</sql><sql name="SQL 5">SELECT distinct
order by DatetimeCrawled desc, TimeStamp desc</sql><sql name="SQL 5">SELECT distinct
blogs.*
, blogname || '.tumblr.com'
FROM
Blogs
inner JOIN
Notes on notes.noteBlogName = blogs.BlogName
WHERE
HasBeenOutput = 0
HasBeenOutput = 0 and type = 'reblog'
order by
Notes.Type desc,
DateAdded desc
TL 20250119.7z
BIN
View File
Binary file not shown.
TL 20250218.7z
BIN
View File
Binary file not shown.
TL 20250529.7z
BIN
View File
Binary file not shown.
TL 20251212.7z
BIN
View File
Binary file not shown.
URLNotesGrabberCORE/DataAccess.cs
+145 -15
View File
@@ -13,6 +13,7 @@ using Microsoft.Extensions.Diagnostics.Latency;
using System.Collections;
using Microsoft.Extensions.Configuration;
using System.IO;
using System.Data;
namespace URLNotesGrabberCORE
{
@@ -68,6 +69,30 @@ namespace URLNotesGrabberCORE
internal class DataAccess
{
private static IConfiguration? _configuration;
private static HashSet<long>? _postIdsToExclude;
static DataAccess()
{
_configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.Build();
var excludeSetting = _configuration["appSettings:PostIDToExclude"];
if (!string.IsNullOrWhiteSpace(excludeSetting))
{
_postIdsToExclude = excludeSetting.Split(',')
.Select(s => long.TryParse(s.Trim(), out var id) ? id : (long?)null)
.Where(id => id.HasValue)
.Select(id => id.Value)
.ToHashSet();
}
else
{
_postIdsToExclude = new HashSet<long>();
}
}
public static string Q(string input)
{
return "'" + input.Replace("'", "''") + "'";
@@ -124,6 +149,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
}
finally
@@ -156,6 +182,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName")
Console.WriteLine(ex.Message);
}
@@ -203,10 +230,14 @@ namespace URLNotesGrabberCORE
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
int rowsInserted = 0;
try
{
rowsInserted = command.ExecuteNonQuery();
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex.Message);
@@ -215,13 +246,14 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
string updateSql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
command.ExecuteNonQuery();
updateCommand.ExecuteNonQuery();
}
catch (Exception ex2)
{
// Breakpoint here
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex2.Message);
@@ -234,6 +266,23 @@ namespace URLNotesGrabberCORE
///
}
}
// Only update HasBeenOutput and DateAdded if a new post was inserted
if (rowsInserted == 1)
{
try
{
string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded WHERE BlogName = @BlogName";
using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
{
updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateBlogCommand.ExecuteNonQuery();
}
}
catch { }
}
}
finally
{
connection.Close();
@@ -260,6 +309,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
//Console.WriteLine(ex.Message);
}
finally
@@ -275,14 +325,14 @@ namespace URLNotesGrabberCORE
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
connection2.Open();
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
{
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
@@ -291,11 +341,27 @@ namespace URLNotesGrabberCORE
command.Parameters.AddWithValue("@Type", type ?? string.Empty);
command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
int rowsInserted = command.ExecuteNonQuery();
// Only update HasBeenOutput if a new note was inserted
if (rowsInserted == 1)
{
try
{
string updateSql = "UPDATE Blogs SET HasBeenOutput = 0 WHERE BlogName = @BlogName";
using (var updateCommand = new SQLiteCommand(updateSql, connection2))
{
updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
updateCommand.ExecuteNonQuery();
}
}
catch { }
}
}
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
@@ -304,7 +370,7 @@ namespace URLNotesGrabberCORE
}
finally
{
connection.Close();
connection2.Close();
}
return false;
}
@@ -344,7 +410,7 @@ namespace URLNotesGrabberCORE
if (withoutNotesOnly)
{
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] > datetime('now', 'localtime', '-1 day') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
}
// Filter by NotesGatheredDateTime if beforeDate is provided
@@ -361,7 +427,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey();
Console.Write(">" ); //Console.ReadKey();
//Thread.Sleep(250);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -376,10 +442,58 @@ namespace URLNotesGrabberCORE
long lastNoteTimestamp;
long notesGatheredTimestamp;
blog = reader.GetString(0); // Assuming Title is the second column
postID = reader.GetInt64(1); // Assuming Id is the first column
lastNoteTimestamp = reader.GetInt64(2); // Assuming Id is the first column
notesGatheredTimestamp = reader.GetInt64(3); // Assuming Id is the first column
// Diagnostic logging before each cast
//for (int i = 0; i < reader.FieldCount; i++)
//{
// var value = reader.GetValue(i);
// var type = reader.GetFieldType(i);
// Console.WriteLine($"[GetPosts] Column {i}: Name={reader.GetName(i)}, Value={value}, Type={type}");
//}
try
{
blog = Convert.ToString(reader.GetValue(0));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 0 to string: {ex.Message}");
throw;
}
try
{
postID = Convert.ToInt64(reader.GetValue(1));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 1 to Int64: {ex.Message}");
throw;
}
try
{
lastNoteTimestamp = Convert.ToInt64(reader.GetValue(2));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 2 to Int64: {ex.Message}");
throw;
}
try
{
notesGatheredTimestamp = Convert.ToInt64(reader.GetValue(3));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 3 to Int64: {ex.Message}");
throw;
}
// Exclude if postID is in the exclusion list
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
continue;
post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
posts.Add(post);
@@ -389,6 +503,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -431,6 +546,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -481,6 +597,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
}
finally
@@ -533,6 +650,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
}
finally
@@ -569,6 +687,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -612,6 +731,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -655,6 +775,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -689,6 +810,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -719,6 +841,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -747,6 +870,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -781,6 +905,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
@@ -855,6 +980,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -881,6 +1007,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -911,6 +1038,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -955,6 +1083,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
}
@@ -994,6 +1123,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
}
URLNotesGrabberCORE/Program.cs
+9 -3
View File
@@ -448,7 +448,7 @@ namespace URLNotesGrabberCORE
{
int APICount = DataAccess.GetAPICount();
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
Thread.Sleep(3000);
//Thread.Sleep(3000);
var allNotes = new List<dynamic>();
int page = 1;
@@ -478,10 +478,10 @@ namespace URLNotesGrabberCORE
}
if (response.statusCode == "TooManyRequests")
{
for (int s = 0; s <= response.retryInSeconds; s += 15)
for (int s = 0; s <= response.retryInSeconds; s += 60)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
Thread.Sleep(15000);
Thread.Sleep(60000);
}
return response.statusCode;
}
@@ -589,6 +589,12 @@ namespace URLNotesGrabberCORE
else
{
Console.WriteLine("GrabNotes Result: " + status);
//if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue
if (status != "RateLimitExceeded")
{
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
}
}
// Re-fetch the updated list after processing the current post
URLNotesGrabberCORE/appsettings.json
+2 -1
View File
@@ -7,7 +7,8 @@
"PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
"PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
"PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot",
"PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218"
},
"TumblrApi": {
"ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",
run_500_times.bat
+52
View File
@@ -0,0 +1,52 @@
@echo off
REM Batch file to run URLNotesGrabberCORE 500 times in a loop
setlocal enabledelayedexpansion
REM Set the path to the executable
REM Update this path if your executable is in a different location
set APP_PATH=URLNotesGrabberCORE.exe
REM Check if the executable exists
if not exist "%APP_PATH%" (
echo Error: %APP_PATH% not found in the current directory.
echo Please ensure the executable is in the same directory as this batch file,
echo or update the APP_PATH variable with the correct path.
pause
exit /b 1
)
REM Loop counter
set ITERATIONS=500
set COUNTER=0
echo Starting to run %APP_PATH% %ITERATIONS% times...
echo.
:LOOP
set /a COUNTER+=1
echo [%COUNTER%/%ITERATIONS%] Running iteration %COUNTER%...
echo Started at: %date% %time%
REM Run the application with -replies option
call "%APP_PATH%" -replies
REM Check if the application ran successfully
if errorlevel 1 (
echo Warning: Application exited with error code !ERRORLEVEL! on iteration %COUNTER%
) else (
echo Iteration %COUNTER% completed successfully.
)
echo Completed at: %date% %time%
echo.
REM Check if we've reached 500 iterations
if %COUNTER% lss %ITERATIONS% (
goto LOOP
)
echo.
echo Completed all %ITERATIONS% iterations!
echo.
pause