.

RERUN.sqbpro

@@ -1,10 +1,10 @@
-<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Notes" custom_title="0" dock_id="4" table="4,5:mainNotes"/><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000010100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00ffffff000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="70"/></column_widths><filter_values><column index="1" value="&lt;&gt;1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values><column index="4" value="/^((?!reblog).)*$/"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="19" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="2" value="=1"/><column index="4" value="=0"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
+<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="0" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Posts" custom_title="0" dock_id="4" table="4,5:mainPosts"/><dock_state state="000000ff00000000fd00000001000000020000077400000365fc0100000005fb000000160064006f0063006b00420072006f00770073006500310100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500320100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500330100000000000004a10000000000000000fb000000160064006f0063006b00420072006f00770073006500350100000000000005f40000000000000000fc0000000000000774000001eb00fffffffa000000000100000002fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff000001eb00fffffffb000000160064006f0063006b00420072006f00770073006500340000000000ffffffff0000000000000000000007740000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="3" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="95"/><column index="3" value="54"/><column index="4" value="156"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Notes" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="4" mode="1"/></sort><column_widths><column index="1" value="207"/><column index="2" value="151"/><column index="3" value="263"/><column index="4" value="127"/><column index="5" value="59"/><column index="6" value="300"/><column index="7" value="191"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table><table schema="main" name="Posts" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="5" mode="1"/></sort><column_widths><column index="1" value="247"/><column index="2" value="151"/><column index="3" value="206"/><column index="4" value="300"/><column index="5" value="110"/><column index="6" value="191"/><column index="7" value="267"/><column index="8" value="116"/><column index="9" value="300"/><column index="10" value="300"/><column index="11" value="119"/><column index="12" value="255"/><column index="13" value="300"/><column index="14" value="71"/><column index="15" value="300"/><column index="16" value="300"/><column index="17" value="59"/><column index="18" value="113"/><column index="19" value="151"/><column index="20" value="300"/><column index="21" value="148"/><column index="22" value="300"/><column index="23" value="300"/><column index="24" value="59"/></column_widths><filter_values><column index="7" value="=1"/><column index="4" value="&lt;&gt;1"/></filter_values><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1">UPDATE Posts
 SET HasNotesGathered = 0
 WHERE (BlogName, PostID) IN (
     SELECT p.BlogName, p.PostID
     FROM Posts p
     WHERE p.HasNotesGathered = 1
-	AND P.notesGatheredDatetime &lt; 1771798752
+	AND P.notesGatheredDatetime &lt; 1774294520
     AND EXISTS (
         SELECT 1 
         FROM Notes n 
@@ -12,7 +12,7 @@ WHERE (BlogName, PostID) IN (
           AND n.RootBlogName = p.BlogName
           --AND n.Type NOT IN ('reblog', 'reply')
     )
-    ORDER BY p.NotesGatheredDateTime ASC
+    ORDER BY P.PostDate ASC
     --LIMIT 500
 );</sql><sql name="SQL 2">select *
 from Blogs
@@ -21,520 +21,27 @@ where HasBeenOutput = 0
 AND
 blogname in
 (
-'pleasantcrusadebeard-blog',
+'teaberrybee',
-'little-ms-spooky',
+'reddevilgoddesstoo',
-'skyhawkmkiv',
+'waywardog13',
-'setgendertono',
+'wzjustbrowsing-blog',
-'sumetal',
+'lewerta',
-'bytorsnwdog',
+'nudenymph',
-'lastchild-dissapointment',
+'caylachief'
-'meesh33699',
+
-'zspin',
-'bagel-bird',
-'ivegotaheartonfire',
-'keiths-dolls',
-'realmeisnotreal-blog',
-'crazy-ones-world',
-'mistermaf',
-'mooncat163',
-'keepmeoutofyourthoughts',
-'realityisnotwhatsreal',
-'thestuffedanimalqueen',
-'afaceinside',
-'iwalkedthroughthedoorwithyou',
-'kcytna',
-'witchymama-c',
-'prof-ashraf',
-'dawson800',
-'omarindigo',
-'hassanelazizy',
-'ruqyia',
-'misterflesheater',
-'chobei',
-'myromance',
-'gsm007g',
-'gstopngo',
-'cloclomonster',
-'sizzle-with-drizzle',
-'random-glob-person',
-'iblufire',
-'kirah69',
-'ilovescaredysquirrel2',
-'destinationhighervibration',
-'allpaintingnowaiting',
-'iamaplussizedluckylemon',
-'shydeaf',
-'daddyryankimberly7482-blog',
-'tooiconic',
-'coldsightsws',
-'atreyu',
-'thesaddemon666',
-'exploringthrupics',
-'pepepsicaryo',
-'fangornprincess',
-'imcal3b',
-'xelderxemox',
-'adeadlydull',
-'delicatecowboi',
-'dom-india',
-'dootsie',
-'ronaldb',
-'qvite--vexing',
-'maccloud1592',
-'mitymo13',
-'moocowofdoom',
-'native-rt',
-'captron101',
-'downworlder69',
-'prince-of-the-rodeo',
-'matisse-things',
-'sadcucky2',
-'udynamicchroniclesml',
-'uinspiringthoughtsqn',
-'jleitzs1',
-'hopefulmisanthrope',
-'kissmewithyourmouth',
-'idkawtd',
-'mrweasel2017',
-'ertzsogood315',
-'wet-panda',
-'floridaboy995',
-'marlborom6969',
-'justus1977posts',
-'rox-80-blog',
-'totalmayhemmmm',
-'rottenlittleshit',
-'coolj90',
-'bowserhightower',
-'singleguy43',
-'spiceyginger3',
-'wunderltd',
-'lilwh0r3xo',
-'hereigoagainonmyones',
-'vitalenuancer',
-'potionhero',
-'hamburgerelitist-blog',
-'0h-my-glob-blog-blog',
-'gabiitrompetero',
-'buttcheekdiaries',
-'suxametonii',
-'kristenkittycat',
-'tu-jefa',
-'slothalgae',
-'sufridakahlo',
-'mierdabsurda-blog',
-'darkamelot',
-'frankencum',
-'cuchimiaw',
-'the-bands-we-love',
-'pelandobananas',
-'amiquetuputavida',
-'nivyii',
-'nomedarisa',
-'andres-hmz',
-'h4r0ld',
-'heyniebla',
-'cuntraception-blog',
-'o11-59',
-'zoomscua',
-'laanatomiadelodio',
-'not---your-princess-anymore',
-'visitmehigh-blog',
-'worello-blog',
-'nextonagin',
-'limiare',
-'justicespank',
-'kx0g',
-'adorable-lesbians',
-'mauricioriachuelos',
-'the-acoustic-sweet-spot',
-'hahahollllaaa-blog',
-'fruhling-in-paris',
-'secre7lyfamous',
-'unexhausted-blog',
-'lizamaphone-blog',
-'chaoswitchquelaag',
-'franjmartin',
-'assigned-frog-at-birth',
-'nocturnalday',
-'nosequehueahagoaqui-blog',
-'greggskii',
-'unhumanrooster',
-'yaggotorres',
-'bitelero-blog',
-'goodfunwithahandgun',
-'anarkopiper',
-'tonyclarks',
-'empireof-dirt',
-'stop-breathe-smile-xo-blog',
-'jimmy-is-nerd',
-'fcukingjokes-blog',
-'nancy-pancy',
-'pantiesnifer-blog',
-'protocrypto',
-'cheekygoober',
-'chochelius',
-'hologrma',
-'cassiecupcake-blog',
-'k2m206-blog',
-'leedeegaga',
-'luisfo',
-'thekylainkylaisfantastical-blog',
-'peoresnada182-blog',
-'fofellie',
-'katpuss-blog',
-'meditacion-etilika',
-'martinespiral-blog-blog-blog',
-'20591',
-'swords-dance',
-'cokethekid',
-'theinternationalcanary-blog',
-'bitchevaporate-blog',
-'veamos1je2-blog',
-'maid450',
-'charquicandenews-blog',
-'poopydoo',
-'follancianas-blog',
-'wordsinmyskin-blog',
-'binarianworld',
-'wittyoriginalurl',
-'heavy-lies-the-cr0wn',
-'ajencerrado',
-'rufus101289',
-'silvermenthol',
-'tutube',
-'selfish-wishes',
-'xslt',
-'thesleepypikachu101',
-'luminisciente',
-'uriskpuck',
-'bermont',
-'duarted',
-'thebasedfive',
-'errormacro-blog',
-'blueswicks-blog',
-'tardeymal-blog',
-'brujorojasdeprecated',
-'eldetijuana',
-'antiguoshimalayas',
-'frogitsme',
-'jagg19',
-'darkfowar',
-'higanbanathan',
-'giioh',
-'underarrowr',
-'alguienyatienemiurl',
-'ortegita',
-'fuckyeahprocrastinator-blog-blog',
-'hitokiricl-blog',
-'kingofb',
-'coffencing',
-'xnothingx',
-'rock-feel-live',
-'international-industries',
-'theanatomyofahangover',
-'middleschools-blog',
-'rizalwithouttheovercoat',
-'hdbskhdhak',
-'orchid-nebula',
-'sibuenono',
-'monrre',
-'quetedije',
-'justonea',
-'tunalunatic',
-'angusbeat',
-'pelusaur',
-'obsonivfx',
-'lunnalien',
-'shablagoo',
-'lainteligenciamepersigue',
-'lancelottnsfw',
-'blueberrynachos',
-'50shadesofbrownie',
-'texas-wandering',
-'ssgtawesome',
-'genesis-does',
-'misadventures-of-lauren',
-'w00ble12-blog',
-'tylerzyazul',
-'glaciesdraco',
-'fullmetalvenom',
-'lifeguardinthesharktank',
-'godinbirdsandsataninlongwords',
-'livelonglovelife',
-'letlive-inthedepths',
-'funeralbelle1',
-'sleepingpeelz',
-'cutie-with-a-b00ty',
-'chokerjoker',
-'stuffofjosh',
-'chasescott',
-'thecutestlittlebunbunfairy',
-'thewalkingscandal',
-'valloir',
-'akiraratsuraart',
-'longlifenerdking',
-'bertau28-blog',
-'littlemissmissy',
-'icecreamandeviscerations',
-'emtvacation',
-'maemismile',
-'coffeestainedcalamity',
-'honestlytrulymaybe',
-'mrsrayiito',
-'violenceinjupiter',
-'atttchooooluieieiieiie',
-'coimbrafilho',
-'jagneta',
-'gloamvalley',
-'railheaven',
-'1974volvo',
-'isukatgam1ng',
-'matted33',
-'becausetrains',
-'worldwiderails',
-'charlesdiasme',
-'searre',
-'railway-express',
-'polaris395',
-'solitude-now',
-'nim-z',
-'kingofcrete',
-'lorde-of-the-rails',
-'ptb8751',
-'capewolfe',
-'tarandur',
-'chrunnous',
-'elitekulturen',
-'shornd',
-'garethgraves',
-'thehylianinthetardis',
-'meubrucar',
-'poroussoul',
-'thefakeray',
-'theninthdoctortrainlord',
-'maninkilt',
-'northcountryman',
-'restrictedapproach',
-'hein-eken',
-'qstation',
-'impureheart',
-'edward-snowdenhands',
-'streamlinedtrains',
-'rychillacases',
-'wncrails',
-'curious-ever',
-'miamiextra',
-'soft-but-ready',
-'bellarose357',
-'gnostix1',
-'armoredwalrus',
-'greatnorthernofficial',
-'jahnwy',
-'maschio61',
-'boonatoot',
-'qsai',
-'whitedogblog',
-'allaboutwheels',
-'theoca',
-'exdraghunt',
-'take-me-away-quietly',
-'people-places-things-etc',
-'timbo5404',
-'jeremylawson',
-'henkara',
-'stressedndepressedlol',
-'siderods',
-'littlewestern',
-'identifyingtrainsinposts',
-'amisplacedalphabet',
-'belle-aesthetics',
-'american-railroading',
-'firecooking',
-'coolduucks',
-'pajjorimre',
-'stillnotspamdog',
-'legendarycashshoekid',
-'dungeon-unionism',
-'dracobibliobibulus',
-'greatwesternway',
-'metalpancam',
-'espee-southernpacifc',
-'itsongecau-blog',
-'gimme-choco',
-'mr-wizard-2',
-'cupcakesinwonderland80',
-'kryptakronickudalite',
-'darling-nothing-isokay',
-'miss-countess-von-fingerbang',
-'violets-n-lavender',
-'impostergir007',
-'ashleyandred',
-'bizarresalmiac',
-'themightyseaunicorn',
-'terminalfish',
-'sanguine----queen',
-'versaceking',
-'become-one-with-noodle',
-'afuture-animator',
-'munch-the-moon',
-'fleshkiller',
-'dancing-cockroaches',
-'kyuubicore',
-'rollsnoice',
-'redrum-all-around-me',
-'stone425',
-'daydreambeliever1994',
-'keefers323',
-'raspberri-iced-tea',
-'dukethebear85',
-'thistooisyuri-ttv',
-'sadond',
-'noodlekid',
-'eggmaster6',
-'raybuggybug',
-'imaginestillback',
-'satan-is-hungry',
-'maripos-ita',
-'myloveisforever56',
-'canarysings',
-'heikotheimperial',
-'somewhereinthefog',
-'the-beautifultragedy',
-'chronicallydramatic',
-'selectedambientwerks',
-'thottyvondoggy',
-'letsrevince',
-'purpleshewolf',
-'3p3rsz3l',
-'thirstykurapika',
-'over-exxposed',
-'chubaccas',
-'celadonisle',
-'lesbiandumbbitch',
-'silver-and-gold27',
-'teagrounebulous',
-'miscreantfromnarnia',
-'pajamas24-7',
-'edgeprincess30',
-'chicxconpelorosa',
-'badbitchandco',
-'a-hot-cuppa',
-'ohyesizzy',
-'inhuman-place-human-monsters',
-'andthentheywilleatthestars',
-'essencse',
-'bard-of-time',
-'elementale',
-'riddleseekers',
-'drowningincolorfultears',
-'alienk1ng',
-'clarified',
-'flannel-penguin',
-'jestercult',
-'tigaren',
-'kingoftheclowns',
-'yougotofastforme',
-'geekylittlej',
-'whyinthewild',
-'yahenni',
-'pepperstrawberry',
-'herecomesthasexy-boom',
-'jovannidanda',
-'svtudio',
-'grown-man-cries',
-'reganpaige',
-'more-silver-than-gold',
-'excaliburunleashed',
-'a-literal-cat',
-'kelp-kelp',
-'halcyon79',
-'moonodyssey1994',
-'flamedanser',
-'flaming-ball-of-basil',
-'df-is-going-on',
-'grandmapattytabs',
-'rainyfdayz',
-'goodfellowss',
-'perchs-place',
-'saberomega',
-'polarspur',
-'unprincessleah',
-'robophantom',
-'f1uffy-turtle',
-'whitetrashgrandpa',
-'the-perplexed-spoon',
-'litpit9000',
-'kreepykate',
-'ohthenovelty',
-'jaidskywalker',
-'not-just-another-juno',
-'peaches-008',
-'space-transgressor',
-'makeshiftdhole',
-'imschlamm',
-'post-hummus-leigh',
-'na-ta-ri',
-'violasarenottheanswer',
-'geekitygeek',
-'colagummy',
-'surprise-bisexual',
-'the-mad-loki',
-'guava-jelly420-blog',
-'curiousmsmeg',
-'boredwithmyself71',
-'petronyx',
-'boabeyyy',
-'the-taste-of-stardust',
-'bbgirla',
-'postergirlwithnoposter',
-'becckali-blog',
-'personspeople',
-'inluvwithagirl',
-'voodoo-mamajuju',
-'hauntedbiscuit',
-'aimforthestars15',
-'thealphabagel',
-'wolfpupwannabe',
-'fairy-faun',
-'cheetah-reblogs-shit',
-'tropicalwave',
-'populusnix',
-'the-glitter-acetylcholine',
-'levis-pajamas',
-'strawberrycreamcheese',
-'exterminate-ak',
-'amoderntragedy',
-'rat-in-impossible-location',
-'incdrop',
-'roguegoblin',
-'official-yimyams',
-'foodscarf',
-'databent',
-'doctorcorbys-junkdrawer',
-'angiric',
-'silkw0rm',
-'themarionettequintet',
-'sayitwithbullets',
-'unfortunatemizeria',
-'smallbabywerewolf',
-'mystical-space-witch',
-'snepdragon',
-'rainbow-smite'
 )</sql><sql name="SQL 3*">select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch')
 from Notes
 where --type like 'r%' and 
 DatetimeCrawled &lt;&gt; '2/12/26 12am'
-order by DatetimeCrawled desc, TimeStamp desc</sql><sql name="SQL 5">select *
+order by DatetimeCrawled desc, TimeStamp desc</sql><sql name="SQL 5">SELECT distinct
---
-from notes where RootBlogName = 'vaguelydoomed'</sql><sql name="SQL 5">SELECT distinct
 	blogs.*
+	, blogname || '.tumblr.com'
 FROM
 	Blogs
 		inner JOIN
 	Notes on notes.noteBlogName = blogs.BlogName
 WHERE
-	HasBeenOutput = 0
+	HasBeenOutput = 0 and type = 'reblog'
 order by
 	Notes.Type desc,
 	DateAdded desc

URLNotesGrabberCORE/DataAccess.cs

@@ -13,6 +13,7 @@ using Microsoft.Extensions.Diagnostics.Latency;
 using System.Collections;
 using Microsoft.Extensions.Configuration;
 using System.IO;
+using System.Data;
 
 namespace URLNotesGrabberCORE
 {
@@ -68,6 +69,30 @@ namespace URLNotesGrabberCORE
 
     internal class DataAccess
     {
+        private static IConfiguration? _configuration;
+        private static HashSet<long>? _postIdsToExclude;
+
+        static DataAccess()
+        {
+            _configuration = new ConfigurationBuilder()
+                .SetBasePath(Directory.GetCurrentDirectory())
+                .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
+                .Build();
+            var excludeSetting = _configuration["appSettings:PostIDToExclude"];
+            if (!string.IsNullOrWhiteSpace(excludeSetting))
+            {
+                _postIdsToExclude = excludeSetting.Split(',')
+                    .Select(s => long.TryParse(s.Trim(), out var id) ? id : (long?)null)
+                    .Where(id => id.HasValue)
+                    .Select(id => id.Value)
+                    .ToHashSet();
+            }
+            else
+            {
+                _postIdsToExclude = new HashSet<long>();
+            }
+        }
+
         public static string Q(string input)
         {
             return "'" + input.Replace("'", "''") + "'";
@@ -124,6 +149,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
             }
             finally
@@ -156,6 +182,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName")
                     Console.WriteLine(ex.Message);
             }
@@ -203,35 +230,57 @@ namespace URLNotesGrabberCORE
                     Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
-                command.ExecuteNonQuery();
+                int rowsInserted = 0;
-            }
+                try
-            catch (Exception ex)
+                {
-            {
+                    rowsInserted = command.ExecuteNonQuery();
-                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                }
+                catch (Exception ex)
+                {
+                    // Breakpoint here
+                    if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    {
+                        Console.WriteLine(ex.Message);
+                        ///
+                        try
+                        {
+                            connection.Open();
+
+                            string updateSql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
+                            SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
+
+                            updateCommand.ExecuteNonQuery();
+                        }
+                        catch (Exception ex2)
+                        {
+                            // Breakpoint here
+                            if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                            {
+                                Console.WriteLine(ex2.Message);
+                            }
+                        }
+                        finally
+                        {
+                            connection.Close();
+                        }
+                        ///
+                    }
+                }
+
+                // Only update HasBeenOutput and DateAdded if a new post was inserted
+                if (rowsInserted == 1)
                 {
-                    Console.WriteLine(ex.Message);
-                    ///
                     try
                     {
-                        connection.Open();
+                        string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded WHERE BlogName = @BlogName";
-
+                        using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
-                        string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
-                        SQLiteCommand command = new SQLiteCommand(sql, connection);
-
-                        command.ExecuteNonQuery();
-                    }
-                    catch (Exception ex2)
-                    {
-                        if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                         {
-                            Console.WriteLine(ex2.Message);
+                            updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
+                            updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
+                            updateBlogCommand.ExecuteNonQuery();
                         }
                     }
-                    finally
+                    catch { }
-                    {
-                        connection.Close();
-                    }
-                    ///
                 }
             }
             finally
@@ -260,6 +309,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 //Console.WriteLine(ex.Message);
             }
             finally
@@ -275,14 +325,14 @@ namespace URLNotesGrabberCORE
 
             Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
 
-            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
 
             try
             {
-                connection.Open();
+                connection2.Open();
 
                 string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)";
-                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
                 {
                     command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
                     command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
@@ -291,11 +341,27 @@ namespace URLNotesGrabberCORE
                     command.Parameters.AddWithValue("@Type", type ?? string.Empty);
                     command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
 
-                    command.ExecuteNonQuery();
+                    int rowsInserted = command.ExecuteNonQuery();
+
+                    // Only update HasBeenOutput if a new note was inserted
+                    if (rowsInserted == 1)
+                    {
+                        try
+                        {
+                            string updateSql = "UPDATE Blogs SET HasBeenOutput = 0 WHERE BlogName = @BlogName";
+                            using (var updateCommand = new SQLiteCommand(updateSql, connection2))
+                            {
+                                updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
+                                updateCommand.ExecuteNonQuery();
+                            }
+                        }
+                        catch { }
+                    }
                 }
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
                 {
                     Console.WriteLine(ex.Message);
@@ -304,7 +370,7 @@ namespace URLNotesGrabberCORE
             }
             finally
             {
-                connection.Close();
+                connection2.Close();
             }
             return false;
         }
@@ -330,7 +396,7 @@ namespace URLNotesGrabberCORE
                 string sql = "SELECT " +
                              "  MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
                              "  Posts.PostID, " + Environment.NewLine +
-                             " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
+                             "  1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
                              "  Posts.NotesGatheredDatetime, " + Environment.NewLine +
                              "  CNT.CNT " + Environment.NewLine +
                              "FROM " + Environment.NewLine +
@@ -344,7 +410,7 @@ namespace URLNotesGrabberCORE
                 if (withoutNotesOnly)
                 {
                     sql += " and HasNotesGathered = 0 " + Environment.NewLine;
-                    sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] > datetime('now', 'localtime', '-1 day') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
+                    sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
                 }
 
                 // Filter by NotesGatheredDateTime if beforeDate is provided
@@ -361,7 +427,7 @@ namespace URLNotesGrabberCORE
 
                 Console.WriteLine(withoutNotesOnly);
                 Console.WriteLine(sql);
-                Console.Write(">"); //Console.ReadKey();
+                Console.Write(">" ); //Console.ReadKey();
                 //Thread.Sleep(250);
 
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -376,10 +442,58 @@ namespace URLNotesGrabberCORE
                             long lastNoteTimestamp;
                             long notesGatheredTimestamp;
 
-                            blog = reader.GetString(0); // Assuming Title is the second column
+                            // Diagnostic logging before each cast
-                            postID = reader.GetInt64(1); // Assuming Id is the first column
+                            //for (int i = 0; i < reader.FieldCount; i++)
-                            lastNoteTimestamp = reader.GetInt64(2); // Assuming Id is the first column
+                            //{
-                            notesGatheredTimestamp = reader.GetInt64(3); // Assuming Id is the first column
+                            //    var value = reader.GetValue(i);
+                            //    var type = reader.GetFieldType(i);
+                            //    Console.WriteLine($"[GetPosts] Column {i}: Name={reader.GetName(i)}, Value={value}, Type={type}");
+                            //}
+
+                            try
+                            {
+                                blog = Convert.ToString(reader.GetValue(0));
+                            }
+                            catch (Exception ex)
+                            {
+                                // Breakpoint here
+                                Console.WriteLine($"[GetPosts] Error casting column 0 to string: {ex.Message}");
+                                throw;
+                            }
+                            try
+                            {
+                                postID = Convert.ToInt64(reader.GetValue(1));
+                            }
+                            catch (Exception ex)
+                            {
+                                // Breakpoint here
+                                Console.WriteLine($"[GetPosts] Error casting column 1 to Int64: {ex.Message}");
+                                throw;
+                            }
+                            try
+                            {
+                                lastNoteTimestamp = Convert.ToInt64(reader.GetValue(2));
+                            }
+                            catch (Exception ex)
+                            {
+                                // Breakpoint here
+                                Console.WriteLine($"[GetPosts] Error casting column 2 to Int64: {ex.Message}");
+                                throw;
+                            }
+                            try
+                            {
+                                notesGatheredTimestamp = Convert.ToInt64(reader.GetValue(3));
+                            }
+                            catch (Exception ex)
+                            {
+                                // Breakpoint here
+                                Console.WriteLine($"[GetPosts] Error casting column 3 to Int64: {ex.Message}");
+                                throw;
+                            }
+
+                            // Exclude if postID is in the exclusion list
+                            if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
+                                continue;
 
                             post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
                             posts.Add(post);
@@ -389,6 +503,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -431,6 +546,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -481,6 +597,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
             }
             finally
@@ -533,6 +650,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
             }
             finally
@@ -569,6 +687,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -612,6 +731,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -655,6 +775,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -689,6 +810,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -719,6 +841,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -747,6 +870,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -781,6 +905,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
                 {
                     Console.WriteLine(ex.Message);
@@ -855,6 +980,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -881,6 +1007,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -911,6 +1038,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
                     Console.WriteLine(ex.Message);
             }
@@ -955,6 +1083,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
                 Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
             }
@@ -994,6 +1123,7 @@ namespace URLNotesGrabberCORE
             }
             catch (Exception ex)
             {
+                // Breakpoint here
                 Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
                 Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
             }

URLNotesGrabberCORE/Program.cs

@@ -448,7 +448,7 @@ namespace URLNotesGrabberCORE
             {
                 int APICount = DataAccess.GetAPICount();
                 Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
-                Thread.Sleep(3000);
+                //Thread.Sleep(3000);
 
                 var allNotes = new List<dynamic>();
                 int page = 1;
@@ -478,10 +478,10 @@ namespace URLNotesGrabberCORE
                 }
                 if (response.statusCode == "TooManyRequests")
                 {
-                    for (int s = 0; s <= response.retryInSeconds; s += 15)
+                    for (int s = 0; s <= response.retryInSeconds; s += 60)
                     {
                         Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
-                        Thread.Sleep(15000);
+                        Thread.Sleep(60000);
                     }
                     return response.statusCode;
                 }
@@ -589,6 +589,12 @@ namespace URLNotesGrabberCORE
                         else
                         {
                             Console.WriteLine("GrabNotes Result: " + status);
+
+                            //if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue
+                            if (status != "RateLimitExceeded")
+                            {
+                                DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                            }
                         }
 
                         // Re-fetch the updated list after processing the current post

URLNotesGrabberCORE/appsettings.json

@@ -7,7 +7,8 @@
     "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
     "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
     "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
-    "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
+    "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot",
+    "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218"
   },
   "TumblrApi": {
     "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",

run_500_times.bat

@@ -0,0 +1,52 @@
+@echo off
+REM Batch file to run URLNotesGrabberCORE 500 times in a loop
+
+setlocal enabledelayedexpansion
+
+REM Set the path to the executable
+REM Update this path if your executable is in a different location
+set APP_PATH=URLNotesGrabberCORE.exe
+
+REM Check if the executable exists
+if not exist "%APP_PATH%" (
+    echo Error: %APP_PATH% not found in the current directory.
+    echo Please ensure the executable is in the same directory as this batch file,
+    echo or update the APP_PATH variable with the correct path.
+    pause
+    exit /b 1
+)
+
+REM Loop counter
+set ITERATIONS=500
+set COUNTER=0
+
+echo Starting to run %APP_PATH% %ITERATIONS% times...
+echo.
+
+:LOOP
+set /a COUNTER+=1
+echo [%COUNTER%/%ITERATIONS%] Running iteration %COUNTER%...
+echo Started at: %date% %time%
+
+REM Run the application with -replies option
+call "%APP_PATH%" -replies
+
+REM Check if the application ran successfully
+if errorlevel 1 (
+    echo Warning: Application exited with error code !ERRORLEVEL! on iteration %COUNTER%
+) else (
+    echo Iteration %COUNTER% completed successfully.
+)
+
+echo Completed at: %date% %time%
+echo.
+
+REM Check if we've reached 500 iterations
+if %COUNTER% lss %ITERATIONS% (
+    goto LOOP
+)
+
+echo.
+echo Completed all %ITERATIONS% iterations!
+echo.
+pause