diff --git a/20260215.7z b/20260215.7z new file mode 100644 index 0000000..aa350c4 Binary files /dev/null and b/20260215.7z differ diff --git a/RERUN.sqbpro b/RERUN.sqbpro index 7c2224b..08bed12 100644 --- a/RERUN.sqbpro +++ b/RERUN.sqbpro @@ -1,10 +1,10 @@ -
UPDATE Posts +
UPDATE Posts SET HasNotesGathered = 0 WHERE (BlogName, PostID) IN ( SELECT p.BlogName, p.PostID FROM Posts p WHERE p.HasNotesGathered = 1 - AND P.notesGatheredDatetime < 1771798752 + AND P.notesGatheredDatetime < 1774294520 AND EXISTS ( SELECT 1 FROM Notes n @@ -12,7 +12,7 @@ WHERE (BlogName, PostID) IN ( AND n.RootBlogName = p.BlogName --AND n.Type NOT IN ('reblog', 'reply') ) - ORDER BY p.NotesGatheredDateTime ASC + ORDER BY P.PostDate ASC --LIMIT 500 );select * from Blogs @@ -21,520 +21,27 @@ where HasBeenOutput = 0 AND blogname in ( -'pleasantcrusadebeard-blog', -'little-ms-spooky', -'skyhawkmkiv', -'setgendertono', -'sumetal', -'bytorsnwdog', -'lastchild-dissapointment', -'meesh33699', -'zspin', -'bagel-bird', -'ivegotaheartonfire', -'keiths-dolls', -'realmeisnotreal-blog', -'crazy-ones-world', -'mistermaf', -'mooncat163', -'keepmeoutofyourthoughts', -'realityisnotwhatsreal', -'thestuffedanimalqueen', -'afaceinside', -'iwalkedthroughthedoorwithyou', -'kcytna', -'witchymama-c', -'prof-ashraf', -'dawson800', -'omarindigo', -'hassanelazizy', -'ruqyia', -'misterflesheater', -'chobei', -'myromance', -'gsm007g', -'gstopngo', -'cloclomonster', -'sizzle-with-drizzle', -'random-glob-person', -'iblufire', -'kirah69', -'ilovescaredysquirrel2', -'destinationhighervibration', -'allpaintingnowaiting', -'iamaplussizedluckylemon', -'shydeaf', -'daddyryankimberly7482-blog', -'tooiconic', -'coldsightsws', -'atreyu', -'thesaddemon666', -'exploringthrupics', -'pepepsicaryo', -'fangornprincess', -'imcal3b', -'xelderxemox', -'adeadlydull', -'delicatecowboi', -'dom-india', -'dootsie', -'ronaldb', -'qvite--vexing', -'maccloud1592', -'mitymo13', -'moocowofdoom', -'native-rt', -'captron101', -'downworlder69', -'prince-of-the-rodeo', -'matisse-things', -'sadcucky2', -'udynamicchroniclesml', -'uinspiringthoughtsqn', -'jleitzs1', -'hopefulmisanthrope', -'kissmewithyourmouth', -'idkawtd', -'mrweasel2017', -'ertzsogood315', -'wet-panda', -'floridaboy995', -'marlborom6969', -'justus1977posts', -'rox-80-blog', -'totalmayhemmmm', -'rottenlittleshit', -'coolj90', -'bowserhightower', -'singleguy43', -'spiceyginger3', -'wunderltd', -'lilwh0r3xo', -'hereigoagainonmyones', -'vitalenuancer', -'potionhero', -'hamburgerelitist-blog', -'0h-my-glob-blog-blog', -'gabiitrompetero', -'buttcheekdiaries', -'suxametonii', -'kristenkittycat', -'tu-jefa', -'slothalgae', -'sufridakahlo', -'mierdabsurda-blog', -'darkamelot', -'frankencum', -'cuchimiaw', -'the-bands-we-love', -'pelandobananas', -'amiquetuputavida', -'nivyii', -'nomedarisa', -'andres-hmz', -'h4r0ld', -'heyniebla', -'cuntraception-blog', -'o11-59', -'zoomscua', -'laanatomiadelodio', -'not---your-princess-anymore', -'visitmehigh-blog', -'worello-blog', -'nextonagin', -'limiare', -'justicespank', -'kx0g', -'adorable-lesbians', -'mauricioriachuelos', -'the-acoustic-sweet-spot', -'hahahollllaaa-blog', -'fruhling-in-paris', -'secre7lyfamous', -'unexhausted-blog', -'lizamaphone-blog', -'chaoswitchquelaag', -'franjmartin', -'assigned-frog-at-birth', -'nocturnalday', -'nosequehueahagoaqui-blog', -'greggskii', -'unhumanrooster', -'yaggotorres', -'bitelero-blog', -'goodfunwithahandgun', -'anarkopiper', -'tonyclarks', -'empireof-dirt', -'stop-breathe-smile-xo-blog', -'jimmy-is-nerd', -'fcukingjokes-blog', -'nancy-pancy', -'pantiesnifer-blog', -'protocrypto', -'cheekygoober', -'chochelius', -'hologrma', -'cassiecupcake-blog', -'k2m206-blog', -'leedeegaga', -'luisfo', -'thekylainkylaisfantastical-blog', -'peoresnada182-blog', -'fofellie', -'katpuss-blog', -'meditacion-etilika', -'martinespiral-blog-blog-blog', -'20591', -'swords-dance', -'cokethekid', -'theinternationalcanary-blog', -'bitchevaporate-blog', -'veamos1je2-blog', -'maid450', -'charquicandenews-blog', -'poopydoo', -'follancianas-blog', -'wordsinmyskin-blog', -'binarianworld', -'wittyoriginalurl', -'heavy-lies-the-cr0wn', -'ajencerrado', -'rufus101289', -'silvermenthol', -'tutube', -'selfish-wishes', -'xslt', -'thesleepypikachu101', -'luminisciente', -'uriskpuck', -'bermont', -'duarted', -'thebasedfive', -'errormacro-blog', -'blueswicks-blog', -'tardeymal-blog', -'brujorojasdeprecated', -'eldetijuana', -'antiguoshimalayas', -'frogitsme', -'jagg19', -'darkfowar', -'higanbanathan', -'giioh', -'underarrowr', -'alguienyatienemiurl', -'ortegita', -'fuckyeahprocrastinator-blog-blog', -'hitokiricl-blog', -'kingofb', -'coffencing', -'xnothingx', -'rock-feel-live', -'international-industries', -'theanatomyofahangover', -'middleschools-blog', -'rizalwithouttheovercoat', -'hdbskhdhak', -'orchid-nebula', -'sibuenono', -'monrre', -'quetedije', -'justonea', -'tunalunatic', -'angusbeat', -'pelusaur', -'obsonivfx', -'lunnalien', -'shablagoo', -'lainteligenciamepersigue', -'lancelottnsfw', -'blueberrynachos', -'50shadesofbrownie', -'texas-wandering', -'ssgtawesome', -'genesis-does', -'misadventures-of-lauren', -'w00ble12-blog', -'tylerzyazul', -'glaciesdraco', -'fullmetalvenom', -'lifeguardinthesharktank', -'godinbirdsandsataninlongwords', -'livelonglovelife', -'letlive-inthedepths', -'funeralbelle1', -'sleepingpeelz', -'cutie-with-a-b00ty', -'chokerjoker', -'stuffofjosh', -'chasescott', -'thecutestlittlebunbunfairy', -'thewalkingscandal', -'valloir', -'akiraratsuraart', -'longlifenerdking', -'bertau28-blog', -'littlemissmissy', -'icecreamandeviscerations', -'emtvacation', -'maemismile', -'coffeestainedcalamity', -'honestlytrulymaybe', -'mrsrayiito', -'violenceinjupiter', -'atttchooooluieieiieiie', -'coimbrafilho', -'jagneta', -'gloamvalley', -'railheaven', -'1974volvo', -'isukatgam1ng', -'matted33', -'becausetrains', -'worldwiderails', -'charlesdiasme', -'searre', -'railway-express', -'polaris395', -'solitude-now', -'nim-z', -'kingofcrete', -'lorde-of-the-rails', -'ptb8751', -'capewolfe', -'tarandur', -'chrunnous', -'elitekulturen', -'shornd', -'garethgraves', -'thehylianinthetardis', -'meubrucar', -'poroussoul', -'thefakeray', -'theninthdoctortrainlord', -'maninkilt', -'northcountryman', -'restrictedapproach', -'hein-eken', -'qstation', -'impureheart', -'edward-snowdenhands', -'streamlinedtrains', -'rychillacases', -'wncrails', -'curious-ever', -'miamiextra', -'soft-but-ready', -'bellarose357', -'gnostix1', -'armoredwalrus', -'greatnorthernofficial', -'jahnwy', -'maschio61', -'boonatoot', -'qsai', -'whitedogblog', -'allaboutwheels', -'theoca', -'exdraghunt', -'take-me-away-quietly', -'people-places-things-etc', -'timbo5404', -'jeremylawson', -'henkara', -'stressedndepressedlol', -'siderods', -'littlewestern', -'identifyingtrainsinposts', -'amisplacedalphabet', -'belle-aesthetics', -'american-railroading', -'firecooking', -'coolduucks', -'pajjorimre', -'stillnotspamdog', -'legendarycashshoekid', -'dungeon-unionism', -'dracobibliobibulus', -'greatwesternway', -'metalpancam', -'espee-southernpacifc', -'itsongecau-blog', -'gimme-choco', -'mr-wizard-2', -'cupcakesinwonderland80', -'kryptakronickudalite', -'darling-nothing-isokay', -'miss-countess-von-fingerbang', -'violets-n-lavender', -'impostergir007', -'ashleyandred', -'bizarresalmiac', -'themightyseaunicorn', -'terminalfish', -'sanguine----queen', -'versaceking', -'become-one-with-noodle', -'afuture-animator', -'munch-the-moon', -'fleshkiller', -'dancing-cockroaches', -'kyuubicore', -'rollsnoice', -'redrum-all-around-me', -'stone425', -'daydreambeliever1994', -'keefers323', -'raspberri-iced-tea', -'dukethebear85', -'thistooisyuri-ttv', -'sadond', -'noodlekid', -'eggmaster6', -'raybuggybug', -'imaginestillback', -'satan-is-hungry', -'maripos-ita', -'myloveisforever56', -'canarysings', -'heikotheimperial', -'somewhereinthefog', -'the-beautifultragedy', -'chronicallydramatic', -'selectedambientwerks', -'thottyvondoggy', -'letsrevince', -'purpleshewolf', -'3p3rsz3l', -'thirstykurapika', -'over-exxposed', -'chubaccas', -'celadonisle', -'lesbiandumbbitch', -'silver-and-gold27', -'teagrounebulous', -'miscreantfromnarnia', -'pajamas24-7', -'edgeprincess30', -'chicxconpelorosa', -'badbitchandco', -'a-hot-cuppa', -'ohyesizzy', -'inhuman-place-human-monsters', -'andthentheywilleatthestars', -'essencse', -'bard-of-time', -'elementale', -'riddleseekers', -'drowningincolorfultears', -'alienk1ng', -'clarified', -'flannel-penguin', -'jestercult', -'tigaren', -'kingoftheclowns', -'yougotofastforme', -'geekylittlej', -'whyinthewild', -'yahenni', -'pepperstrawberry', -'herecomesthasexy-boom', -'jovannidanda', -'svtudio', -'grown-man-cries', -'reganpaige', -'more-silver-than-gold', -'excaliburunleashed', -'a-literal-cat', -'kelp-kelp', -'halcyon79', -'moonodyssey1994', -'flamedanser', -'flaming-ball-of-basil', -'df-is-going-on', -'grandmapattytabs', -'rainyfdayz', -'goodfellowss', -'perchs-place', -'saberomega', -'polarspur', -'unprincessleah', -'robophantom', -'f1uffy-turtle', -'whitetrashgrandpa', -'the-perplexed-spoon', -'litpit9000', -'kreepykate', -'ohthenovelty', -'jaidskywalker', -'not-just-another-juno', -'peaches-008', -'space-transgressor', -'makeshiftdhole', -'imschlamm', -'post-hummus-leigh', -'na-ta-ri', -'violasarenottheanswer', -'geekitygeek', -'colagummy', -'surprise-bisexual', -'the-mad-loki', -'guava-jelly420-blog', -'curiousmsmeg', -'boredwithmyself71', -'petronyx', -'boabeyyy', -'the-taste-of-stardust', -'bbgirla', -'postergirlwithnoposter', -'becckali-blog', -'personspeople', -'inluvwithagirl', -'voodoo-mamajuju', -'hauntedbiscuit', -'aimforthestars15', -'thealphabagel', -'wolfpupwannabe', -'fairy-faun', -'cheetah-reblogs-shit', -'tropicalwave', -'populusnix', -'the-glitter-acetylcholine', -'levis-pajamas', -'strawberrycreamcheese', -'exterminate-ak', -'amoderntragedy', -'rat-in-impossible-location', -'incdrop', -'roguegoblin', -'official-yimyams', -'foodscarf', -'databent', -'doctorcorbys-junkdrawer', -'angiric', -'silkw0rm', -'themarionettequintet', -'sayitwithbullets', -'unfortunatemizeria', -'smallbabywerewolf', -'mystical-space-witch', -'snepdragon', -'rainbow-smite' +'teaberrybee', +'reddevilgoddesstoo', +'waywardog13', +'wzjustbrowsing-blog', +'lewerta', +'nudenymph', +'caylachief' + )select RootBlogName, PostID, NoteBlogName || '.tumblr.com' as NoteBlogName, DatetimeCrawled, TimeStamp, type, RootBlogName || '.tumblr.com/post/' || postid, datetime(timestamp, 'unixepoch') from Notes where --type like 'r%' and DatetimeCrawled <> '2/12/26 12am' -order by DatetimeCrawled desc, TimeStamp descselect * --- -from notes where RootBlogName = 'vaguelydoomed'SELECT distinct +order by DatetimeCrawled desc, TimeStamp descSELECT distinct blogs.* + , blogname || '.tumblr.com' FROM Blogs inner JOIN Notes on notes.noteBlogName = blogs.BlogName WHERE - HasBeenOutput = 0 + HasBeenOutput = 0 and type = 'reblog' order by Notes.Type desc, DateAdded desc diff --git a/TL 20250119.7z b/TL 20250119.7z new file mode 100644 index 0000000..32484cc Binary files /dev/null and b/TL 20250119.7z differ diff --git a/TL 20250218.7z b/TL 20250218.7z new file mode 100644 index 0000000..6b63a1e Binary files /dev/null and b/TL 20250218.7z differ diff --git a/TL 20250529.7z b/TL 20250529.7z new file mode 100644 index 0000000..e7e6f0f Binary files /dev/null and b/TL 20250529.7z differ diff --git a/TL 20251212.7z b/TL 20251212.7z new file mode 100644 index 0000000..1d704bb Binary files /dev/null and b/TL 20251212.7z differ diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index aadfaf5..8e7e3e3 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -13,6 +13,7 @@ using Microsoft.Extensions.Diagnostics.Latency; using System.Collections; using Microsoft.Extensions.Configuration; using System.IO; +using System.Data; namespace URLNotesGrabberCORE { @@ -68,6 +69,30 @@ namespace URLNotesGrabberCORE internal class DataAccess { + private static IConfiguration? _configuration; + private static HashSet? _postIdsToExclude; + + static DataAccess() + { + _configuration = new ConfigurationBuilder() + .SetBasePath(Directory.GetCurrentDirectory()) + .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true) + .Build(); + var excludeSetting = _configuration["appSettings:PostIDToExclude"]; + if (!string.IsNullOrWhiteSpace(excludeSetting)) + { + _postIdsToExclude = excludeSetting.Split(',') + .Select(s => long.TryParse(s.Trim(), out var id) ? id : (long?)null) + .Where(id => id.HasValue) + .Select(id => id.Value) + .ToHashSet(); + } + else + { + _postIdsToExclude = new HashSet(); + } + } + public static string Q(string input) { return "'" + input.Replace("'", "''") + "'"; @@ -124,6 +149,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here Console.WriteLine($"Error checking/creating replyText column: {ex.Message}"); } finally @@ -156,6 +182,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName") Console.WriteLine(ex.Message); } @@ -203,35 +230,57 @@ namespace URLNotesGrabberCORE Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")"; SQLiteCommand command = new SQLiteCommand(sql, connection); - command.ExecuteNonQuery(); - } - catch (Exception ex) - { - if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + int rowsInserted = 0; + try + { + rowsInserted = command.ExecuteNonQuery(); + } + catch (Exception ex) + { + // Breakpoint here + if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + { + Console.WriteLine(ex.Message); + /// + try + { + connection.Open(); + + string updateSql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; + SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); + + updateCommand.ExecuteNonQuery(); + } + catch (Exception ex2) + { + // Breakpoint here + if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + { + Console.WriteLine(ex2.Message); + } + } + finally + { + connection.Close(); + } + /// + } + } + + // Only update HasBeenOutput and DateAdded if a new post was inserted + if (rowsInserted == 1) { - Console.WriteLine(ex.Message); - /// try { - connection.Open(); - - string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; - SQLiteCommand command = new SQLiteCommand(sql, connection); - - command.ExecuteNonQuery(); - } - catch (Exception ex2) - { - if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") + string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded WHERE BlogName = @BlogName"; + using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection)) { - Console.WriteLine(ex2.Message); + updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName); + updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + updateBlogCommand.ExecuteNonQuery(); } } - finally - { - connection.Close(); - } - /// + catch { } } } finally @@ -260,6 +309,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here //Console.WriteLine(ex.Message); } finally @@ -275,14 +325,14 @@ namespace URLNotesGrabberCORE Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); - SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); try { - connection.Open(); + connection2.Open(); string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)"; - using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + using (SQLiteCommand command = new SQLiteCommand(sql, connection2)) { command.Parameters.AddWithValue("@rootBlogName", rootBlogName); command.Parameters.AddWithValue("@noteBlogName", noteBlogName); @@ -291,11 +341,27 @@ namespace URLNotesGrabberCORE command.Parameters.AddWithValue("@Type", type ?? string.Empty); command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); - command.ExecuteNonQuery(); + int rowsInserted = command.ExecuteNonQuery(); + + // Only update HasBeenOutput if a new note was inserted + if (rowsInserted == 1) + { + try + { + string updateSql = "UPDATE Blogs SET HasBeenOutput = 0 WHERE BlogName = @BlogName"; + using (var updateCommand = new SQLiteCommand(updateSql, connection2)) + { + updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName); + updateCommand.ExecuteNonQuery(); + } + } + catch { } + } } } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") { Console.WriteLine(ex.Message); @@ -304,7 +370,7 @@ namespace URLNotesGrabberCORE } finally { - connection.Close(); + connection2.Close(); } return false; } @@ -330,7 +396,7 @@ namespace URLNotesGrabberCORE string sql = "SELECT " + " MAX(Posts.BlogName) as BlogName, " + Environment.NewLine + " Posts.PostID, " + Environment.NewLine + - " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine + + " 1925013599 as LatestNoteTimestamp, " + Environment.NewLine + " Posts.NotesGatheredDatetime, " + Environment.NewLine + " CNT.CNT " + Environment.NewLine + "FROM " + Environment.NewLine + @@ -344,7 +410,7 @@ namespace URLNotesGrabberCORE if (withoutNotesOnly) { sql += " and HasNotesGathered = 0 " + Environment.NewLine; - sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] > datetime('now', 'localtime', '-1 day') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine; + sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine; } // Filter by NotesGatheredDateTime if beforeDate is provided @@ -361,7 +427,7 @@ namespace URLNotesGrabberCORE Console.WriteLine(withoutNotesOnly); Console.WriteLine(sql); - Console.Write(">"); //Console.ReadKey(); + Console.Write(">" ); //Console.ReadKey(); //Thread.Sleep(250); using (SQLiteCommand command = new SQLiteCommand(sql, connection)) @@ -376,10 +442,58 @@ namespace URLNotesGrabberCORE long lastNoteTimestamp; long notesGatheredTimestamp; - blog = reader.GetString(0); // Assuming Title is the second column - postID = reader.GetInt64(1); // Assuming Id is the first column - lastNoteTimestamp = reader.GetInt64(2); // Assuming Id is the first column - notesGatheredTimestamp = reader.GetInt64(3); // Assuming Id is the first column + // Diagnostic logging before each cast + //for (int i = 0; i < reader.FieldCount; i++) + //{ + // var value = reader.GetValue(i); + // var type = reader.GetFieldType(i); + // Console.WriteLine($"[GetPosts] Column {i}: Name={reader.GetName(i)}, Value={value}, Type={type}"); + //} + + try + { + blog = Convert.ToString(reader.GetValue(0)); + } + catch (Exception ex) + { + // Breakpoint here + Console.WriteLine($"[GetPosts] Error casting column 0 to string: {ex.Message}"); + throw; + } + try + { + postID = Convert.ToInt64(reader.GetValue(1)); + } + catch (Exception ex) + { + // Breakpoint here + Console.WriteLine($"[GetPosts] Error casting column 1 to Int64: {ex.Message}"); + throw; + } + try + { + lastNoteTimestamp = Convert.ToInt64(reader.GetValue(2)); + } + catch (Exception ex) + { + // Breakpoint here + Console.WriteLine($"[GetPosts] Error casting column 2 to Int64: {ex.Message}"); + throw; + } + try + { + notesGatheredTimestamp = Convert.ToInt64(reader.GetValue(3)); + } + catch (Exception ex) + { + // Breakpoint here + Console.WriteLine($"[GetPosts] Error casting column 3 to Int64: {ex.Message}"); + throw; + } + + // Exclude if postID is in the exclusion list + if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID)) + continue; post = new Tuple(blog, postID, lastNoteTimestamp, notesGatheredTimestamp); posts.Add(post); @@ -389,6 +503,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -431,6 +546,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -481,6 +597,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here Console.WriteLine($"Error getting replies with missing text: {ex.Message}"); } finally @@ -533,6 +650,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here Console.WriteLine($"Error getting replies with filled text: {ex.Message}"); } finally @@ -569,6 +687,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -612,6 +731,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -655,6 +775,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -689,6 +810,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -719,6 +841,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -747,6 +870,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -781,6 +905,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") { Console.WriteLine(ex.Message); @@ -855,6 +980,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -881,6 +1007,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -911,6 +1038,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } @@ -955,6 +1083,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}"); Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}"); } @@ -994,6 +1123,7 @@ namespace URLNotesGrabberCORE } catch (Exception ex) { + // Breakpoint here Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}"); Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}"); } diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 3c6fc5a..441260d 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -448,7 +448,7 @@ namespace URLNotesGrabberCORE { int APICount = DataAccess.GetAPICount(); Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}"); - Thread.Sleep(3000); + //Thread.Sleep(3000); var allNotes = new List(); int page = 1; @@ -478,10 +478,10 @@ namespace URLNotesGrabberCORE } if (response.statusCode == "TooManyRequests") { - for (int s = 0; s <= response.retryInSeconds; s += 15) + for (int s = 0; s <= response.retryInSeconds; s += 60) { Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString()); - Thread.Sleep(15000); + Thread.Sleep(60000); } return response.statusCode; } @@ -589,6 +589,12 @@ namespace URLNotesGrabberCORE else { Console.WriteLine("GrabNotes Result: " + status); + + //if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue + if (status != "RateLimitExceeded") + { + DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); + } } // Re-fetch the updated list after processing the current post diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index 5378080..f8845df 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -7,7 +7,8 @@ "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt", "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt", "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db", - "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot" + "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot", + "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218" }, "TumblrApi": { "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3", diff --git a/run_500_times.bat b/run_500_times.bat new file mode 100644 index 0000000..fc949f7 --- /dev/null +++ b/run_500_times.bat @@ -0,0 +1,52 @@ +@echo off +REM Batch file to run URLNotesGrabberCORE 500 times in a loop + +setlocal enabledelayedexpansion + +REM Set the path to the executable +REM Update this path if your executable is in a different location +set APP_PATH=URLNotesGrabberCORE.exe + +REM Check if the executable exists +if not exist "%APP_PATH%" ( + echo Error: %APP_PATH% not found in the current directory. + echo Please ensure the executable is in the same directory as this batch file, + echo or update the APP_PATH variable with the correct path. + pause + exit /b 1 +) + +REM Loop counter +set ITERATIONS=500 +set COUNTER=0 + +echo Starting to run %APP_PATH% %ITERATIONS% times... +echo. + +:LOOP +set /a COUNTER+=1 +echo [%COUNTER%/%ITERATIONS%] Running iteration %COUNTER%... +echo Started at: %date% %time% + +REM Run the application with -replies option +call "%APP_PATH%" -replies + +REM Check if the application ran successfully +if errorlevel 1 ( + echo Warning: Application exited with error code !ERRORLEVEL! on iteration %COUNTER% +) else ( + echo Iteration %COUNTER% completed successfully. +) + +echo Completed at: %date% %time% +echo. + +REM Check if we've reached 500 iterations +if %COUNTER% lss %ITERATIONS% ( + goto LOOP +) + +echo. +echo Completed all %ITERATIONS% iterations! +echo. +pause