diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index 61b6032..0c9f9c5 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -71,6 +71,8 @@ namespace URLNotesGrabberCORE { private static IConfiguration? _configuration; private static HashSet? _postIdsToExclude; + private static readonly object _importPragmaLock = new object(); + private static Tuple? _savedImportPragmas; static DataAccess() { @@ -106,6 +108,95 @@ namespace URLNotesGrabberCORE return dateTime; } + public static void EnableImportModePragmas(string DBPath = @"TL.db") + { + lock (_importPragmaLock) + { + if (_savedImportPragmas != null) + return; + + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + try + { + connection.Open(); + + string currentJournalMode = "delete"; + string currentSynchronous = "2"; + + using (var cmd = new SQLiteCommand("PRAGMA journal_mode;", connection)) + { + currentJournalMode = Convert.ToString(cmd.ExecuteScalar()) ?? "delete"; + } + + using (var cmd = new SQLiteCommand("PRAGMA synchronous;", connection)) + { + currentSynchronous = Convert.ToString(cmd.ExecuteScalar()) ?? "2"; + } + + _savedImportPragmas = new Tuple(currentJournalMode, currentSynchronous); + + using (var cmd = new SQLiteCommand("PRAGMA journal_mode=WAL;", connection)) + { + cmd.ExecuteNonQuery(); + } + + using (var cmd = new SQLiteCommand("PRAGMA synchronous=NORMAL;", connection)) + { + cmd.ExecuteNonQuery(); + } + + Console.WriteLine($"[SQLite Import Mode] Enabled | journal_mode=WAL | synchronous=NORMAL (previous: journal_mode={currentJournalMode}, synchronous={currentSynchronous})"); + } + catch (Exception ex) + { + Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}"); + } + finally + { + connection.Close(); + } + } + } + + public static void RestoreImportModePragmas(string DBPath = @"TL.db") + { + lock (_importPragmaLock) + { + if (_savedImportPragmas == null) + return; + + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + try + { + connection.Open(); + + string priorJournalMode = _savedImportPragmas.Item1; + string priorSynchronous = _savedImportPragmas.Item2; + + using (var cmd = new SQLiteCommand($"PRAGMA journal_mode={priorJournalMode};", connection)) + { + cmd.ExecuteNonQuery(); + } + + using (var cmd = new SQLiteCommand($"PRAGMA synchronous={priorSynchronous};", connection)) + { + cmd.ExecuteNonQuery(); + } + + Console.WriteLine($"[SQLite Import Mode] Restored | journal_mode={priorJournalMode} | synchronous={priorSynchronous}"); + } + catch (Exception ex) + { + Console.WriteLine($"[SQLite Import Mode] Failed to restore pragmas: {ex.Message}"); + } + finally + { + _savedImportPragmas = null; + connection.Close(); + } + } + } + public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); @@ -220,12 +311,13 @@ namespace URLNotesGrabberCORE connection.Open(); // Insert BlogName and DateAdded (current UTC datetime) - string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @ByLikes)"; + string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, DateCreated, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @DateCreated, @ByLikes)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@BlogName", blogName); command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); command.Parameters.AddWithValue("@ByLikes", byLikes ? 1 : 0); command.ExecuteNonQuery(); } @@ -277,12 +369,13 @@ namespace URLNotesGrabberCORE Answer, Title, DateModified, + DateCreated, RootBlogName, RootURL, HasImage, ByLikes ) VALUES (" + - Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")"; + Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")"; SQLiteCommand command = new SQLiteCommand(sql, connection); int rowsInserted = 0; @@ -301,7 +394,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string updateSql = "UPDATE Posts SET hasImage = " + hasImage + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; + string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); updateCommand.ExecuteNonQuery(); @@ -355,10 +448,11 @@ namespace URLNotesGrabberCORE // Use INSERT OR IGNORE to avoid UNIQUE constraint errors when the date row already exists. // Also explicitly initialize APICount to 0 in case the table has no default. - string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount) values(@date, 0)"; + string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount, DateCreated) values(@date, 0, @DateCreated)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString()); + command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); command.ExecuteNonQuery(); } } @@ -378,15 +472,13 @@ namespace URLNotesGrabberCORE //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { } - Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); - SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); try { connection2.Open(); - string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified)"; + string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection2)) { command.Parameters.AddWithValue("@rootBlogName", rootBlogName); @@ -396,9 +488,23 @@ namespace URLNotesGrabberCORE command.Parameters.AddWithValue("@Type", type ?? string.Empty); command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); + command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); int rowsInserted = command.ExecuteNonQuery(); + if (rowsInserted == 1) + { + ConsoleColor previousColor = Console.ForegroundColor; + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); + Console.ForegroundColor = previousColor; + //Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output + } + else + { + Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); + } + // Only update HasBeenOutput if a new note was inserted if (rowsInserted == 1) { @@ -462,12 +568,12 @@ namespace URLNotesGrabberCORE " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + " LEFT OUTER JOIN " + Environment.NewLine + " ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " + - "WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine; + "WHERE NotFound = 0 " + Environment.NewLine; if (withoutNotesOnly) { sql += " and HasNotesGathered = 0 " + Environment.NewLine; - sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine; + sql += "OR ( NotFound = 0 AND Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine; } // Filter by NotesGatheredDateTime if beforeDate is provided @@ -885,6 +991,45 @@ namespace URLNotesGrabberCORE } return blogs; } + public static IEnumerable> GetAllPostTextColumns(string DBPath = @"TL.db") + { + SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); + connection.Open(); + try + { + string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; + using (SQLiteCommand command = new SQLiteCommand(sql, connection)) + { + using (SQLiteDataReader reader = command.ExecuteReader()) + { + while (reader.Read()) + { + List rowTexts = new List(); + for (int i = 0; i < reader.FieldCount; i++) + { + if (!reader.IsDBNull(i)) + { + var val = reader.GetValue(i); + if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".") + { + rowTexts.Add(str); + } + } + } + if (rowTexts.Count > 0) + { + yield return rowTexts; + } + } + } + } + } + finally + { + connection.Close(); + connection.Dispose(); + } + } #endregion Gets #region Updates @@ -899,7 +1044,7 @@ namespace URLNotesGrabberCORE connection.Open(); //string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID"; - string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID"; + string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds()); @@ -932,7 +1077,7 @@ namespace URLNotesGrabberCORE connection.Open(); - string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID"; + string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(NotFound, 0) <> 1"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")); @@ -961,7 +1106,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID"; + string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(postDate, '') <> @postDate"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty); @@ -996,7 +1141,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID"; + string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID AND IFNULL(timestamp, 0) <> @timestamp"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@timestamp", timestamp); @@ -1056,7 +1201,30 @@ namespace URLNotesGrabberCORE sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, "; sql += "hasImage = @hasImage, "; sql += "ByLikes = @byLikes "; - sql += " WHERE BlogName = @BlogName AND PostID = @PostID"; + sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND ("; + sql += "IFNULL(postDate, '') <> @postDate OR "; + sql += "IFNULL(reblogURL, '') <> @reblogURL OR "; + sql += "IFNULL(postURL, '') <> @postURL OR "; + sql += "IFNULL(slug, '') <> @slug OR "; + sql += "IFNULL(reblogKey, '') <> @reblogKey OR "; + sql += "IFNULL(reblogName, '') <> @reblogName OR "; + sql += "IFNULL(summary, '') <> @summary OR "; + sql += "IFNULL(quote, '') <> @quote OR "; + sql += "IFNULL(body, '') <> @body OR "; + sql += "IFNULL(tags, '') <> @tags OR "; + sql += "IFNULL(link, '') <> @link OR "; + sql += "IFNULL(photoURL, '') <> @photoURL OR "; + sql += "IFNULL(photoCaption, '') <> @photoCaption OR "; + sql += "IFNULL(downloadedFiles, '') <> @downloadedFiles OR "; + sql += "IFNULL(audioCaption, '') <> @audioCaption OR "; + sql += "IFNULL(question, '') <> @question OR "; + sql += "IFNULL(answer, '') <> @answer OR "; + sql += "IFNULL(title, '') <> @title OR "; + sql += "IFNULL(hasImage, 0) <> @hasImage OR "; + sql += "IFNULL(ByLikes, 0) <> @byLikes OR "; + sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR "; + sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)"; + sql += ")"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { @@ -1110,7 +1278,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName"; + string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName AND IFNULL(HasBeenOutput, 0) <> 1"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@BlogName", blogName); @@ -1136,7 +1304,7 @@ namespace URLNotesGrabberCORE try { connection.Open(); - string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name"; + string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name AND (IFNULL(LikesPulled, 0) <> @pulled OR IFNULL(LikesCursor, 0) <> @cursor)"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@pulled", likesPulled); @@ -1198,7 +1366,7 @@ namespace URLNotesGrabberCORE connection.Open(); //string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; - string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'"; + string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@replyText", replyText ?? "."); @@ -1211,7 +1379,7 @@ namespace URLNotesGrabberCORE if (rowsAffected == 0) { - Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}"); + Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)"); Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}"); Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}"); } @@ -1241,7 +1409,7 @@ namespace URLNotesGrabberCORE { connection.Open(); - string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'"; + string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { command.Parameters.AddWithValue("@replyText", replyText ?? "."); @@ -1252,7 +1420,7 @@ namespace URLNotesGrabberCORE if (rowsAffected == 0) { - Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}"); + Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}/{postID} (rows not found or values unchanged)"); Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}"); Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}"); } @@ -1417,6 +1585,11 @@ namespace URLNotesGrabberCORE myDeserializedClass = deserializedResult; myDeserializedClass.rawJson = myJsonResponse; + if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404) + { + myDeserializedClass.statusCode = "NotFound"; + } + // If the response JSON indicates a 429 (Too Many Requests) via meta.status or message, // treat it like a rate-limited response and attempt to read Retry headers. bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429; diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index bbfa6c3..86b1a69 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -6,6 +6,7 @@ using System.Configuration; using System.Threading; using Microsoft.Extensions.Diagnostics.Latency; using static System.Runtime.InteropServices.JavaScript.JSType; +using System.Text.RegularExpressions; namespace URLNotesGrabberCORE { @@ -22,6 +23,7 @@ namespace URLNotesGrabberCORE var settings = config.GetSection("appSettings"); string apiSectionName = "TumblrApi"; + string startFromBlogName = string.Empty; List filteredArgs = new List(); for (int i = 0; i < args.Length; i++) { @@ -54,6 +56,21 @@ namespace URLNotesGrabberCORE continue; } + if (string.Equals(args[i], "-start", StringComparison.OrdinalIgnoreCase) || + string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase)) + { + if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1])) + { + startFromBlogName = args[i + 1].Trim(); + i++; + } + else + { + Console.WriteLine("--Missing blog name after -start/--start. Ignoring.--"); + } + continue; + } + filteredArgs.Add(args[i]); } @@ -61,32 +78,45 @@ namespace URLNotesGrabberCORE APIAccess.SetApiConfigSection(apiSectionName); // Setup Dual Logging - string logPath = "console_output.log"; - if (File.Exists(logPath)) + bool enableFileLogging = settings.GetValue("EnableFileLogging", true); + if (enableFileLogging) { - string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); - string archiveDirectory = "logs"; - - // Ensure the archive directory exists - if (!Directory.Exists(archiveDirectory)) + string logPath = "console_output.log"; + if (File.Exists(logPath)) { - Directory.CreateDirectory(archiveDirectory); + string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss"); + string archiveDirectory = "logs"; + + // Ensure the archive directory exists + if (!Directory.Exists(archiveDirectory)) + { + Directory.CreateDirectory(archiveDirectory); + } + + string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log"); + File.Move(logPath, newPath); } - string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log"); - File.Move(logPath, newPath); + StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true }; + DualLogger dualLogger = new DualLogger(Console.Out, fileWriter); + Console.SetOut(dualLogger); } - StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true }; - DualLogger dualLogger = new DualLogger(Console.Out, fileWriter); - Console.SetOut(dualLogger); - List contains = settings.GetValue("ContainsList").Split(',').ToList(); + bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false); if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB { int postsAdded = 0; - TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, ref postsAdded); + try + { + DataAccess.EnableImportModePragmas(); + TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports); + } + finally + { + DataAccess.RestoreImportModePragmas(); + } Console.WriteLine($"Total posts added: {postsAdded}"); } else @@ -116,10 +146,14 @@ namespace URLNotesGrabberCORE Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param"); + Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file"); + Console.WriteLine("-api3\t Use TumblrApi3 settings from appsettings.json"); Console.WriteLine("-api4\t Use TumblrApi4 settings from appsettings.json"); + Console.WriteLine("-start [blogname]\t Start traversal alphabetically at this blog name"); + Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)"); break; @@ -127,7 +161,15 @@ namespace URLNotesGrabberCORE case "-parse": string blogNameToParse = args[1]; int postsAdded = 0; - TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse); + try + { + DataAccess.EnableImportModePragmas(); + TraverseDirectory(settings.GetValue("PathInput"), settings.GetValue("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse, startFromBlogName, logTraversalRecordImports); + } + finally + { + DataAccess.RestoreImportModePragmas(); + } Console.WriteLine($"Total posts added: {postsAdded}"); break; @@ -237,6 +279,10 @@ namespace URLNotesGrabberCORE CollectLikes(likeBlog, contains).GetAwaiter().GetResult(); break; + case "-urldump": + DumpUrls(settings.GetValue("PathOutputUrls")); + break; + default: Console.WriteLine("** Unknown Command ** " + args[0]); break; @@ -320,6 +366,45 @@ namespace URLNotesGrabberCORE } } + static void DumpUrls(string? outPath) + { + if (string.IsNullOrWhiteSpace(outPath)) + { + Console.WriteLine("--PathOutputUrls is not configured in appsettings.json--"); + return; + } + + Console.WriteLine($"Starting URL extraction to {outPath}..."); + HashSet uniqueUrls = new HashSet(StringComparer.OrdinalIgnoreCase); + Regex urlRegex = new Regex(@"https?://[^\s""'<>]+", RegexOptions.IgnoreCase | RegexOptions.Compiled); + + int rowsProcessed = 0; + foreach (var texts in DataAccess.GetAllPostTextColumns()) + { + rowsProcessed++; + if (rowsProcessed % 10000 == 0) + { + Console.WriteLine($"Scanned {rowsProcessed} rows... found {uniqueUrls.Count} unique URLs so far."); + } + + foreach (var text in texts) + { + var matches = urlRegex.Matches(text); + foreach (Match match in matches) + { + uniqueUrls.Add(match.Value); + } + } + } + + Console.WriteLine($"Scan complete. Sorting and saving {uniqueUrls.Count} distinct URLs..."); + var sortedUrls = uniqueUrls.ToList(); + sortedUrls.Sort(); + + File.WriteAllLines(outPath, sortedUrls); + Console.WriteLine($"Saved URLs to {outPath}"); + } + protected static bool ContainsAny(string input, List contains) { if (string.IsNullOrEmpty(input)) return false; @@ -335,6 +420,20 @@ namespace URLNotesGrabberCORE return false; } + protected static string NormalizeBlogFolderName(string folderName) + { + return folderName + .Replace("_1", "") + .Replace("_2", "") + .Replace("_3", "") + .Replace("_4", "") + .Replace("_5", "") + .Replace("_6", "") + .Replace("_7", "") + .Replace("_8", "") + .Replace("_9", ""); + } + static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp) { try @@ -494,6 +593,44 @@ namespace URLNotesGrabberCORE } } + static ConsoleColor GetMatchedCounterColor(long matchedCount) + { + if (matchedCount <= 0) + return ConsoleColor.White; + + return ((matchedCount - 1) % 3) switch + { + 0 => ConsoleColor.Red, + 1 => ConsoleColor.Green, + _ => ConsoleColor.Blue + }; + } + + static void WriteLikesTotalsLine(string blogName, string label, long parsedForBlog, long matchedForBlog, int likedCountForBlog) + { + if (likedCountForBlog > 0) + { + double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0; + double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0; + + Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: "); + var previousColor = Console.ForegroundColor; + Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog); + Console.Write(matchedForBlog); + Console.ForegroundColor = previousColor; + Console.WriteLine($"/{likedCountForBlog} ({matchedPct:F2}%)"); + } + else + { + Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog} | Matched: "); + var previousColor = Console.ForegroundColor; + Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog); + Console.Write(matchedForBlog); + Console.ForegroundColor = previousColor; + Console.WriteLine(); + } + } + static async Task CollectLikes(string specificBlog, List contains) { try @@ -691,16 +828,7 @@ namespace URLNotesGrabberCORE } } - if (likedCountForBlog > 0) - { - double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0; - double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0; - Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)"); - } - else - { - Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}"); - } + WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog); // Determine the next BeforeCursor. long nextCursor = 0; @@ -728,16 +856,7 @@ namespace URLNotesGrabberCORE await Task.Delay(1000); // 1-second delay between pages } - if (likedCountForBlog > 0) - { - double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0; - double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0; - Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)"); - } - else - { - Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}"); - } + WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog); } Console.WriteLine("Likes collection complete."); @@ -753,6 +872,11 @@ namespace URLNotesGrabberCORE { try { + static bool IsNotFound(Root? r) + { + return (r?.meta != null && r.meta.status == 404) || string.Equals(r?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase); + } + int APICount = DataAccess.GetAPICount(); Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}"); //Thread.Sleep(3000); @@ -765,9 +889,9 @@ namespace URLNotesGrabberCORE var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult(); // Handle 404 and error codes - if (response?.meta != null && response.meta.status == 404) + if (IsNotFound(response)) { - Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)"); + Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}"); DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); Thread.Sleep(1000); return "NotFound"; @@ -777,12 +901,6 @@ namespace URLNotesGrabberCORE Console.WriteLine("##### Response is null - API Failure? ###"); return "FAILURE"; } - if (response.statusCode == "NotFound") - { - Thread.Sleep(1000); - DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); - return response.statusCode; - } if (response.statusCode == "TooManyRequests") { for (int s = 0; s <= response.retryInSeconds; s += 60) @@ -801,11 +919,23 @@ namespace URLNotesGrabberCORE if (response?.response == null) { + if (IsNotFound(response)) + { + Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (empty response payload)"); + DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); + return "NotFound"; + } Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}"); return "FAILURE"; } if (response.response.notes == null) { + if (IsNotFound(response)) + { + Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (notes payload missing)"); + DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); + return "NotFound"; + } Console.WriteLine("##### Notes Null - WHY? ###"); return "FAILURE"; } @@ -831,6 +961,12 @@ namespace URLNotesGrabberCORE // Fetch next page response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult(); + if (IsNotFound(response)) + { + Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}"); + DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); + return "NotFound"; + } } Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}"); @@ -893,15 +1029,14 @@ namespace URLNotesGrabberCORE { DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); } + else if (status == "NotFound") + { + Console.WriteLine("GrabNotes Result: NotFound"); + DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); + } else { Console.WriteLine("GrabNotes Result: " + status); - - //if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue - if (status != "RateLimitExceeded") - { - DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2); - } } // Re-fetch the updated list after processing the current post @@ -916,203 +1051,235 @@ namespace URLNotesGrabberCORE } - static void TraverseDirectory(string path, string outPath, List contains, ref int postsAdded, string blogName = "") + static void TraverseDirectory(string path, string outPath, List contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false) { - // Get all directories in the current directory and sort them alphabetically - var directories = Directory.GetDirectories(path); - Array.Sort(directories, StringComparer.InvariantCulture); - - foreach (var directory in directories) - { - Console.WriteLine("Directory: " + directory); - TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories - } + DateTime directoryStart = DateTime.Now; + int directoryRecordsImported = 0; + Console.WriteLine($"[Directory Start] {path} | {directoryStart:yyyy-MM-dd HH:mm:ss.fff}"); try { - bool headerWasWritten = false; - // Process all files in the current directory - foreach (var file in Directory.GetFiles(path)) - { - if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)) - { - try - { - var urls = new List(); - var reblog = new ReblogRecord(); + // Get all directories in the current directory and sort them alphabetically + var directories = Directory.GetDirectories(path); + Array.Sort(directories, StringComparer.InvariantCulture); - foreach (string line in File.ReadLines(file)) + foreach (var directory in directories) + { + Console.WriteLine("Directory: " + directory); + TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName, startFromBlogName, logRecordImports); // Recursively traverse subdirectories + } + + try + { + bool headerWasWritten = false; + // Process all files in the current directory + foreach (var file in Directory.GetFiles(path)) + { + string normalizedDirectoryName = NormalizeBlogFolderName(new DirectoryInfo(path).Name); + bool isAtOrAfterStart = string.IsNullOrWhiteSpace(startFromBlogName) || string.Compare(normalizedDirectoryName, startFromBlogName, StringComparison.OrdinalIgnoreCase) >= 0; + + if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) + && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0) + && isAtOrAfterStart) + { + try { - if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) + var urls = new List(); + var reblog = new ReblogRecord(); + + foreach (string line in File.ReadLines(file)) { - if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") + if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) { - if (!reblog.reblogURL.Contains("deactivated") - && reblog.reblogURL.Length != 0 - && ContainsAny(reblog.reblogURL, contains)) + if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") + { + if (!reblog.reblogURL.Contains("deactivated") + && reblog.reblogURL.Length != 0 + && ContainsAny(reblog.reblogURL, contains)) + { + DirectoryInfo currentDir = new DirectoryInfo(path); + if (!headerWasWritten) + { + headerWasWritten = true; + } + string curDir = NormalizeBlogFolderName(currentDir.Name); + urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); + + var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew(); + DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, + reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, + reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, + reblog.title, false); + recordImportStopwatch.Stop(); + + postsAdded++; + directoryRecordsImported++; + if (logRecordImports) + Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}"); + + // Output hyperlink and post date + //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}"); + } + } + reblog = new ReblogRecord(); + reblog.postID = line.Substring(9).Trim(); + } + if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase)) + { + //reblog = new ReblogRecord(); + + reblog.reblogURL = line.Substring(12).Trim(); + } + if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase)) + { + reblog.reblogName = line.Substring(13).Trim(); + } + if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) + { + reblog.downloadedFiles = line.Substring(17).Trim(); + } + if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase)) + { + reblog.reblogKey = line.Substring(11).Trim(); + } + if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase)) + { + reblog.date = line.Substring(6).Trim(); + } + if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase)) + { + reblog.body = line.Substring(6).Trim(); + } + if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase)) + { + reblog.postURL = line.Substring(10).Trim(); + } + if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase)) + { + reblog.answer = line.Substring(8).Trim(); + } + if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase)) + { + reblog.audioCaption = line.Substring(15).Trim(); + } + if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase)) + { + reblog.blogName = line.Substring(11).Trim(); + } + if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase)) + { + reblog.link = line.Substring(6).Trim(); + } + if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase)) + { + reblog.photoCaption = line.Substring(15).Trim(); + } + if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase)) + { + reblog.photoURL = line.Substring(11).Trim(); + } + if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase)) + { + reblog.question = line.Substring(10).Trim(); + } + if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase)) + { + reblog.quote = line.Substring(7).Trim(); + } + if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase)) + { + reblog.slug = line.Substring(6).Trim(); + } + if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase)) + { + reblog.summary = line.Substring(9).Trim(); + } + if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase)) + { + reblog.tags = line.Substring(6).Trim(); + } + if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase)) + { + reblog.title = line.Substring(7).Trim(); + } + + if ((reblog.downloadedFiles != "." + || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0) + || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") + { + if ((ContainsAny(reblog.downloadedFiles, contains) + || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0) + || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) + //&& !reblog.reblogURL.Contains("deactivated") + || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase)) { DirectoryInfo currentDir = new DirectoryInfo(path); if (!headerWasWritten) { headerWasWritten = true; } - string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); + //if (reblog.reblogURL.Contains("/blog/private") + // || reblog.body.Contains("/blog/private")) + //{ + // Console.WriteLine(reblog.postID); + // Console.WriteLine(reblog.postURL); + // Console.WriteLine(reblog.date); + // Console.WriteLine(reblog.body); + // Console.WriteLine(reblog.reblogKey); + // Console.WriteLine(reblog.reblogURL); + // Console.WriteLine(reblog.reblogName); + // Console.WriteLine(reblog.downloadedFiles); + //} + string curDir = NormalizeBlogFolderName(currentDir.Name); urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); + + var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew(); DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, - reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, - reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, - reblog.title, false); + reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, + reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, + reblog.title, true); + recordImportStopwatch.Stop(); + postsAdded++; + directoryRecordsImported++; + if (logRecordImports) + Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}"); // Output hyperlink and post date //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}"); } } - reblog = new ReblogRecord(); - reblog.postID = line.Substring(9).Trim(); } - if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase)) - { - //reblog = new ReblogRecord(); + urls.Sort(); - reblog.reblogURL = line.Substring(12).Trim(); - } - if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase)) - { - reblog.reblogName = line.Substring(13).Trim(); - } - if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) - { - reblog.downloadedFiles = line.Substring(17).Trim(); - } - if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase)) - { - reblog.reblogKey = line.Substring(11).Trim(); - } - if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase)) - { - reblog.date = line.Substring(6).Trim(); - } - if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase)) - { - reblog.body = line.Substring(6).Trim(); - } - if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase)) - { - reblog.postURL = line.Substring(10).Trim(); - } - if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase)) - { - reblog.answer = line.Substring(8).Trim(); - } - if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase)) - { - reblog.audioCaption = line.Substring(15).Trim(); - } - if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase)) - { - reblog.blogName = line.Substring(11).Trim(); - } - if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase)) - { - reblog.link = line.Substring(6).Trim(); - } - if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase)) - { - reblog.photoCaption = line.Substring(15).Trim(); - } - if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase)) - { - reblog.photoURL = line.Substring(11).Trim(); - } - if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase)) - { - reblog.question = line.Substring(10).Trim(); - } - if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase)) - { - reblog.quote = line.Substring(7).Trim(); - } - if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase)) - { - reblog.slug = line.Substring(6).Trim(); - } - if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase)) - { - reblog.summary = line.Substring(9).Trim(); - } - if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase)) - { - reblog.tags = line.Substring(6).Trim(); - } - if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase)) - { - reblog.title = line.Substring(7).Trim(); - } - if ((reblog.downloadedFiles != "." - || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0) - || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") - { - if ((ContainsAny(reblog.downloadedFiles, contains) - || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0) - || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) - //&& !reblog.reblogURL.Contains("deactivated") - || path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase)) - { - DirectoryInfo currentDir = new DirectoryInfo(path); - if (!headerWasWritten) - { - headerWasWritten = true; - } - //if (reblog.reblogURL.Contains("/blog/private") - // || reblog.body.Contains("/blog/private")) - //{ - // Console.WriteLine(reblog.postID); - // Console.WriteLine(reblog.postURL); - // Console.WriteLine(reblog.date); - // Console.WriteLine(reblog.body); - // Console.WriteLine(reblog.reblogKey); - // Console.WriteLine(reblog.reblogURL); - // Console.WriteLine(reblog.reblogName); - // Console.WriteLine(reblog.downloadedFiles); - //} - string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", ""); - urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID)); - DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, - reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, - reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, - reblog.title, true); - postsAdded++; - - // Output hyperlink and post date - //Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}"); - } - } + //using (StreamWriter sw = new StreamWriter(outPath, true)) + //{ + // foreach (string line in urls.Distinct()) + // { + // sw.WriteLine(line); + // } + //} + } + catch (Exception e) + { + Console.WriteLine("The file could not be read:"); + Console.WriteLine(e.Message); } - urls.Sort(); - - - //using (StreamWriter sw = new StreamWriter(outPath, true)) - //{ - // foreach (string line in urls.Distinct()) - // { - // sw.WriteLine(line); - // } - //} - } - catch (Exception e) - { - Console.WriteLine("The file could not be read:"); - Console.WriteLine(e.Message); } + // Add your file processing logic here } - // Add your file processing logic here + } + catch (Exception ex) + { + Console.WriteLine($"An error occurred: {ex.Message}"); } } - catch (Exception ex) + finally { - Console.WriteLine($"An error occurred: {ex.Message}"); + DateTime directoryEnd = DateTime.Now; + TimeSpan elapsed = directoryEnd - directoryStart; + Console.WriteLine($"[Directory End] {path} | {directoryEnd:yyyy-MM-dd HH:mm:ss.fff} | Duration: {elapsed:hh\\:mm\\:ss\\.fff} | Records Imported: {directoryRecordsImported}"); } } diff --git a/URLNotesGrabberCORE/Properties/launchSettings.json b/URLNotesGrabberCORE/Properties/launchSettings.json index ed139f1..f489d4e 100644 --- a/URLNotesGrabberCORE/Properties/launchSettings.json +++ b/URLNotesGrabberCORE/Properties/launchSettings.json @@ -2,7 +2,7 @@ "profiles": { "URLNotesGrabberCORE": { "commandName": "Project", - "commandLineArgs": "-likes timothywrite" + "commandLineArgs": "-collect 1 -api4" } } } \ No newline at end of file diff --git a/URLNotesGrabberCORE/appsettings.json b/URLNotesGrabberCORE/appsettings.json index 69358fd..b3f3494 100644 --- a/URLNotesGrabberCORE/appsettings.json +++ b/URLNotesGrabberCORE/appsettings.json @@ -6,9 +6,12 @@ "PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt", "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt", "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt", + "PathOutputUrls": "u:\\jim\\Documents\\Web Copies\\blogs\\GetUrls.txt", "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db", "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,h4rdspot", - "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218" + "PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218", + "EnableFileLogging": false, + "LogTraversalRecordImports": false }, "TumblrApi": { "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",