jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote master into local master

Browse Source
This commit is contained in:
jim
2026-07-22 12:12:33 -05:00
parent 3e2b287737 16147b273e
commit f9e1d2100b
3 changed files with 110 additions and 157 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+1
View File
@@ -11,6 +11,7 @@
- `ResponseNotes.cs`: Tumblr API response models - `ResponseNotes.cs`: Tumblr API response models
- Round-robin API key rotation with rate-limit tracking - Round-robin API key rotation with rate-limit tracking
- Automatic console color assignment per API key for output differentiation - Automatic console color assignment per API key for output differentiation
- No-argument mode (`Program.TraverseDirectory`) ingests `.txt` blog export files into `Posts` via `DataAccess.AddPost`. Recognized field prefixes live in `TraverseDirectoryFieldPrefixes`; `Body:` and `Downloaded files:` collect every following line up to the next recognized prefix (multi-line values). `RootURL` is populated from a `Reblog root url:` line the same way it's populated from the API-based `--likes` flow — both paths converge on `DataAccess.AddPost`'s `rootURL` parameter, which `UpdatePost` only overwrites when the incoming value is non-empty (existing `RootURL` is preserved otherwise)
## Developer Guidelines ## Developer Guidelines
URLNotesGrabberCORE/DataAccess.cs
+35 -134
View File
@@ -37,6 +37,7 @@ namespace URLNotesGrabberCORE
public string reblogKey; public string reblogKey;
public string reblogName; public string reblogName;
public string reblogURL; public string reblogURL;
public string rootURL;
public string slug; public string slug;
public string summary; public string summary;
public string tags; public string tags;
@@ -60,6 +61,7 @@ namespace URLNotesGrabberCORE
reblogKey = "."; reblogKey = ".";
reblogName = "."; reblogName = ".";
reblogURL = "."; reblogURL = ".";
rootURL = ".";
slug = "."; slug = ".";
summary = "."; summary = ".";
tags = "."; tags = ".";
@@ -139,7 +141,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas != null) if (_savedImportPragmas != null)
return; return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -175,10 +177,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}"); Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
} }
@@ -190,7 +188,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas == null) if (_savedImportPragmas == null)
return; return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -217,7 +215,6 @@ namespace URLNotesGrabberCORE
finally finally
{ {
_savedImportPragmas = null; _savedImportPragmas = null;
connection.Close();
} }
} }
} }
@@ -252,7 +249,7 @@ namespace URLNotesGrabberCORE
public static void EnsureReplyTextColumnExists(string? DBPath = null) public static void EnsureReplyTextColumnExists(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -296,16 +293,12 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}"); Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null) public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -391,10 +384,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}"); Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
#region Adds #region Adds
@@ -518,8 +507,12 @@ namespace URLNotesGrabberCORE
{ {
if (ownsConnection) connection.Open(); if (ownsConnection) connection.Open();
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateCommand.Parameters.AddWithValue("@blogName", blogName);
updateCommand.Parameters.AddWithValue("@postID", postID);
updateCommand.ExecuteNonQuery(); updateCommand.ExecuteNonQuery();
} }
@@ -565,7 +558,7 @@ namespace URLNotesGrabberCORE
public static void AddAPICount(string? DBPath = null) public static void AddAPICount(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -586,10 +579,6 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
//Console.WriteLine(ex.Message); //Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null) public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
@@ -598,7 +587,7 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { } //try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { }
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -657,10 +646,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine("^^^^^ - SHORTCUT"); Console.WriteLine("^^^^^ - SHORTCUT");
} }
} }
finally
{
connection2.Close();
}
return false; return false;
} }
#endregion Adds #endregion Adds
@@ -676,7 +661,7 @@ namespace URLNotesGrabberCORE
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null) public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>(); List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
try try
@@ -854,17 +839,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static List<Tuple<string, long>> GetReplies(string? DBPath = null) public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>(); List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try try
@@ -898,17 +879,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50) public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>(); List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try try
@@ -949,17 +926,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
Console.WriteLine($"Error getting replies with missing text: {ex.Message}"); Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null) public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>(); List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
try try
@@ -1011,17 +984,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
Console.WriteLine($"Error getting replies with filled text: {ex.Message}"); Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static int GetAPICount(string? DBPath = null) public static int GetAPICount(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int count = 0; int count = 0;
try { AddAPICount(); } catch { } try { AddAPICount(); } catch { }
@@ -1050,17 +1019,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return count; return count;
} }
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null) public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>(); List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
try try
@@ -1136,17 +1101,13 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error fetching blogs for likes: {ex.Message}"); Console.WriteLine($"Error fetching blogs for likes: {ex.Message}");
} }
finally
{
connection.Close();
}
return blogs; return blogs;
} }
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null) public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>(); List<string> blogs = new List<string>();
try try
@@ -1181,17 +1142,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return blogs; return blogs;
} }
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null) public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>(); List<string> blogs = new List<string>();
try try
@@ -1226,19 +1183,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return blogs; return blogs;
} }
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null) public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open(); connection.Open();
try
{
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection)) using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
@@ -1266,12 +1217,6 @@ namespace URLNotesGrabberCORE
} }
} }
} }
finally
{
connection.Close();
connection.Dispose();
}
}
#endregion Gets #endregion Gets
#region Updates #region Updates
@@ -1280,7 +1225,7 @@ namespace URLNotesGrabberCORE
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null) public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1303,16 +1248,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null) public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1336,10 +1277,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run ----- // ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
@@ -1451,7 +1388,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1478,10 +1415,6 @@ namespace URLNotesGrabberCORE
return true; return true;
} }
} }
finally
{
connection.Close();
}
return false; return false;
} }
@@ -1597,7 +1530,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogOutput(string blogName, string? DBPath = null) public static void UpdateBlogOutput(string blogName, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1617,16 +1550,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null) public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -1644,10 +1573,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error updating blog likes status: {ex.Message}"); Console.WriteLine($"Error updating blog likes status: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we // Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
@@ -1657,7 +1582,7 @@ namespace URLNotesGrabberCORE
{ {
if (newestTimestamp <= 0) return; if (newestTimestamp <= 0) return;
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -1677,10 +1602,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}"); Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the // Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
@@ -1688,7 +1609,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null) public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -1711,16 +1632,12 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}"); Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
public static int UpdateAPICount(string? DBPath = null) public static int UpdateAPICount(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int APICount = DataAccess.GetAPICount(); int APICount = DataAccess.GetAPICount();
APICount++; APICount++;
@@ -1743,10 +1660,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return APICount; return APICount;
} }
@@ -1754,7 +1667,7 @@ namespace URLNotesGrabberCORE
public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null) public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int rowsAffected = 0; int rowsAffected = 0;
// A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel. // A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
@@ -1797,17 +1710,13 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}"); Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}"); Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
} }
finally
{
connection.Close();
}
return rowsAffected; return rowsAffected;
} }
public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null) public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1843,10 +1752,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}"); Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
return 0; return 0;
} }
finally
{
connection.Close();
}
} }
#endregion Updates #endregion Updates
@@ -1857,7 +1762,7 @@ namespace URLNotesGrabberCORE
public static void EnsureTTFileHelperColumnsExist(string? DBPath = null) public static void EnsureTTFileHelperColumnsExist(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1903,10 +1808,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}"); Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export. // INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
@@ -2731,7 +2632,7 @@ namespace URLNotesGrabberCORE
{ {
minRetrySeconds = 0; minRetrySeconds = 0;
if (!_usePool || _overrideKey != null) return false; if (!_usePool || _overrideKey != null) return false;
if (_keys.Count <= 1) return true; if (_keys.Count == 0) return false;
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
bool all = true; bool all = true;
URLNotesGrabberCORE/Program.cs
+60 -9
View File
@@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration;
using System.Configuration; using System.Configuration;
using System.Threading; using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency; using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
namespace URLNotesGrabberCORE namespace URLNotesGrabberCORE
@@ -140,7 +139,10 @@ namespace URLNotesGrabberCORE
Console.SetOut(dualLogger); Console.SetOut(dualLogger);
} }
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList(); string? containsListSetting = settings.GetValue<string>("ContainsList");
if (string.IsNullOrEmpty(containsListSetting))
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
List<string> contains = containsListSetting.Split(',').ToList();
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false); bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
@@ -175,6 +177,12 @@ namespace URLNotesGrabberCORE
break; break;
case "--parse": case "--parse":
if (args.Length < 2)
{
Console.WriteLine("Usage: --parse <blogname>");
exitCode = 2;
break;
}
string blogNameToParse = args[1]; string blogNameToParse = args[1];
int postsAdded = 0; int postsAdded = 0;
try try
@@ -283,7 +291,7 @@ namespace URLNotesGrabberCORE
case "--blogsO": //collect notes from all posts case "--blogsO": //collect notes from all posts
int from = 1, to = 999999, top = 100; int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null) if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{ {
from = int.Parse(args[1]); from = int.Parse(args[1]);
to = int.Parse(args[2]); to = int.Parse(args[2]);
@@ -293,6 +301,7 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine("--Expected FROM TO--"); Console.WriteLine("--Expected FROM TO--");
exitCode = 2; exitCode = 2;
break;
} }
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top); WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break; break;
@@ -300,7 +309,7 @@ namespace URLNotesGrabberCORE
case "--bop": //collect notes from all posts case "--bop": //collect notes from all posts
from = 1; to = 999999; top = 100; from = 1; to = 999999; top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null) if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{ {
from = int.Parse(args[1]); from = int.Parse(args[1]);
to = int.Parse(args[2]); to = int.Parse(args[2]);
@@ -310,6 +319,7 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine("--Expected FROM TO--"); Console.WriteLine("--Expected FROM TO--");
exitCode = 2; exitCode = 2;
break;
} }
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top); WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break; break;
@@ -1412,6 +1422,25 @@ if (shouldInsert)
} }
// Field prefixes TraverseDirectory recognizes as the start of a new record field.
// Used to know where a multi-line Body/Downloaded files value ends.
private static readonly string[] TraverseDirectoryFieldPrefixes = new[]
{
"Post id:", "Reblog url:", "Reblog name:", "Reblog root url:", "Downloaded files:",
"Reblog key:", "Date:", "Body:", "Post url:", "Answer:", "Audio Caption:", "Blog Name:",
"Link:", "Photo Caption:", "Photo url:", "Question:", "Quote:", "Slug:", "Summary:",
"Tags:", "Title:"
};
private static bool IsTraverseDirectoryFieldLine(string line)
{
foreach (var prefix in TraverseDirectoryFieldPrefixes)
{
if (line.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) return true;
}
return false;
}
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false) static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
{ {
DateTime directoryStart = DateTime.Now; DateTime directoryStart = DateTime.Now;
@@ -1448,8 +1477,10 @@ if (shouldInsert)
var urls = new List<string>(); var urls = new List<string>();
var reblog = new ReblogRecord(); var reblog = new ReblogRecord();
foreach (string line in File.ReadLines(file)) string[] fileLines = File.ReadAllLines(file);
for (int lineIndex = 0; lineIndex < fileLines.Length; lineIndex++)
{ {
string line = fileLines[lineIndex];
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
{ {
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
@@ -1470,7 +1501,7 @@ if (shouldInsert)
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, false); reblog.title, false, rootURL: reblog.rootURL);
recordImportStopwatch.Stop(); recordImportStopwatch.Stop();
postsAdded++; postsAdded++;
@@ -1495,9 +1526,21 @@ if (shouldInsert)
{ {
reblog.reblogName = line.Substring(13).Trim(); reblog.reblogName = line.Substring(13).Trim();
} }
if (line.StartsWith(@"Reblog root url:", StringComparison.OrdinalIgnoreCase))
{
reblog.rootURL = line.Substring(16).Trim();
}
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.downloadedFiles = line.Substring(17).Trim(); var valueLines = new List<string> { line.Substring(17).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.downloadedFiles = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
} }
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
{ {
@@ -1509,7 +1552,15 @@ if (shouldInsert)
} }
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.body = line.Substring(6).Trim(); var valueLines = new List<string> { line.Substring(6).Trim() };
int nextLineIndex = lineIndex + 1;
while (nextLineIndex < fileLines.Length && !IsTraverseDirectoryFieldLine(fileLines[nextLineIndex]))
{
valueLines.Add(fileLines[nextLineIndex]);
nextLineIndex++;
}
reblog.body = string.Join("\n", valueLines).Trim();
lineIndex = nextLineIndex - 1;
} }
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase)) if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
{ {
@@ -1598,7 +1649,7 @@ if (shouldInsert)
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey, DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL, reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer, reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, true); reblog.title, true, rootURL: reblog.rootURL);
recordImportStopwatch.Stop(); recordImportStopwatch.Stop();
postsAdded++; postsAdded++;