jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote master into txt-validation branch

Browse Source
This commit is contained in:
jim
2026-07-16 10:42:28 -05:00
parent 21a5525094 f541ec4260
commit 5361bb78b8
2 changed files with 61 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+47 -148
View File
@@ -141,7 +141,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas != null) if (_savedImportPragmas != null)
return; return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -177,10 +177,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}"); Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
} }
@@ -192,7 +188,7 @@ namespace URLNotesGrabberCORE
if (_savedImportPragmas == null) if (_savedImportPragmas == null)
return; return;
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -219,7 +215,6 @@ namespace URLNotesGrabberCORE
finally finally
{ {
_savedImportPragmas = null; _savedImportPragmas = null;
connection.Close();
} }
} }
} }
@@ -254,7 +249,7 @@ namespace URLNotesGrabberCORE
public static void EnsureReplyTextColumnExists(string? DBPath = null) public static void EnsureReplyTextColumnExists(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -298,16 +293,12 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}"); Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null) public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -393,10 +384,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}"); Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
#region Adds #region Adds
@@ -520,8 +507,12 @@ namespace URLNotesGrabberCORE
{ {
if (ownsConnection) connection.Open(); if (ownsConnection) connection.Open();
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection); using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateCommand.Parameters.AddWithValue("@blogName", blogName);
updateCommand.Parameters.AddWithValue("@postID", postID);
updateCommand.ExecuteNonQuery(); updateCommand.ExecuteNonQuery();
} }
@@ -567,7 +558,7 @@ namespace URLNotesGrabberCORE
public static void AddAPICount(string? DBPath = null) public static void AddAPICount(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -588,10 +579,6 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
//Console.WriteLine(ex.Message); //Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null) public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
@@ -600,7 +587,7 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { } //try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, false, DBPath); } catch { } try { AddBlog(noteBlogName, false, DBPath); } catch { }
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -659,10 +646,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine("^^^^^ - SHORTCUT"); Console.WriteLine("^^^^^ - SHORTCUT");
} }
} }
finally
{
connection2.Close();
}
return false; return false;
} }
#endregion Adds #endregion Adds
@@ -678,7 +661,7 @@ namespace URLNotesGrabberCORE
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null) public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>(); List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
try try
@@ -856,17 +839,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static List<Tuple<string, long>> GetReplies(string? DBPath = null) public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>(); List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try try
@@ -900,17 +879,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50) public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>(); List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try try
@@ -951,17 +926,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
Console.WriteLine($"Error getting replies with missing text: {ex.Message}"); Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null) public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>(); List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
try try
@@ -1013,17 +984,13 @@ namespace URLNotesGrabberCORE
// Breakpoint here // Breakpoint here
Console.WriteLine($"Error getting replies with filled text: {ex.Message}"); Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
} }
finally
{
connection.Close();
}
return posts; return posts;
} }
public static int GetAPICount(string? DBPath = null) public static int GetAPICount(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int count = 0; int count = 0;
try { AddAPICount(); } catch { } try { AddAPICount(); } catch { }
@@ -1052,17 +1019,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return count; return count;
} }
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null) public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>(); List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
try try
@@ -1138,17 +1101,13 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error fetching blogs for likes: {ex.Message}"); Console.WriteLine($"Error fetching blogs for likes: {ex.Message}");
} }
finally
{
connection.Close();
}
return blogs; return blogs;
} }
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null) public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>(); List<string> blogs = new List<string>();
try try
@@ -1183,17 +1142,13 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return blogs; return blogs;
} }
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null) public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>(); List<string> blogs = new List<string>();
try try
@@ -1228,51 +1183,39 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return blogs; return blogs;
} }
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null) public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
connection.Open(); connection.Open();
try string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'"; using (SQLiteDataReader reader = command.ExecuteReader())
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
using (SQLiteDataReader reader = command.ExecuteReader()) while (reader.Read())
{ {
while (reader.Read()) List<string> rowTexts = new List<string>();
for (int i = 0; i < reader.FieldCount; i++)
{ {
List<string> rowTexts = new List<string>(); if (!reader.IsDBNull(i))
for (int i = 0; i < reader.FieldCount; i++)
{ {
if (!reader.IsDBNull(i)) var val = reader.GetValue(i);
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
{ {
var val = reader.GetValue(i); rowTexts.Add(str);
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
{
rowTexts.Add(str);
}
} }
} }
if (rowTexts.Count > 0) }
{ if (rowTexts.Count > 0)
yield return rowTexts; {
} yield return rowTexts;
} }
} }
} }
} }
finally
{
connection.Close();
connection.Dispose();
}
} }
#endregion Gets #endregion Gets
@@ -1282,7 +1225,7 @@ namespace URLNotesGrabberCORE
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null) public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1305,16 +1248,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null) public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1338,10 +1277,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run ----- // ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
@@ -1453,7 +1388,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type); Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1480,10 +1415,6 @@ namespace URLNotesGrabberCORE
return true; return true;
} }
} }
finally
{
connection.Close();
}
return false; return false;
} }
@@ -1599,7 +1530,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogOutput(string blogName, string? DBPath = null) public static void UpdateBlogOutput(string blogName, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1619,16 +1550,12 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
} }
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null) public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -1646,10 +1573,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error updating blog likes status: {ex.Message}"); Console.WriteLine($"Error updating blog likes status: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we // Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
@@ -1659,7 +1582,7 @@ namespace URLNotesGrabberCORE
{ {
if (newestTimestamp <= 0) return; if (newestTimestamp <= 0) return;
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -1679,10 +1602,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}"); Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the // Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
@@ -1690,7 +1609,7 @@ namespace URLNotesGrabberCORE
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null) public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
connection.Open(); connection.Open();
@@ -1713,16 +1632,12 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}"); Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
public static int UpdateAPICount(string? DBPath = null) public static int UpdateAPICount(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int APICount = DataAccess.GetAPICount(); int APICount = DataAccess.GetAPICount();
APICount++; APICount++;
@@ -1745,10 +1660,6 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
} }
finally
{
connection.Close();
}
return APICount; return APICount;
} }
@@ -1756,7 +1667,7 @@ namespace URLNotesGrabberCORE
public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null) public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
int rowsAffected = 0; int rowsAffected = 0;
// A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel. // A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
@@ -1799,17 +1710,13 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}"); Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}"); Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
} }
finally
{
connection.Close();
}
return rowsAffected; return rowsAffected;
} }
public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null) public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1845,10 +1752,6 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}"); Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
return 0; return 0;
} }
finally
{
connection.Close();
}
} }
#endregion Updates #endregion Updates
@@ -1859,7 +1762,7 @@ namespace URLNotesGrabberCORE
public static void EnsureTTFileHelperColumnsExist(string? DBPath = null) public static void EnsureTTFileHelperColumnsExist(string? DBPath = null)
{ {
DBPath ??= GetDefaultDbPath(); DBPath ??= GetDefaultDbPath();
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -1905,10 +1808,6 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}"); Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}");
} }
finally
{
connection.Close();
}
} }
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export. // INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
@@ -2729,7 +2628,7 @@ namespace URLNotesGrabberCORE
{ {
minRetrySeconds = 0; minRetrySeconds = 0;
if (!_usePool || _overrideKey != null) return false; if (!_usePool || _overrideKey != null) return false;
if (_keys.Count <= 1) return true; if (_keys.Count == 0) return false;
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds(); var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
bool all = true; bool all = true;
URLNotesGrabberCORE/Program.cs
+14 -4
View File
@@ -5,7 +5,6 @@ using Microsoft.Extensions.Configuration;
using System.Configuration; using System.Configuration;
using System.Threading; using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency; using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
namespace URLNotesGrabberCORE namespace URLNotesGrabberCORE
@@ -140,7 +139,10 @@ namespace URLNotesGrabberCORE
Console.SetOut(dualLogger); Console.SetOut(dualLogger);
} }
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList(); string? containsListSetting = settings.GetValue<string>("ContainsList");
if (string.IsNullOrEmpty(containsListSetting))
throw new InvalidOperationException("ContainsList is not configured in appsettings.json");
List<string> contains = containsListSetting.Split(',').ToList();
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false); bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
@@ -175,6 +177,12 @@ namespace URLNotesGrabberCORE
break; break;
case "--parse": case "--parse":
if (args.Length < 2)
{
Console.WriteLine("Usage: --parse <blogname>");
exitCode = 2;
break;
}
string blogNameToParse = args[1]; string blogNameToParse = args[1];
int postsAdded = 0; int postsAdded = 0;
try try
@@ -283,7 +291,7 @@ namespace URLNotesGrabberCORE
case "--blogsO": //collect notes from all posts case "--blogsO": //collect notes from all posts
int from = 1, to = 999999, top = 100; int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null) if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{ {
from = int.Parse(args[1]); from = int.Parse(args[1]);
to = int.Parse(args[2]); to = int.Parse(args[2]);
@@ -293,6 +301,7 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine("--Expected FROM TO--"); Console.WriteLine("--Expected FROM TO--");
exitCode = 2; exitCode = 2;
break;
} }
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top); WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break; break;
@@ -300,7 +309,7 @@ namespace URLNotesGrabberCORE
case "--bop": //collect notes from all posts case "--bop": //collect notes from all posts
from = 1; to = 999999; top = 100; from = 1; to = 999999; top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null) if (args.Length >= 4 && args[1] is not null && args[2] is not null && args[3] is not null)
{ {
from = int.Parse(args[1]); from = int.Parse(args[1]);
to = int.Parse(args[2]); to = int.Parse(args[2]);
@@ -310,6 +319,7 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine("--Expected FROM TO--"); Console.WriteLine("--Expected FROM TO--");
exitCode = 2; exitCode = 2;
break;
} }
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top); WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break; break;