jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add support for fetching and storing reply text

Browse Source 
- Add database migration and update logic for replyText column in Notes
- Implement batch collection of missing reply text via Tumblr API
- Add new API integration to fetch reply text for replies
- Enhance DataAccess with methods to query/update replyText
- Update CLI: -replies now collects and stores reply text
- Improve logging (archive logs/), error handling, and output
- Remove TL.db from source control
This commit is contained in:
jim
2026-02-12 23:12:54 -06:00
parent 0e5fb124a0
commit bc9985a6f0
5 changed files with 635 additions and 208 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+333 -27
View File
@@ -81,6 +81,57 @@ namespace URLNotesGrabberCORE
return dateTime;
}
public static void EnsureReplyTextColumnExists(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
// Check if replyText column exists
string checkSql = "PRAGMA table_info(Notes);";
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
bool columnExists = false;
while (reader.Read())
{
string columnName = reader.GetString(1);
if (columnName.Equals("replyText", StringComparison.OrdinalIgnoreCase))
{
columnExists = true;
break;
}
}
if (!columnExists)
{
// Add the column if it doesn't exist
connection.Close();
connection.Open();
string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';";
using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection))
{
addCommand.ExecuteNonQuery();
Console.WriteLine("[Migration] Added replyText column to Notes table");
}
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
}
finally
{
connection.Close();
}
}
#region Adds
public static void AddBlog(string blogName, string DBPath = @"TL.db")
{
@@ -93,10 +144,14 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "INSERT OR IGNORE INTO Blogs (BlogName) values('" + blogName + "')";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
// Insert BlogName and DateAdded (current UTC datetime)
string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded) VALUES (@BlogName, @DateAdded)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@BlogName", blogName);
command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
// Console.WriteLine("+ " + blogName);
}
catch (Exception ex)
@@ -226,7 +281,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type)";
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
@@ -234,6 +289,7 @@ namespace URLNotesGrabberCORE
command.Parameters.AddWithValue("@PostID", postID);
command.Parameters.AddWithValue("@TimeStamp", timestamp);
command.Parameters.AddWithValue("@Type", type ?? string.Empty);
command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
}
@@ -272,9 +328,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "SELECT " +
" Posts.BlogName, " + Environment.NewLine +
" MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
" Posts.PostID, " + Environment.NewLine +
" Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
" 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
" Posts.NotesGatheredDatetime, " + Environment.NewLine +
" CNT.CNT " + Environment.NewLine +
"FROM " + Environment.NewLine +
@@ -283,7 +339,7 @@ namespace URLNotesGrabberCORE
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine;
"WHERE NotFound = 0 AND HasImage = 1 AND DownloadedFiles <> '.' " + Environment.NewLine;
if (withoutNotesOnly)
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
@@ -298,12 +354,12 @@ namespace URLNotesGrabberCORE
sql += "GROUP BY " + Environment.NewLine +
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
"ORDER BY " + Environment.NewLine +
" notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
" notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine;
Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey();
Thread.Sleep(250);
//Console.WriteLine(withoutNotesOnly);
//Console.WriteLine(sql);
//Console.Write(">"); //Console.ReadKey();
//Thread.Sleep(250);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -382,6 +438,107 @@ namespace URLNotesGrabberCORE
return posts;
}
public static List<Tuple<string, long>> GetRepliesWithMissingText(string DBPath = @"TL.db", int limit = 50)
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
try
{
connection.Open();
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
MAX(Notes.timestamp) as LatestTimestamp
FROM Notes
WHERE Notes.type = 'reply'
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
GROUP BY Notes.RootBlogName, Notes.PostID
ORDER BY LatestTimestamp ASC
LIMIT @limit";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@limit", limit);
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
Tuple<string, long> post = default;
string blog = null;
long id = 0;
blog = reader.GetString(reader.GetOrdinal("blogName"));
id = reader.GetInt64(reader.GetOrdinal("PostID"));
post = new Tuple<string, long>(blog, id);
posts.Add(post);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string DBPath = @"TL.db", int limit = 1)
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
try
{
connection.Open();
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
MAX(Notes.timestamp) as LatestTimestamp
FROM Notes
WHERE Notes.type = 'reply'
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
GROUP BY Notes.RootBlogName, Notes.PostID
ORDER BY LatestTimestamp ASC
LIMIT @limit";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@limit", limit);
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
Tuple<string, long, long> post = default;
string blog = null;
long id = 0;
long timestamp = 0;
blog = reader.GetString(reader.GetOrdinal("blogName"));
id = reader.GetInt64(reader.GetOrdinal("PostID"));
timestamp = reader.GetInt64(reader.GetOrdinal("LatestTimestamp"));
timestamp++;
post = new Tuple<string, long, long>(blog, id, timestamp);
posts.Add(post);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
}
finally
{
connection.Close();
}
return posts;
}
public static int GetAPICount(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -517,7 +674,8 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
//string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE PostID = @PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
@@ -760,6 +918,87 @@ namespace URLNotesGrabberCORE
return APICount;
}
public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
string sql = "UPDATE Notes SET replyText = @replyText WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
command.Parameters.AddWithValue("@PostID", postID);
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
command.Parameters.AddWithValue("@TimeStamp", timestamp);
int rowsAffected = command.ExecuteNonQuery();
if (rowsAffected == 0)
{
Console.WriteLine($"[UpdateNoteReplyText] WARNING: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)}");
Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
}
else
{
Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}");
}
}
}
catch (Exception ex)
{
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
}
finally
{
connection.Close();
}
}
public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply'";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
command.Parameters.AddWithValue("@PostID", postID);
int rowsAffected = command.ExecuteNonQuery();
if (rowsAffected == 0)
{
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] WARNING: No rows updated for {rootBlogName}/{postID}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
}
else
{
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'");
}
}
}
catch (Exception ex)
{
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
}
finally
{
connection.Close();
}
}
#endregion Updates
}
@@ -784,30 +1023,33 @@ namespace URLNotesGrabberCORE
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
throw new InvalidOperationException("ConsumerSecret is not configured");
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
throw new InvalidOperationException("OAuthToken is not configured");
throw new InvalidOperationException("OAuthToken is not configured");
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
throw new InvalidOperationException("OAuthTokenSecret is not configured");
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
{
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]";
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all";
URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
if (!string.IsNullOrEmpty(timestamp))
{
URL += "&before_timestamp=" + timestamp;
await Task.Delay(100);
}
var client = new RestClient(URL);
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
consumerSecret: ConsumerSecret,
token: OAuthToken,
tokenSecret: OAuthTokenSecret,
OAuthSignatureMethod.HmacSha1
);
// Create a new RestClient for each request to ensure fresh OAuth signatures
using (var client = new RestClient(URL))
{
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
consumerSecret: ConsumerSecret,
token: OAuthToken,
tokenSecret: OAuthTokenSecret,
OAuthSignatureMethod.HmacSha1
);
client.Authenticator = oAuth1;
var request = new RestRequest(URL, Method.Get);
var response = await client.ExecuteAsync(request);
client.Authenticator = oAuth1;
var request = new RestRequest(URL, Method.Get);
var response = await client.ExecuteAsync(request);
var myJsonResponse = response.Content ?? string.Empty;
Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
@@ -962,11 +1204,75 @@ namespace URLNotesGrabberCORE
myDeserializedClass.statusCode = "TooManyRequests";
}
}
}
}
}
}
return myDeserializedClass;
}
}
public static async Task<PostsRoot> GrabPostWithReplies(string blog, long postID, long timestamp)
{
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}&notes_info=true&before_timestamp={timestamp}";
// Create a new RestClient for each request to ensure fresh OAuth signatures
using (var client = new RestClient(URL))
{
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
consumerSecret: ConsumerSecret,
token: OAuthToken,
tokenSecret: OAuthTokenSecret,
OAuthSignatureMethod.HmacSha1
);
client.Authenticator = oAuth1;
var request = new RestRequest(URL, Method.Get);
var response = await client.ExecuteAsync(request);
var myJsonResponse = response.Content ?? string.Empty;
Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
var myDeserializedClass = new PostsRoot();
// Check if response is successful and contains JSON
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
{
string? statusStr = null;
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
Console.WriteLine($"[Reply API] {statusStr}\t{response?.StatusDescription}");
if (!string.IsNullOrEmpty(statusStr))
myDeserializedClass.statusCode = statusStr;
return myDeserializedClass;
}
// Check if response is HTML (error page) instead of JSON
if (myJsonResponse.TrimStart().StartsWith("<"))
{
Console.WriteLine($"[Reply API] Received HTML response (likely error page): {response?.StatusCode}");
string? statusStr = null;
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
if (!string.IsNullOrEmpty(statusStr))
myDeserializedClass.statusCode = statusStr;
return myDeserializedClass;
}
try
{
var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
if (deserializedResult != null)
{
myDeserializedClass = deserializedResult;
myDeserializedClass.rawJson = myJsonResponse;
}
}
catch (Exception ex)
{
Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
}
return myDeserializedClass;
}
}
}
}