jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance filtering, SQL, and error handling logic

Browse Source 
- Added `beforeDate` parameter to filter posts by `NotesGatheredDateTime`.
- Improved SQL queries with additional joins and conditions.
- Refactored exception handling for better resource cleanup.
- Enhanced string comparisons to support case-insensitivity.
- Added `GetReplies` method for fetching replies.
- Improved handling of `ReblogRecord` data and filtering logic.
- Updated `-collect` command to support optional date filtering.
- Adjusted `launchSettings.json` for testing with specific parameters.
- Improved logging and error reporting in API and database operations.
This commit is contained in:
jim
2025-12-11 23:19:19 -06:00
parent e55d2b0e29
commit 0e5fb124a0
4 changed files with 150 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+47 -40
View File
@@ -145,7 +145,7 @@ namespace URLNotesGrabberCORE
Title,
HasImage
) VALUES (" +
Q(blogName) + ", " + postID +", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
@@ -156,27 +156,27 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(ex.Message);
///
try
{
connection.Open();
try
{
connection.Open();
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex2)
command.ExecuteNonQuery();
}
catch (Exception ex2)
{
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex2.Message);
}
Console.WriteLine(ex2.Message);
}
finally
{
connection.Close();
}
///
}
finally
{
connection.Close();
}
///
}
}
finally
@@ -188,7 +188,7 @@ namespace URLNotesGrabberCORE
public static void AddAPICount(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
@@ -243,7 +243,7 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
Console.WriteLine("^^^^^ - SHORTCUT");
Console.WriteLine("^^^^^ - SHORTCUT");
}
}
finally
@@ -262,7 +262,7 @@ namespace URLNotesGrabberCORE
/// <param name="withoutNotesOnly"></param>
/// <param name="DBPath"></param>
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
@@ -271,7 +271,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "SELECT " +
string sql = "SELECT " +
" Posts.BlogName, " + Environment.NewLine +
" Posts.PostID, " + Environment.NewLine +
" Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
@@ -279,16 +279,23 @@ namespace URLNotesGrabberCORE
" CNT.CNT " + Environment.NewLine +
"FROM " + Environment.NewLine +
" Posts " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine;
if (withoutNotesOnly)
sql += " and HasNotesGathered = 0 " + Environment.NewLine ;
sql += "GROUP BY " + Environment.NewLine +
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
// Filter by NotesGatheredDateTime if beforeDate is provided
if (beforeDate.HasValue)
{
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
}
sql += "GROUP BY " + Environment.NewLine +
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
"ORDER BY " + Environment.NewLine +
" notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
@@ -296,7 +303,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey();
// Thread.Sleep(1000);
Thread.Sleep(250);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -332,7 +339,7 @@ namespace URLNotesGrabberCORE
}
return posts;
}
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -395,7 +402,7 @@ namespace URLNotesGrabberCORE
{
while (reader.Read())
{
count = reader.GetInt32(0); // Assuming Id is the first column
count = reader.GetInt32(0); // Assuming Id is the first column
}
}
}
@@ -635,10 +642,10 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Posts SET ";
string sql = "UPDATE Posts SET ";
sql += "postDate = @postDate, ";
sql += "reblogURL = @reblogURL, ";
sql += "postURL = @postURL, ";
sql += "postURL = @postURL, ";
sql += "slug = @slug, ";
sql += "reblogKey = @reblogKey, ";
sql += "reblogName = @reblogName, ";
@@ -752,8 +759,8 @@ namespace URLNotesGrabberCORE
}
return APICount;
}
#endregion Updates
}
#endregion Updates
}
internal class APIAccess
@@ -772,13 +779,13 @@ namespace URLNotesGrabberCORE
throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json");
}
private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
throw new InvalidOperationException("ConsumerKey is not configured");
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
throw new InvalidOperationException("ConsumerSecret is not configured");
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
throw new InvalidOperationException("OAuthToken is not configured");
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
throw new InvalidOperationException("OAuthTokenSecret is not configured");
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
@@ -864,8 +871,8 @@ namespace URLNotesGrabberCORE
}
}
}
catch (Exception ex)
{
catch (Exception ex)
{
Console.WriteLine($"Failed JSON: {myJsonResponse}");
Console.WriteLine(ex.ToString());
@@ -939,7 +946,7 @@ namespace URLNotesGrabberCORE
if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs;
}
foundRateLimitHeader = true;
foundRateLimitHeader = true;
}
}
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
URLNotesGrabberCORE/Program.cs
+78 -51
View File
@@ -56,7 +56,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -71,7 +71,7 @@ namespace URLNotesGrabberCORE
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
break;
case "-test":
#region Manual test
@@ -148,7 +148,7 @@ namespace URLNotesGrabberCORE
break;
}
}
Console.WriteLine("Test completed successfully");
#endregion
@@ -168,13 +168,15 @@ namespace URLNotesGrabberCORE
case "-collect": //collect notes from all posts
bool withoutNotesOnly = true;
DateTime? beforeDate = null;
if(args.Length != 2)
if (args.Length < 2)
{
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
break;
}
// Parse withoutNotesOnly flag
if (args.Length > 1 && args[1] is not null)
{
if (args[1] == "1")
@@ -190,7 +192,23 @@ namespace URLNotesGrabberCORE
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
// Parse optional beforeDate parameter
if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
{
if (DateTime.TryParse(args[2], out DateTime parsedDate))
{
beforeDate = parsedDate;
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
}
else
{
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
break;
}
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
break;
case "-blogsR": //collect notes from all posts
@@ -249,7 +267,7 @@ namespace URLNotesGrabberCORE
static void WritePostBlogsToFile(string outPath)
{
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -341,7 +359,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
// If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return
if (response?.meta != null && response.meta.status == 404)
{
@@ -391,10 +409,10 @@ namespace URLNotesGrabberCORE
return "FAILURE";
}
else
{
while (response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
{
while (response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
@@ -416,24 +434,28 @@ namespace URLNotesGrabberCORE
return "Success";
}
catch(Exception ex) {
Console.WriteLine(ex.ToString()); }
catch (Exception ex)
{
Console.WriteLine(ex.ToString());
}
return "UNKNOWN";
}
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
{
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
PermitLimit = 300,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
QueueLimit = 1,
Window = TimeSpan.FromMinutes(1),
SegmentsPerWindow = 60,
AutoReplenishment = true} );
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
{
PermitLimit = 300,
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
QueueLimit = 1,
Window = TimeSpan.FromMinutes(1),
SegmentsPerWindow = 60,
AutoReplenishment = true
});
try
{
@@ -455,7 +477,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return;
}
if(status == "Success")
if (status == "Success")
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
else
Console.WriteLine("GrabNotes Result: " + status);
@@ -471,6 +493,10 @@ namespace URLNotesGrabberCORE
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
{
//if(!ContainsAny(path, contains))
//{
// return;
//}
// Get all directories in the current directory and sort them alphabetically
var directories = Directory.GetDirectories(path);
Array.Sort(directories, StringComparer.InvariantCulture);
@@ -492,7 +518,7 @@ namespace URLNotesGrabberCORE
// Process all files in the current directory
foreach (var file in Directory.GetFiles(path))
{
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
{
//Console.WriteLine($"=====File: {file}");
@@ -508,8 +534,8 @@ namespace URLNotesGrabberCORE
foreach (string line in File.ReadLines(file))
{
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
if (line.StartsWith(@"Post id:"))
//if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase))
{ // if there were no files, let's still collect post info
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
{
@@ -547,94 +573,95 @@ namespace URLNotesGrabberCORE
reblog = new ReblogRecord();
reblog.postID = line.Substring(9).Trim();
}
if (line.StartsWith(@"Reblog url:"))
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
{
//reblog = new ReblogRecord();
reblog.reblogURL = line.Substring(12).Trim();
}
if (line.StartsWith(@"Reblog name:"))
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
{
reblog.reblogName = line.Substring(13).Trim();
}
if (line.StartsWith(@"Downloaded files:"))
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
{
reblog.downloadedFiles = line.Substring(17).Trim();
}
if (line.StartsWith(@"Reblog key:"))
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
{
reblog.reblogKey = line.Substring(11).Trim();
}
if (line.StartsWith(@"Date:"))
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
{
reblog.date = line.Substring(6).Trim();
}
if (line.StartsWith(@"Body:"))
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
{
reblog.body = line.Substring(6).Trim();
}
if (line.StartsWith(@"Post url:"))
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
{
reblog.postURL = line.Substring(10).Trim();
}
if (line.StartsWith(@"Answer:"))
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
{
reblog.answer = line.Substring(8).Trim();
}
if (line.StartsWith(@"Audio Caption:"))
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
{
reblog.audioCaption = line.Substring(15).Trim();
}
if (line.StartsWith(@"Blog Name:"))
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
{
reblog.blogName = line.Substring(11).Trim();
}
if (line.StartsWith(@"Link:"))
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
{
reblog.link = line.Substring(6).Trim();
}
if (line.StartsWith(@"Photo Caption:"))
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
{
reblog.photoCaption = line.Substring(15).Trim();
}
if (line.StartsWith(@"Photo url:"))
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
{
reblog.photoURL = line.Substring(11).Trim();
}
if (line.StartsWith(@"Question:"))
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
{
reblog.question = line.Substring(10).Trim();
}
if (line.StartsWith(@"Quote:"))
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
{
reblog.quote = line.Substring(7).Trim();
}
if (line.StartsWith(@"Slug:"))
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
{
reblog.slug = line.Substring(6).Trim();
}
if (line.StartsWith(@"Summary:"))
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
{
reblog.summary = line.Substring(9).Trim();
}
if (line.StartsWith(@"Tags:"))
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
{
reblog.tags = line.Substring(6).Trim();
}
if (line.StartsWith(@"Title:"))
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
{
reblog.title = line.Substring(7).Trim();
}
if ((reblog.downloadedFiles != "."
|| reblog.reblogURL.Contains("/blog/private")
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
{
if ((ContainsAny(reblog.downloadedFiles, contains)
|| reblog.reblogURL.Contains("/blog/private")
|| reblog.body.Contains("/blog/private"))
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
// && !ContainsAny(reblog.reblogURL, contains)
&& !reblog.reblogURL.Contains(@"deactivated"))
&& !reblog.reblogURL.Contains(@"deactivated")
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
URLNotesGrabberCORE/Properties/launchSettings.json
+1 -1
View File
@@ -2,7 +2,7 @@
"profiles": {
"URLNotesGrabberCORE": {
"commandName": "Project",
"commandLineArgs": "-collect 0"
"commandLineArgs": "-test zomb-eh 802499389473079296"
}
}
}