Enhance filtering, SQL, and error handling logic
- Added `beforeDate` parameter to filter posts by `NotesGatheredDateTime`. - Improved SQL queries with additional joins and conditions. - Refactored exception handling for better resource cleanup. - Enhanced string comparisons to support case-insensitivity. - Added `GetReplies` method for fetching replies. - Improved handling of `ReblogRecord` data and filtering logic. - Updated `-collect` command to support optional date filtering. - Adjusted `launchSettings.json` for testing with specific parameters. - Improved logging and error reporting in API and database operations.
This commit is contained in:
@@ -145,7 +145,7 @@ namespace URLNotesGrabberCORE
|
||||
Title,
|
||||
HasImage
|
||||
) VALUES (" +
|
||||
Q(blogName) + ", " + postID +", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
|
||||
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
|
||||
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||
|
||||
command.ExecuteNonQuery();
|
||||
@@ -156,27 +156,27 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
Console.WriteLine(ex.Message);
|
||||
///
|
||||
try
|
||||
{
|
||||
connection.Open();
|
||||
try
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
|
||||
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
|
||||
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
||||
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
catch (Exception ex2)
|
||||
command.ExecuteNonQuery();
|
||||
}
|
||||
catch (Exception ex2)
|
||||
{
|
||||
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
||||
{
|
||||
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
||||
{
|
||||
Console.WriteLine(ex2.Message);
|
||||
}
|
||||
Console.WriteLine(ex2.Message);
|
||||
}
|
||||
finally
|
||||
{
|
||||
connection.Close();
|
||||
}
|
||||
///
|
||||
}
|
||||
finally
|
||||
{
|
||||
connection.Close();
|
||||
}
|
||||
///
|
||||
}
|
||||
}
|
||||
finally
|
||||
@@ -188,7 +188,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
public static void AddAPICount(string DBPath = @"TL.db")
|
||||
{
|
||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
|
||||
try
|
||||
{
|
||||
@@ -243,7 +243,7 @@ namespace URLNotesGrabberCORE
|
||||
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
|
||||
{
|
||||
Console.WriteLine(ex.Message);
|
||||
Console.WriteLine("^^^^^ - SHORTCUT");
|
||||
Console.WriteLine("^^^^^ - SHORTCUT");
|
||||
}
|
||||
}
|
||||
finally
|
||||
@@ -262,7 +262,7 @@ namespace URLNotesGrabberCORE
|
||||
/// <param name="withoutNotesOnly"></param>
|
||||
/// <param name="DBPath"></param>
|
||||
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
|
||||
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
|
||||
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db")
|
||||
{
|
||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
|
||||
@@ -271,7 +271,7 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
string sql = "SELECT " +
|
||||
string sql = "SELECT " +
|
||||
" Posts.BlogName, " + Environment.NewLine +
|
||||
" Posts.PostID, " + Environment.NewLine +
|
||||
" Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
|
||||
@@ -279,16 +279,23 @@ namespace URLNotesGrabberCORE
|
||||
" CNT.CNT " + Environment.NewLine +
|
||||
"FROM " + Environment.NewLine +
|
||||
" Posts " + Environment.NewLine +
|
||||
" LEFT OUTER JOIN " + Environment.NewLine +
|
||||
" LEFT OUTER JOIN " + Environment.NewLine +
|
||||
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
|
||||
" LEFT OUTER JOIN " + Environment.NewLine +
|
||||
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
|
||||
"WHERE NotFound = 0 " + Environment.NewLine;
|
||||
|
||||
if (withoutNotesOnly)
|
||||
sql += " and HasNotesGathered = 0 " + Environment.NewLine ;
|
||||
|
||||
sql += "GROUP BY " + Environment.NewLine +
|
||||
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
|
||||
|
||||
// Filter by NotesGatheredDateTime if beforeDate is provided
|
||||
if (beforeDate.HasValue)
|
||||
{
|
||||
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
||||
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
|
||||
}
|
||||
|
||||
sql += "GROUP BY " + Environment.NewLine +
|
||||
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
|
||||
"ORDER BY " + Environment.NewLine +
|
||||
" notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
|
||||
@@ -296,7 +303,7 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine(withoutNotesOnly);
|
||||
Console.WriteLine(sql);
|
||||
Console.Write(">"); //Console.ReadKey();
|
||||
// Thread.Sleep(1000);
|
||||
Thread.Sleep(250);
|
||||
|
||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||
{
|
||||
@@ -332,7 +339,7 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
return posts;
|
||||
}
|
||||
|
||||
|
||||
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
|
||||
{
|
||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
@@ -395,7 +402,7 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
while (reader.Read())
|
||||
{
|
||||
count = reader.GetInt32(0); // Assuming Id is the first column
|
||||
count = reader.GetInt32(0); // Assuming Id is the first column
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -635,10 +642,10 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
connection.Open();
|
||||
|
||||
string sql = "UPDATE Posts SET ";
|
||||
string sql = "UPDATE Posts SET ";
|
||||
sql += "postDate = @postDate, ";
|
||||
sql += "reblogURL = @reblogURL, ";
|
||||
sql += "postURL = @postURL, ";
|
||||
sql += "postURL = @postURL, ";
|
||||
sql += "slug = @slug, ";
|
||||
sql += "reblogKey = @reblogKey, ";
|
||||
sql += "reblogName = @reblogName, ";
|
||||
@@ -752,8 +759,8 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
|
||||
return APICount;
|
||||
}
|
||||
#endregion Updates
|
||||
}
|
||||
#endregion Updates
|
||||
}
|
||||
|
||||
internal class APIAccess
|
||||
@@ -772,13 +779,13 @@ namespace URLNotesGrabberCORE
|
||||
throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json");
|
||||
}
|
||||
|
||||
private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
|
||||
private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
|
||||
throw new InvalidOperationException("ConsumerKey is not configured");
|
||||
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
|
||||
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
|
||||
throw new InvalidOperationException("ConsumerSecret is not configured");
|
||||
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
|
||||
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
|
||||
throw new InvalidOperationException("OAuthToken is not configured");
|
||||
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
|
||||
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
|
||||
throw new InvalidOperationException("OAuthTokenSecret is not configured");
|
||||
|
||||
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
|
||||
@@ -864,8 +871,8 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Failed JSON: {myJsonResponse}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
|
||||
@@ -939,7 +946,7 @@ namespace URLNotesGrabberCORE
|
||||
if (myDeserializedClass.retryInSeconds < secs)
|
||||
myDeserializedClass.retryInSeconds = secs;
|
||||
}
|
||||
foundRateLimitHeader = true;
|
||||
foundRateLimitHeader = true;
|
||||
}
|
||||
}
|
||||
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
||||
|
||||
Reference in New Issue
Block a user