Enhance filtering, SQL, and error handling logic
- Added `beforeDate` parameter to filter posts by `NotesGatheredDateTime`. - Improved SQL queries with additional joins and conditions. - Refactored exception handling for better resource cleanup. - Enhanced string comparisons to support case-insensitivity. - Added `GetReplies` method for fetching replies. - Improved handling of `ReblogRecord` data and filtering logic. - Updated `-collect` command to support optional date filtering. - Adjusted `launchSettings.json` for testing with specific parameters. - Improved logging and error reporting in API and database operations.
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="1" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Blogs" custom_title="0" dock_id="1" table="4,5:mainBlogs"/><dock_state state="000000ff00000000fd0000000100000002000005f40000031dfc0100000001fb000000160064006f0063006b00420072006f00770073006500310100000000000005f4000000fb00ffffff000002130000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="0" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="48"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1*">SELECT ␍
|
||||
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="" readonly="1" foreign_keys="" case_sensitive_like="" temp_store="" wal_autocheckpoint="" synchronous=""/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/></tab_structure><tab_browse><table title="." custom_title="0" dock_id="4" table="0,0:"/><dock_state state="000000ff00000000fd0000000100000002000005f40000030ffc0100000002fb000000160064006f0063006b00420072006f00770073006500310100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011700ffffff000005f40000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings/></tab_browse><tab_sql><sql name="SQL 1">SELECT
|
||||
BlogName || '.tumblr.com/post/' || postID,
|
||||
|
||||
datetime(NotesGatheredDateTime, 'unixepoch'), *
|
||||
@@ -9,11 +9,16 @@ WHERE
|
||||
ORDER BY
|
||||
postdate desc</sql><sql name="SQL 2*">SELECT
|
||||
datetime(TimeStamp, 'unixepoch'),
|
||||
RootBlogName || '.tumblr.com/post/' || postid,␍
|
||||
RootBlogName || '.tumblr.com/post/' || N.postid,
|
||||
*,
|
||||
NoteBlogName || '.tumblr.com'
|
||||
FROM
|
||||
Notes␍
|
||||
WHERE RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin')␍
|
||||
Notes N␍
|
||||
inner JOIN␍
|
||||
Posts P on P.PostID = N.PostID and P.BlogName = N.RootBlogName
|
||||
WHERE RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')␍
|
||||
and type like 'r%'␍
|
||||
and RootBlogName = 'zomb-eh'␍
|
||||
and P.HasImage = 1
|
||||
ORDER BY
|
||||
TimeStamp desc</sql><current_tab id="0"/></tab_sql></sqlb_project>
|
||||
TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project>
|
||||
|
||||
@@ -262,7 +262,7 @@ namespace URLNotesGrabberCORE
|
||||
/// <param name="withoutNotesOnly"></param>
|
||||
/// <param name="DBPath"></param>
|
||||
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
|
||||
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
|
||||
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db")
|
||||
{
|
||||
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
||||
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
|
||||
@@ -288,6 +288,13 @@ namespace URLNotesGrabberCORE
|
||||
if (withoutNotesOnly)
|
||||
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
|
||||
|
||||
// Filter by NotesGatheredDateTime if beforeDate is provided
|
||||
if (beforeDate.HasValue)
|
||||
{
|
||||
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
||||
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
|
||||
}
|
||||
|
||||
sql += "GROUP BY " + Environment.NewLine +
|
||||
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
|
||||
"ORDER BY " + Environment.NewLine +
|
||||
@@ -296,7 +303,7 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine(withoutNotesOnly);
|
||||
Console.WriteLine(sql);
|
||||
Console.Write(">"); //Console.ReadKey();
|
||||
// Thread.Sleep(1000);
|
||||
Thread.Sleep(250);
|
||||
|
||||
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
||||
{
|
||||
|
||||
@@ -56,7 +56,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
|
||||
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
|
||||
|
||||
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
||||
|
||||
@@ -168,13 +168,15 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
case "-collect": //collect notes from all posts
|
||||
bool withoutNotesOnly = true;
|
||||
DateTime? beforeDate = null;
|
||||
|
||||
if(args.Length != 2)
|
||||
if (args.Length < 2)
|
||||
{
|
||||
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
|
||||
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
|
||||
break;
|
||||
}
|
||||
|
||||
// Parse withoutNotesOnly flag
|
||||
if (args.Length > 1 && args[1] is not null)
|
||||
{
|
||||
if (args[1] == "1")
|
||||
@@ -190,7 +192,23 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
|
||||
}
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
|
||||
|
||||
// Parse optional beforeDate parameter
|
||||
if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
|
||||
{
|
||||
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
||||
{
|
||||
beforeDate = parsedDate;
|
||||
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
|
||||
break;
|
||||
|
||||
case "-blogsR": //collect notes from all posts
|
||||
@@ -416,24 +434,28 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
return "Success";
|
||||
}
|
||||
catch(Exception ex) {
|
||||
Console.WriteLine(ex.ToString()); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
|
||||
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
|
||||
static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
|
||||
{
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true} );
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
@@ -471,6 +493,10 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
|
||||
{
|
||||
//if(!ContainsAny(path, contains))
|
||||
//{
|
||||
// return;
|
||||
//}
|
||||
// Get all directories in the current directory and sort them alphabetically
|
||||
var directories = Directory.GetDirectories(path);
|
||||
Array.Sort(directories, StringComparer.InvariantCulture);
|
||||
@@ -492,7 +518,7 @@ namespace URLNotesGrabberCORE
|
||||
// Process all files in the current directory
|
||||
foreach (var file in Directory.GetFiles(path))
|
||||
{
|
||||
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
|
||||
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
{
|
||||
//Console.WriteLine($"=====File: {file}");
|
||||
|
||||
@@ -508,8 +534,8 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
foreach (string line in File.ReadLines(file))
|
||||
{
|
||||
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
||||
if (line.StartsWith(@"Post id:"))
|
||||
//if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
||||
if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase))
|
||||
{ // if there were no files, let's still collect post info
|
||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
||||
{
|
||||
@@ -547,94 +573,95 @@ namespace URLNotesGrabberCORE
|
||||
reblog = new ReblogRecord();
|
||||
reblog.postID = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog url:"))
|
||||
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
//reblog = new ReblogRecord();
|
||||
|
||||
reblog.reblogURL = line.Substring(12).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog name:"))
|
||||
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogName = line.Substring(13).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Downloaded files:"))
|
||||
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.downloadedFiles = line.Substring(17).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog key:"))
|
||||
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogKey = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Date:"))
|
||||
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.date = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Body:"))
|
||||
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.body = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Post url:"))
|
||||
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.postURL = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Answer:"))
|
||||
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.answer = line.Substring(8).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Audio Caption:"))
|
||||
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.audioCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Blog Name:"))
|
||||
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.blogName = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Link:"))
|
||||
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.link = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo Caption:"))
|
||||
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo url:"))
|
||||
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoURL = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Question:"))
|
||||
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.question = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Quote:"))
|
||||
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.quote = line.Substring(7).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Slug:"))
|
||||
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.slug = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Summary:"))
|
||||
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.summary = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Tags:"))
|
||||
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.tags = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Title:"))
|
||||
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.title = line.Substring(7).Trim();
|
||||
}
|
||||
|
||||
if ((reblog.downloadedFiles != "."
|
||||
|| reblog.reblogURL.Contains("/blog/private")
|
||||
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
{
|
||||
if ((ContainsAny(reblog.downloadedFiles, contains)
|
||||
|| reblog.reblogURL.Contains("/blog/private")
|
||||
|| reblog.body.Contains("/blog/private"))
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
// && !ContainsAny(reblog.reblogURL, contains)
|
||||
&& !reblog.reblogURL.Contains(@"deactivated"))
|
||||
&& !reblog.reblogURL.Contains(@"deactivated")
|
||||
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
if (!headerWasWritten)
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"profiles": {
|
||||
"URLNotesGrabberCORE": {
|
||||
"commandName": "Project",
|
||||
"commandLineArgs": "-collect 0"
|
||||
"commandLineArgs": "-test zomb-eh 802499389473079296"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user