Enhance filtering, SQL, and error handling logic
- Added `beforeDate` parameter to filter posts by `NotesGatheredDateTime`. - Improved SQL queries with additional joins and conditions. - Refactored exception handling for better resource cleanup. - Enhanced string comparisons to support case-insensitivity. - Added `GetReplies` method for fetching replies. - Improved handling of `ReblogRecord` data and filtering logic. - Updated `-collect` command to support optional date filtering. - Adjusted `launchSettings.json` for testing with specific parameters. - Improved logging and error reporting in API and database operations.
This commit is contained in:
@@ -56,7 +56,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
||||
|
||||
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
|
||||
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
|
||||
|
||||
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
||||
|
||||
@@ -71,7 +71,7 @@ namespace URLNotesGrabberCORE
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
|
||||
|
||||
break;
|
||||
|
||||
|
||||
case "-test":
|
||||
#region Manual test
|
||||
|
||||
@@ -148,7 +148,7 @@ namespace URLNotesGrabberCORE
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Console.WriteLine("Test completed successfully");
|
||||
#endregion
|
||||
|
||||
@@ -168,13 +168,15 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
case "-collect": //collect notes from all posts
|
||||
bool withoutNotesOnly = true;
|
||||
DateTime? beforeDate = null;
|
||||
|
||||
if(args.Length != 2)
|
||||
if (args.Length < 2)
|
||||
{
|
||||
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
|
||||
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
|
||||
break;
|
||||
}
|
||||
|
||||
// Parse withoutNotesOnly flag
|
||||
if (args.Length > 1 && args[1] is not null)
|
||||
{
|
||||
if (args[1] == "1")
|
||||
@@ -190,7 +192,23 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
|
||||
}
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
|
||||
|
||||
// Parse optional beforeDate parameter
|
||||
if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
|
||||
{
|
||||
if (DateTime.TryParse(args[2], out DateTime parsedDate))
|
||||
{
|
||||
beforeDate = parsedDate;
|
||||
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
|
||||
break;
|
||||
|
||||
case "-blogsR": //collect notes from all posts
|
||||
@@ -249,7 +267,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
static void WritePostBlogsToFile(string outPath)
|
||||
{
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
|
||||
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
@@ -341,7 +359,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
|
||||
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
|
||||
|
||||
|
||||
// If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return
|
||||
if (response?.meta != null && response.meta.status == 404)
|
||||
{
|
||||
@@ -391,10 +409,10 @@ namespace URLNotesGrabberCORE
|
||||
return "FAILURE";
|
||||
}
|
||||
else
|
||||
{
|
||||
while (response.response != null
|
||||
&& response.response._links != null
|
||||
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
|
||||
{
|
||||
while (response.response != null
|
||||
&& response.response._links != null
|
||||
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
|
||||
{
|
||||
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
||||
if (response.response.notes is null)
|
||||
@@ -416,24 +434,28 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
return "Success";
|
||||
}
|
||||
catch(Exception ex) {
|
||||
Console.WriteLine(ex.ToString()); }
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
|
||||
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
|
||||
static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
|
||||
{
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
|
||||
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true} );
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
try
|
||||
{
|
||||
@@ -455,7 +477,7 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
return;
|
||||
}
|
||||
if(status == "Success")
|
||||
if (status == "Success")
|
||||
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
||||
else
|
||||
Console.WriteLine("GrabNotes Result: " + status);
|
||||
@@ -471,6 +493,10 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
|
||||
{
|
||||
//if(!ContainsAny(path, contains))
|
||||
//{
|
||||
// return;
|
||||
//}
|
||||
// Get all directories in the current directory and sort them alphabetically
|
||||
var directories = Directory.GetDirectories(path);
|
||||
Array.Sort(directories, StringComparer.InvariantCulture);
|
||||
@@ -492,7 +518,7 @@ namespace URLNotesGrabberCORE
|
||||
// Process all files in the current directory
|
||||
foreach (var file in Directory.GetFiles(path))
|
||||
{
|
||||
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
|
||||
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
{
|
||||
//Console.WriteLine($"=====File: {file}");
|
||||
|
||||
@@ -508,8 +534,8 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
foreach (string line in File.ReadLines(file))
|
||||
{
|
||||
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
||||
if (line.StartsWith(@"Post id:"))
|
||||
//if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
||||
if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase))
|
||||
{ // if there were no files, let's still collect post info
|
||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
||||
{
|
||||
@@ -547,94 +573,95 @@ namespace URLNotesGrabberCORE
|
||||
reblog = new ReblogRecord();
|
||||
reblog.postID = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog url:"))
|
||||
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
//reblog = new ReblogRecord();
|
||||
|
||||
reblog.reblogURL = line.Substring(12).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog name:"))
|
||||
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogName = line.Substring(13).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Downloaded files:"))
|
||||
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.downloadedFiles = line.Substring(17).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog key:"))
|
||||
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogKey = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Date:"))
|
||||
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.date = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Body:"))
|
||||
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.body = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Post url:"))
|
||||
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.postURL = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Answer:"))
|
||||
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.answer = line.Substring(8).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Audio Caption:"))
|
||||
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.audioCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Blog Name:"))
|
||||
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.blogName = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Link:"))
|
||||
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.link = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo Caption:"))
|
||||
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo url:"))
|
||||
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoURL = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Question:"))
|
||||
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.question = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Quote:"))
|
||||
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.quote = line.Substring(7).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Slug:"))
|
||||
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.slug = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Summary:"))
|
||||
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.summary = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Tags:"))
|
||||
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.tags = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Title:"))
|
||||
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.title = line.Substring(7).Trim();
|
||||
}
|
||||
|
||||
if ((reblog.downloadedFiles != "."
|
||||
|| reblog.reblogURL.Contains("/blog/private")
|
||||
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
{
|
||||
if ((ContainsAny(reblog.downloadedFiles, contains)
|
||||
|| reblog.reblogURL.Contains("/blog/private")
|
||||
|| reblog.body.Contains("/blog/private"))
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
// && !ContainsAny(reblog.reblogURL, contains)
|
||||
&& !reblog.reblogURL.Contains(@"deactivated"))
|
||||
&& !reblog.reblogURL.Contains(@"deactivated")
|
||||
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
if (!headerWasWritten)
|
||||
|
||||
Reference in New Issue
Block a user