jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Enhance filtering, SQL, and error handling logic

Browse Source 
- Added `beforeDate` parameter to filter posts by `NotesGatheredDateTime`.
- Improved SQL queries with additional joins and conditions.
- Refactored exception handling for better resource cleanup.
- Enhanced string comparisons to support case-insensitivity.
- Added `GetReplies` method for fetching replies.
- Improved handling of `ReblogRecord` data and filtering logic.
- Updated `-collect` command to support optional date filtering.
- Adjusted `launchSettings.json` for testing with specific parameters.
- Improved logging and error reporting in API and database operations.
This commit is contained in:
jim
2025-12-11 23:19:19 -06:00
parent e55d2b0e29
commit 0e5fb124a0
4 changed files with 150 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TL.sqbpro
+24 -19
View File
@@ -1,19 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="1" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Blogs" custom_title="0" dock_id="1" table="4,5:mainBlogs"/><dock_state state="000000ff00000000fd0000000100000002000005f40000031dfc0100000001fb000000160064006f0063006b00420072006f00770073006500310100000000000005f4000000fb00ffffff000002130000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="0" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="48"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1*">SELECT ␍ <?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="" readonly="1" foreign_keys="" case_sensitive_like="" temp_store="" wal_autocheckpoint="" synchronous=""/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/></tab_structure><tab_browse><table title="." custom_title="0" dock_id="4" table="0,0:"/><dock_state state="000000ff00000000fd0000000100000002000005f40000030ffc0100000002fb000000160064006f0063006b00420072006f00770073006500310100000000000005f40000000000000000fb000000160064006f0063006b00420072006f00770073006500340100000000ffffffff0000011700ffffff000005f40000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings/></tab_browse><tab_sql><sql name="SQL 1">SELECT
BlogName || '.tumblr.com/post/' || postID,␍ BlogName || '.tumblr.com/post/' || postID,
␍
datetime(NotesGatheredDateTime, 'unixepoch'), *␍ datetime(NotesGatheredDateTime, 'unixepoch'), *
FROM␍ FROM
Posts␍ Posts
WHERE␍ WHERE
NotesGatheredDateTime &lt;&gt; 0␍ NotesGatheredDateTime &lt;&gt; 0
ORDER BY␍ ORDER BY
postdate desc</sql><sql name="SQL 2*">SELECT ␍ postdate desc</sql><sql name="SQL 2*">SELECT
datetime(TimeStamp, 'unixepoch'),␍ datetime(TimeStamp, 'unixepoch'),
RootBlogName || '.tumblr.com/post/' || postid,␍ RootBlogName || '.tumblr.com/post/' || N.postid,
*,␍ *,
NoteBlogName || '.tumblr.com'␍ NoteBlogName || '.tumblr.com'
FROM␍ FROM
Notes␍ Notes N␍
WHERE RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin')␍ inner JOIN␍
ORDER BY␍ Posts P on P.PostID = N.PostID and P.BlogName = N.RootBlogName
TimeStamp desc</sql><current_tab id="0"/></tab_sql></sqlb_project> WHERE RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin', 'vvenus-child')␍
and type like 'r%'␍
and RootBlogName = 'zomb-eh'␍
and P.HasImage = 1
ORDER BY
TimeStamp desc</sql><current_tab id="1"/></tab_sql></sqlb_project>
URLNotesGrabberCORE/DataAccess.cs
+47 -40
View File
@@ -145,7 +145,7 @@ namespace URLNotesGrabberCORE
Title, Title,
HasImage HasImage
) VALUES (" + ) VALUES (" +
Q(blogName) + ", " + postID +", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")"; Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection); SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery(); command.ExecuteNonQuery();
@@ -156,27 +156,27 @@ namespace URLNotesGrabberCORE
{ {
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
/// ///
try try
{ {
connection.Open(); connection.Open();
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'"; string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection); SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery(); command.ExecuteNonQuery();
} }
catch (Exception ex2) catch (Exception ex2)
{
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{ {
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex2.Message);
{
Console.WriteLine(ex2.Message);
}
} }
finally }
{ finally
connection.Close(); {
} connection.Close();
/// }
///
} }
} }
finally finally
@@ -188,7 +188,7 @@ namespace URLNotesGrabberCORE
public static void AddAPICount(string DBPath = @"TL.db") public static void AddAPICount(string DBPath = @"TL.db")
{ {
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try try
{ {
@@ -243,7 +243,7 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{ {
Console.WriteLine(ex.Message); Console.WriteLine(ex.Message);
Console.WriteLine("^^^^^ - SHORTCUT"); Console.WriteLine("^^^^^ - SHORTCUT");
} }
} }
finally finally
@@ -262,7 +262,7 @@ namespace URLNotesGrabberCORE
/// <param name="withoutNotesOnly"></param> /// <param name="withoutNotesOnly"></param>
/// <param name="DBPath"></param> /// <param name="DBPath"></param>
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns> /// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db") public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string DBPath = @"TL.db")
{ {
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>(); List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
@@ -271,7 +271,7 @@ namespace URLNotesGrabberCORE
{ {
connection.Open(); connection.Open();
string sql = "SELECT " + string sql = "SELECT " +
" Posts.BlogName, " + Environment.NewLine + " Posts.BlogName, " + Environment.NewLine +
" Posts.PostID, " + Environment.NewLine + " Posts.PostID, " + Environment.NewLine +
" Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine + " Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
@@ -279,16 +279,23 @@ namespace URLNotesGrabberCORE
" CNT.CNT " + Environment.NewLine + " CNT.CNT " + Environment.NewLine +
"FROM " + Environment.NewLine + "FROM " + Environment.NewLine +
" Posts " + Environment.NewLine + " Posts " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine + " LEFT OUTER JOIN " + Environment.NewLine +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine + " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine + " LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " + " ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine; "WHERE NotFound = 0 " + Environment.NewLine;
if (withoutNotesOnly) if (withoutNotesOnly)
sql += " and HasNotesGathered = 0 " + Environment.NewLine ; sql += " and HasNotesGathered = 0 " + Environment.NewLine;
sql += "GROUP BY " + Environment.NewLine + // Filter by NotesGatheredDateTime if beforeDate is provided
if (beforeDate.HasValue)
{
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
}
sql += "GROUP BY " + Environment.NewLine +
" Posts.BlogName, Posts.PostID " + Environment.NewLine + " Posts.BlogName, Posts.PostID " + Environment.NewLine +
"ORDER BY " + Environment.NewLine + "ORDER BY " + Environment.NewLine +
" notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine; " notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
@@ -296,7 +303,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(withoutNotesOnly); Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql); Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey(); Console.Write(">"); //Console.ReadKey();
// Thread.Sleep(1000); Thread.Sleep(250);
using (SQLiteCommand command = new SQLiteCommand(sql, connection)) using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{ {
@@ -332,7 +339,7 @@ namespace URLNotesGrabberCORE
} }
return posts; return posts;
} }
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db") public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
{ {
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -395,7 +402,7 @@ namespace URLNotesGrabberCORE
{ {
while (reader.Read()) while (reader.Read())
{ {
count = reader.GetInt32(0); // Assuming Id is the first column count = reader.GetInt32(0); // Assuming Id is the first column
} }
} }
} }
@@ -635,10 +642,10 @@ namespace URLNotesGrabberCORE
{ {
connection.Open(); connection.Open();
string sql = "UPDATE Posts SET "; string sql = "UPDATE Posts SET ";
sql += "postDate = @postDate, "; sql += "postDate = @postDate, ";
sql += "reblogURL = @reblogURL, "; sql += "reblogURL = @reblogURL, ";
sql += "postURL = @postURL, "; sql += "postURL = @postURL, ";
sql += "slug = @slug, "; sql += "slug = @slug, ";
sql += "reblogKey = @reblogKey, "; sql += "reblogKey = @reblogKey, ";
sql += "reblogName = @reblogName, "; sql += "reblogName = @reblogName, ";
@@ -752,8 +759,8 @@ namespace URLNotesGrabberCORE
} }
return APICount; return APICount;
} }
#endregion Updates #endregion Updates
} }
internal class APIAccess internal class APIAccess
@@ -772,13 +779,13 @@ namespace URLNotesGrabberCORE
throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json"); throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json");
} }
private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ?? private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
throw new InvalidOperationException("ConsumerKey is not configured"); throw new InvalidOperationException("ConsumerKey is not configured");
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ?? private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
throw new InvalidOperationException("ConsumerSecret is not configured"); throw new InvalidOperationException("ConsumerSecret is not configured");
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ?? private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
throw new InvalidOperationException("OAuthToken is not configured"); throw new InvalidOperationException("OAuthToken is not configured");
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ?? private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
throw new InvalidOperationException("OAuthTokenSecret is not configured"); throw new InvalidOperationException("OAuthTokenSecret is not configured");
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null) public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
@@ -864,8 +871,8 @@ namespace URLNotesGrabberCORE
} }
} }
} }
catch (Exception ex) catch (Exception ex)
{ {
Console.WriteLine($"Failed JSON: {myJsonResponse}"); Console.WriteLine($"Failed JSON: {myJsonResponse}");
Console.WriteLine(ex.ToString()); Console.WriteLine(ex.ToString());
@@ -939,7 +946,7 @@ namespace URLNotesGrabberCORE
if (myDeserializedClass.retryInSeconds < secs) if (myDeserializedClass.retryInSeconds < secs)
myDeserializedClass.retryInSeconds = secs; myDeserializedClass.retryInSeconds = secs;
} }
foundRateLimitHeader = true; foundRateLimitHeader = true;
} }
} }
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0") if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
URLNotesGrabberCORE/Program.cs
+78 -51
View File
@@ -56,7 +56,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file"); Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes"); Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes. Optional datetime parameter to filter by NotesGatheredDateTime");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file "); Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
@@ -71,7 +71,7 @@ namespace URLNotesGrabberCORE
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse); TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
break; break;
case "-test": case "-test":
#region Manual test #region Manual test
@@ -148,7 +148,7 @@ namespace URLNotesGrabberCORE
break; break;
} }
} }
Console.WriteLine("Test completed successfully"); Console.WriteLine("Test completed successfully");
#endregion #endregion
@@ -168,13 +168,15 @@ namespace URLNotesGrabberCORE
case "-collect": //collect notes from all posts case "-collect": //collect notes from all posts
bool withoutNotesOnly = true; bool withoutNotesOnly = true;
DateTime? beforeDate = null;
if(args.Length != 2) if (args.Length < 2)
{ {
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--"); Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1) [OPTIONAL: BEFOREDATE]--");
break; break;
} }
// Parse withoutNotesOnly flag
if (args.Length > 1 && args[1] is not null) if (args.Length > 1 && args[1] is not null)
{ {
if (args[1] == "1") if (args[1] == "1")
@@ -190,7 +192,23 @@ namespace URLNotesGrabberCORE
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]); Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
} }
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
// Parse optional beforeDate parameter
if (args.Length >= 3 && !string.IsNullOrEmpty(args[2]))
{
if (DateTime.TryParse(args[2], out DateTime parsedDate))
{
beforeDate = parsedDate;
Console.WriteLine($"Filter: Collecting notes for posts with NotesGatheredDateTime < {beforeDate}");
}
else
{
Console.WriteLine($"ERROR: Invalid date format '{args[2]}'");
break;
}
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly, beforeDate);
break; break;
case "-blogsR": //collect notes from all posts case "-blogsR": //collect notes from all posts
@@ -249,7 +267,7 @@ namespace URLNotesGrabberCORE
static void WritePostBlogsToFile(string outPath) static void WritePostBlogsToFile(string outPath)
{ {
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(); List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
using (StreamWriter sw = new StreamWriter(outPath, true)) using (StreamWriter sw = new StreamWriter(outPath, true))
{ {
@@ -341,7 +359,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount); Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult(); var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
// If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return // If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return
if (response?.meta != null && response.meta.status == 404) if (response?.meta != null && response.meta.status == 404)
{ {
@@ -391,10 +409,10 @@ namespace URLNotesGrabberCORE
return "FAILURE"; return "FAILURE";
} }
else else
{ {
while (response.response != null while (response.response != null
&& response.response._links != null && response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3) && long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
{ {
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult(); response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null) if (response.response.notes is null)
@@ -416,24 +434,28 @@ namespace URLNotesGrabberCORE
return "Success"; return "Success";
} }
catch(Exception ex) { catch (Exception ex)
Console.WriteLine(ex.ToString()); } {
Console.WriteLine(ex.ToString());
}
return "UNKNOWN"; return "UNKNOWN";
} }
static async void CollectNotes(string outPath, bool withoutNotesOnly = true) static async void CollectNotes(string outPath, bool withoutNotesOnly = true, DateTime? beforeDate = null)
{ {
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly); List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions { RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
PermitLimit = 300, {
QueueProcessingOrder = QueueProcessingOrder.OldestFirst, PermitLimit = 300,
QueueLimit = 1, QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
Window = TimeSpan.FromMinutes(1), QueueLimit = 1,
SegmentsPerWindow = 60, Window = TimeSpan.FromMinutes(1),
AutoReplenishment = true} ); SegmentsPerWindow = 60,
AutoReplenishment = true
});
try try
{ {
@@ -455,7 +477,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available"); Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
return; return;
} }
if(status == "Success") if (status == "Success")
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2); DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
else else
Console.WriteLine("GrabNotes Result: " + status); Console.WriteLine("GrabNotes Result: " + status);
@@ -471,6 +493,10 @@ namespace URLNotesGrabberCORE
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "") static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
{ {
//if(!ContainsAny(path, contains))
//{
// return;
//}
// Get all directories in the current directory and sort them alphabetically // Get all directories in the current directory and sort them alphabetically
var directories = Directory.GetDirectories(path); var directories = Directory.GetDirectories(path);
Array.Sort(directories, StringComparer.InvariantCulture); Array.Sort(directories, StringComparer.InvariantCulture);
@@ -492,7 +518,7 @@ namespace URLNotesGrabberCORE
// Process all files in the current directory // Process all files in the current directory
foreach (var file in Directory.GetFiles(path)) foreach (var file in Directory.GetFiles(path))
{ {
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == "")) if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
{ {
//Console.WriteLine($"=====File: {file}"); //Console.WriteLine($"=====File: {file}");
@@ -508,8 +534,8 @@ namespace URLNotesGrabberCORE
foreach (string line in File.ReadLines(file)) foreach (string line in File.ReadLines(file))
{ {
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated")) //if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
if (line.StartsWith(@"Post id:")) if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase))
{ // if there were no files, let's still collect post info { // if there were no files, let's still collect post info
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".") if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
{ {
@@ -547,94 +573,95 @@ namespace URLNotesGrabberCORE
reblog = new ReblogRecord(); reblog = new ReblogRecord();
reblog.postID = line.Substring(9).Trim(); reblog.postID = line.Substring(9).Trim();
} }
if (line.StartsWith(@"Reblog url:")) if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
{ {
//reblog = new ReblogRecord(); //reblog = new ReblogRecord();
reblog.reblogURL = line.Substring(12).Trim(); reblog.reblogURL = line.Substring(12).Trim();
} }
if (line.StartsWith(@"Reblog name:")) if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.reblogName = line.Substring(13).Trim(); reblog.reblogName = line.Substring(13).Trim();
} }
if (line.StartsWith(@"Downloaded files:")) if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.downloadedFiles = line.Substring(17).Trim(); reblog.downloadedFiles = line.Substring(17).Trim();
} }
if (line.StartsWith(@"Reblog key:")) if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.reblogKey = line.Substring(11).Trim(); reblog.reblogKey = line.Substring(11).Trim();
} }
if (line.StartsWith(@"Date:")) if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.date = line.Substring(6).Trim(); reblog.date = line.Substring(6).Trim();
} }
if (line.StartsWith(@"Body:")) if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.body = line.Substring(6).Trim(); reblog.body = line.Substring(6).Trim();
} }
if (line.StartsWith(@"Post url:")) if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.postURL = line.Substring(10).Trim(); reblog.postURL = line.Substring(10).Trim();
} }
if (line.StartsWith(@"Answer:")) if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.answer = line.Substring(8).Trim(); reblog.answer = line.Substring(8).Trim();
} }
if (line.StartsWith(@"Audio Caption:")) if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.audioCaption = line.Substring(15).Trim(); reblog.audioCaption = line.Substring(15).Trim();
} }
if (line.StartsWith(@"Blog Name:")) if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.blogName = line.Substring(11).Trim(); reblog.blogName = line.Substring(11).Trim();
} }
if (line.StartsWith(@"Link:")) if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.link = line.Substring(6).Trim(); reblog.link = line.Substring(6).Trim();
} }
if (line.StartsWith(@"Photo Caption:")) if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.photoCaption = line.Substring(15).Trim(); reblog.photoCaption = line.Substring(15).Trim();
} }
if (line.StartsWith(@"Photo url:")) if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.photoURL = line.Substring(11).Trim(); reblog.photoURL = line.Substring(11).Trim();
} }
if (line.StartsWith(@"Question:")) if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.question = line.Substring(10).Trim(); reblog.question = line.Substring(10).Trim();
} }
if (line.StartsWith(@"Quote:")) if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.quote = line.Substring(7).Trim(); reblog.quote = line.Substring(7).Trim();
} }
if (line.StartsWith(@"Slug:")) if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.slug = line.Substring(6).Trim(); reblog.slug = line.Substring(6).Trim();
} }
if (line.StartsWith(@"Summary:")) if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.summary = line.Substring(9).Trim(); reblog.summary = line.Substring(9).Trim();
} }
if (line.StartsWith(@"Tags:")) if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.tags = line.Substring(6).Trim(); reblog.tags = line.Substring(6).Trim();
} }
if (line.StartsWith(@"Title:")) if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
{ {
reblog.title = line.Substring(7).Trim(); reblog.title = line.Substring(7).Trim();
} }
if ((reblog.downloadedFiles != "." if ((reblog.downloadedFiles != "."
|| reblog.reblogURL.Contains("/blog/private") || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".") || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
{ {
if ((ContainsAny(reblog.downloadedFiles, contains) if ((ContainsAny(reblog.downloadedFiles, contains)
|| reblog.reblogURL.Contains("/blog/private") || (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|| reblog.body.Contains("/blog/private")) || (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
// && !ContainsAny(reblog.reblogURL, contains) // && !ContainsAny(reblog.reblogURL, contains)
&& !reblog.reblogURL.Contains(@"deactivated")) && !reblog.reblogURL.Contains(@"deactivated")
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
{ {
DirectoryInfo currentDir = new DirectoryInfo(path); DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten) if (!headerWasWritten)
URLNotesGrabberCORE/Properties/launchSettings.json
+1 -1
View File
@@ -2,7 +2,7 @@
"profiles": { "profiles": {
"URLNotesGrabberCORE": { "URLNotesGrabberCORE": {
"commandName": "Project", "commandName": "Project",
"commandLineArgs": "-collect 0" "commandLineArgs": "-test zomb-eh 802499389473079296"
} }
} }
} }