jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
jim
2026-03-26 00:35:24 -05:00
parent 15b484bdf8
commit 4e87daf2b2
10 changed files with 242 additions and 546 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+165 -35
View File
@@ -13,6 +13,7 @@ using Microsoft.Extensions.Diagnostics.Latency;
using System.Collections;
using Microsoft.Extensions.Configuration;
using System.IO;
using System.Data;
namespace URLNotesGrabberCORE
{
@@ -68,6 +69,30 @@ namespace URLNotesGrabberCORE
internal class DataAccess
{
private static IConfiguration? _configuration;
private static HashSet<long>? _postIdsToExclude;
static DataAccess()
{
_configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.Build();
var excludeSetting = _configuration["appSettings:PostIDToExclude"];
if (!string.IsNullOrWhiteSpace(excludeSetting))
{
_postIdsToExclude = excludeSetting.Split(',')
.Select(s => long.TryParse(s.Trim(), out var id) ? id : (long?)null)
.Where(id => id.HasValue)
.Select(id => id.Value)
.ToHashSet();
}
else
{
_postIdsToExclude = new HashSet<long>();
}
}
public static string Q(string input)
{
return "'" + input.Replace("'", "''") + "'";
@@ -124,6 +149,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
}
finally
@@ -156,6 +182,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName")
Console.WriteLine(ex.Message);
}
@@ -203,35 +230,57 @@ namespace URLNotesGrabberCORE
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
int rowsInserted = 0;
try
{
rowsInserted = command.ExecuteNonQuery();
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex.Message);
///
try
{
connection.Open();
string updateSql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
updateCommand.ExecuteNonQuery();
}
catch (Exception ex2)
{
// Breakpoint here
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex2.Message);
}
}
finally
{
connection.Close();
}
///
}
}
// Only update HasBeenOutput and DateAdded if a new post was inserted
if (rowsInserted == 1)
{
Console.WriteLine(ex.Message);
///
try
{
connection.Open();
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex2)
{
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded WHERE BlogName = @BlogName";
using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
{
Console.WriteLine(ex2.Message);
updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
updateBlogCommand.ExecuteNonQuery();
}
}
finally
{
connection.Close();
}
///
catch { }
}
}
finally
@@ -260,6 +309,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
//Console.WriteLine(ex.Message);
}
finally
@@ -275,14 +325,14 @@ namespace URLNotesGrabberCORE
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
connection2.Open();
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled)";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
{
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
@@ -291,11 +341,27 @@ namespace URLNotesGrabberCORE
command.Parameters.AddWithValue("@Type", type ?? string.Empty);
command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
command.ExecuteNonQuery();
int rowsInserted = command.ExecuteNonQuery();
// Only update HasBeenOutput if a new note was inserted
if (rowsInserted == 1)
{
try
{
string updateSql = "UPDATE Blogs SET HasBeenOutput = 0 WHERE BlogName = @BlogName";
using (var updateCommand = new SQLiteCommand(updateSql, connection2))
{
updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
updateCommand.ExecuteNonQuery();
}
}
catch { }
}
}
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
@@ -304,7 +370,7 @@ namespace URLNotesGrabberCORE
}
finally
{
connection.Close();
connection2.Close();
}
return false;
}
@@ -330,7 +396,7 @@ namespace URLNotesGrabberCORE
string sql = "SELECT " +
" MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
" Posts.PostID, " + Environment.NewLine +
" 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
" 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
" Posts.NotesGatheredDatetime, " + Environment.NewLine +
" CNT.CNT " + Environment.NewLine +
"FROM " + Environment.NewLine +
@@ -344,7 +410,7 @@ namespace URLNotesGrabberCORE
if (withoutNotesOnly)
{
sql += " and HasNotesGathered = 0 " + Environment.NewLine;
sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] > datetime('now', 'localtime', '-1 day') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
sql += "OR ( Posts.blogname = 'zomb-eh' and [notesGatheredDateTime] < datetime('now', 'localtime', '-3 days') and ( ( postdate > '1/1/26' or ( HasNotesGathered = 1 and PostDate > '8/1/23' ) ) ) )" + Environment.NewLine;
}
// Filter by NotesGatheredDateTime if beforeDate is provided
@@ -361,7 +427,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey();
Console.Write(">" ); //Console.ReadKey();
//Thread.Sleep(250);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
@@ -376,10 +442,58 @@ namespace URLNotesGrabberCORE
long lastNoteTimestamp;
long notesGatheredTimestamp;
blog = reader.GetString(0); // Assuming Title is the second column
postID = reader.GetInt64(1); // Assuming Id is the first column
lastNoteTimestamp = reader.GetInt64(2); // Assuming Id is the first column
notesGatheredTimestamp = reader.GetInt64(3); // Assuming Id is the first column
// Diagnostic logging before each cast
//for (int i = 0; i < reader.FieldCount; i++)
//{
// var value = reader.GetValue(i);
// var type = reader.GetFieldType(i);
// Console.WriteLine($"[GetPosts] Column {i}: Name={reader.GetName(i)}, Value={value}, Type={type}");
//}
try
{
blog = Convert.ToString(reader.GetValue(0));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 0 to string: {ex.Message}");
throw;
}
try
{
postID = Convert.ToInt64(reader.GetValue(1));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 1 to Int64: {ex.Message}");
throw;
}
try
{
lastNoteTimestamp = Convert.ToInt64(reader.GetValue(2));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 2 to Int64: {ex.Message}");
throw;
}
try
{
notesGatheredTimestamp = Convert.ToInt64(reader.GetValue(3));
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[GetPosts] Error casting column 3 to Int64: {ex.Message}");
throw;
}
// Exclude if postID is in the exclusion list
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
continue;
post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
posts.Add(post);
@@ -389,6 +503,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -431,6 +546,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -481,6 +597,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
}
finally
@@ -533,6 +650,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
}
finally
@@ -569,6 +687,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -612,6 +731,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -655,6 +775,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -689,6 +810,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -719,6 +841,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -747,6 +870,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -781,6 +905,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
@@ -855,6 +980,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -881,6 +1007,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -911,6 +1038,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
@@ -955,6 +1083,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
}
@@ -994,6 +1123,7 @@ namespace URLNotesGrabberCORE
}
catch (Exception ex)
{
// Breakpoint here
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
}
URLNotesGrabberCORE/Program.cs
+9 -3
View File
@@ -448,7 +448,7 @@ namespace URLNotesGrabberCORE
{
int APICount = DataAccess.GetAPICount();
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
Thread.Sleep(3000);
//Thread.Sleep(3000);
var allNotes = new List<dynamic>();
int page = 1;
@@ -478,10 +478,10 @@ namespace URLNotesGrabberCORE
}
if (response.statusCode == "TooManyRequests")
{
for (int s = 0; s <= response.retryInSeconds; s += 15)
for (int s = 0; s <= response.retryInSeconds; s += 60)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
Thread.Sleep(15000);
Thread.Sleep(60000);
}
return response.statusCode;
}
@@ -589,6 +589,12 @@ namespace URLNotesGrabberCORE
else
{
Console.WriteLine("GrabNotes Result: " + status);
//if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue
if (status != "RateLimitExceeded")
{
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
}
}
// Re-fetch the updated list after processing the current post
URLNotesGrabberCORE/appsettings.json
+2 -1
View File
@@ -7,7 +7,8 @@
"PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
"PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
"PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot",
"PostIDToExclude": "730084470076686336,726309940037287936,639974102120235008,184045126218"
},
"TumblrApi": {
"ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",