jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Initialize URLNotesGrabberCORE project with core application structure, data access, and configuration.

Browse Source
This commit is contained in:
jim
2025-11-19 15:51:52 -06:00
parent 3c44cbe671
commit 64ef0e9079
16 changed files with 846 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+309 -51
View File
@@ -3,6 +3,9 @@ using Microsoft.Extensions.Http.Resilience;
using Microsoft.Extensions;
using Microsoft.Extensions.Configuration;
using System.Configuration;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace URLNotesGrabberCORE
{
@@ -18,16 +21,51 @@ namespace URLNotesGrabberCORE
.Build();
var settings = config.GetSection("appSettings");
// Setup Dual Logging
string logPath = "console_output.log";
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
Console.SetOut(dualLogger);
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
{
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
}
else
{
switch (args[0])
{
case "-n":
case "-?":
Console.WriteLine("\t Parse .txt files to find blogs");
Console.WriteLine("-?\t Usage help");
Console.WriteLine("-parse\t Parse .txt files with specified blogname");
Console.WriteLine("-test\t Calls API for given blogname and postID");
Console.WriteLine("-posts\t For each Post in DB, write blogname to file");
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
break;
case "-parse":
string blogNameToParse = args[1];
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
break;
case "-test":
#region Manual test
string blogName = args[1];
@@ -60,31 +98,50 @@ namespace URLNotesGrabberCORE
break;
case "-p": //write post's blogs to file
case "-post":
TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
break;
case "-posts": //write post's blogs to file
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
break;
case "-b": //write blogs to file
case "-blogs": //write blogs to file
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
break;
case "-pn": //collect notes from all posts
case "-collect": //collect notes from all posts
bool withoutNotesOnly = true;
if (args[1] is not null)
if(args.Length != 2)
{
bool.TryParse(args[1], out withoutNotesOnly);
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
break;
}
if (args.Length > 1 && args[1] is not null)
{
if (args[1] == "1")
{
withoutNotesOnly = true;
Console.WriteLine("Parsed");
}
else
{
withoutNotesOnly = false;
Console.WriteLine("--NOT Parsed");
}
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
break;
case "-br": //collect notes from all posts
case "-blogsR": //collect notes from all posts
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
break;
case "-bo": //collect notes from all posts
case "-blogsO": //collect notes from all posts
int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
@@ -100,7 +157,23 @@ namespace URLNotesGrabberCORE
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
case "-pr": //colection posts with replies
case "-bop": //collect notes from all posts
from = 1; to = 999999; top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
top = int.Parse(args[3]);
}
else
{
Console.WriteLine("--Expected FROM TO--");
}
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
case "-replies": //colection posts with replies
try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
@@ -110,6 +183,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("** Unknown Command ** " + args[0]);
break;
}
}
@@ -119,7 +193,7 @@ namespace URLNotesGrabberCORE
static void WritePostBlogsToFile(string outPath)
{
List<Tuple<string, long, string>> posts = DataAccess.GetPosts();
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -130,6 +204,7 @@ namespace URLNotesGrabberCORE
}
blogs = blogs.Distinct().ToList();
blogs.Sort();
blogs.Reverse();
foreach (var blog in blogs)
{
@@ -143,6 +218,24 @@ namespace URLNotesGrabberCORE
{
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
blogs.Sort();
blogs.Reverse();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
foreach (var blog in blogs)
{
Console.WriteLine(blog);
sw.WriteLine(blog + ".tumblr.com");
DataAccess.UpdateBlogOutput(blog);
}
}
}
static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
{
List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
blogs.Sort();
blogs.Reverse();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -163,8 +256,8 @@ namespace URLNotesGrabberCORE
{
foreach (var post in posts)
{
Console.WriteLine("{0}\t{1}", post.Item1, post.Item2);
sw.WriteLine("{0}\t{1}", post.Item1 + ".tumblr.com", post.Item2);
Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
}
}
}
@@ -184,25 +277,25 @@ namespace URLNotesGrabberCORE
return false;
}
static async Task<string> GrabNotes(Tuple<string, long, string> post)
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
{
try
{
int APICount = DataAccess.GetAPICount();
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3).GetAwaiter().GetResult();
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
if (response.statusCode == "NotFound")
{
Thread.Sleep(2000);
Thread.Sleep(1000);
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
return response.statusCode;
}
if (response.statusCode == "TooManyRequests")
{
for( int s = 0; s <= response.retryInSeconds; s+=15)
for (int s = 0; s <= response.retryInSeconds; s += 15)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
Thread.Sleep(15000);
@@ -210,7 +303,7 @@ namespace URLNotesGrabberCORE
return response.statusCode;
}
if (response.response != null)
if (response.response != null && response.response.notes != null)
{
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
@@ -221,30 +314,43 @@ namespace URLNotesGrabberCORE
break;
}
}
if (response.response == null)
else if (response.response == null)
{
Console.WriteLine("response.response == null");
Console.WriteLine("##### Response Null - API Failure? ###");
return "FAILURE";
}
while ( response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= long.Parse(post.Item3))
else if (response.response.notes == null)
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
Console.WriteLine("##### Notes Null - WHY? ###");
return "FAILURE";
}
else
{
if (response.response == null)
{
Console.WriteLine("response.response.notes == null");
return "NULL NOTES";
Console.WriteLine("response.response == null");
return "FAILURE";
}
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
while (response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
{
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
{
Console.WriteLine("response.response.notes == null");
return "NULL NOTES";
}
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
}
@@ -258,7 +364,7 @@ namespace URLNotesGrabberCORE
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
{
List<Tuple<string, long, string>> posts = DataAccess.GetPosts(withoutNotesOnly);
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
PermitLimit = 300,
@@ -279,6 +385,8 @@ namespace URLNotesGrabberCORE
using RateLimitLease lease = limiter.AttemptAcquire(1);
if (lease.IsAcquired)
{
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
Thread.Sleep(1000);
status = await GrabNotes(post);
}
else
@@ -300,7 +408,7 @@ namespace URLNotesGrabberCORE
}
static void TraverseDirectory(string path, string outPath, List<string> contains)
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
{
// Get all directories in the current directory and sort them alphabetically
var directories = Directory.GetDirectories(path);
@@ -308,13 +416,14 @@ namespace URLNotesGrabberCORE
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("=====" + path);
//sw.WriteLine("=====" + path);
//}
foreach (var directory in directories)
{
Console.WriteLine("Directory: " + directory);
TraverseDirectory(directory, outPath, contains); // Recursively traverse subdirectories
TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
}
try
{
@@ -322,13 +431,13 @@ namespace URLNotesGrabberCORE
// Process all files in the current directory
foreach (var file in Directory.GetFiles(path))
{
if (file.EndsWith(".txt"))
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
{
//Console.WriteLine($"=====File: {file}");
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("---" + file);
//sw.WriteLine("---" + file);
//}
try
@@ -340,7 +449,40 @@ namespace URLNotesGrabberCORE
{
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
if (line.StartsWith(@"Post id:"))
{
{ // if there were no files, let's still collect post info
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
{
if (!reblog.reblogURL.Contains(@"deactivated")
&& reblog.reblogURL.Length != 0
&& ContainsAny(reblog.reblogURL, contains))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
//if (reblog.reblogURL.Contains("/blog/private")
// || reblog.body.Contains("/blog/private"))
//{
// Console.WriteLine(reblog.postID);
// Console.WriteLine(reblog.postURL);
// Console.WriteLine(reblog.date);
// Console.WriteLine(reblog.body);
// Console.WriteLine(reblog.reblogKey);
// Console.WriteLine(reblog.reblogURL);
// Console.WriteLine(reblog.reblogName);
// Console.WriteLine(reblog.downloadedFiles);
//}
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, false);
}
}
reblog = new ReblogRecord();
reblog.postID = line.Substring(9).Trim();
}
@@ -366,25 +508,96 @@ namespace URLNotesGrabberCORE
{
reblog.date = line.Substring(6).Trim();
}
if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
if (line.StartsWith(@"Body:"))
{
if (ContainsAny(reblog.downloadedFiles, contains) && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated"))
reblog.body = line.Substring(6).Trim();
}
if (line.StartsWith(@"Post url:"))
{
reblog.postURL = line.Substring(10).Trim();
}
if (line.StartsWith(@"Answer:"))
{
reblog.answer = line.Substring(8).Trim();
}
if (line.StartsWith(@"Audio Caption:"))
{
reblog.audioCaption = line.Substring(15).Trim();
}
if (line.StartsWith(@"Blog Name:"))
{
reblog.blogName = line.Substring(11).Trim();
}
if (line.StartsWith(@"Link:"))
{
reblog.link = line.Substring(6).Trim();
}
if (line.StartsWith(@"Photo Caption:"))
{
reblog.photoCaption = line.Substring(15).Trim();
}
if (line.StartsWith(@"Photo url:"))
{
reblog.photoURL = line.Substring(11).Trim();
}
if (line.StartsWith(@"Question:"))
{
reblog.question = line.Substring(10).Trim();
}
if (line.StartsWith(@"Quote:"))
{
reblog.quote = line.Substring(7).Trim();
}
if (line.StartsWith(@"Slug:"))
{
reblog.slug = line.Substring(6).Trim();
}
if (line.StartsWith(@"Summary:"))
{
reblog.summary = line.Substring(9).Trim();
}
if (line.StartsWith(@"Tags:"))
{
reblog.tags = line.Substring(6).Trim();
}
if (line.StartsWith(@"Title:"))
{
reblog.title = line.Substring(7).Trim();
}
if ((reblog.downloadedFiles != "."
|| reblog.reblogURL.Contains("/blog/private")
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
{
if ((ContainsAny(reblog.downloadedFiles, contains)
|| reblog.reblogURL.Contains("/blog/private")
|| reblog.body.Contains("/blog/private"))
// && !ContainsAny(reblog.reblogURL, contains)
&& !reblog.reblogURL.Contains(@"deactivated"))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
//Console.WriteLine(reblog.postID);
//Console.WriteLine(reblog.date);
//Console.WriteLine(reblog.reblogKey);
//Console.WriteLine(reblog.reblogURL);
//Console.WriteLine(reblog.reblogName);
//Console.WriteLine(reblog.downloadedFiles);
//if (reblog.reblogURL.Contains("/blog/private")
// || reblog.body.Contains("/blog/private"))
//{
// Console.WriteLine(reblog.postID);
// Console.WriteLine(reblog.postURL);
// Console.WriteLine(reblog.date);
// Console.WriteLine(reblog.body);
// Console.WriteLine(reblog.reblogKey);
// Console.WriteLine(reblog.reblogURL);
// Console.WriteLine(reblog.reblogName);
// Console.WriteLine(reblog.downloadedFiles);
//}
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date );
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, true);
}
}
}
@@ -413,5 +626,50 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"An error occurred: {ex.Message}");
}
}
static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
{
var directories = Directory.GetDirectories(path);
Array.Sort(directories, StringComparer.InvariantCulture);
foreach (var directory in directories)
{
//Console.WriteLine("Directory: " + directory);
TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
}
try
{
bool headerWasWritten = false;
foreach (var file in Directory.GetFiles(path))
{
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
{
try
{
var urls = new List<string>();
var reblog = new ReblogRecord();
foreach (string line in File.ReadLines(file))
{
if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
{
Console.WriteLine(file);
}
}
}
catch (Exception e)
{
Console.WriteLine("The file could not be read:");
Console.WriteLine(e.Message);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.Message}");
}
}
}
}