jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
jim
2024-11-15 09:17:53 -06:00
parent 5cddf2a427
commit 3c44cbe671
5 changed files with 248 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/Program.cs
+130 -73
View File
@@ -18,38 +18,45 @@ namespace URLNotesGrabberCORE
.Build();
var settings = config.GetSection("appSettings");
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
{
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
}
else
switch(args[0])
switch (args[0])
{
case "-n": //Manual test
case "-n":
#region Manual test
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
string blogName = args[1];
long postID = long.Parse(args[2]);
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
while (response.response._links != null)
{
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
while (response.response._links != null)
{
response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
#endregion
break;
@@ -62,23 +69,57 @@ namespace URLNotesGrabberCORE
break;
case "-pn": //collect notes from all posts
CollectNotes(settings.GetValue<string>("PathOutput"));
bool withoutNotesOnly = true;
if (args[1] is not null)
{
bool.TryParse(args[1], out withoutNotesOnly);
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
break;
case "-br": //collect notes from all posts
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
break;
}
case "-bo": //collect notes from all posts
int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
top = int.Parse(args[3]);
}
else
{
Console.WriteLine("--Expected FROM TO--");
}
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
case "-pr": //colection posts with replies
try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
break;
default:
Console.WriteLine("** Unknown Command ** " + args[0]);
break;
}
System.Console.WriteLine("<fin>:/");
System.Console.ReadKey();
//System.Console.ReadKey();
}
static void WritePostBlogsToFile(string outPath)
{
List<Tuple<string, long>> posts = DataAccess.GetPosts();
List<Tuple<string, long, string>> posts = DataAccess.GetPosts();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -98,9 +139,9 @@ namespace URLNotesGrabberCORE
}
}
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false)
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
{
List<string> blogs = DataAccess.GetBlogs(reblogsOnly);
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
blogs.Sort();
using (StreamWriter sw = new StreamWriter(outPath, true))
@@ -109,24 +150,31 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(blog);
sw.WriteLine(blog + ".tumblr.com");
DataAccess.UpdatePostBlogOutput(blog);
DataAccess.UpdateBlogOutput(blog);
}
}
}
protected static bool ContainsAny(string input)
static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
{
List<Tuple<string, long>> posts = DataAccess.GetReplies();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
foreach (var post in posts)
{
Console.WriteLine("{0}\t{1}", post.Item1, post.Item2);
sw.WriteLine("{0}\t{1}", post.Item1 + ".tumblr.com", post.Item2);
}
}
}
protected static bool ContainsAny(string input, List<string> contains)
{
if (string.IsNullOrEmpty(input)) return false;
var items = new List<string>();
items.Add("zombaee");
items.Add("zomb-eh");
items.Add("ahzombae");
items.Add("thebugandme");
items.Add("lovingbabybug");
items.Add("swarthyvillain");
foreach (string item in items)
foreach (string item in contains)
{
if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
{
@@ -136,28 +184,28 @@ namespace URLNotesGrabberCORE
return false;
}
static async Task<string> GrabNotes(Tuple<string, long> post)
static async Task<string> GrabNotes(Tuple<string, long, string> post)
{
try
{
int APICount = DataAccess.GetAPICount();
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now);
var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult();
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
if (response.statusCode == "NotFound")
{
Thread.Sleep(1000);
Thread.Sleep(2000);
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
return response.statusCode;
}
if (response.statusCode == "TooManyRequests")
{
for( int s = 0; s <= response.retryInSeconds; s++)
for( int s = 0; s <= response.retryInSeconds; s+=15)
{
Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s);
Thread.Sleep(1000);
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
Thread.Sleep(15000);
}
return response.statusCode;
}
@@ -174,13 +222,21 @@ namespace URLNotesGrabberCORE
}
}
if (response.response == null)
{
Console.WriteLine("response.response == null");
return "FAILURE";
}
while (response.response != null && response.response._links != null)
while ( response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= long.Parse(post.Item3))
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
{
Console.WriteLine("response.response.notes == null");
return "NULL NOTES";
}
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
@@ -200,9 +256,9 @@ namespace URLNotesGrabberCORE
return "UNKNOWN";
}
static async void CollectNotes(string outPath)
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
{
List<Tuple<string, long>> posts = DataAccess.GetPosts(true);
List<Tuple<string, long, string>> posts = DataAccess.GetPosts(withoutNotesOnly);
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
PermitLimit = 300,
@@ -244,20 +300,21 @@ namespace URLNotesGrabberCORE
}
static void TraverseDirectory(string path, string outPath)
static void TraverseDirectory(string path, string outPath, List<string> contains)
{
// Get all directories in the current directory and sort them alphabetically
var directories = Directory.GetDirectories(path);
Array.Sort(directories, StringComparer.InvariantCulture);
using (StreamWriter sw = new StreamWriter(outPath, true))
{
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("=====" + path);
}
//}
foreach (var directory in directories)
{
Console.WriteLine("Directory: " + directory);
TraverseDirectory(directory, outPath); // Recursively traverse subdirectories
TraverseDirectory(directory, outPath, contains); // Recursively traverse subdirectories
}
try
{
@@ -267,12 +324,12 @@ namespace URLNotesGrabberCORE
{
if (file.EndsWith(".txt"))
{
Console.WriteLine($"=====File: {file}");
//Console.WriteLine($"=====File: {file}");
using (StreamWriter sw = new StreamWriter(outPath, true))
{
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("---" + file);
}
//}
try
{
@@ -312,35 +369,35 @@ namespace URLNotesGrabberCORE
if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
{
if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated"))
if (ContainsAny(reblog.downloadedFiles, contains) && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated"))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
Console.WriteLine(reblog.postID);
Console.WriteLine(reblog.date);
Console.WriteLine(reblog.reblogKey);
Console.WriteLine(reblog.reblogURL);
Console.WriteLine(reblog.reblogName);
Console.WriteLine(reblog.downloadedFiles);
//Console.WriteLine(reblog.postID);
//Console.WriteLine(reblog.date);
//Console.WriteLine(reblog.reblogKey);
//Console.WriteLine(reblog.reblogURL);
//Console.WriteLine(reblog.reblogName);
//Console.WriteLine(reblog.downloadedFiles);
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL );
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date );
}
}
}
urls.Sort();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
foreach (string line in urls.Distinct())
{
sw.WriteLine(line);
}
}
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
// foreach (string line in urls.Distinct())
// {
// sw.WriteLine(line);
// }
//}
}
catch (Exception e)
{