Add project files.
This commit is contained in:
@@ -0,0 +1,345 @@
|
||||
using System.Threading.RateLimiting;
|
||||
using Microsoft.Extensions.Http.Resilience;
|
||||
using Microsoft.Extensions;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using System.Configuration;
|
||||
|
||||
namespace URLNotesGrabberCORE
|
||||
{
|
||||
internal class Program
|
||||
{
|
||||
|
||||
static void Main(string[] args)
|
||||
{
|
||||
IConfiguration config = new ConfigurationBuilder()
|
||||
.SetBasePath(Directory.GetCurrentDirectory())
|
||||
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
|
||||
.AddCommandLine(args)
|
||||
.Build();
|
||||
var settings = config.GetSection("appSettings");
|
||||
|
||||
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
||||
{
|
||||
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
|
||||
}
|
||||
else if (args[0] == "-n") //Manual test
|
||||
{
|
||||
/*var response = */
|
||||
//var url = await CreateProductAsync(product);
|
||||
//var t = await GrabNotes();
|
||||
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
|
||||
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
|
||||
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
|
||||
while (response.response._links != null)
|
||||
{
|
||||
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
||||
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
}
|
||||
//var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList();
|
||||
//notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList();
|
||||
|
||||
//using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
|
||||
//{
|
||||
// foreach (var note in notes)
|
||||
// {
|
||||
// using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
|
||||
// }
|
||||
//}
|
||||
|
||||
}
|
||||
else if (args[0] == "-p") //write post's blogs to file
|
||||
{
|
||||
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
|
||||
}
|
||||
else if (args[0] == "-b") //write blogs to file
|
||||
{
|
||||
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
|
||||
}
|
||||
else if (args[0] == "-pn") //collect notes from all posts
|
||||
{
|
||||
CollectNotes(settings.GetValue<string>("PathOutput"));
|
||||
}
|
||||
else if (args[0] == "-br") //collect notes from all posts
|
||||
{
|
||||
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
|
||||
}
|
||||
|
||||
|
||||
|
||||
System.Console.WriteLine("<fin>:/");
|
||||
System.Console.ReadKey();
|
||||
}
|
||||
|
||||
static void WritePostBlogsToFile(string outPath)
|
||||
{
|
||||
List<Tuple<string, long>> posts = DataAccess.GetPosts();
|
||||
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
List<string> blogs = new List<string>();
|
||||
foreach (var post in posts)
|
||||
{
|
||||
blogs.Add(post.Item1);
|
||||
}
|
||||
blogs = blogs.Distinct().ToList();
|
||||
blogs.Sort();
|
||||
|
||||
foreach (var blog in blogs)
|
||||
{
|
||||
Console.WriteLine(blog);
|
||||
sw.WriteLine(blog + ".tumblr.com");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false)
|
||||
{
|
||||
List<string> blogs = DataAccess.GetBlogs(reblogsOnly);
|
||||
blogs.Sort();
|
||||
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
foreach (var blog in blogs)
|
||||
{
|
||||
Console.WriteLine(blog);
|
||||
sw.WriteLine(blog + ".tumblr.com");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected static bool ContainsAny(string input)
|
||||
{
|
||||
if (string.IsNullOrEmpty(input)) return false;
|
||||
var items = new List<string>();
|
||||
items.Add("zombaee");
|
||||
items.Add("zomb-eh");
|
||||
items.Add("ahzombae");
|
||||
items.Add("thebugandme");
|
||||
items.Add("lovingbabybug");
|
||||
items.Add("swarthyvillain");
|
||||
|
||||
|
||||
foreach (string item in items)
|
||||
{
|
||||
if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
static async Task GrabNotes(Tuple<string, long> post)
|
||||
{
|
||||
try
|
||||
{
|
||||
int APICount = DataAccess.GetAPICount();
|
||||
|
||||
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now);
|
||||
var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult();
|
||||
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
|
||||
|
||||
if (response.statusCode == "NotFound")
|
||||
return;
|
||||
if (response.statusCode == "TooManyRequests")
|
||||
return;
|
||||
|
||||
if (response.response != null)
|
||||
{
|
||||
Console.WriteLine("Notes\t" + response.response.notes.Count);
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (response.response == null)
|
||||
throw new Exception("Response is null");
|
||||
|
||||
while (response.response != null && response.response._links != null)
|
||||
{
|
||||
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
||||
|
||||
Console.WriteLine("Notes\t" + response.response.notes.Count);
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch(Exception ex) { Console.WriteLine(ex.ToString()); }
|
||||
}
|
||||
|
||||
static async void CollectNotes(string outPath)
|
||||
{
|
||||
List<Tuple<string, long>> posts = DataAccess.GetPosts(true);
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true} );
|
||||
|
||||
try
|
||||
{
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
foreach (var post in posts)
|
||||
{
|
||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||
if (lease.IsAcquired)
|
||||
{
|
||||
await GrabNotes(post);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
return;
|
||||
}
|
||||
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static void TraverseDirectory(string path, string outPath)
|
||||
{
|
||||
// Get all directories in the current directory and sort them alphabetically
|
||||
var directories = Directory.GetDirectories(path);
|
||||
Array.Sort(directories, StringComparer.InvariantCulture);
|
||||
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
//sw.WriteLine("=====" + path);
|
||||
}
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
Console.WriteLine("Directory: " + directory);
|
||||
TraverseDirectory(directory, outPath); // Recursively traverse subdirectories
|
||||
}
|
||||
try
|
||||
{
|
||||
bool headerWasWritten = false;
|
||||
// Process all files in the current directory
|
||||
foreach (var file in Directory.GetFiles(path))
|
||||
{
|
||||
if (file.EndsWith(".txt"))
|
||||
{
|
||||
Console.WriteLine($"=====File: {file}");
|
||||
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
//sw.WriteLine("---" + file);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var urls = new List<string>();
|
||||
var reblog = new ReblogRecord();
|
||||
|
||||
foreach (string line in File.ReadLines(file))
|
||||
{
|
||||
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
||||
if (line.StartsWith(@"Post id:"))
|
||||
{
|
||||
reblog = new ReblogRecord();
|
||||
reblog.postID = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog url:"))
|
||||
{
|
||||
//reblog = new ReblogRecord();
|
||||
|
||||
reblog.reblogURL = line.Substring(12).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog name:"))
|
||||
{
|
||||
reblog.reblogName = line.Substring(13).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Downloaded files:"))
|
||||
{
|
||||
reblog.downloadedFiles = line.Substring(17).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog key:"))
|
||||
{
|
||||
reblog.reblogKey = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Date:"))
|
||||
{
|
||||
reblog.date = line.Substring(6).Trim();
|
||||
}
|
||||
|
||||
if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
{
|
||||
if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated"))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
if (!headerWasWritten)
|
||||
{
|
||||
headerWasWritten = true;
|
||||
}
|
||||
Console.WriteLine(reblog.postID);
|
||||
Console.WriteLine(reblog.date);
|
||||
Console.WriteLine(reblog.reblogKey);
|
||||
Console.WriteLine(reblog.reblogURL);
|
||||
Console.WriteLine(reblog.reblogName);
|
||||
Console.WriteLine(reblog.downloadedFiles);
|
||||
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
|
||||
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
||||
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL );
|
||||
}
|
||||
}
|
||||
}
|
||||
urls.Sort();
|
||||
|
||||
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
foreach (string line in urls.Distinct())
|
||||
{
|
||||
sw.WriteLine(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("The file could not be read:");
|
||||
Console.WriteLine(e.Message);
|
||||
}
|
||||
}
|
||||
// Add your file processing logic here
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"An error occurred: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user