718 lines
34 KiB
C#
718 lines
34 KiB
C#
using System.Threading.RateLimiting;
|
|
using Microsoft.Extensions.Http.Resilience;
|
|
using Microsoft.Extensions;
|
|
using Microsoft.Extensions.Configuration;
|
|
using System.Configuration;
|
|
using System.Threading;
|
|
using Microsoft.Extensions.Diagnostics.Latency;
|
|
using static System.Runtime.InteropServices.JavaScript.JSType;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
internal class Program
|
|
{
|
|
|
|
static void Main(string[] args)
|
|
{
|
|
IConfiguration config = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: false)
|
|
.AddCommandLine(args)
|
|
.Build();
|
|
var settings = config.GetSection("appSettings");
|
|
|
|
// Setup Dual Logging
|
|
string logPath = "console_output.log";
|
|
if (File.Exists(logPath))
|
|
{
|
|
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
|
|
string newPath = $"console_output_{timestamp}.log";
|
|
File.Move(logPath, newPath);
|
|
}
|
|
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
|
|
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
|
|
Console.SetOut(dualLogger);
|
|
|
|
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
|
|
|
|
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
|
{
|
|
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
|
|
}
|
|
else
|
|
{
|
|
switch (args[0])
|
|
{
|
|
case "-?":
|
|
Console.WriteLine("\t Parse .txt files to find blogs");
|
|
|
|
Console.WriteLine("-?\t Usage help");
|
|
|
|
Console.WriteLine("-parse\t Parse .txt files with specified blogname");
|
|
|
|
Console.WriteLine("-test\t Calls API for given blogname and postID");
|
|
|
|
Console.WriteLine("-posts\t For each Post in DB, write blogname to file");
|
|
|
|
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
|
|
|
|
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
|
|
|
|
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
|
|
|
|
Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
|
|
|
|
Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
|
|
|
|
break;
|
|
case "-parse":
|
|
string blogNameToParse = args[1];
|
|
|
|
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
|
|
|
|
break;
|
|
case "-test":
|
|
#region Manual test
|
|
|
|
string blogName = args[1];
|
|
long postID = long.Parse(args[2]);
|
|
|
|
var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
|
|
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
|
|
|
|
// Check for API errors
|
|
if (response.statusCode == "NotFound")
|
|
{
|
|
Console.WriteLine($"ERROR: Post not found - {blogName}/{postID}");
|
|
break;
|
|
}
|
|
if (response.statusCode == "TooManyRequests")
|
|
{
|
|
Console.WriteLine($"ERROR: Rate limited - retry in {response.retryInSeconds} seconds");
|
|
break;
|
|
}
|
|
if (!string.IsNullOrEmpty(response.statusCode))
|
|
{
|
|
Console.WriteLine($"ERROR: API returned status code: {response.statusCode}");
|
|
break;
|
|
}
|
|
|
|
// Check for null response
|
|
if (response.response == null)
|
|
{
|
|
Console.WriteLine("ERROR: API response is null");
|
|
break;
|
|
}
|
|
|
|
// Check for null notes
|
|
if (response.response.notes == null)
|
|
{
|
|
Console.WriteLine("ERROR: Notes collection is null (post may have 0 notes)");
|
|
break;
|
|
}
|
|
|
|
Console.WriteLine($"Processing {response.response.notes.Count} notes...");
|
|
foreach (var note in response.response.notes)
|
|
{
|
|
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
|
|
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
|
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
|
break;
|
|
}
|
|
|
|
while (response.response._links != null)
|
|
{
|
|
response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
|
|
|
if (response.response?.notes == null)
|
|
{
|
|
Console.WriteLine("ERROR: Notes collection became null during pagination");
|
|
break;
|
|
}
|
|
|
|
Console.WriteLine($"Processing {response.response.notes.Count} more notes...");
|
|
foreach (var note in response.response.notes)
|
|
{
|
|
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
|
|
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
|
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
|
break;
|
|
}
|
|
}
|
|
|
|
Console.WriteLine("Test completed successfully");
|
|
#endregion
|
|
|
|
break;
|
|
|
|
case "-post":
|
|
TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
|
|
break;
|
|
|
|
case "-posts": //write post's blogs to file
|
|
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
|
|
break;
|
|
|
|
case "-blogs": //write blogs to file
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
|
|
break;
|
|
|
|
case "-collect": //collect notes from all posts
|
|
bool withoutNotesOnly = true;
|
|
|
|
if(args.Length != 2)
|
|
{
|
|
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
|
|
break;
|
|
}
|
|
|
|
if (args.Length > 1 && args[1] is not null)
|
|
{
|
|
if (args[1] == "1")
|
|
{
|
|
withoutNotesOnly = true;
|
|
Console.WriteLine("Parsed");
|
|
}
|
|
else
|
|
{
|
|
withoutNotesOnly = false;
|
|
Console.WriteLine("--NOT Parsed");
|
|
}
|
|
|
|
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
|
|
}
|
|
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
|
|
break;
|
|
|
|
case "-blogsR": //collect notes from all posts
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
|
|
break;
|
|
|
|
case "-blogsO": //collect notes from all posts
|
|
int from = 1, to = 999999, top = 100;
|
|
|
|
if (args[1] is not null && args[2] is not null && args[3] is not null)
|
|
{
|
|
from = int.Parse(args[1]);
|
|
to = int.Parse(args[2]);
|
|
top = int.Parse(args[3]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Expected FROM TO--");
|
|
}
|
|
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
|
break;
|
|
|
|
case "-bop": //collect notes from all posts
|
|
from = 1; to = 999999; top = 100;
|
|
|
|
if (args[1] is not null && args[2] is not null && args[3] is not null)
|
|
{
|
|
from = int.Parse(args[1]);
|
|
to = int.Parse(args[2]);
|
|
top = int.Parse(args[3]);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("--Expected FROM TO--");
|
|
}
|
|
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
|
break;
|
|
|
|
case "-replies": //colection posts with replies
|
|
try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
|
|
WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
|
|
|
|
break;
|
|
|
|
default:
|
|
Console.WriteLine("** Unknown Command ** " + args[0]);
|
|
break;
|
|
}
|
|
}
|
|
|
|
|
|
|
|
System.Console.WriteLine("<fin>:/");
|
|
//System.Console.ReadKey();
|
|
}
|
|
|
|
static void WritePostBlogsToFile(string outPath)
|
|
{
|
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
List<string> blogs = new List<string>();
|
|
foreach (var post in posts)
|
|
{
|
|
blogs.Add(post.Item1);
|
|
}
|
|
blogs = blogs.Distinct().ToList();
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
|
|
{
|
|
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
DataAccess.UpdateBlogOutput(blog);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
|
|
{
|
|
List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
|
|
blogs.Sort();
|
|
blogs.Reverse();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var blog in blogs)
|
|
{
|
|
Console.WriteLine(blog);
|
|
sw.WriteLine(blog + ".tumblr.com");
|
|
DataAccess.UpdateBlogOutput(blog);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
|
|
{
|
|
List<Tuple<string, long>> posts = DataAccess.GetReplies();
|
|
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var post in posts)
|
|
{
|
|
Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
|
|
sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
|
|
}
|
|
}
|
|
}
|
|
|
|
protected static bool ContainsAny(string input, List<string> contains)
|
|
{
|
|
if (string.IsNullOrEmpty(input)) return false;
|
|
|
|
|
|
foreach (string item in contains)
|
|
{
|
|
if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
|
|
{
|
|
try
|
|
{
|
|
int APICount = DataAccess.GetAPICount();
|
|
|
|
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
|
|
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
|
|
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
|
|
|
|
if (response.statusCode == "NotFound")
|
|
{
|
|
Thread.Sleep(1000);
|
|
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
|
return response.statusCode;
|
|
}
|
|
if (response.statusCode == "TooManyRequests")
|
|
{
|
|
for (int s = 0; s <= response.retryInSeconds; s += 15)
|
|
{
|
|
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
|
|
Thread.Sleep(15000);
|
|
}
|
|
return response.statusCode;
|
|
}
|
|
|
|
if (response.response != null && response.response.notes != null)
|
|
{
|
|
Console.WriteLine("Notes\t" + response.response.notes.Count);
|
|
foreach (var note in response.response.notes)
|
|
{
|
|
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
|
|
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
|
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
|
break;
|
|
}
|
|
}
|
|
else if (response.response == null)
|
|
{
|
|
Console.WriteLine("##### Response Null - API Failure? ###");
|
|
Console.WriteLine($"Raw JSON: {response.rawJson}");
|
|
return "FAILURE";
|
|
}
|
|
else if (response.response.notes == null)
|
|
{
|
|
Console.WriteLine("##### Notes Null - WHY? ###");
|
|
return "FAILURE";
|
|
}
|
|
else
|
|
{
|
|
while (response.response != null
|
|
&& response.response._links != null
|
|
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
|
|
{
|
|
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
|
if (response.response.notes is null)
|
|
{
|
|
Console.WriteLine("response.response.notes == null");
|
|
return "NULL NOTES";
|
|
}
|
|
|
|
Console.WriteLine("Notes\t" + response.response.notes.Count);
|
|
foreach (var note in response.response.notes)
|
|
{
|
|
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
|
|
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
|
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return "Success";
|
|
}
|
|
catch(Exception ex) { Console.WriteLine(ex.ToString()); }
|
|
|
|
|
|
return "UNKNOWN";
|
|
}
|
|
|
|
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
|
|
{
|
|
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
|
|
|
|
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
|
|
PermitLimit = 300,
|
|
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
|
QueueLimit = 1,
|
|
Window = TimeSpan.FromMinutes(1),
|
|
SegmentsPerWindow = 60,
|
|
AutoReplenishment = true} );
|
|
|
|
try
|
|
{
|
|
using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
{
|
|
foreach (var post in posts)
|
|
{
|
|
string status;
|
|
|
|
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
|
if (lease.IsAcquired)
|
|
{
|
|
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
|
|
Thread.Sleep(1000);
|
|
status = await GrabNotes(post);
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
|
return;
|
|
}
|
|
if(status == "Success")
|
|
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
|
else
|
|
Console.WriteLine("GrabNotes Result: " + status);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
}
|
|
|
|
|
|
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
|
|
{
|
|
// Get all directories in the current directory and sort them alphabetically
|
|
var directories = Directory.GetDirectories(path);
|
|
Array.Sort(directories, StringComparer.InvariantCulture);
|
|
|
|
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
//{
|
|
//sw.WriteLine("=====" + path);
|
|
//}
|
|
|
|
foreach (var directory in directories)
|
|
{
|
|
|
|
Console.WriteLine("Directory: " + directory);
|
|
TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
|
|
}
|
|
try
|
|
{
|
|
bool headerWasWritten = false;
|
|
// Process all files in the current directory
|
|
foreach (var file in Directory.GetFiles(path))
|
|
{
|
|
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
|
|
{
|
|
//Console.WriteLine($"=====File: {file}");
|
|
|
|
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
//{
|
|
//sw.WriteLine("---" + file);
|
|
//}
|
|
|
|
try
|
|
{
|
|
var urls = new List<string>();
|
|
var reblog = new ReblogRecord();
|
|
|
|
foreach (string line in File.ReadLines(file))
|
|
{
|
|
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
|
if (line.StartsWith(@"Post id:"))
|
|
{ // if there were no files, let's still collect post info
|
|
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
|
{
|
|
if (!reblog.reblogURL.Contains(@"deactivated")
|
|
&& reblog.reblogURL.Length != 0
|
|
&& ContainsAny(reblog.reblogURL, contains))
|
|
{
|
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
if (!headerWasWritten)
|
|
{
|
|
headerWasWritten = true;
|
|
}
|
|
|
|
//if (reblog.reblogURL.Contains("/blog/private")
|
|
// || reblog.body.Contains("/blog/private"))
|
|
//{
|
|
// Console.WriteLine(reblog.postID);
|
|
// Console.WriteLine(reblog.postURL);
|
|
// Console.WriteLine(reblog.date);
|
|
// Console.WriteLine(reblog.body);
|
|
// Console.WriteLine(reblog.reblogKey);
|
|
// Console.WriteLine(reblog.reblogURL);
|
|
// Console.WriteLine(reblog.reblogName);
|
|
// Console.WriteLine(reblog.downloadedFiles);
|
|
//}
|
|
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
|
|
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
reblog.title, false);
|
|
}
|
|
}
|
|
|
|
reblog = new ReblogRecord();
|
|
reblog.postID = line.Substring(9).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog url:"))
|
|
{
|
|
//reblog = new ReblogRecord();
|
|
|
|
reblog.reblogURL = line.Substring(12).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog name:"))
|
|
{
|
|
reblog.reblogName = line.Substring(13).Trim();
|
|
}
|
|
if (line.StartsWith(@"Downloaded files:"))
|
|
{
|
|
reblog.downloadedFiles = line.Substring(17).Trim();
|
|
}
|
|
if (line.StartsWith(@"Reblog key:"))
|
|
{
|
|
reblog.reblogKey = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Date:"))
|
|
{
|
|
reblog.date = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Body:"))
|
|
{
|
|
reblog.body = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Post url:"))
|
|
{
|
|
reblog.postURL = line.Substring(10).Trim();
|
|
}
|
|
if (line.StartsWith(@"Answer:"))
|
|
{
|
|
reblog.answer = line.Substring(8).Trim();
|
|
}
|
|
if (line.StartsWith(@"Audio Caption:"))
|
|
{
|
|
reblog.audioCaption = line.Substring(15).Trim();
|
|
}
|
|
if (line.StartsWith(@"Blog Name:"))
|
|
{
|
|
reblog.blogName = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Link:"))
|
|
{
|
|
reblog.link = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Photo Caption:"))
|
|
{
|
|
reblog.photoCaption = line.Substring(15).Trim();
|
|
}
|
|
if (line.StartsWith(@"Photo url:"))
|
|
{
|
|
reblog.photoURL = line.Substring(11).Trim();
|
|
}
|
|
if (line.StartsWith(@"Question:"))
|
|
{
|
|
reblog.question = line.Substring(10).Trim();
|
|
}
|
|
if (line.StartsWith(@"Quote:"))
|
|
{
|
|
reblog.quote = line.Substring(7).Trim();
|
|
}
|
|
if (line.StartsWith(@"Slug:"))
|
|
{
|
|
reblog.slug = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Summary:"))
|
|
{
|
|
reblog.summary = line.Substring(9).Trim();
|
|
}
|
|
if (line.StartsWith(@"Tags:"))
|
|
{
|
|
reblog.tags = line.Substring(6).Trim();
|
|
}
|
|
if (line.StartsWith(@"Title:"))
|
|
{
|
|
reblog.title = line.Substring(7).Trim();
|
|
}
|
|
|
|
if ((reblog.downloadedFiles != "."
|
|
|| reblog.reblogURL.Contains("/blog/private")
|
|
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
|
{
|
|
if ((ContainsAny(reblog.downloadedFiles, contains)
|
|
|| reblog.reblogURL.Contains("/blog/private")
|
|
|| reblog.body.Contains("/blog/private"))
|
|
// && !ContainsAny(reblog.reblogURL, contains)
|
|
&& !reblog.reblogURL.Contains(@"deactivated"))
|
|
{
|
|
DirectoryInfo currentDir = new DirectoryInfo(path);
|
|
if (!headerWasWritten)
|
|
{
|
|
headerWasWritten = true;
|
|
}
|
|
//if (reblog.reblogURL.Contains("/blog/private")
|
|
// || reblog.body.Contains("/blog/private"))
|
|
//{
|
|
// Console.WriteLine(reblog.postID);
|
|
// Console.WriteLine(reblog.postURL);
|
|
// Console.WriteLine(reblog.date);
|
|
// Console.WriteLine(reblog.body);
|
|
// Console.WriteLine(reblog.reblogKey);
|
|
// Console.WriteLine(reblog.reblogURL);
|
|
// Console.WriteLine(reblog.reblogName);
|
|
// Console.WriteLine(reblog.downloadedFiles);
|
|
//}
|
|
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
|
|
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
|
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
|
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
|
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
|
reblog.title, true);
|
|
}
|
|
}
|
|
}
|
|
urls.Sort();
|
|
|
|
|
|
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
|
//{
|
|
// foreach (string line in urls.Distinct())
|
|
// {
|
|
// sw.WriteLine(line);
|
|
// }
|
|
//}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine("The file could not be read:");
|
|
Console.WriteLine(e.Message);
|
|
}
|
|
}
|
|
// Add your file processing logic here
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"An error occurred: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
|
|
static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
|
|
{
|
|
var directories = Directory.GetDirectories(path);
|
|
Array.Sort(directories, StringComparer.InvariantCulture);
|
|
|
|
foreach (var directory in directories)
|
|
{
|
|
//Console.WriteLine("Directory: " + directory);
|
|
TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
|
|
}
|
|
try
|
|
{
|
|
bool headerWasWritten = false;
|
|
foreach (var file in Directory.GetFiles(path))
|
|
{
|
|
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
|
|
{
|
|
try
|
|
{
|
|
var urls = new List<string>();
|
|
var reblog = new ReblogRecord();
|
|
|
|
foreach (string line in File.ReadLines(file))
|
|
{
|
|
if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
|
|
{
|
|
Console.WriteLine(file);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception e)
|
|
{
|
|
Console.WriteLine("The file could not be read:");
|
|
Console.WriteLine(e.Message);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"An error occurred: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
}
|