Add support for fetching and storing reply text
- Add database migration and update logic for replyText column in Notes - Implement batch collection of missing reply text via Tumblr API - Add new API integration to fetch reply text for replies - Enhance DataAccess with methods to query/update replyText - Update CLI: -replies now collects and stores reply text - Improve logging (archive logs/), error handling, and output - Remove TL.db from source control
This commit is contained in:
+268
-180
@@ -26,9 +26,18 @@ namespace URLNotesGrabberCORE
|
||||
if (File.Exists(logPath))
|
||||
{
|
||||
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
|
||||
string newPath = $"console_output_{timestamp}.log";
|
||||
string archiveDirectory = "logs";
|
||||
|
||||
// Ensure the archive directory exists
|
||||
if (!Directory.Exists(archiveDirectory))
|
||||
{
|
||||
Directory.CreateDirectory(archiveDirectory);
|
||||
}
|
||||
|
||||
string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
|
||||
File.Move(logPath, newPath);
|
||||
}
|
||||
|
||||
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
|
||||
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
|
||||
Console.SetOut(dualLogger);
|
||||
@@ -37,7 +46,9 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
||||
{
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
|
||||
int postsAdded = 0;
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded);
|
||||
Console.WriteLine($"Total posts added: {postsAdded}");
|
||||
}
|
||||
else
|
||||
{
|
||||
@@ -62,96 +73,19 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
|
||||
|
||||
Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
|
||||
Console.WriteLine("-replies\t Fetch and update missing reply text for all replies in database");
|
||||
|
||||
break;
|
||||
|
||||
case "-parse":
|
||||
string blogNameToParse = args[1];
|
||||
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
|
||||
|
||||
int postsAdded = 0;
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse);
|
||||
Console.WriteLine($"Total posts added: {postsAdded}");
|
||||
break;
|
||||
|
||||
|
||||
case "-test":
|
||||
#region Manual test
|
||||
|
||||
string blogName = args[1];
|
||||
long postID = long.Parse(args[2]);
|
||||
|
||||
var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
|
||||
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
|
||||
|
||||
// If the API returned a 404 inside the JSON `meta` block, mark the post NotFound
|
||||
if (response?.meta != null && response.meta.status == 404)
|
||||
{
|
||||
Console.WriteLine($"ERROR: Post not found - {blogName}/{postID} (meta.status=404)");
|
||||
DataAccess.UpdatePostMarkNotFound(blogName, postID);
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for API errors
|
||||
if (response.statusCode == "NotFound")
|
||||
{
|
||||
Console.WriteLine($"ERROR: Post not found - {blogName}/{postID}");
|
||||
break;
|
||||
}
|
||||
if (response.statusCode == "TooManyRequests")
|
||||
{
|
||||
Console.WriteLine($"ERROR: Rate limited - retry in {response.retryInSeconds} seconds");
|
||||
break;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(response.statusCode))
|
||||
{
|
||||
Console.WriteLine($"ERROR: API returned status code: {response.statusCode}");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for null response
|
||||
if (response.response == null)
|
||||
{
|
||||
Console.WriteLine("ERROR: API response is null");
|
||||
break;
|
||||
}
|
||||
|
||||
// Check for null notes
|
||||
if (response.response.notes == null)
|
||||
{
|
||||
Console.WriteLine("ERROR: Notes collection is null (post may have 0 notes)");
|
||||
break;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Processing {response.response.notes.Count} notes...");
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
|
||||
while (response.response._links != null)
|
||||
{
|
||||
response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
||||
|
||||
if (response.response?.notes == null)
|
||||
{
|
||||
Console.WriteLine("ERROR: Notes collection became null during pagination");
|
||||
break;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Processing {response.response.notes.Count} more notes...");
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Test completed successfully");
|
||||
#endregion
|
||||
|
||||
Console.WriteLine("Test command not implemented");
|
||||
break;
|
||||
|
||||
case "-post":
|
||||
@@ -168,7 +102,7 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
case "-collect": //collect notes from all posts
|
||||
bool withoutNotesOnly = true;
|
||||
DateTime? beforeDate = null;
|
||||
DateTime? beforeDate = DateTime.Now;
|
||||
|
||||
if (args.Length < 2)
|
||||
{
|
||||
@@ -247,10 +181,8 @@ namespace URLNotesGrabberCORE
|
||||
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
|
||||
break;
|
||||
|
||||
case "-replies": //colection posts with replies
|
||||
try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
|
||||
WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
|
||||
|
||||
case "-replies": //update reply text
|
||||
CollectMissingReplyText().GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
default:
|
||||
@@ -351,16 +283,181 @@ namespace URLNotesGrabberCORE
|
||||
return false;
|
||||
}
|
||||
|
||||
static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
|
||||
{
|
||||
try
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Fetching reply text for {blogName}/{postID}/{timestamp}");
|
||||
// Add 2-second delay before API call to avoid server-side rate limiting
|
||||
await Task.Delay(2000);
|
||||
var postsResponse = await APIAccess.GrabPostWithReplies(blogName, postID, timestamp);
|
||||
|
||||
if (postsResponse?.response?.posts == null || postsResponse.response.posts.Count == 0)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] No posts found in response for {blogName}/{postID}");
|
||||
Console.WriteLine($"[Reply Text] Response Status Code: {postsResponse?.statusCode ?? "N/A"}");
|
||||
Console.WriteLine($"[Reply Text] Response.response is null: {postsResponse?.response == null}");
|
||||
if (postsResponse?.response != null)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Posts count: {postsResponse.response.posts?.Count ?? 0}");
|
||||
}
|
||||
if (!string.IsNullOrEmpty(postsResponse?.rawJson))
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Raw Response JSON: {postsResponse.rawJson}");
|
||||
}
|
||||
|
||||
// Mark all replies for this post with '?' to indicate API returned no posts
|
||||
Console.WriteLine($"[Reply Text] Marking all replies for {blogName}/{postID} with '?' due to no posts in response");
|
||||
DataAccess.UpdateAllNoteReplyTextForPost(blogName, postID, "?");
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the first (and should be only) post
|
||||
var post = postsResponse.response.posts.FirstOrDefault();
|
||||
if (post?.notes == null)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] No notes found in post {blogName}/{postID}");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine($"[Reply Text] Found {post.notes.Count} total notes for {blogName}/{postID}");
|
||||
|
||||
// Update each reply with its text
|
||||
int replyCount = 0;
|
||||
int skippedCount = 0;
|
||||
foreach (var note in post.notes)
|
||||
{
|
||||
if (note.type == "reply")
|
||||
{
|
||||
if (!string.IsNullOrEmpty(note.reply_text))
|
||||
{
|
||||
DataAccess.UpdateNoteReplyText(blogName, postID, note.blog_name, note.timestamp, note.reply_text);
|
||||
replyCount++;
|
||||
|
||||
// Output the reply text being stored
|
||||
string displayText = note.reply_text.Length > 100
|
||||
? note.reply_text.Substring(0, 100) + "..."
|
||||
: note.reply_text;
|
||||
Console.WriteLine($" [{note.blog_name}] {displayText}");
|
||||
}
|
||||
else
|
||||
{
|
||||
skippedCount++;
|
||||
Console.WriteLine($"[Reply Text] Skipped reply from {note.blog_name} - empty reply_text");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (replyCount > 0)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Updated {replyCount} reply texts for {blogName}/{postID}");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] No reply text found for {blogName}/{postID} (skipped: {skippedCount})");
|
||||
}
|
||||
|
||||
// Mark any remaining replies with '.' as '?' to indicate they were processed but had no text
|
||||
Console.WriteLine($"[Reply Text] Marking any remaining replies with '.' as '?' for {blogName}/{postID}");
|
||||
//int cleanupCount = DataAccess.UpdateRemainingDefaultReplyText(blogName, postID, ".", "?");
|
||||
//if (cleanupCount > 0)
|
||||
//{
|
||||
// Console.WriteLine($"[Reply Text] Cleaned up {cleanupCount} remaining replies for {blogName}/{postID}");
|
||||
//}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"[Reply Text] Error fetching reply text for {blogName}/{postID}: {ex.Message}");
|
||||
Console.WriteLine($"[Reply Text] StackTrace: {ex.StackTrace}");
|
||||
}
|
||||
}
|
||||
|
||||
static async Task CollectMissingReplyText()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Ensure the replyText column exists
|
||||
DataAccess.EnsureReplyTextColumnExists();
|
||||
|
||||
Console.WriteLine("Starting collection of missing reply text...");
|
||||
Console.WriteLine("Processing 10 posts at a time.");
|
||||
|
||||
int batchSize = 1; // Process 10 posts per batch
|
||||
int totalProcessedCount = 0;
|
||||
|
||||
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions
|
||||
{
|
||||
PermitLimit = 300,
|
||||
QueueProcessingOrder = QueueProcessingOrder.OldestFirst,
|
||||
QueueLimit = 1,
|
||||
Window = TimeSpan.FromMinutes(1),
|
||||
SegmentsPerWindow = 60,
|
||||
AutoReplenishment = true
|
||||
});
|
||||
|
||||
while (true && totalProcessedCount < batchSize)
|
||||
{
|
||||
var postsWithFilledReplies = DataAccess.GetRepliesWithFilledText(limit: batchSize);
|
||||
|
||||
if (postsWithFilledReplies.Count == 0)
|
||||
{
|
||||
Console.WriteLine("No more posts with filled reply text found. Collection complete.");
|
||||
break;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Found {postsWithFilledReplies.Count} posts with filled reply text. Processing batch...");
|
||||
int batchProcessedCount = 0;
|
||||
|
||||
foreach (var post in postsWithFilledReplies)
|
||||
{
|
||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||
if (lease.IsAcquired)
|
||||
{
|
||||
Console.WriteLine($"Processing {post.Item1}/{post.Item2}/{post.Item3}");
|
||||
await FetchAndStoreReplyText(post.Item1, post.Item2, post.Item3);
|
||||
batchProcessedCount++;
|
||||
totalProcessedCount++;
|
||||
|
||||
// Add 1 second delay between attempts
|
||||
await Task.Delay(1000);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
Console.WriteLine($"Stopped after processing {totalProcessedCount} posts total");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"Batch complete. Processed {batchProcessedCount} posts in this batch.");
|
||||
}
|
||||
|
||||
Console.WriteLine($"Completed collection of missing reply text. Total processed: {totalProcessedCount} posts.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"Error collecting missing reply text: {ex.Message}");
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
|
||||
{
|
||||
try
|
||||
{
|
||||
int APICount = DataAccess.GetAPICount();
|
||||
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
|
||||
Thread.Sleep(3000);
|
||||
|
||||
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
|
||||
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
|
||||
var allNotes = new List<dynamic>();
|
||||
int page = 1;
|
||||
string beforeTimestamp = post.Item3.ToString();
|
||||
bool hasReplies = false;
|
||||
const int maxPages = 500;
|
||||
var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
|
||||
|
||||
// If the API returned a 404 inside the JSON `meta` block, mark the post NotFound and return
|
||||
// Handle 404 and error codes
|
||||
if (response?.meta != null && response.meta.status == 404)
|
||||
{
|
||||
Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)");
|
||||
@@ -368,8 +465,11 @@ namespace URLNotesGrabberCORE
|
||||
Thread.Sleep(1000);
|
||||
return "NotFound";
|
||||
}
|
||||
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
|
||||
|
||||
if (response == null)
|
||||
{
|
||||
Console.WriteLine("##### Response is null - API Failure? ###");
|
||||
return "FAILURE";
|
||||
}
|
||||
if (response.statusCode == "NotFound")
|
||||
{
|
||||
Thread.Sleep(1000);
|
||||
@@ -386,50 +486,56 @@ namespace URLNotesGrabberCORE
|
||||
return response.statusCode;
|
||||
}
|
||||
|
||||
if (response.response != null && response.response.notes != null)
|
||||
// Pagination loop
|
||||
while (true)
|
||||
{
|
||||
Console.WriteLine("Notes\t" + response.response.notes.Count);
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (response.response == null)
|
||||
{
|
||||
Console.WriteLine("##### Response Null - API Failure? ###");
|
||||
Console.WriteLine($"Raw JSON: {response.rawJson}");
|
||||
return "FAILURE";
|
||||
}
|
||||
else if (response.response.notes == null)
|
||||
{
|
||||
Console.WriteLine("##### Notes Null - WHY? ###");
|
||||
return "FAILURE";
|
||||
}
|
||||
else
|
||||
{
|
||||
while (response.response != null
|
||||
&& response.response._links != null
|
||||
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
|
||||
{
|
||||
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
|
||||
if (response.response.notes is null)
|
||||
{
|
||||
Console.WriteLine("response.response.notes == null");
|
||||
return "NULL NOTES";
|
||||
}
|
||||
int noteCount = response?.response?.notes?.Count ?? 0;
|
||||
Console.WriteLine($"[GrabNotes] Page {page} | before_timestamp={beforeTimestamp} | Notes={noteCount}");
|
||||
|
||||
Console.WriteLine("Notes\t" + response.response.notes.Count);
|
||||
foreach (var note in response.response.notes)
|
||||
{
|
||||
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
|
||||
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
|
||||
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
|
||||
break;
|
||||
}
|
||||
if (response?.response == null)
|
||||
{
|
||||
Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
|
||||
return "FAILURE";
|
||||
}
|
||||
if (response.response.notes == null)
|
||||
{
|
||||
Console.WriteLine("##### Notes Null - WHY? ###");
|
||||
return "FAILURE";
|
||||
}
|
||||
|
||||
// Accumulate notes
|
||||
allNotes.AddRange(response.response.notes);
|
||||
|
||||
// Pagination: check for next
|
||||
var nextLink = response.response._links?.next;
|
||||
if (nextLink == null || string.IsNullOrEmpty(nextLink.query_params?.before_timestamp))
|
||||
{
|
||||
Console.WriteLine($"[GrabNotes] No more pages. Pagination complete after {page} page(s).");
|
||||
break;
|
||||
}
|
||||
|
||||
beforeTimestamp = nextLink.query_params.before_timestamp;
|
||||
page++;
|
||||
if (page > maxPages)
|
||||
{
|
||||
Console.WriteLine($"[GrabNotes] ERROR: Max page limit ({maxPages}) reached for {post.Item1}/{post.Item2}. Aborting further pagination.");
|
||||
break;
|
||||
}
|
||||
|
||||
// Fetch next page
|
||||
response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
|
||||
|
||||
// Process all accumulated notes
|
||||
foreach (var note in allNotes)
|
||||
{
|
||||
note.reblog_parent_blog_name = post.Item1;
|
||||
note.post_id = post.Item2.ToString();
|
||||
if (note.type == "reply")
|
||||
hasReplies = true;
|
||||
DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type);
|
||||
}
|
||||
|
||||
return "Success";
|
||||
@@ -438,8 +544,6 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
Console.WriteLine(ex.ToString());
|
||||
}
|
||||
|
||||
|
||||
return "UNKNOWN";
|
||||
}
|
||||
|
||||
@@ -461,15 +565,15 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
{
|
||||
foreach (var post in posts)
|
||||
while (posts.Count > 0)
|
||||
{
|
||||
var post = posts[0]; // Process the first post in the list
|
||||
string status;
|
||||
|
||||
using RateLimitLease lease = limiter.AttemptAcquire(1);
|
||||
if (lease.IsAcquired)
|
||||
{
|
||||
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
|
||||
// Thread.Sleep(1000);
|
||||
status = await GrabNotes(post);
|
||||
}
|
||||
else
|
||||
@@ -477,10 +581,18 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
|
||||
return;
|
||||
}
|
||||
|
||||
if (status == "Success")
|
||||
{
|
||||
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("GrabNotes Result: " + status);
|
||||
}
|
||||
|
||||
// Re-fetch the updated list after processing the current post
|
||||
posts = DataAccess.GetPosts(withoutNotesOnly, beforeDate);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -491,27 +603,18 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
|
||||
|
||||
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
|
||||
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "")
|
||||
{
|
||||
//if(!ContainsAny(path, contains))
|
||||
//{
|
||||
// return;
|
||||
//}
|
||||
// Get all directories in the current directory and sort them alphabetically
|
||||
var directories = Directory.GetDirectories(path);
|
||||
Array.Sort(directories, StringComparer.InvariantCulture);
|
||||
|
||||
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
//{
|
||||
//sw.WriteLine("=====" + path);
|
||||
//}
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
|
||||
Console.WriteLine("Directory: " + directory);
|
||||
TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
|
||||
TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
bool headerWasWritten = false;
|
||||
@@ -520,13 +623,6 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
{
|
||||
//Console.WriteLine($"=====File: {file}");
|
||||
|
||||
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
//{
|
||||
//sw.WriteLine("---" + file);
|
||||
//}
|
||||
|
||||
try
|
||||
{
|
||||
var urls = new List<string>();
|
||||
@@ -534,12 +630,11 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
foreach (string line in File.ReadLines(file))
|
||||
{
|
||||
//if (line.StartsWith(@"Reblog url: https://", StringComparison.OrdinalIgnoreCase) && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
|
||||
if (line.StartsWith(@"Post id:", StringComparison.OrdinalIgnoreCase))
|
||||
{ // if there were no files, let's still collect post info
|
||||
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
||||
{
|
||||
if (!reblog.reblogURL.Contains(@"deactivated")
|
||||
if (!reblog.reblogURL.Contains("deactivated")
|
||||
&& reblog.reblogURL.Length != 0
|
||||
&& ContainsAny(reblog.reblogURL, contains))
|
||||
{
|
||||
@@ -548,28 +643,18 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
headerWasWritten = true;
|
||||
}
|
||||
|
||||
//if (reblog.reblogURL.Contains("/blog/private")
|
||||
// || reblog.body.Contains("/blog/private"))
|
||||
//{
|
||||
// Console.WriteLine(reblog.postID);
|
||||
// Console.WriteLine(reblog.postURL);
|
||||
// Console.WriteLine(reblog.date);
|
||||
// Console.WriteLine(reblog.body);
|
||||
// Console.WriteLine(reblog.reblogKey);
|
||||
// Console.WriteLine(reblog.reblogURL);
|
||||
// Console.WriteLine(reblog.reblogName);
|
||||
// Console.WriteLine(reblog.downloadedFiles);
|
||||
//}
|
||||
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
|
||||
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
||||
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
||||
reblog.title, false);
|
||||
postsAdded++;
|
||||
|
||||
// Output hyperlink and post date
|
||||
Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
||||
}
|
||||
}
|
||||
|
||||
reblog = new ReblogRecord();
|
||||
reblog.postID = line.Substring(9).Trim();
|
||||
}
|
||||
@@ -659,8 +744,7 @@ namespace URLNotesGrabberCORE
|
||||
if ((ContainsAny(reblog.downloadedFiles, contains)
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
// && !ContainsAny(reblog.reblogURL, contains)
|
||||
&& !reblog.reblogURL.Contains(@"deactivated")
|
||||
//&& !reblog.reblogURL.Contains("deactivated")
|
||||
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
@@ -686,6 +770,10 @@ namespace URLNotesGrabberCORE
|
||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
||||
reblog.title, true);
|
||||
postsAdded++;
|
||||
|
||||
// Output hyperlink and post date
|
||||
Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user