Improve import speed, add URL dump, and CLI options
- Add SQLite import mode pragmas for faster bulk inserts - Implement -urldump command to extract all URLs from posts - Add -start [blogname] CLI option for partial traversal - Support toggling file logging and record import logging via config - Only update DB rows if values change to reduce writes - Add DateCreated fields to relevant tables - Enhance logging and output formatting - Refactor directory traversal for better control and reporting
This commit is contained in:
+383
-216
@@ -6,6 +6,7 @@ using System.Configuration;
|
||||
using System.Threading;
|
||||
using Microsoft.Extensions.Diagnostics.Latency;
|
||||
using static System.Runtime.InteropServices.JavaScript.JSType;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace URLNotesGrabberCORE
|
||||
{
|
||||
@@ -22,6 +23,7 @@ namespace URLNotesGrabberCORE
|
||||
var settings = config.GetSection("appSettings");
|
||||
|
||||
string apiSectionName = "TumblrApi";
|
||||
string startFromBlogName = string.Empty;
|
||||
List<string> filteredArgs = new List<string>();
|
||||
for (int i = 0; i < args.Length; i++)
|
||||
{
|
||||
@@ -54,6 +56,21 @@ namespace URLNotesGrabberCORE
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(args[i], "-start", StringComparison.OrdinalIgnoreCase) ||
|
||||
string.Equals(args[i], "--start", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (i + 1 < args.Length && !string.IsNullOrWhiteSpace(args[i + 1]))
|
||||
{
|
||||
startFromBlogName = args[i + 1].Trim();
|
||||
i++;
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("--Missing blog name after -start/--start. Ignoring.--");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
filteredArgs.Add(args[i]);
|
||||
}
|
||||
|
||||
@@ -61,32 +78,45 @@ namespace URLNotesGrabberCORE
|
||||
APIAccess.SetApiConfigSection(apiSectionName);
|
||||
|
||||
// Setup Dual Logging
|
||||
string logPath = "console_output.log";
|
||||
if (File.Exists(logPath))
|
||||
bool enableFileLogging = settings.GetValue("EnableFileLogging", true);
|
||||
if (enableFileLogging)
|
||||
{
|
||||
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
|
||||
string archiveDirectory = "logs";
|
||||
|
||||
// Ensure the archive directory exists
|
||||
if (!Directory.Exists(archiveDirectory))
|
||||
string logPath = "console_output.log";
|
||||
if (File.Exists(logPath))
|
||||
{
|
||||
Directory.CreateDirectory(archiveDirectory);
|
||||
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss");
|
||||
string archiveDirectory = "logs";
|
||||
|
||||
// Ensure the archive directory exists
|
||||
if (!Directory.Exists(archiveDirectory))
|
||||
{
|
||||
Directory.CreateDirectory(archiveDirectory);
|
||||
}
|
||||
|
||||
string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
|
||||
File.Move(logPath, newPath);
|
||||
}
|
||||
|
||||
string newPath = Path.Combine(archiveDirectory, $"console_output_{timestamp}.log");
|
||||
File.Move(logPath, newPath);
|
||||
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
|
||||
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
|
||||
Console.SetOut(dualLogger);
|
||||
}
|
||||
|
||||
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
|
||||
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
|
||||
Console.SetOut(dualLogger);
|
||||
|
||||
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
|
||||
bool logTraversalRecordImports = settings.GetValue("LogTraversalRecordImports", false);
|
||||
|
||||
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
|
||||
{
|
||||
int postsAdded = 0;
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded);
|
||||
try
|
||||
{
|
||||
DataAccess.EnableImportModePragmas();
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, startFromBlogName: startFromBlogName, logRecordImports: logTraversalRecordImports);
|
||||
}
|
||||
finally
|
||||
{
|
||||
DataAccess.RestoreImportModePragmas();
|
||||
}
|
||||
Console.WriteLine($"Total posts added: {postsAdded}");
|
||||
}
|
||||
else
|
||||
@@ -116,10 +146,14 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
Console.WriteLine("-likes\t Fetch likes for all blogs needing it (LikesPulled=0), or a specific blog via param");
|
||||
|
||||
Console.WriteLine("-urldump\t Scan all posts' text columns and extract suspected URLs to configured file");
|
||||
|
||||
Console.WriteLine("-api3\t Use TumblrApi3 settings from appsettings.json");
|
||||
|
||||
Console.WriteLine("-api4\t Use TumblrApi4 settings from appsettings.json");
|
||||
|
||||
Console.WriteLine("-start [blogname]\t Start traversal alphabetically at this blog name");
|
||||
|
||||
Console.WriteLine("-api [section]\t Use a specific API settings section from appsettings.json (e.g. TumblrApi3)");
|
||||
|
||||
break;
|
||||
@@ -127,7 +161,15 @@ namespace URLNotesGrabberCORE
|
||||
case "-parse":
|
||||
string blogNameToParse = args[1];
|
||||
int postsAdded = 0;
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse);
|
||||
try
|
||||
{
|
||||
DataAccess.EnableImportModePragmas();
|
||||
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, ref postsAdded, blogNameToParse, startFromBlogName, logTraversalRecordImports);
|
||||
}
|
||||
finally
|
||||
{
|
||||
DataAccess.RestoreImportModePragmas();
|
||||
}
|
||||
Console.WriteLine($"Total posts added: {postsAdded}");
|
||||
break;
|
||||
|
||||
@@ -237,6 +279,10 @@ namespace URLNotesGrabberCORE
|
||||
CollectLikes(likeBlog, contains).GetAwaiter().GetResult();
|
||||
break;
|
||||
|
||||
case "-urldump":
|
||||
DumpUrls(settings.GetValue<string>("PathOutputUrls"));
|
||||
break;
|
||||
|
||||
default:
|
||||
Console.WriteLine("** Unknown Command ** " + args[0]);
|
||||
break;
|
||||
@@ -320,6 +366,45 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
static void DumpUrls(string? outPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(outPath))
|
||||
{
|
||||
Console.WriteLine("--PathOutputUrls is not configured in appsettings.json--");
|
||||
return;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Starting URL extraction to {outPath}...");
|
||||
HashSet<string> uniqueUrls = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
Regex urlRegex = new Regex(@"https?://[^\s""'<>]+", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
|
||||
int rowsProcessed = 0;
|
||||
foreach (var texts in DataAccess.GetAllPostTextColumns())
|
||||
{
|
||||
rowsProcessed++;
|
||||
if (rowsProcessed % 10000 == 0)
|
||||
{
|
||||
Console.WriteLine($"Scanned {rowsProcessed} rows... found {uniqueUrls.Count} unique URLs so far.");
|
||||
}
|
||||
|
||||
foreach (var text in texts)
|
||||
{
|
||||
var matches = urlRegex.Matches(text);
|
||||
foreach (Match match in matches)
|
||||
{
|
||||
uniqueUrls.Add(match.Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"Scan complete. Sorting and saving {uniqueUrls.Count} distinct URLs...");
|
||||
var sortedUrls = uniqueUrls.ToList();
|
||||
sortedUrls.Sort();
|
||||
|
||||
File.WriteAllLines(outPath, sortedUrls);
|
||||
Console.WriteLine($"Saved URLs to {outPath}");
|
||||
}
|
||||
|
||||
protected static bool ContainsAny(string input, List<string> contains)
|
||||
{
|
||||
if (string.IsNullOrEmpty(input)) return false;
|
||||
@@ -335,6 +420,20 @@ namespace URLNotesGrabberCORE
|
||||
return false;
|
||||
}
|
||||
|
||||
protected static string NormalizeBlogFolderName(string folderName)
|
||||
{
|
||||
return folderName
|
||||
.Replace("_1", "")
|
||||
.Replace("_2", "")
|
||||
.Replace("_3", "")
|
||||
.Replace("_4", "")
|
||||
.Replace("_5", "")
|
||||
.Replace("_6", "")
|
||||
.Replace("_7", "")
|
||||
.Replace("_8", "")
|
||||
.Replace("_9", "");
|
||||
}
|
||||
|
||||
static async Task FetchAndStoreReplyText(string blogName, long postID, long timestamp)
|
||||
{
|
||||
try
|
||||
@@ -494,6 +593,44 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
static ConsoleColor GetMatchedCounterColor(long matchedCount)
|
||||
{
|
||||
if (matchedCount <= 0)
|
||||
return ConsoleColor.White;
|
||||
|
||||
return ((matchedCount - 1) % 3) switch
|
||||
{
|
||||
0 => ConsoleColor.Red,
|
||||
1 => ConsoleColor.Green,
|
||||
_ => ConsoleColor.Blue
|
||||
};
|
||||
}
|
||||
|
||||
static void WriteLikesTotalsLine(string blogName, string label, long parsedForBlog, long matchedForBlog, int likedCountForBlog)
|
||||
{
|
||||
if (likedCountForBlog > 0)
|
||||
{
|
||||
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
||||
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
||||
|
||||
Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: ");
|
||||
var previousColor = Console.ForegroundColor;
|
||||
Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
|
||||
Console.Write(matchedForBlog);
|
||||
Console.ForegroundColor = previousColor;
|
||||
Console.WriteLine($"/{likedCountForBlog} ({matchedPct:F2}%)");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.Write($"[Likes] {blogName} {label} | Parsed: {parsedForBlog} | Matched: ");
|
||||
var previousColor = Console.ForegroundColor;
|
||||
Console.ForegroundColor = GetMatchedCounterColor(matchedForBlog);
|
||||
Console.Write(matchedForBlog);
|
||||
Console.ForegroundColor = previousColor;
|
||||
Console.WriteLine();
|
||||
}
|
||||
}
|
||||
|
||||
static async Task CollectLikes(string specificBlog, List<string> contains)
|
||||
{
|
||||
try
|
||||
@@ -691,16 +828,7 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
}
|
||||
|
||||
if (likedCountForBlog > 0)
|
||||
{
|
||||
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
||||
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
||||
Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"[Likes] {blogName} Running Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}");
|
||||
}
|
||||
WriteLikesTotalsLine(blogName, "Running Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
||||
|
||||
// Determine the next BeforeCursor.
|
||||
long nextCursor = 0;
|
||||
@@ -728,16 +856,7 @@ namespace URLNotesGrabberCORE
|
||||
await Task.Delay(1000); // 1-second delay between pages
|
||||
}
|
||||
|
||||
if (likedCountForBlog > 0)
|
||||
{
|
||||
double parsedPct = (double)parsedForBlog / likedCountForBlog * 100.0;
|
||||
double matchedPct = (double)matchedForBlog / likedCountForBlog * 100.0;
|
||||
Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog}/{likedCountForBlog} ({parsedPct:F2}%) | Matched: {matchedForBlog}/{likedCountForBlog} ({matchedPct:F2}%)");
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine($"[Likes] {blogName} Final Totals | Parsed: {parsedForBlog} | Matched: {matchedForBlog}");
|
||||
}
|
||||
WriteLikesTotalsLine(blogName, "Final Totals", parsedForBlog, matchedForBlog, likedCountForBlog);
|
||||
}
|
||||
|
||||
Console.WriteLine("Likes collection complete.");
|
||||
@@ -753,6 +872,11 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
try
|
||||
{
|
||||
static bool IsNotFound(Root? r)
|
||||
{
|
||||
return (r?.meta != null && r.meta.status == 404) || string.Equals(r?.statusCode, "NotFound", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
int APICount = DataAccess.GetAPICount();
|
||||
Console.WriteLine($"{post.Item1}\t{post.Item2}\t{DateTime.Now}\t{APICount}");
|
||||
//Thread.Sleep(3000);
|
||||
@@ -765,9 +889,9 @@ namespace URLNotesGrabberCORE
|
||||
var response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
|
||||
|
||||
// Handle 404 and error codes
|
||||
if (response?.meta != null && response.meta.status == 404)
|
||||
if (IsNotFound(response))
|
||||
{
|
||||
Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (meta.status=404)");
|
||||
Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2}");
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
Thread.Sleep(1000);
|
||||
return "NotFound";
|
||||
@@ -777,12 +901,6 @@ namespace URLNotesGrabberCORE
|
||||
Console.WriteLine("##### Response is null - API Failure? ###");
|
||||
return "FAILURE";
|
||||
}
|
||||
if (response.statusCode == "NotFound")
|
||||
{
|
||||
Thread.Sleep(1000);
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
return response.statusCode;
|
||||
}
|
||||
if (response.statusCode == "TooManyRequests")
|
||||
{
|
||||
for (int s = 0; s <= response.retryInSeconds; s += 60)
|
||||
@@ -801,11 +919,23 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
if (response?.response == null)
|
||||
{
|
||||
if (IsNotFound(response))
|
||||
{
|
||||
Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (empty response payload)");
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
return "NotFound";
|
||||
}
|
||||
Console.WriteLine($"##### Response Null - API Failure? ###\nRaw JSON: {response?.rawJson}");
|
||||
return "FAILURE";
|
||||
}
|
||||
if (response.response.notes == null)
|
||||
{
|
||||
if (IsNotFound(response))
|
||||
{
|
||||
Console.WriteLine($"API returned 404 Not Found for {post.Item1}/{post.Item2} (notes payload missing)");
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
return "NotFound";
|
||||
}
|
||||
Console.WriteLine("##### Notes Null - WHY? ###");
|
||||
return "FAILURE";
|
||||
}
|
||||
@@ -831,6 +961,12 @@ namespace URLNotesGrabberCORE
|
||||
|
||||
// Fetch next page
|
||||
response = APIAccess.GrabNotes(post.Item1, post.Item2, beforeTimestamp).GetAwaiter().GetResult();
|
||||
if (IsNotFound(response))
|
||||
{
|
||||
Console.WriteLine($"API returned 404 Not Found during pagination for {post.Item1}/{post.Item2}");
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
return "NotFound";
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine($"[GrabNotes] Total notes accumulated: {allNotes.Count}");
|
||||
@@ -893,15 +1029,14 @@ namespace URLNotesGrabberCORE
|
||||
{
|
||||
DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
|
||||
}
|
||||
else if (status == "NotFound")
|
||||
{
|
||||
Console.WriteLine("GrabNotes Result: NotFound");
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.WriteLine("GrabNotes Result: " + status);
|
||||
|
||||
//if not success, mark as not found to avoid repeated attempts, unless it was a rate limit issue
|
||||
if (status != "RateLimitExceeded")
|
||||
{
|
||||
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
|
||||
}
|
||||
}
|
||||
|
||||
// Re-fetch the updated list after processing the current post
|
||||
@@ -916,203 +1051,235 @@ namespace URLNotesGrabberCORE
|
||||
}
|
||||
|
||||
|
||||
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "")
|
||||
static void TraverseDirectory(string path, string outPath, List<string> contains, ref int postsAdded, string blogName = "", string startFromBlogName = "", bool logRecordImports = false)
|
||||
{
|
||||
// Get all directories in the current directory and sort them alphabetically
|
||||
var directories = Directory.GetDirectories(path);
|
||||
Array.Sort(directories, StringComparer.InvariantCulture);
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
Console.WriteLine("Directory: " + directory);
|
||||
TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName); // Recursively traverse subdirectories
|
||||
}
|
||||
DateTime directoryStart = DateTime.Now;
|
||||
int directoryRecordsImported = 0;
|
||||
Console.WriteLine($"[Directory Start] {path} | {directoryStart:yyyy-MM-dd HH:mm:ss.fff}");
|
||||
|
||||
try
|
||||
{
|
||||
bool headerWasWritten = false;
|
||||
// Process all files in the current directory
|
||||
foreach (var file in Directory.GetFiles(path))
|
||||
{
|
||||
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase) && (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
{
|
||||
try
|
||||
{
|
||||
var urls = new List<string>();
|
||||
var reblog = new ReblogRecord();
|
||||
// Get all directories in the current directory and sort them alphabetically
|
||||
var directories = Directory.GetDirectories(path);
|
||||
Array.Sort(directories, StringComparer.InvariantCulture);
|
||||
|
||||
foreach (string line in File.ReadLines(file))
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
Console.WriteLine("Directory: " + directory);
|
||||
TraverseDirectory(directory, outPath, contains, ref postsAdded, blogName, startFromBlogName, logRecordImports); // Recursively traverse subdirectories
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
bool headerWasWritten = false;
|
||||
// Process all files in the current directory
|
||||
foreach (var file in Directory.GetFiles(path))
|
||||
{
|
||||
string normalizedDirectoryName = NormalizeBlogFolderName(new DirectoryInfo(path).Name);
|
||||
bool isAtOrAfterStart = string.IsNullOrWhiteSpace(startFromBlogName) || string.Compare(normalizedDirectoryName, startFromBlogName, StringComparison.OrdinalIgnoreCase) >= 0;
|
||||
|
||||
if (file.EndsWith(".txt", StringComparison.OrdinalIgnoreCase)
|
||||
&& (string.IsNullOrEmpty(blogName) || path.IndexOf(blogName, StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
&& isAtOrAfterStart)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
||||
var urls = new List<string>();
|
||||
var reblog = new ReblogRecord();
|
||||
|
||||
foreach (string line in File.ReadLines(file))
|
||||
{
|
||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
||||
if (line.StartsWith("Post id:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
if (!reblog.reblogURL.Contains("deactivated")
|
||||
&& reblog.reblogURL.Length != 0
|
||||
&& ContainsAny(reblog.reblogURL, contains))
|
||||
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
|
||||
{
|
||||
if (!reblog.reblogURL.Contains("deactivated")
|
||||
&& reblog.reblogURL.Length != 0
|
||||
&& ContainsAny(reblog.reblogURL, contains))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
if (!headerWasWritten)
|
||||
{
|
||||
headerWasWritten = true;
|
||||
}
|
||||
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
||||
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
||||
|
||||
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
||||
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
||||
reblog.title, false);
|
||||
recordImportStopwatch.Stop();
|
||||
|
||||
postsAdded++;
|
||||
directoryRecordsImported++;
|
||||
if (logRecordImports)
|
||||
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
||||
|
||||
// Output hyperlink and post date
|
||||
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
||||
}
|
||||
}
|
||||
reblog = new ReblogRecord();
|
||||
reblog.postID = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
//reblog = new ReblogRecord();
|
||||
|
||||
reblog.reblogURL = line.Substring(12).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogName = line.Substring(13).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.downloadedFiles = line.Substring(17).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogKey = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.date = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.body = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.postURL = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.answer = line.Substring(8).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.audioCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.blogName = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.link = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoURL = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.question = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.quote = line.Substring(7).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.slug = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.summary = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.tags = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.title = line.Substring(7).Trim();
|
||||
}
|
||||
|
||||
if ((reblog.downloadedFiles != "."
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
{
|
||||
if ((ContainsAny(reblog.downloadedFiles, contains)
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
//&& !reblog.reblogURL.Contains("deactivated")
|
||||
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
if (!headerWasWritten)
|
||||
{
|
||||
headerWasWritten = true;
|
||||
}
|
||||
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
|
||||
//if (reblog.reblogURL.Contains("/blog/private")
|
||||
// || reblog.body.Contains("/blog/private"))
|
||||
//{
|
||||
// Console.WriteLine(reblog.postID);
|
||||
// Console.WriteLine(reblog.postURL);
|
||||
// Console.WriteLine(reblog.date);
|
||||
// Console.WriteLine(reblog.body);
|
||||
// Console.WriteLine(reblog.reblogKey);
|
||||
// Console.WriteLine(reblog.reblogURL);
|
||||
// Console.WriteLine(reblog.reblogName);
|
||||
// Console.WriteLine(reblog.downloadedFiles);
|
||||
//}
|
||||
string curDir = NormalizeBlogFolderName(currentDir.Name);
|
||||
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
||||
|
||||
var recordImportStopwatch = System.Diagnostics.Stopwatch.StartNew();
|
||||
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
||||
reblog.title, false);
|
||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
||||
reblog.title, true);
|
||||
recordImportStopwatch.Stop();
|
||||
|
||||
postsAdded++;
|
||||
directoryRecordsImported++;
|
||||
if (logRecordImports)
|
||||
Console.WriteLine($"[Record Import] {curDir}/{reblog.postID} | {recordImportStopwatch.Elapsed.TotalMilliseconds:F2} ms | DirectoryCount={directoryRecordsImported} | TotalCount={postsAdded}");
|
||||
|
||||
// Output hyperlink and post date
|
||||
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
||||
}
|
||||
}
|
||||
reblog = new ReblogRecord();
|
||||
reblog.postID = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
//reblog = new ReblogRecord();
|
||||
urls.Sort();
|
||||
|
||||
reblog.reblogURL = line.Substring(12).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogName = line.Substring(13).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Downloaded files:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.downloadedFiles = line.Substring(17).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Reblog key:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.reblogKey = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Date:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.date = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Body:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.body = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Post url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.postURL = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Answer:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.answer = line.Substring(8).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Audio Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.audioCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Blog Name:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.blogName = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Link:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.link = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo Caption:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoCaption = line.Substring(15).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Photo url:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.photoURL = line.Substring(11).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Question:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.question = line.Substring(10).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Quote:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.quote = line.Substring(7).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Slug:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.slug = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Summary:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.summary = line.Substring(9).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Tags:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.tags = line.Substring(6).Trim();
|
||||
}
|
||||
if (line.StartsWith(@"Title:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
reblog.title = line.Substring(7).Trim();
|
||||
}
|
||||
|
||||
if ((reblog.downloadedFiles != "."
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
|
||||
{
|
||||
if ((ContainsAny(reblog.downloadedFiles, contains)
|
||||
|| (reblog.reblogURL?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0)
|
||||
|| (reblog.body?.IndexOf("/blog/private", StringComparison.OrdinalIgnoreCase) >= 0))
|
||||
//&& !reblog.reblogURL.Contains("deactivated")
|
||||
|| path.Contains("zomb-eh", StringComparison.InvariantCultureIgnoreCase))
|
||||
{
|
||||
DirectoryInfo currentDir = new DirectoryInfo(path);
|
||||
if (!headerWasWritten)
|
||||
{
|
||||
headerWasWritten = true;
|
||||
}
|
||||
//if (reblog.reblogURL.Contains("/blog/private")
|
||||
// || reblog.body.Contains("/blog/private"))
|
||||
//{
|
||||
// Console.WriteLine(reblog.postID);
|
||||
// Console.WriteLine(reblog.postURL);
|
||||
// Console.WriteLine(reblog.date);
|
||||
// Console.WriteLine(reblog.body);
|
||||
// Console.WriteLine(reblog.reblogKey);
|
||||
// Console.WriteLine(reblog.reblogURL);
|
||||
// Console.WriteLine(reblog.reblogName);
|
||||
// Console.WriteLine(reblog.downloadedFiles);
|
||||
//}
|
||||
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
|
||||
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
|
||||
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
|
||||
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
|
||||
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
|
||||
reblog.title, true);
|
||||
postsAdded++;
|
||||
|
||||
// Output hyperlink and post date
|
||||
//Console.WriteLine($"https://{reblog.reblogName}.tumblr.com/post/{reblog.postID} - {reblog.date}");
|
||||
}
|
||||
}
|
||||
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
//{
|
||||
// foreach (string line in urls.Distinct())
|
||||
// {
|
||||
// sw.WriteLine(line);
|
||||
// }
|
||||
//}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("The file could not be read:");
|
||||
Console.WriteLine(e.Message);
|
||||
}
|
||||
urls.Sort();
|
||||
|
||||
|
||||
//using (StreamWriter sw = new StreamWriter(outPath, true))
|
||||
//{
|
||||
// foreach (string line in urls.Distinct())
|
||||
// {
|
||||
// sw.WriteLine(line);
|
||||
// }
|
||||
//}
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine("The file could not be read:");
|
||||
Console.WriteLine(e.Message);
|
||||
}
|
||||
// Add your file processing logic here
|
||||
}
|
||||
// Add your file processing logic here
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"An error occurred: {ex.Message}");
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
finally
|
||||
{
|
||||
Console.WriteLine($"An error occurred: {ex.Message}");
|
||||
DateTime directoryEnd = DateTime.Now;
|
||||
TimeSpan elapsed = directoryEnd - directoryStart;
|
||||
Console.WriteLine($"[Directory End] {path} | {directoryEnd:yyyy-MM-dd HH:mm:ss.fff} | Duration: {elapsed:hh\\:mm\\:ss\\.fff} | Records Imported: {directoryRecordsImported}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user