jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: Initialize URLNotesGrabberCORE project with core application structure, data access, and configuration.

Browse Source
This commit is contained in:
jim
2025-11-19 15:51:52 -06:00
parent 3c44cbe671
commit 64ef0e9079
16 changed files with 846 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/.vscode/launch.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"args": ["-pn", "0", "999999"]
}
]
}
URLNotesGrabberCORE/DataAccess.cs
+319 -83
View File
@@ -9,32 +9,70 @@ using RestSharp.Authenticators;
using RestSharp;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
using Microsoft.Extensions.Diagnostics.Latency;
using System.Collections;
using Microsoft.Extensions.Configuration;
using System.IO;
namespace URLNotesGrabberCORE
{
class ReblogRecord
{
public string postID;
public string reblogURL;
public string reblogName;
public string downloadedFiles;
public string reblogKey;
public string answer;
public string audioCaption;
public string blogName;
public string body;
public string date;
public string downloadedFiles;
public string link;
public string photoCaption;
public string photoURL;
public string postID;
public string postURL;
public string question;
public string quote;
public string reblogKey;
public string reblogName;
public string reblogURL;
public string slug;
public string summary;
public string tags;
public string title;
public ReblogRecord()
{
postID = ".";
reblogURL = ".";
reblogName = ".";
downloadedFiles = ".";
reblogKey = ".";
answer = ".";
audioCaption = ".";
blogName = ".";
body = ".";
date = ".";
downloadedFiles = ".";
link = ".";
photoCaption = ".";
photoURL = ".";
postID = ".";
postURL = ".";
question = ".";
quote = ".";
reblogKey = ".";
reblogName = ".";
reblogURL = ".";
slug = ".";
summary = ".";
tags = ".";
title = ".";
}
}
internal class DataAccess
{
public static string Q(string input)
{
return "'" + input.Replace("'", "''") + "'";
}
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
{
// Unix timestamp is seconds past epoch
@@ -71,12 +109,11 @@ namespace URLNotesGrabberCORE
connection.Close();
}
}
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db")
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, string DBPath = @"TL.db")
{
try { AddBlog(blogName, DBPath); } catch { }
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, DBPath); } catch { }
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -84,7 +121,30 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')";
string sql = @"INSERT INTO Posts (
BlogName,
PostID,
reblogURL,
PostDate,
PostURL,
Slug,
ReblogKey,
ReblogName,
Summary,
Quote,
Body,
Tags,
Link,
PhotoURL,
PhotoCaption,
DownloadedFiles,
AudioCaption,
Question,
Answer,
Title,
HasImage
) VALUES (" +
Q(blogName) + ", " + postID +", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
@@ -94,6 +154,28 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex.Message);
///
try
{
connection.Open();
string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex2)
{
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex2.Message);
}
}
finally
{
connection.Close();
}
///
}
}
finally
@@ -132,7 +214,7 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, DBPath); } catch { }
Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName);
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -150,9 +232,10 @@ namespace URLNotesGrabberCORE
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
Console.WriteLine("^^^^^ - SHORTCUT");
return true;
Console.WriteLine("^^^^^ - SHORTCUT");
}
else
return UpdateNote(rootBlogName, noteBlogName, postID, timestamp, type, DBPath);
}
finally
{
@@ -163,31 +246,48 @@ namespace URLNotesGrabberCORE
#endregion Adds
#region Gets
public static List<Tuple<string, long, string>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
/// <summary>
///
/// </summary>
/// <param name="withoutNotesOnly"></param>
/// <param name="DBPath"></param>
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, string>> posts = new List<Tuple<string, long, string>>();
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
try
{
connection.Open();
string sql = "SELECT " +
" Posts.BlogName, " +
" Posts.PostID, " +
" Max(Notes.timestamp) as LatestNoteTimestamp " +
"FROM " +
" Posts INNER JOIN " +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " +
"WHERE NotFound = 0 ";
string sql = "SELECT " +
" Posts.BlogName, " + Environment.NewLine +
" Posts.PostID, " + Environment.NewLine +
" Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
" Posts.NotesGatheredDatetime, " + Environment.NewLine +
" CNT.CNT " + Environment.NewLine +
"FROM " + Environment.NewLine +
" Posts " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
" LEFT OUTER JOIN " + Environment.NewLine +
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
"WHERE NotFound = 0 " + Environment.NewLine;
if (withoutNotesOnly)
sql += " and HasNotesGathered = 0 ";
sql += " and HasNotesGathered = 0 " + Environment.NewLine ;
sql += "GROUP BY " +
" Posts.BlogName, Posts.PostID " +
"ORDER BY " +
" Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID";
sql += "GROUP BY " + Environment.NewLine +
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
"ORDER BY " + Environment.NewLine +
" notesgathereddatetime, Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
Console.WriteLine(withoutNotesOnly);
Console.WriteLine(sql);
Console.Write(">"); //Console.ReadKey();
Thread.Sleep(1000);
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
@@ -195,15 +295,18 @@ namespace URLNotesGrabberCORE
{
while (reader.Read())
{
Tuple<string, long, string> post = default;
string blog = null;
long id = 0;
long timestamp;
timestamp = reader.GetInt64(2); // Assuming Id is the first column
id = reader.GetInt64(1); // Assuming Id is the first column
blog = reader.GetString(0); // Assuming Title is the second column
Tuple<string, long, long, long> post = default;
string blog;
long postID = 0;
long lastNoteTimestamp;
long notesGatheredTimestamp;
post = new Tuple<string, long, string>(blog, id, timestamp.ToString());
blog = reader.GetString(0); // Assuming Title is the second column
postID = reader.GetInt64(1); // Assuming Id is the first column
lastNoteTimestamp = reader.GetInt64(2); // Assuming Id is the first column
notesGatheredTimestamp = reader.GetInt64(3); // Assuming Id is the first column
post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
posts.Add(post);
}
}
@@ -343,6 +446,46 @@ namespace URLNotesGrabberCORE
}
return blogs;
}
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
try
{
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
else
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
string blog = null;
blog = reader.GetString(0); // Assuming Title is the second column
blogs.Add(blog);
}
}
}
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return blogs;
}
#endregion Gets
#region Updates
@@ -423,6 +566,87 @@ namespace URLNotesGrabberCORE
}
public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
{
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, DBPath); } catch { }
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = string.Format("UPDATE Notes SET timestamp = '{0}' WHERE rootBlogName = '{1}' AND noteBlogName = '{2}' AND PostID = '{3}'", timestamp, rootBlogName, noteBlogName, postID);
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
{
Console.WriteLine(ex.Message);
Console.WriteLine("^^^^^ - SHORTCUT");
return true;
}
}
finally
{
connection.Close();
}
return false;
}
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Posts ";
sql += "SET postDate = " + Q(postDate) + ", ";
sql += "reblogURL = " + Q(reblogURL) + ", ";
sql += " postURL = " + Q(postURL) + ", ";
sql += " slug = " + Q(slug) + ", ";
sql += " reblogKey = " + Q(reblogKey) + ", ";
sql += " reblogName = " + Q(reblogName) + ", ";
sql += " summary = " + Q(summary) + ", ";
sql += " quote = " + Q(quote) + ", ";
sql += " body = " + Q(body) + ", ";
sql += " tags = " + Q(tags) + ", ";
sql += " link = " + Q(link) + ", ";
sql += " photoURL = " + Q(photoURL) + ", ";
sql += " photoCaption = " + Q(photoCaption) + ", ";
sql += " downloadedFiles = " + Q(downloadedFiles) + ", ";
sql += " audioCaption = " + Q(audioCaption) + ", ";
sql += " question = " + Q(question) + ", ";
sql += " answer = " + Q(answer) + ", ";
sql += " title = " + Q(title) + ", ";
sql += " hasImage = " + hasImage;
sql += " WHERE BlogName = " + Q(blogName) + " AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -483,82 +707,94 @@ namespace URLNotesGrabberCORE
internal class APIAccess
{
public static string Blog { get; set; }
public static string Blog { get; set; } = string.Empty;
private static IConfiguration Configuration { get; set; } = null!;
const string CONSUMER_KEY = "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3";
const string CONSUMER_SECRET = "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG";
const string OAUTH_TOKEN = "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J";
const string OAUTH_TOKEN_SECRET = "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w";
static APIAccess()
{
Configuration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
.Build();
public static async Task<Root> GrabNotes(string blog, long ID, string timestamp = null)
if (Configuration["TumblrApi:ConsumerKey"] == null)
throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json");
}
private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ??
throw new InvalidOperationException("ConsumerKey is not configured");
private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ??
throw new InvalidOperationException("ConsumerSecret is not configured");
private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ??
throw new InvalidOperationException("OAuthToken is not configured");
private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ??
throw new InvalidOperationException("OAuthTokenSecret is not configured");
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
{
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]";
URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
if (timestamp != null)
if (!string.IsNullOrEmpty(timestamp))
{
URL += "&before_timestamp=" + timestamp;
Thread.Sleep(1000);
await Task.Delay(1000);
}
var client = new RestClient(URL);
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY,
consumerSecret: CONSUMER_SECRET,
token: OAUTH_TOKEN,
tokenSecret: OAUTH_TOKEN_SECRET,
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
consumerSecret: ConsumerSecret,
token: OAuthToken,
tokenSecret: OAuthTokenSecret,
OAuthSignatureMethod.HmacSha1
);
//oAuth1.Realm =
client.Authenticator = oAuth1;
var request = new RestRequest(URL, Method.Get);
var response = await client.ExecuteAsync(request);
var response = client.Execute(request);
var myJsonResponse = response.Content;
Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
Root myDeserializedClass = new Root();
var myJsonResponse = response.Content ?? string.Empty;
Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
var myDeserializedClass = new Root();
try
{
myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
//if (myDeserializedClass.response.total_notes != 0)
//{
//DataAccess.AddBlog(blog);
//DataAccess.AddPost(blog, ID);
//}
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
if (deserializedResult != null)
{
myDeserializedClass = deserializedResult;
}
}
catch (Exception ex)
{
Console.WriteLine($"Failed JSON: {myJsonResponse}");
Console.WriteLine(ex.ToString());
if (!response.IsSuccessful)
{
Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription);
Console.WriteLine($"{response.StatusCode}\t{response.StatusDescription}");
myDeserializedClass.statusCode = response.StatusCode.ToString();
bool checkReset = false;
if (myDeserializedClass.statusCode != "NotFound")
if (myDeserializedClass.statusCode != "NotFound" && response.Headers != null)
{
foreach (var header in response.Headers)
{
Console.WriteLine("{0} - {1}", header.Name, header.Value);
if (checkReset && header.Name.Contains("Reset"))
if (header.Name != null && header.Value != null)
{
if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString()))
myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString());
Console.WriteLine($"{header.Name} - {header.Value}");
if (checkReset && header.Name.Contains("Reset"))
{
var headerValue = header.Value.ToString();
if (headerValue != null && int.TryParse(headerValue, out int resetValue))
{
if (myDeserializedClass.retryInSeconds < resetValue)
myDeserializedClass.retryInSeconds = resetValue;
}
}
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
checkReset = true;
else
checkReset = false;
}
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
checkReset = true;
else
checkReset = false;
//if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = "))
//{
// string temp = header.ToString().Substring(59);
// int indexOf = temp.IndexOf(',');
// myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf));
//}
}
}
}
URLNotesGrabberCORE/DualLogger.cs
+53
View File
@@ -0,0 +1,53 @@
using System;
using System.IO;
using System.Text;
namespace URLNotesGrabberCORE
{
public class DualLogger : TextWriter
{
private TextWriter _consoleWriter;
private TextWriter _fileWriter;
public DualLogger(TextWriter consoleWriter, TextWriter fileWriter)
{
_consoleWriter = consoleWriter;
_fileWriter = fileWriter;
}
public override Encoding Encoding => _consoleWriter.Encoding;
public override void Write(char value)
{
_consoleWriter.Write(value);
_fileWriter.Write(value);
}
public override void Write(string value)
{
_consoleWriter.Write(value);
_fileWriter.Write(value);
}
public override void WriteLine(string value)
{
_consoleWriter.WriteLine(value);
_fileWriter.WriteLine(value);
}
public override void Flush()
{
_consoleWriter.Flush();
_fileWriter.Flush();
}
protected override void Dispose(bool disposing)
{
if (disposing)
{
_fileWriter.Dispose();
}
base.Dispose(disposing);
}
}
}
URLNotesGrabberCORE/EmptyArrayOrObjectConverter.cs
+29
View File
@@ -0,0 +1,29 @@
using System;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
namespace URLNotesGrabberCORE
{
public class EmptyArrayOrObjectConverter<T> : JsonConverter
{
public override bool CanConvert(Type objectType)
{
return objectType == typeof(T);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
{
JToken token = JToken.Load(reader);
if (token.Type == JTokenType.Array)
{
return null;
}
return token.ToObject<T>(serializer);
}
public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
{
serializer.Serialize(writer, value);
}
}
}
URLNotesGrabberCORE/Program.cs
+309 -51
View File
@@ -3,6 +3,9 @@ using Microsoft.Extensions.Http.Resilience;
using Microsoft.Extensions;
using Microsoft.Extensions.Configuration;
using System.Configuration;
using System.Threading;
using Microsoft.Extensions.Diagnostics.Latency;
using static System.Runtime.InteropServices.JavaScript.JSType;
namespace URLNotesGrabberCORE
{
@@ -18,16 +21,51 @@ namespace URLNotesGrabberCORE
.Build();
var settings = config.GetSection("appSettings");
// Setup Dual Logging
string logPath = "console_output.log";
StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
Console.SetOut(dualLogger);
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
{
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
}
else
{
switch (args[0])
{
case "-n":
case "-?":
Console.WriteLine("\t Parse .txt files to find blogs");
Console.WriteLine("-?\t Usage help");
Console.WriteLine("-parse\t Parse .txt files with specified blogname");
Console.WriteLine("-test\t Calls API for given blogname and postID");
Console.WriteLine("-posts\t For each Post in DB, write blogname to file");
Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
break;
case "-parse":
string blogNameToParse = args[1];
TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
break;
case "-test":
#region Manual test
string blogName = args[1];
@@ -60,31 +98,50 @@ namespace URLNotesGrabberCORE
break;
case "-p": //write post's blogs to file
case "-post":
TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
break;
case "-posts": //write post's blogs to file
WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
break;
case "-b": //write blogs to file
case "-blogs": //write blogs to file
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
break;
case "-pn": //collect notes from all posts
case "-collect": //collect notes from all posts
bool withoutNotesOnly = true;
if (args[1] is not null)
if(args.Length != 2)
{
bool.TryParse(args[1], out withoutNotesOnly);
Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
break;
}
if (args.Length > 1 && args[1] is not null)
{
if (args[1] == "1")
{
withoutNotesOnly = true;
Console.WriteLine("Parsed");
}
else
{
withoutNotesOnly = false;
Console.WriteLine("--NOT Parsed");
}
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
break;
case "-br": //collect notes from all posts
case "-blogsR": //collect notes from all posts
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
break;
case "-bo": //collect notes from all posts
case "-blogsO": //collect notes from all posts
int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
@@ -100,7 +157,23 @@ namespace URLNotesGrabberCORE
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
case "-pr": //colection posts with replies
case "-bop": //collect notes from all posts
from = 1; to = 999999; top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
top = int.Parse(args[3]);
}
else
{
Console.WriteLine("--Expected FROM TO--");
}
WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
case "-replies": //colection posts with replies
try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
@@ -110,6 +183,7 @@ namespace URLNotesGrabberCORE
Console.WriteLine("** Unknown Command ** " + args[0]);
break;
}
}
@@ -119,7 +193,7 @@ namespace URLNotesGrabberCORE
static void WritePostBlogsToFile(string outPath)
{
List<Tuple<string, long, string>> posts = DataAccess.GetPosts();
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -130,6 +204,7 @@ namespace URLNotesGrabberCORE
}
blogs = blogs.Distinct().ToList();
blogs.Sort();
blogs.Reverse();
foreach (var blog in blogs)
{
@@ -143,6 +218,24 @@ namespace URLNotesGrabberCORE
{
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
blogs.Sort();
blogs.Reverse();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
foreach (var blog in blogs)
{
Console.WriteLine(blog);
sw.WriteLine(blog + ".tumblr.com");
DataAccess.UpdateBlogOutput(blog);
}
}
}
static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
{
List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
blogs.Sort();
blogs.Reverse();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -163,8 +256,8 @@ namespace URLNotesGrabberCORE
{
foreach (var post in posts)
{
Console.WriteLine("{0}\t{1}", post.Item1, post.Item2);
sw.WriteLine("{0}\t{1}", post.Item1 + ".tumblr.com", post.Item2);
Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
}
}
}
@@ -184,25 +277,25 @@ namespace URLNotesGrabberCORE
return false;
}
static async Task<string> GrabNotes(Tuple<string, long, string> post)
static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
{
try
{
int APICount = DataAccess.GetAPICount();
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3).GetAwaiter().GetResult();
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
if (response.statusCode == "NotFound")
{
Thread.Sleep(2000);
Thread.Sleep(1000);
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
return response.statusCode;
}
if (response.statusCode == "TooManyRequests")
{
for( int s = 0; s <= response.retryInSeconds; s+=15)
for (int s = 0; s <= response.retryInSeconds; s += 15)
{
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
Thread.Sleep(15000);
@@ -210,7 +303,7 @@ namespace URLNotesGrabberCORE
return response.statusCode;
}
if (response.response != null)
if (response.response != null && response.response.notes != null)
{
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
@@ -221,30 +314,43 @@ namespace URLNotesGrabberCORE
break;
}
}
if (response.response == null)
else if (response.response == null)
{
Console.WriteLine("response.response == null");
Console.WriteLine("##### Response Null - API Failure? ###");
return "FAILURE";
}
while ( response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= long.Parse(post.Item3))
else if (response.response.notes == null)
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
Console.WriteLine("##### Notes Null - WHY? ###");
return "FAILURE";
}
else
{
if (response.response == null)
{
Console.WriteLine("response.response.notes == null");
return "NULL NOTES";
Console.WriteLine("response.response == null");
return "FAILURE";
}
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
while (response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
{
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
{
Console.WriteLine("response.response.notes == null");
return "NULL NOTES";
}
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = post.Item1; note.post_id = post.Item2.ToString();
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
}
@@ -258,7 +364,7 @@ namespace URLNotesGrabberCORE
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
{
List<Tuple<string, long, string>> posts = DataAccess.GetPosts(withoutNotesOnly);
List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
PermitLimit = 300,
@@ -279,6 +385,8 @@ namespace URLNotesGrabberCORE
using RateLimitLease lease = limiter.AttemptAcquire(1);
if (lease.IsAcquired)
{
Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
Thread.Sleep(1000);
status = await GrabNotes(post);
}
else
@@ -300,7 +408,7 @@ namespace URLNotesGrabberCORE
}
static void TraverseDirectory(string path, string outPath, List<string> contains)
static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
{
// Get all directories in the current directory and sort them alphabetically
var directories = Directory.GetDirectories(path);
@@ -308,13 +416,14 @@ namespace URLNotesGrabberCORE
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("=====" + path);
//sw.WriteLine("=====" + path);
//}
foreach (var directory in directories)
{
Console.WriteLine("Directory: " + directory);
TraverseDirectory(directory, outPath, contains); // Recursively traverse subdirectories
TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
}
try
{
@@ -322,13 +431,13 @@ namespace URLNotesGrabberCORE
// Process all files in the current directory
foreach (var file in Directory.GetFiles(path))
{
if (file.EndsWith(".txt"))
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
{
//Console.WriteLine($"=====File: {file}");
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("---" + file);
//sw.WriteLine("---" + file);
//}
try
@@ -340,7 +449,40 @@ namespace URLNotesGrabberCORE
{
//if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
if (line.StartsWith(@"Post id:"))
{
{ // if there were no files, let's still collect post info
if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
{
if (!reblog.reblogURL.Contains(@"deactivated")
&& reblog.reblogURL.Length != 0
&& ContainsAny(reblog.reblogURL, contains))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
//if (reblog.reblogURL.Contains("/blog/private")
// || reblog.body.Contains("/blog/private"))
//{
// Console.WriteLine(reblog.postID);
// Console.WriteLine(reblog.postURL);
// Console.WriteLine(reblog.date);
// Console.WriteLine(reblog.body);
// Console.WriteLine(reblog.reblogKey);
// Console.WriteLine(reblog.reblogURL);
// Console.WriteLine(reblog.reblogName);
// Console.WriteLine(reblog.downloadedFiles);
//}
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, false);
}
}
reblog = new ReblogRecord();
reblog.postID = line.Substring(9).Trim();
}
@@ -366,25 +508,96 @@ namespace URLNotesGrabberCORE
{
reblog.date = line.Substring(6).Trim();
}
if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
if (line.StartsWith(@"Body:"))
{
if (ContainsAny(reblog.downloadedFiles, contains) && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated"))
reblog.body = line.Substring(6).Trim();
}
if (line.StartsWith(@"Post url:"))
{
reblog.postURL = line.Substring(10).Trim();
}
if (line.StartsWith(@"Answer:"))
{
reblog.answer = line.Substring(8).Trim();
}
if (line.StartsWith(@"Audio Caption:"))
{
reblog.audioCaption = line.Substring(15).Trim();
}
if (line.StartsWith(@"Blog Name:"))
{
reblog.blogName = line.Substring(11).Trim();
}
if (line.StartsWith(@"Link:"))
{
reblog.link = line.Substring(6).Trim();
}
if (line.StartsWith(@"Photo Caption:"))
{
reblog.photoCaption = line.Substring(15).Trim();
}
if (line.StartsWith(@"Photo url:"))
{
reblog.photoURL = line.Substring(11).Trim();
}
if (line.StartsWith(@"Question:"))
{
reblog.question = line.Substring(10).Trim();
}
if (line.StartsWith(@"Quote:"))
{
reblog.quote = line.Substring(7).Trim();
}
if (line.StartsWith(@"Slug:"))
{
reblog.slug = line.Substring(6).Trim();
}
if (line.StartsWith(@"Summary:"))
{
reblog.summary = line.Substring(9).Trim();
}
if (line.StartsWith(@"Tags:"))
{
reblog.tags = line.Substring(6).Trim();
}
if (line.StartsWith(@"Title:"))
{
reblog.title = line.Substring(7).Trim();
}
if ((reblog.downloadedFiles != "."
|| reblog.reblogURL.Contains("/blog/private")
|| reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
{
if ((ContainsAny(reblog.downloadedFiles, contains)
|| reblog.reblogURL.Contains("/blog/private")
|| reblog.body.Contains("/blog/private"))
// && !ContainsAny(reblog.reblogURL, contains)
&& !reblog.reblogURL.Contains(@"deactivated"))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
//Console.WriteLine(reblog.postID);
//Console.WriteLine(reblog.date);
//Console.WriteLine(reblog.reblogKey);
//Console.WriteLine(reblog.reblogURL);
//Console.WriteLine(reblog.reblogName);
//Console.WriteLine(reblog.downloadedFiles);
//if (reblog.reblogURL.Contains("/blog/private")
// || reblog.body.Contains("/blog/private"))
//{
// Console.WriteLine(reblog.postID);
// Console.WriteLine(reblog.postURL);
// Console.WriteLine(reblog.date);
// Console.WriteLine(reblog.body);
// Console.WriteLine(reblog.reblogKey);
// Console.WriteLine(reblog.reblogURL);
// Console.WriteLine(reblog.reblogName);
// Console.WriteLine(reblog.downloadedFiles);
//}
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date );
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
reblog.title, true);
}
}
}
@@ -413,5 +626,50 @@ namespace URLNotesGrabberCORE
Console.WriteLine($"An error occurred: {ex.Message}");
}
}
static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
{
var directories = Directory.GetDirectories(path);
Array.Sort(directories, StringComparer.InvariantCulture);
foreach (var directory in directories)
{
//Console.WriteLine("Directory: " + directory);
TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
}
try
{
bool headerWasWritten = false;
foreach (var file in Directory.GetFiles(path))
{
if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
{
try
{
var urls = new List<string>();
var reblog = new ReblogRecord();
foreach (string line in File.ReadLines(file))
{
if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
{
Console.WriteLine(file);
}
}
}
catch (Exception e)
{
Console.WriteLine("The file could not be read:");
Console.WriteLine(e.Message);
}
}
}
}
catch (Exception ex)
{
Console.WriteLine($"An error occurred: {ex.Message}");
}
}
}
}
URLNotesGrabberCORE/Properties/launchSettings.json
+1 -1
View File
@@ -2,7 +2,7 @@
"profiles": {
"URLNotesGrabberCORE": {
"commandName": "Project",
"commandLineArgs": "-pn true 31"
"commandLineArgs": "-collect 1"
}
}
}
URLNotesGrabberCORE/ResponseNotes.cs
+1
View File
@@ -74,6 +74,7 @@ namespace URLNotesGrabberCORE
public class Root
{
public Meta meta { get; set; }
[JsonConverter(typeof(EmptyArrayOrObjectConverter<Response>))]
public Response response { get; set; }
public string statusCode { get; set; }
URLNotesGrabberCORE/TL.db
BIN
View File
Binary file not shown.
URLNotesGrabberCORE/TL.db-journal
BIN
View File
Binary file not shown.
URLNotesGrabberCORE/TLDB.7z
BIN
View File
Binary file not shown.
URLNotesGrabberCORE/URLNotesGrabberCORE.csproj
+2
View File
@@ -7,6 +7,8 @@
<Nullable>enable</Nullable>
</PropertyGroup>
<PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'" />
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
URLNotesGrabberCORE/URLNotesGrabberCORE.sln
+24
View File
@@ -0,0 +1,24 @@
Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 17
VisualStudioVersion = 17.5.2.0
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "URLNotesGrabberCORE", "URLNotesGrabberCORE.csproj", "{008F0DF4-6E08-F5FD-D644-B9B999F533E7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {C1F768CD-5EA9-444F-87CF-923EE287ACDB}
EndGlobalSection
EndGlobal
URLNotesGrabberCORE/appsettings.json
+6
View File
@@ -8,5 +8,11 @@
"PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
"PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
},
"TumblrApi": {
"ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",
"ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG",
"OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J",
"OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w"
}
}