jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
jim
2024-11-15 09:17:53 -06:00
parent 5cddf2a427
commit 3c44cbe671
5 changed files with 248 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+111 -21
View File
@@ -8,6 +8,7 @@ using RestSharp.Authenticators.OAuth;
using RestSharp.Authenticators;
using RestSharp;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
namespace URLNotesGrabberCORE
{
@@ -34,6 +35,14 @@ namespace URLNotesGrabberCORE
internal class DataAccess
{
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
{
// Unix timestamp is seconds past epoch
DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
return dateTime;
}
#region Adds
public static void AddBlog(string blogName, string DBPath = @"TL.db")
{
@@ -50,6 +59,7 @@ namespace URLNotesGrabberCORE
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
Console.WriteLine("+ " + blogName);
}
catch (Exception ex)
{
@@ -63,9 +73,10 @@ namespace URLNotesGrabberCORE
}
public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db")
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db")
{
try { AddBlog(blogName, DBPath); } catch { }
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -73,7 +84,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')";
string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
@@ -81,8 +92,10 @@ namespace URLNotesGrabberCORE
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex.Message);
}
}
finally
{
connection.Close();
@@ -119,6 +132,8 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, DBPath); } catch { }
Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -148,7 +163,64 @@ namespace URLNotesGrabberCORE
#endregion Adds
#region Gets
public static List<Tuple<string, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
public static List<Tuple<string, long, string>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, string>> posts = new List<Tuple<string, long, string>>();
try
{
connection.Open();
string sql = "SELECT " +
" Posts.BlogName, " +
" Posts.PostID, " +
" Max(Notes.timestamp) as LatestNoteTimestamp " +
"FROM " +
" Posts INNER JOIN " +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " +
"WHERE NotFound = 0 ";
if (withoutNotesOnly)
sql += " and HasNotesGathered = 0 ";
sql += "GROUP BY " +
" Posts.BlogName, Posts.PostID " +
"ORDER BY " +
" Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
Tuple<string, long, string> post = default;
string blog = null;
long id = 0;
long timestamp;
timestamp = reader.GetInt64(2); // Assuming Id is the first column
id = reader.GetInt64(1); // Assuming Id is the first column
blog = reader.GetString(0); // Assuming Title is the second column
post = new Tuple<string, long, string>(blog, id, timestamp.ToString());
posts.Add(post);
}
}
}
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
@@ -157,16 +229,8 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "SELECT " +
" MIN(blogName) as blogName, " +
" postID " +
"FROM" +
" Posts ";
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
if (withoutNotesOnly)
sql += " WHERE HasNotesGathered = 0 and NotFound = 0";
sql += " GROUP BY postID ORDER BY reblogURL, PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
@@ -176,8 +240,9 @@ namespace URLNotesGrabberCORE
Tuple<string, long> post = default;
string blog = null;
long id = 0;
id = reader.GetInt64(1); // Assuming Id is the first column
blog = reader.GetString(0); // Assuming Title is the second column
id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
post = new Tuple<string, long>(blog, id);
posts.Add(post);
@@ -240,7 +305,7 @@ namespace URLNotesGrabberCORE
}
public static List<string> GetBlogs(bool reblogsOnly, string DBPath = @"TL.db")
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
@@ -250,9 +315,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName";
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
else
sql = "SELECT * FROM Blogs ORDER BY BlogName";
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
@@ -291,7 +356,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
@@ -333,7 +398,32 @@ namespace URLNotesGrabberCORE
}
public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db")
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -407,7 +497,7 @@ namespace URLNotesGrabberCORE
if (timestamp != null)
{
URL += "&before_timestamp=" + timestamp;
Thread.Sleep(500);
Thread.Sleep(1000);
}
var client = new RestClient(URL);
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY,
@@ -425,7 +515,7 @@ namespace URLNotesGrabberCORE
var response = client.Execute(request);
var myJsonResponse = response.Content;
Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
Root myDeserializedClass = new Root();
try
URLNotesGrabberCORE/Program.cs
+113 -56
View File
@@ -18,21 +18,27 @@ namespace URLNotesGrabberCORE
.Build();
var settings = config.GetSection("appSettings");
List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
{
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
}
else
switch(args[0])
switch (args[0])
{
case "-n": //Manual test
case "-n":
#region Manual test
var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
string blogName = args[1];
long postID = long.Parse(args[2]);
var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
@@ -40,16 +46,17 @@ namespace URLNotesGrabberCORE
while (response.response._links != null)
{
response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
foreach (var note in response.response.notes)
{
note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
//notes.Add(new Tuple<string, string>(note.blog_url, note.type));
if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
break;
}
}
#endregion
break;
@@ -62,23 +69,57 @@ namespace URLNotesGrabberCORE
break;
case "-pn": //collect notes from all posts
CollectNotes(settings.GetValue<string>("PathOutput"));
bool withoutNotesOnly = true;
if (args[1] is not null)
{
bool.TryParse(args[1], out withoutNotesOnly);
Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
}
CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
break;
case "-br": //collect notes from all posts
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
break;
case "-bo": //collect notes from all posts
int from = 1, to = 999999, top = 100;
if (args[1] is not null && args[2] is not null && args[3] is not null)
{
from = int.Parse(args[1]);
to = int.Parse(args[2]);
top = int.Parse(args[3]);
}
else
{
Console.WriteLine("--Expected FROM TO--");
}
WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
break;
case "-pr": //colection posts with replies
try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
break;
default:
Console.WriteLine("** Unknown Command ** " + args[0]);
break;
}
System.Console.WriteLine("<fin>:/");
System.Console.ReadKey();
//System.Console.ReadKey();
}
static void WritePostBlogsToFile(string outPath)
{
List<Tuple<string, long>> posts = DataAccess.GetPosts();
List<Tuple<string, long, string>> posts = DataAccess.GetPosts();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
@@ -98,9 +139,9 @@ namespace URLNotesGrabberCORE
}
}
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false)
static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
{
List<string> blogs = DataAccess.GetBlogs(reblogsOnly);
List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
blogs.Sort();
using (StreamWriter sw = new StreamWriter(outPath, true))
@@ -109,24 +150,31 @@ namespace URLNotesGrabberCORE
{
Console.WriteLine(blog);
sw.WriteLine(blog + ".tumblr.com");
DataAccess.UpdatePostBlogOutput(blog);
DataAccess.UpdateBlogOutput(blog);
}
}
}
protected static bool ContainsAny(string input)
static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
{
List<Tuple<string, long>> posts = DataAccess.GetReplies();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
foreach (var post in posts)
{
Console.WriteLine("{0}\t{1}", post.Item1, post.Item2);
sw.WriteLine("{0}\t{1}", post.Item1 + ".tumblr.com", post.Item2);
}
}
}
protected static bool ContainsAny(string input, List<string> contains)
{
if (string.IsNullOrEmpty(input)) return false;
var items = new List<string>();
items.Add("zombaee");
items.Add("zomb-eh");
items.Add("ahzombae");
items.Add("thebugandme");
items.Add("lovingbabybug");
items.Add("swarthyvillain");
foreach (string item in items)
foreach (string item in contains)
{
if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
{
@@ -136,28 +184,28 @@ namespace URLNotesGrabberCORE
return false;
}
static async Task<string> GrabNotes(Tuple<string, long> post)
static async Task<string> GrabNotes(Tuple<string, long, string> post)
{
try
{
int APICount = DataAccess.GetAPICount();
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now);
var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult();
Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3).GetAwaiter().GetResult();
List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
if (response.statusCode == "NotFound")
{
Thread.Sleep(1000);
Thread.Sleep(2000);
DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
return response.statusCode;
}
if (response.statusCode == "TooManyRequests")
{
for( int s = 0; s <= response.retryInSeconds; s++)
for( int s = 0; s <= response.retryInSeconds; s+=15)
{
Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s);
Thread.Sleep(1000);
Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
Thread.Sleep(15000);
}
return response.statusCode;
}
@@ -174,13 +222,21 @@ namespace URLNotesGrabberCORE
}
}
if (response.response == null)
{
Console.WriteLine("response.response == null");
return "FAILURE";
}
while (response.response != null && response.response._links != null)
while ( response.response != null
&& response.response._links != null
&& long.Parse(response.response._links.next.query_params.before_timestamp) >= long.Parse(post.Item3))
{
response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
if (response.response.notes is null)
{
Console.WriteLine("response.response.notes == null");
return "NULL NOTES";
}
Console.WriteLine("Notes\t" + response.response.notes.Count);
foreach (var note in response.response.notes)
@@ -200,9 +256,9 @@ namespace URLNotesGrabberCORE
return "UNKNOWN";
}
static async void CollectNotes(string outPath)
static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
{
List<Tuple<string, long>> posts = DataAccess.GetPosts(true);
List<Tuple<string, long, string>> posts = DataAccess.GetPosts(withoutNotesOnly);
RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
PermitLimit = 300,
@@ -244,20 +300,21 @@ namespace URLNotesGrabberCORE
}
static void TraverseDirectory(string path, string outPath)
static void TraverseDirectory(string path, string outPath, List<string> contains)
{
// Get all directories in the current directory and sort them alphabetically
var directories = Directory.GetDirectories(path);
Array.Sort(directories, StringComparer.InvariantCulture);
using (StreamWriter sw = new StreamWriter(outPath, true))
{
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("=====" + path);
}
//}
foreach (var directory in directories)
{
Console.WriteLine("Directory: " + directory);
TraverseDirectory(directory, outPath); // Recursively traverse subdirectories
TraverseDirectory(directory, outPath, contains); // Recursively traverse subdirectories
}
try
{
@@ -267,12 +324,12 @@ namespace URLNotesGrabberCORE
{
if (file.EndsWith(".txt"))
{
Console.WriteLine($"=====File: {file}");
//Console.WriteLine($"=====File: {file}");
using (StreamWriter sw = new StreamWriter(outPath, true))
{
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
//sw.WriteLine("---" + file);
}
//}
try
{
@@ -312,35 +369,35 @@ namespace URLNotesGrabberCORE
if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
{
if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated"))
if (ContainsAny(reblog.downloadedFiles, contains) && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated"))
{
DirectoryInfo currentDir = new DirectoryInfo(path);
if (!headerWasWritten)
{
headerWasWritten = true;
}
Console.WriteLine(reblog.postID);
Console.WriteLine(reblog.date);
Console.WriteLine(reblog.reblogKey);
Console.WriteLine(reblog.reblogURL);
Console.WriteLine(reblog.reblogName);
Console.WriteLine(reblog.downloadedFiles);
//Console.WriteLine(reblog.postID);
//Console.WriteLine(reblog.date);
//Console.WriteLine(reblog.reblogKey);
//Console.WriteLine(reblog.reblogURL);
//Console.WriteLine(reblog.reblogName);
//Console.WriteLine(reblog.downloadedFiles);
string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL );
DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date );
}
}
}
urls.Sort();
using (StreamWriter sw = new StreamWriter(outPath, true))
{
foreach (string line in urls.Distinct())
{
sw.WriteLine(line);
}
}
//using (StreamWriter sw = new StreamWriter(outPath, true))
//{
// foreach (string line in urls.Distinct())
// {
// sw.WriteLine(line);
// }
//}
}
catch (Exception e)
{
URLNotesGrabberCORE/Properties/launchSettings.json
+1 -1
View File
@@ -2,7 +2,7 @@
"profiles": {
"URLNotesGrabberCORE": {
"commandName": "Project",
"commandLineArgs": "-pn"
"commandLineArgs": "-pn true 31"
}
}
}
URLNotesGrabberCORE/TL.db
BIN
View File
Binary file not shown.
URLNotesGrabberCORE/appsettings.json
+3 -1
View File
@@ -5,6 +5,8 @@
"PathOutputJSON": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotesJSON.txt",
"PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt",
"PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
"PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db"
"PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
"PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
"ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
}
}