jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

.

Browse Source
This commit is contained in:
jim
2024-11-15 09:17:53 -06:00
parent 5cddf2a427
commit 3c44cbe671
5 changed files with 248 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+111 -21
View File
@@ -8,6 +8,7 @@ using RestSharp.Authenticators.OAuth;
using RestSharp.Authenticators;
using RestSharp;
using Newtonsoft.Json;
using Newtonsoft.Json.Converters;
namespace URLNotesGrabberCORE
{
@@ -34,6 +35,14 @@ namespace URLNotesGrabberCORE
internal class DataAccess
{
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
{
// Unix timestamp is seconds past epoch
DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
return dateTime;
}
#region Adds
public static void AddBlog(string blogName, string DBPath = @"TL.db")
{
@@ -50,6 +59,7 @@ namespace URLNotesGrabberCORE
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
Console.WriteLine("+ " + blogName);
}
catch (Exception ex)
{
@@ -63,9 +73,10 @@ namespace URLNotesGrabberCORE
}
public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db")
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db")
{
try { AddBlog(blogName, DBPath); } catch { }
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -73,7 +84,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')";
string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')";
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
@@ -81,7 +92,9 @@ namespace URLNotesGrabberCORE
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
{
Console.WriteLine(ex.Message);
}
}
finally
{
@@ -119,6 +132,8 @@ namespace URLNotesGrabberCORE
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
try { AddBlog(noteBlogName, DBPath); } catch { }
Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName);
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
@@ -148,7 +163,64 @@ namespace URLNotesGrabberCORE
#endregion Adds
#region Gets
public static List<Tuple<string, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
public static List<Tuple<string, long, string>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long, string>> posts = new List<Tuple<string, long, string>>();
try
{
connection.Open();
string sql = "SELECT " +
" Posts.BlogName, " +
" Posts.PostID, " +
" Max(Notes.timestamp) as LatestNoteTimestamp " +
"FROM " +
" Posts INNER JOIN " +
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " +
"WHERE NotFound = 0 ";
if (withoutNotesOnly)
sql += " and HasNotesGathered = 0 ";
sql += "GROUP BY " +
" Posts.BlogName, Posts.PostID " +
"ORDER BY " +
" Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
{
while (reader.Read())
{
Tuple<string, long, string> post = default;
string blog = null;
long id = 0;
long timestamp;
timestamp = reader.GetInt64(2); // Assuming Id is the first column
id = reader.GetInt64(1); // Assuming Id is the first column
blog = reader.GetString(0); // Assuming Title is the second column
post = new Tuple<string, long, string>(blog, id, timestamp.ToString());
posts.Add(post);
}
}
}
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
return posts;
}
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
@@ -157,16 +229,8 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "SELECT " +
" MIN(blogName) as blogName, " +
" postID " +
"FROM" +
" Posts ";
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
if (withoutNotesOnly)
sql += " WHERE HasNotesGathered = 0 and NotFound = 0";
sql += " GROUP BY postID ORDER BY reblogURL, PostID";
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
@@ -176,8 +240,9 @@ namespace URLNotesGrabberCORE
Tuple<string, long> post = default;
string blog = null;
long id = 0;
id = reader.GetInt64(1); // Assuming Id is the first column
blog = reader.GetString(0); // Assuming Title is the second column
id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
post = new Tuple<string, long>(blog, id);
posts.Add(post);
@@ -240,7 +305,7 @@ namespace URLNotesGrabberCORE
}
public static List<string> GetBlogs(bool reblogsOnly, string DBPath = @"TL.db")
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
List<string> blogs = new List<string>();
@@ -250,9 +315,9 @@ namespace URLNotesGrabberCORE
connection.Open();
string sql = "";
if (reblogsOnly)
sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName";
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
else
sql = "SELECT * FROM Blogs ORDER BY BlogName";
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
{
using (SQLiteDataReader reader = command.ExecuteReader())
@@ -291,7 +356,7 @@ namespace URLNotesGrabberCORE
{
connection.Open();
string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
@@ -333,7 +398,32 @@ namespace URLNotesGrabberCORE
}
public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db")
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
try
{
connection.Open();
string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
SQLiteCommand command = new SQLiteCommand(sql, connection);
command.ExecuteNonQuery();
}
catch (Exception ex)
{
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
Console.WriteLine(ex.Message);
}
finally
{
connection.Close();
}
}
public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
{
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
@@ -407,7 +497,7 @@ namespace URLNotesGrabberCORE
if (timestamp != null)
{
URL += "&before_timestamp=" + timestamp;
Thread.Sleep(500);
Thread.Sleep(1000);
}
var client = new RestClient(URL);
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY,
@@ -425,7 +515,7 @@ namespace URLNotesGrabberCORE
var response = client.Execute(request);
var myJsonResponse = response.Content;
Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
Root myDeserializedClass = new Root();
try