571 lines
21 KiB
C#
571 lines
21 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using System.Data.SQLite;
|
|
using RestSharp.Authenticators.OAuth;
|
|
using RestSharp.Authenticators;
|
|
using RestSharp;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Converters;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
|
|
class ReblogRecord
|
|
{
|
|
public string postID;
|
|
public string reblogURL;
|
|
public string reblogName;
|
|
public string downloadedFiles;
|
|
public string reblogKey;
|
|
public string date;
|
|
|
|
public ReblogRecord()
|
|
{
|
|
postID = ".";
|
|
reblogURL = ".";
|
|
reblogName = ".";
|
|
downloadedFiles = ".";
|
|
reblogKey = ".";
|
|
date = ".";
|
|
}
|
|
}
|
|
|
|
internal class DataAccess
|
|
{
|
|
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
|
|
{
|
|
// Unix timestamp is seconds past epoch
|
|
DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
|
dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
|
|
return dateTime;
|
|
}
|
|
|
|
#region Adds
|
|
public static void AddBlog(string blogName, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
if (blogName.Contains("deact"))
|
|
return;
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "INSERT INTO Blogs (BlogName) values('" + blogName + "')";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
Console.WriteLine("+ " + blogName);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db")
|
|
{
|
|
try { AddBlog(blogName, DBPath); } catch { }
|
|
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
|
|
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static void AddAPICount(string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "INSERT INTO DailyAPICount (Date) values('" + DateTime.Today.ToShortDateString() + "')";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
//Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
|
|
{
|
|
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
|
|
try { AddBlog(noteBlogName, DBPath); } catch { }
|
|
|
|
Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName);
|
|
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values('" + rootBlogName + "', '" + noteBlogName + "', " + postID + ", " + timestamp + ", '" + type + "')";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
Console.WriteLine("^^^^^ - SHORTCUT");
|
|
return true;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return false;
|
|
}
|
|
#endregion Adds
|
|
|
|
#region Gets
|
|
public static List<Tuple<string, long, string>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long, string>> posts = new List<Tuple<string, long, string>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT " +
|
|
" Posts.BlogName, " +
|
|
" Posts.PostID, " +
|
|
" Max(Notes.timestamp) as LatestNoteTimestamp " +
|
|
"FROM " +
|
|
" Posts INNER JOIN " +
|
|
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " +
|
|
"WHERE NotFound = 0 ";
|
|
|
|
if (withoutNotesOnly)
|
|
sql += " and HasNotesGathered = 0 ";
|
|
|
|
sql += "GROUP BY " +
|
|
" Posts.BlogName, Posts.PostID " +
|
|
"ORDER BY " +
|
|
" Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long, string> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
long timestamp;
|
|
timestamp = reader.GetInt64(2); // Assuming Id is the first column
|
|
id = reader.GetInt64(1); // Assuming Id is the first column
|
|
blog = reader.GetString(0); // Assuming Title is the second column
|
|
|
|
post = new Tuple<string, long, string>(blog, id, timestamp.ToString());
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return posts;
|
|
}
|
|
public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
|
|
id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
|
|
blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
|
|
|
|
post = new Tuple<string, long>(blog, id);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
|
|
public static int GetAPICount(string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
int count = 0;
|
|
|
|
try { AddAPICount(); } catch { }
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT " +
|
|
" APICount " +
|
|
"FROM " +
|
|
" DailyAPICount " +
|
|
" WHERE [Date] = '" +
|
|
DateTime.Today.ToShortDateString() + "'";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
count = reader.GetInt32(0); // Assuming Id is the first column
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return count;
|
|
}
|
|
|
|
|
|
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<string> blogs = new List<string>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = "";
|
|
if (reblogsOnly)
|
|
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
|
|
else
|
|
sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top;
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string blog = null;
|
|
blog = reader.GetString(0); // Assuming Title is the second column
|
|
|
|
blogs.Add(blog);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return blogs;
|
|
}
|
|
#endregion Gets
|
|
|
|
#region Updates
|
|
|
|
|
|
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
|
|
public static int UpdateAPICount(string DBPath = @"TL.db")
|
|
{
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
int APICount = DataAccess.GetAPICount();
|
|
APICount++;
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE DailyAPICount SET APICount = " + APICount +
|
|
" WHERE [Date] = '" +
|
|
DateTime.Today.ToShortDateString() + "'";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
|
|
return APICount;
|
|
}
|
|
#endregion Updates
|
|
}
|
|
|
|
internal class APIAccess
|
|
{
|
|
public static string Blog { get; set; }
|
|
|
|
const string CONSUMER_KEY = "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3";
|
|
const string CONSUMER_SECRET = "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG";
|
|
const string OAUTH_TOKEN = "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J";
|
|
const string OAUTH_TOKEN_SECRET = "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w";
|
|
|
|
public static async Task<Root> GrabNotes(string blog, long ID, string timestamp = null)
|
|
{
|
|
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]";
|
|
URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
|
|
if (timestamp != null)
|
|
{
|
|
URL += "&before_timestamp=" + timestamp;
|
|
Thread.Sleep(1000);
|
|
}
|
|
var client = new RestClient(URL);
|
|
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY,
|
|
consumerSecret: CONSUMER_SECRET,
|
|
token: OAUTH_TOKEN,
|
|
tokenSecret: OAUTH_TOKEN_SECRET,
|
|
OAuthSignatureMethod.HmacSha1
|
|
);
|
|
//oAuth1.Realm =
|
|
|
|
client.Authenticator = oAuth1;
|
|
|
|
var request = new RestRequest(URL, Method.Get);
|
|
|
|
var response = client.Execute(request);
|
|
|
|
var myJsonResponse = response.Content;
|
|
Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
|
|
Root myDeserializedClass = new Root();
|
|
|
|
try
|
|
{
|
|
myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
|
|
|
//if (myDeserializedClass.response.total_notes != 0)
|
|
//{
|
|
//DataAccess.AddBlog(blog);
|
|
//DataAccess.AddPost(blog, ID);
|
|
//}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine(ex.ToString());
|
|
|
|
if (!response.IsSuccessful)
|
|
{
|
|
Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription);
|
|
myDeserializedClass.statusCode = response.StatusCode.ToString();
|
|
bool checkReset = false;
|
|
|
|
if (myDeserializedClass.statusCode != "NotFound")
|
|
{
|
|
foreach (var header in response.Headers)
|
|
{
|
|
Console.WriteLine("{0} - {1}", header.Name, header.Value);
|
|
if (checkReset && header.Name.Contains("Reset"))
|
|
{
|
|
if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString()))
|
|
myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString());
|
|
}
|
|
if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
|
|
checkReset = true;
|
|
else
|
|
checkReset = false;
|
|
|
|
//if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = "))
|
|
//{
|
|
// string temp = header.ToString().Substring(59);
|
|
// int indexOf = temp.IndexOf(',');
|
|
// myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf));
|
|
//}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
}
|