using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Data.SQLite; using RestSharp.Authenticators.OAuth; using RestSharp.Authenticators; using RestSharp; using Newtonsoft.Json; using Newtonsoft.Json.Converters; namespace URLNotesGrabberCORE { class ReblogRecord { public string postID; public string reblogURL; public string reblogName; public string downloadedFiles; public string reblogKey; public string date; public ReblogRecord() { postID = "."; reblogURL = "."; reblogName = "."; downloadedFiles = "."; reblogKey = "."; date = "."; } } internal class DataAccess { public static DateTime UnixTimeStampToDateTime(double unixTimeStamp) { // Unix timestamp is seconds past epoch DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime(); return dateTime; } #region Adds public static void AddBlog(string blogName, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); if (blogName.Contains("deact")) return; try { connection.Open(); string sql = "INSERT INTO Blogs (BlogName) values('" + blogName + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); Console.WriteLine("+ " + blogName); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db") { try { AddBlog(blogName, DBPath); } catch { } try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { } SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") { Console.WriteLine(ex.Message); } } finally { connection.Close(); } } public static void AddAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "INSERT INTO DailyAPICount (Date) values('" + DateTime.Today.ToShortDateString() + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { //Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db") { //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, DBPath); } catch { } Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName); SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values('" + rootBlogName + "', '" + noteBlogName + "', " + postID + ", " + timestamp + ", '" + type + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") { Console.WriteLine(ex.Message); Console.WriteLine("^^^^^ - SHORTCUT"); return true; } } finally { connection.Close(); } return false; } #endregion Adds #region Gets public static List> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try { connection.Open(); string sql = "SELECT " + " Posts.BlogName, " + " Posts.PostID, " + " Max(Notes.timestamp) as LatestNoteTimestamp " + "FROM " + " Posts INNER JOIN " + " Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + "WHERE NotFound = 0 "; if (withoutNotesOnly) sql += " and HasNotesGathered = 0 "; sql += "GROUP BY " + " Posts.BlogName, Posts.PostID " + "ORDER BY " + " Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { Tuple post = default; string blog = null; long id = 0; long timestamp; timestamp = reader.GetInt64(2); // Assuming Id is the first column id = reader.GetInt64(1); // Assuming Id is the first column blog = reader.GetString(0); // Assuming Title is the second column post = new Tuple(blog, id, timestamp.ToString()); posts.Add(post); } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return posts; } public static List> GetReplies(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try { connection.Open(); string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { Tuple post = default; string blog = null; long id = 0; id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column post = new Tuple(blog, id); posts.Add(post); } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return posts; } public static int GetAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int count = 0; try { AddAPICount(); } catch { } try { connection.Open(); string sql = "SELECT " + " APICount " + "FROM " + " DailyAPICount " + " WHERE [Date] = '" + DateTime.Today.ToShortDateString() + "'"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { count = reader.GetInt32(0); // Assuming Id is the first column } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return count; } public static List GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); try { connection.Open(); string sql = ""; if (reblogsOnly) sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top; else sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName ORDER BY count(*) desc, BlogName LIMIT " + top; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { string blog = null; blog = reader.GetString(0); // Assuming Title is the second column blogs.Add(blog); } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return blogs; } #endregion Gets #region Updates public static void UpdatePostMarkNotesCollected(string blogName, long postID, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static int UpdateAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int APICount = DataAccess.GetAPICount(); APICount++; try { connection.Open(); string sql = "UPDATE DailyAPICount SET APICount = " + APICount + " WHERE [Date] = '" + DateTime.Today.ToShortDateString() + "'"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return APICount; } #endregion Updates } internal class APIAccess { public static string Blog { get; set; } const string CONSUMER_KEY = "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3"; const string CONSUMER_SECRET = "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG"; const string OAUTH_TOKEN = "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J"; const string OAUTH_TOKEN_SECRET = "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w"; public static async Task GrabNotes(string blog, long ID, string timestamp = null) { var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]"; URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString()); if (timestamp != null) { URL += "&before_timestamp=" + timestamp; Thread.Sleep(1000); } var client = new RestClient(URL); var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY, consumerSecret: CONSUMER_SECRET, token: OAUTH_TOKEN, tokenSecret: OAUTH_TOKEN_SECRET, OAuthSignatureMethod.HmacSha1 ); //oAuth1.Realm = client.Authenticator = oAuth1; var request = new RestRequest(URL, Method.Get); var response = client.Execute(request); var myJsonResponse = response.Content; Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount()); Root myDeserializedClass = new Root(); try { myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); //if (myDeserializedClass.response.total_notes != 0) //{ //DataAccess.AddBlog(blog); //DataAccess.AddPost(blog, ID); //} } catch (Exception ex) { Console.WriteLine(ex.ToString()); if (!response.IsSuccessful) { Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription); myDeserializedClass.statusCode = response.StatusCode.ToString(); bool checkReset = false; if (myDeserializedClass.statusCode != "NotFound") { foreach (var header in response.Headers) { Console.WriteLine("{0} - {1}", header.Name, header.Value); if (checkReset && header.Name.Contains("Reset")) { if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString())) myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString()); } if (header.Name.Contains("Remaining") && header.Value.ToString() == "0") checkReset = true; else checkReset = false; //if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = ")) //{ // string temp = header.ToString().Substring(59); // int indexOf = temp.IndexOf(','); // myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf)); //} } } } } return myDeserializedClass; } } }