using System; using System.Collections.Generic; using System.Linq; using System.Text; using System.Threading.Tasks; using System.Data.SQLite; using RestSharp.Authenticators.OAuth; using RestSharp.Authenticators; using RestSharp; using Newtonsoft.Json; namespace URLNotesGrabberCORE { class ReblogRecord { public string postID; public string reblogURL; public string reblogName; public string downloadedFiles; public string reblogKey; public string date; public ReblogRecord() { postID = "."; reblogURL = "."; reblogName = "."; downloadedFiles = "."; reblogKey = "."; date = "."; } } internal class DataAccess { #region Adds public static void AddBlog(string blogName, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); if (blogName.Contains("deact")) return; try { connection.Open(); string sql = "INSERT INTO Blogs (BlogName) values('" + blogName + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db") { try { AddBlog(blogName, DBPath); } catch { } SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void AddAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "INSERT INTO DailyAPICount (Date) values('" + DateTime.Today.ToShortDateString() + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { //Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db") { //try { AddPost(rootBlogName, postID, DBPath); } catch { } try { AddBlog(noteBlogName, DBPath); } catch { } SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "INSERT INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type) values('" + rootBlogName + "', '" + noteBlogName + "', " + postID + ", " + timestamp + ", '" + type + "')"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName") { Console.WriteLine(ex.Message); Console.WriteLine("^^^^^ - SHORTCUT"); return true; } } finally { connection.Close(); } return false; } #endregion Adds #region Gets public static List> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List> posts = new List>(); try { connection.Open(); string sql = "SELECT " + " MIN(blogName) as blogName, " + " postID " + "FROM" + " Posts "; if (withoutNotesOnly) sql += " WHERE HasNotesGathered = 0 and NotFound = 0"; sql += " GROUP BY postID ORDER BY reblogURL, PostID"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { Tuple post = default; string blog = null; long id = 0; id = reader.GetInt64(1); // Assuming Id is the first column blog = reader.GetString(0); // Assuming Title is the second column post = new Tuple(blog, id); posts.Add(post); } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return posts; } public static int GetAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int count = 0; try { AddAPICount(); } catch { } try { connection.Open(); string sql = "SELECT " + " APICount " + "FROM " + " DailyAPICount " + " WHERE [Date] = '" + DateTime.Today.ToShortDateString() + "'"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { count = reader.GetInt32(0); // Assuming Id is the first column } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return count; } public static List GetBlogs(bool reblogsOnly, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); List blogs = new List(); try { connection.Open(); string sql = ""; if (reblogsOnly) sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0 ORDER BY BlogName"; else sql = "SELECT * FROM Blogs ORDER BY BlogName"; using (SQLiteCommand command = new SQLiteCommand(sql, connection)) { using (SQLiteDataReader reader = command.ExecuteReader()) { while (reader.Read()) { string blog = null; blog = reader.GetString(0); // Assuming Title is the second column blogs.Add(blog); } } } } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return blogs; } #endregion Gets #region Updates public static void UpdatePostMarkNotesCollected(string blogName, long postID, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); try { connection.Open(); string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } } public static int UpdateAPICount(string DBPath = @"TL.db") { SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath); int APICount = DataAccess.GetAPICount(); APICount++; try { connection.Open(); string sql = "UPDATE DailyAPICount SET APICount = " + APICount + " WHERE [Date] = '" + DateTime.Today.ToShortDateString() + "'"; SQLiteCommand command = new SQLiteCommand(sql, connection); command.ExecuteNonQuery(); } catch (Exception ex) { if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID") Console.WriteLine(ex.Message); } finally { connection.Close(); } return APICount; } #endregion Updates } internal class APIAccess { public static string Blog { get; set; } const string CONSUMER_KEY = "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3"; const string CONSUMER_SECRET = "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG"; const string OAUTH_TOKEN = "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J"; const string OAUTH_TOKEN_SECRET = "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w"; public static async Task GrabNotes(string blog, long ID, string timestamp = null) { var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]"; URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString()); if (timestamp != null) { URL += "&before_timestamp=" + timestamp; Thread.Sleep(500); } var client = new RestClient(URL); var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY, consumerSecret: CONSUMER_SECRET, token: OAUTH_TOKEN, tokenSecret: OAUTH_TOKEN_SECRET, OAuthSignatureMethod.HmacSha1 ); //oAuth1.Realm = client.Authenticator = oAuth1; var request = new RestRequest(URL, Method.Get); var response = client.Execute(request); var myJsonResponse = response.Content; Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount()); Root myDeserializedClass = new Root(); try { myDeserializedClass = JsonConvert.DeserializeObject(myJsonResponse); //if (myDeserializedClass.response.total_notes != 0) //{ //DataAccess.AddBlog(blog); //DataAccess.AddPost(blog, ID); //} } catch (Exception ex) { Console.WriteLine(ex.ToString()); if (!response.IsSuccessful) { Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription); myDeserializedClass.statusCode = response.StatusCode.ToString(); bool checkReset = false; if (myDeserializedClass.statusCode != "NotFound") { foreach (var header in response.Headers) { Console.WriteLine("{0} - {1}", header.Name, header.Value); if (checkReset && header.Name.Contains("Reset")) { if (myDeserializedClass.retryInSeconds < int.Parse(header.Value.ToString())) myDeserializedClass.retryInSeconds = int.Parse(header.Value.ToString()); } if (header.Name.Contains("Remaining") && header.Value.ToString() == "0") checkReset = true; else checkReset = false; //if(header.ToString().Contains("X-Ratelimit-Perday-Reset, Value = ")) //{ // string temp = header.ToString().Substring(59); // int indexOf = temp.IndexOf(','); // myDeserializedClass.retryInSeconds = int.Parse(temp.Substring(0, indexOf)); //} } } } } return myDeserializedClass; } } }