feat: Initialize URLNotesGrabberCORE project with core application structure, data access, and configuration.

.github/copilot-instructions.md

@@ -0,0 +1,60 @@
+# AI Development Instructions for URLNotesGrabberCORE
+
+This document provides essential context for AI agents working with URLNotesGrabberCORE, a .NET Core application designed to process and analyze blog notes and interactions.
+
+## Project Architecture
+
+### Core Components
+- **Program.cs**: Entry point and command handler for various operations (parsing, testing, collecting notes)
+- **DataAccess.cs**: SQLite database operations for storing blog and note data
+- **ResponseNotes.cs**: Data models for API responses
+- **appsettings.json**: Configuration for paths and application settings
+
+### Data Flow
+1. Input text files are processed from configured input path
+2. Blog data is extracted and stored in SQLite database
+3. API calls collect notes/interactions for each blog post
+4. Results are written to configured output paths
+
+## Key Development Workflows
+
+### Building and Running
+```powershell
+dotnet build
+dotnet run            # Process all files in input directory
+dotnet run -- -parse [blogname]  # Process specific blog
+dotnet run -- -test [blogname] [postID]  # Test API for specific post
+```
+
+### Command-Line Interface
+- `-parse [blogname]`: Parse text files for specific blog
+- `-test [blogname] [postID]`: Test API note collection
+- `-posts`: Export post blogs to file
+- `-blogs`: Export blog list to file
+- `-collect`: Collect notes for all posts in DB
+- `-blogsR`: Export reply blogs to file
+- `-blogsO [start] [stop]`: Export blogs within range
+
+## Project Conventions
+
+### Configuration
+- All paths and settings are managed in `appsettings.json`
+- Command-line args override config file settings
+- SQLite database is used for persistent storage
+
+### API Integration
+- Uses RestSharp for API calls with OAuth authentication
+- Rate limiting implemented via `System.Threading.RateLimiting`
+- Resilient HTTP handling with `Microsoft.Extensions.Http.Resilience`
+
+### Data Models
+- `ReblogRecord`: Core data structure for blog post information
+- `Note`: Represents interaction data from API responses
+- Default field values are "." (period) to handle null cases
+
+## Integration Points
+- SQLite Database: Primary data store (`System.Data.SQLite`)
+- REST API: External blog platform API (OAuth authentication)
+- File System: Input/Output for text file processing
+
+For questions or clarifications, please refer to the codebase or request updates to these instructions.

.vscode/launch.json

@@ -0,0 +1,11 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "args": ["-pn"], // <--- ADD OR MODIFY THIS LINE
+        }
+    ]
+}

TL.sqbpro

@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?><sqlb_project><db path="D:/NextCloud/C#/URLNotesGrabberCORE/URLNotesGrabberCORE/bin/Debug/net8.0/TL.db" readonly="1" foreign_keys="1" case_sensitive_like="0" temp_store="0" wal_autocheckpoint="1000" synchronous="2"/><attached/><window><main_tabs open="structure browser pragmas query" current="3"/></window><tab_structure><column_width id="0" width="300"/><column_width id="1" width="0"/><column_width id="2" width="100"/><column_width id="3" width="3571"/><column_width id="4" width="0"/><expanded_item id="0" parent="1"/><expanded_item id="1" parent="1"/><expanded_item id="2" parent="1"/><expanded_item id="3" parent="1"/></tab_structure><tab_browse><table title="Blogs" custom_title="0" dock_id="1" table="4,5:mainBlogs"/><dock_state state="000000ff00000000fd0000000100000002000005f40000031dfc0100000001fb000000160064006f0063006b00420072006f00770073006500310100000000000005f4000000fb00ffffff000002130000000000000004000000040000000800000008fc00000000"/><default_encoding codec=""/><browse_table_settings><table schema="main" name="Blogs" show_row_id="0" encoding="" plot_x_axis="" unlock_view_pk="_rowid_" freeze_columns="0"><sort><column index="0" mode="1"/></sort><column_widths><column index="1" value="257"/><column index="2" value="48"/></column_widths><filter_values/><conditional_formats/><row_id_formats/><display_formats/><hidden_columns/><plot_y_axes/><global_filter/></table></browse_table_settings></tab_browse><tab_sql><sql name="SQL 1*">SELECT	␍
+	BlogName || '.tumblr.com/post/' || postID,␍
+	␍
+	datetime(NotesGatheredDateTime, 'unixepoch'), *␍
+FROM␍
+	Posts␍
+WHERE␍
+	NotesGatheredDateTime &lt;&gt; 0␍
+ORDER BY␍
+	postdate desc</sql><sql name="SQL 2*">SELECT	␍
+	datetime(TimeStamp, 'unixepoch'),␍
+	RootBlogName || '.tumblr.com/post/' || postid,␍
+	*,␍
+	NoteBlogName || '.tumblr.com'␍
+FROM␍
+	Notes␍
+WHERE	RootBlogName NOT IN ('xlittle-ghost', 'glimmerin-darlin')␍
+ORDER BY␍
+	TimeStamp desc</sql><current_tab id="0"/></tab_sql></sqlb_project>

URLNotesGrabberCORE/.vscode/launch.json

@@ -0,0 +1,12 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "args": ["-pn", "0", "999999"]
+        }
+
+    ]
+}

URLNotesGrabberCORE/DataAccess.cs

@@ -8,32 +8,79 @@ using RestSharp.Authenticators.OAuth;
 using RestSharp.Authenticators;
 using RestSharp;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
+using Microsoft.Extensions.Diagnostics.Latency;
+using System.Collections;
+using Microsoft.Extensions.Configuration;
+using System.IO;
 
 namespace URLNotesGrabberCORE
 {
 
     class ReblogRecord
     {
-        public string postID;
+        public string answer;
-        public string reblogURL;
+        public string audioCaption;
-        public string reblogName;
+        public string blogName;
-        public string downloadedFiles;
+        public string body;
-        public string reblogKey;
         public string date;
+        public string downloadedFiles;
+        public string link;
+        public string photoCaption;
+        public string photoURL;
+        public string postID;
+        public string postURL;
+        public string question;
+        public string quote;
+        public string reblogKey;
+        public string reblogName;
+        public string reblogURL;
+        public string slug;
+        public string summary;
+        public string tags;
+        public string title;
 
         public ReblogRecord()
         {
-            postID = ".";
+            answer = ".";
-            reblogURL = ".";
+            audioCaption = ".";
-            reblogName = ".";
+            blogName = ".";
-            downloadedFiles = ".";
+            body = ".";
-            reblogKey = ".";
             date = ".";
+            downloadedFiles = ".";
+            link = ".";
+            photoCaption = ".";
+            photoURL = ".";
+            postID = ".";
+            postURL = ".";
+            question = ".";
+            quote = ".";
+            reblogKey = ".";
+            reblogName = ".";
+            reblogURL = ".";
+            slug = ".";
+            summary = ".";
+            tags = ".";
+            title = ".";
         }
+
     }
 
     internal class DataAccess
     {
+        public static string Q(string input)
+        {
+            return "'" + input.Replace("'", "''") + "'";
+        }
+
+        public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
+        {
+            // Unix timestamp is seconds past epoch
+            DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
+            dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
+            return dateTime;
+        }
+
         #region Adds
         public static void AddBlog(string blogName, string DBPath = @"TL.db")
         {
@@ -50,6 +97,7 @@ namespace URLNotesGrabberCORE
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
+                Console.WriteLine("+ " + blogName);
             }
             catch (Exception ex)
             {
@@ -61,11 +109,11 @@ namespace URLNotesGrabberCORE
                 connection.Close();
             }
         }
-
+        public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, string DBPath = @"TL.db")
-
-        public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db")
         {
             try { AddBlog(blogName, DBPath); } catch { }
+            try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
+            try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, DBPath); } catch { }
 
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
@@ -73,7 +121,30 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')";
+                string sql = @"INSERT INTO Posts (
+                                BlogName,
+                                PostID,
+                                reblogURL,
+                                PostDate,
+                                PostURL,
+                                Slug,
+                                ReblogKey,
+                                ReblogName,
+                                Summary,
+                                Quote,
+                                Body,
+                                Tags,
+                                Link,
+                                PhotoURL,
+                                PhotoCaption,
+                                DownloadedFiles,
+                                AudioCaption,
+                                Question,
+                                Answer,
+                                Title,
+                                HasImage
+                                ) VALUES (" +
+                    Q(blogName) + ", " + postID +", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + hasImage + ")";
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
@@ -81,7 +152,31 @@ namespace URLNotesGrabberCORE
             catch (Exception ex)
             {
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                {
                     Console.WriteLine(ex.Message);
+                    ///
+                        try
+                        {
+                            connection.Open();
+
+                            string sql = "UPDATE Posts SET hasImage = " + hasImage + "WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
+                            SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                            command.ExecuteNonQuery();
+                        }
+                        catch (Exception ex2)
+                        {
+                            if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                            {
+                                Console.WriteLine(ex2.Message);
+                            }
+                        }
+                        finally
+                        {
+                            connection.Close();
+                        }
+                        ///
+                }
             }
             finally
             {
@@ -116,9 +211,11 @@ namespace URLNotesGrabberCORE
 
         public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
         {
-            try { AddPost(rootBlogName, postID, DBPath); } catch { }
+            //try { AddPost(rootBlogName, postID, DBPath); } catch { }
             try { AddBlog(noteBlogName, DBPath); } catch { }
 
+            Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
+
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
             try
@@ -136,8 +233,9 @@ namespace URLNotesGrabberCORE
                 {
                     Console.WriteLine(ex.Message);
                     Console.WriteLine("^^^^^ - SHORTCUT");                    
-                    return true;
                 }
+                else
+                    return UpdateNote(rootBlogName, noteBlogName, postID, timestamp, type, DBPath);
             }
             finally
             {
@@ -148,7 +246,84 @@ namespace URLNotesGrabberCORE
         #endregion Adds
 
         #region Gets
-        public static List<Tuple<string, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
+
+        /// <summary>
+        /// 
+        /// </summary>
+        /// <param name="withoutNotesOnly"></param>
+        /// <param name="DBPath"></param>
+        /// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
+        public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
+
+            try
+            {
+                connection.Open();
+
+                string sql = "SELECT " + 
+                             "  Posts.BlogName, " + Environment.NewLine +
+                             "  Posts.PostID, " + Environment.NewLine +
+                             "  Max(IFNULL(Notes.timestamp, 1925013599)) as LatestNoteTimestamp, " + Environment.NewLine +
+                             "  Posts.NotesGatheredDatetime, " + Environment.NewLine +
+                             "  CNT.CNT " + Environment.NewLine +
+                             "FROM " + Environment.NewLine +
+                             "   Posts " + Environment.NewLine +
+                             "      LEFT OUTER JOIN " + Environment.NewLine + 
+                             "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
+                             "      LEFT OUTER JOIN " + Environment.NewLine +
+                             "   (      select BlogName, count(PostID) as CNT from Posts group by BlogName)  CNT on CNT.blogName = Posts.BlogName " +
+                             "WHERE NotFound = 0 " + Environment.NewLine;
+
+                if (withoutNotesOnly)
+                    sql += " and HasNotesGathered = 0 "  + Environment.NewLine ; 
+                
+                sql +=       "GROUP BY " + Environment.NewLine +
+                             "   Posts.BlogName, Posts.PostID " + Environment.NewLine +
+                             "ORDER BY " + Environment.NewLine +
+                             " notesgathereddatetime,  Posts.PostDate DESC, Posts.BlogName, Posts.PostID" + Environment.NewLine;
+
+                Console.WriteLine(withoutNotesOnly);
+                Console.WriteLine(sql);
+                Console.Write(">"); //Console.ReadKey();
+                Thread.Sleep(1000);
+
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            Tuple<string, long, long, long> post = default;
+                            string blog;
+                            long postID = 0;
+                            long lastNoteTimestamp;
+                            long notesGatheredTimestamp;
+
+                            blog = reader.GetString(0); // Assuming Title is the second column
+                            postID = reader.GetInt64(1); // Assuming Id is the first column
+                            lastNoteTimestamp = reader.GetInt64(2); // Assuming Id is the first column
+                            notesGatheredTimestamp = reader.GetInt64(3); // Assuming Id is the first column
+
+                            post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
+                            posts.Add(post);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return posts;
+        }
+        public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
@@ -157,16 +332,8 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "SELECT " +
+                string sql = "SELECT	distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
-                             " MIN(blogName) as blogName, " +
-                             " postID " +
-                             "FROM" +
-                             " Posts ";
 
-                if (withoutNotesOnly)
-                    sql += " WHERE HasNotesGathered = 0";
-
-                sql += " GROUP BY  postID ORDER BY reblogURL, PostID";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     using (SQLiteDataReader reader = command.ExecuteReader())
@@ -176,8 +343,9 @@ namespace URLNotesGrabberCORE
                             Tuple<string, long> post = default;
                             string blog = null;
                             long id = 0;
-                            id = reader.GetInt64(1); // Assuming Id is the first column
+
-                            blog = reader.GetString(0); // Assuming Title is the second column
+                            id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
+                            blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
 
                             post = new Tuple<string, long>(blog, id);
                             posts.Add(post);
@@ -240,7 +408,7 @@ namespace URLNotesGrabberCORE
         }
 
 
-        public static List<string> GetBlogs(bool reblogsOnly, string DBPath = @"TL.db")
+        public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             List<string> blogs = new List<string>();
@@ -250,9 +418,49 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog'  ORDER BY BlogName";
+                    sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName  ORDER BY count(*) desc, BlogName LIMIT " + top;
                 else
-                    sql = "SELECT * FROM Blogs ORDER BY BlogName";
+                    sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName  ORDER BY count(*) desc, BlogName LIMIT " + top;
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            string blog = null;
+                            blog = reader.GetString(0); // Assuming Title is the second column
+
+                            blogs.Add(blog);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return blogs;
+        }
+
+
+        public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<string> blogs = new List<string>();
+
+            try
+            {
+                connection.Open();
+                string sql = "";
+                if (reblogsOnly)
+                    sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName  ORDER BY count(*) desc, BlogName LIMIT " + top;
+                else
+                    sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where HasBeenOutput = 0 group by NoteBlogName  ORDER BY count(*) desc, BlogName LIMIT " + top;
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     using (SQLiteDataReader reader = command.ExecuteReader())
@@ -291,7 +499,163 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
+        public static void UpdatePostMarkNotFound(string blogName, long postID, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Posts SET NotFound = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
+        public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
+        public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string DBPath = @"TL.db")
+        {
+            //try { AddPost(rootBlogName, postID, DBPath); } catch { }
+            try { AddBlog(noteBlogName, DBPath); } catch { }
+
+            Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
+
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = string.Format("UPDATE Notes SET timestamp = '{0}' WHERE rootBlogName = '{1}' AND noteBlogName = '{2}' AND PostID = '{3}'", timestamp, rootBlogName, noteBlogName, postID);
+                SQLiteCommand command = new SQLiteCommand(sql, connection); 
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
+                {
+                    Console.WriteLine(ex.Message);
+                    Console.WriteLine("^^^^^ - SHORTCUT");
+                    return true;
+                }
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return false;
+        }
+
+
+        public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql  =  "UPDATE Posts ";
+                sql += "SET postDate = " + Q(postDate) + ", ";
+                sql += "reblogURL = " + Q(reblogURL) + ", ";
+                sql += " postURL = " + Q(postURL) + ", "; 
+                sql += " slug = " + Q(slug) + ", ";
+                sql += " reblogKey = " + Q(reblogKey) + ", ";
+                sql += " reblogName = " + Q(reblogName) + ", ";
+                sql += " summary = " + Q(summary) + ", ";
+                sql += " quote = " + Q(quote) + ", ";
+                sql += " body = " + Q(body) + ", ";
+                sql += " tags = " + Q(tags) + ", ";
+                sql += " link = " + Q(link) + ", ";
+                sql += " photoURL = " + Q(photoURL) + ", ";
+                sql += " photoCaption = " + Q(photoCaption) + ", ";
+                sql += " downloadedFiles = " + Q(downloadedFiles) + ", ";
+                sql += " audioCaption = " + Q(audioCaption) + ", ";
+                sql += " question = " + Q(question) + ", ";
+                sql += " answer = " + Q(answer) + ", ";
+                sql += " title = " + Q(title) + ", ";
+                sql += " hasImage = " + hasImage;
+                sql         += " WHERE BlogName = " + Q(blogName) + " AND PostID = " + postID;
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
+        public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Blogs SET HasBeenOutput = 1 WHERE BlogName = '" + blogName + "'";
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
@@ -343,59 +707,96 @@ namespace URLNotesGrabberCORE
 
     internal class APIAccess
     {
-        public static string Blog { get; set; }
+        public static string Blog { get; set; } = string.Empty;
+        private static IConfiguration Configuration { get; set; } = null!;
 
-        const string CONSUMER_KEY = "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3";
+        static APIAccess()
-        const string CONSUMER_SECRET = "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG";
+        {
-        const string OAUTH_TOKEN = "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J";
+            Configuration = new ConfigurationBuilder()
-        const string OAUTH_TOKEN_SECRET = "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w";
+                .SetBasePath(Directory.GetCurrentDirectory())
+                .AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
+                .Build();
 
-        public static async Task<Root> GrabNotes(string blog, long ID, string timestamp = null)
+            if (Configuration["TumblrApi:ConsumerKey"] == null)
+                throw new InvalidOperationException("TumblrApi configuration is missing in appsettings.json");
+        }
+
+        private static string ConsumerKey => Configuration["TumblrApi:ConsumerKey"] ?? 
+            throw new InvalidOperationException("ConsumerKey is not configured");
+        private static string ConsumerSecret => Configuration["TumblrApi:ConsumerSecret"] ?? 
+            throw new InvalidOperationException("ConsumerSecret is not configured");
+        private static string OAuthToken => Configuration["TumblrApi:OAuthToken"] ?? 
+            throw new InvalidOperationException("OAuthToken is not configured");
+        private static string OAuthTokenSecret => Configuration["TumblrApi:OAuthTokenSecret"] ?? 
+            throw new InvalidOperationException("OAuthTokenSecret is not configured");
+
+        public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
         {
             var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]";
             URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
-            if (timestamp != null)
+            if (!string.IsNullOrEmpty(timestamp))
             {
                 URL += "&before_timestamp=" + timestamp;
-                Thread.Sleep(500);
+                await Task.Delay(1000);
             }
             var client = new RestClient(URL);
-            var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY,
+            var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
-                                                            consumerSecret: CONSUMER_SECRET,
+                                                            consumerSecret: ConsumerSecret,
-                                                            token: OAUTH_TOKEN,
+                                                            token: OAuthToken,
-                                                            tokenSecret: OAUTH_TOKEN_SECRET,
+                                                            tokenSecret: OAuthTokenSecret,
                                                             OAuthSignatureMethod.HmacSha1
                                                             );
-            //oAuth1.Realm = 
 
             client.Authenticator = oAuth1;
-
             var request = new RestRequest(URL, Method.Get);
+            var response = await client.ExecuteAsync(request);
 
-            var response = client.Execute(request);
+            var myJsonResponse = response.Content ?? string.Empty;
-
+            Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
-            var myJsonResponse = response.Content;
+            var myDeserializedClass = new Root();
-            Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
-            Root myDeserializedClass = new Root();
 
             try
             {
-                myDeserializedClass = JsonConvert.DeserializeObject<Root>(myJsonResponse);
+                var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
-
+                if (deserializedResult != null)
-                if (myDeserializedClass.response.total_notes != 0)
                 {
-                    DataAccess.AddBlog(blog);
+                    myDeserializedClass = deserializedResult;
-                    //DataAccess.AddPost(blog, ID);
                 }
             }
             catch (Exception ex) 
             { 
+                Console.WriteLine($"Failed JSON: {myJsonResponse}");
                 Console.WriteLine(ex.ToString());
 
                 if (!response.IsSuccessful)
                 {
-                    Console.WriteLine(response.StatusCode + '\t' + response.StatusDescription);
+                    Console.WriteLine($"{response.StatusCode}\t{response.StatusDescription}");
                     myDeserializedClass.statusCode = response.StatusCode.ToString();
+                    bool checkReset = false;
+
+                    if (myDeserializedClass.statusCode != "NotFound" && response.Headers != null)
+                    {
+                        foreach (var header in response.Headers)
+                        {
+                            if (header.Name != null && header.Value != null)
+                            {
+                                Console.WriteLine($"{header.Name}  -   {header.Value}");
+                                if (checkReset && header.Name.Contains("Reset"))
+                                {
+                                    var headerValue = header.Value.ToString();
+                                    if (headerValue != null && int.TryParse(headerValue, out int resetValue))
+                                    {
+                                        if (myDeserializedClass.retryInSeconds < resetValue)
+                                            myDeserializedClass.retryInSeconds = resetValue;
+                                    }
+                                }
+                                if (header.Name.Contains("Remaining") && header.Value.ToString() == "0")
+                                    checkReset = true;
+                                else
+                                    checkReset = false;
+                            }
+                        }
+                    }
                 }
             }
 

URLNotesGrabberCORE/DualLogger.cs

@@ -0,0 +1,53 @@
+using System;
+using System.IO;
+using System.Text;
+
+namespace URLNotesGrabberCORE
+{
+    public class DualLogger : TextWriter
+    {
+        private TextWriter _consoleWriter;
+        private TextWriter _fileWriter;
+
+        public DualLogger(TextWriter consoleWriter, TextWriter fileWriter)
+        {
+            _consoleWriter = consoleWriter;
+            _fileWriter = fileWriter;
+        }
+
+        public override Encoding Encoding => _consoleWriter.Encoding;
+
+        public override void Write(char value)
+        {
+            _consoleWriter.Write(value);
+            _fileWriter.Write(value);
+        }
+
+        public override void Write(string value)
+        {
+            _consoleWriter.Write(value);
+            _fileWriter.Write(value);
+        }
+
+        public override void WriteLine(string value)
+        {
+            _consoleWriter.WriteLine(value);
+            _fileWriter.WriteLine(value);
+        }
+
+        public override void Flush()
+        {
+            _consoleWriter.Flush();
+            _fileWriter.Flush();
+        }
+
+        protected override void Dispose(bool disposing)
+        {
+            if (disposing)
+            {
+                _fileWriter.Dispose();
+            }
+            base.Dispose(disposing);
+        }
+    }
+}

URLNotesGrabberCORE/EmptyArrayOrObjectConverter.cs

@@ -0,0 +1,29 @@
+using System;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+namespace URLNotesGrabberCORE
+{
+    public class EmptyArrayOrObjectConverter<T> : JsonConverter
+    {
+        public override bool CanConvert(Type objectType)
+        {
+            return objectType == typeof(T);
+        }
+
+        public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer)
+        {
+            JToken token = JToken.Load(reader);
+            if (token.Type == JTokenType.Array)
+            {
+                return null;
+            }
+            return token.ToObject<T>(serializer);
+        }
+
+        public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer)
+        {
+            serializer.Serialize(writer, value);
+        }
+    }
+}

URLNotesGrabberCORE/Program.cs

@@ -3,6 +3,9 @@ using Microsoft.Extensions.Http.Resilience;
 using Microsoft.Extensions;
 using Microsoft.Extensions.Configuration;
 using System.Configuration;
+using System.Threading;
+using Microsoft.Extensions.Diagnostics.Latency;
+using static System.Runtime.InteropServices.JavaScript.JSType;
 
 namespace URLNotesGrabberCORE
 {
@@ -18,21 +21,62 @@ namespace URLNotesGrabberCORE
                                     .Build();
             var settings = config.GetSection("appSettings");
 
+            // Setup Dual Logging
+            string logPath = "console_output.log";
+            StreamWriter fileWriter = new StreamWriter(logPath, append: false) { AutoFlush = true };
+            DualLogger dualLogger = new DualLogger(Console.Out, fileWriter);
+            Console.SetOut(dualLogger);
+
+            List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
+
             if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
             {
-                TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
+                TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
             }
-            else if (args[0] == "-n") //Manual test 
+            else
             {
-                /*var response = */
+                switch (args[0])
-                //var url = await CreateProductAsync(product);
+                {
-                //var t = await GrabNotes();
+                    case "-?":
-                var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
+                        Console.WriteLine("\t Parse .txt files to find blogs");
+
+                        Console.WriteLine("-?\t Usage help");
+
+                        Console.WriteLine("-parse\t Parse .txt files with specified blogname");
+
+                        Console.WriteLine("-test\t Calls API for given blogname and postID");
+
+                        Console.WriteLine("-posts\t For each Post in DB, write blogname to file");
+
+                        Console.WriteLine("-blogs\t For each Blog in DB, write blogname to file");
+
+                        Console.WriteLine("-collect\t For each Post in DB, hit API to collect Notes");
+
+                        Console.WriteLine("-blogsR\t For each Note that is a REPLY, write blogname to file ");
+
+                        Console.WriteLine("-blogsO\t For each Blog in DB, write blogname to file, but limit via a passed start and stop range ");
+
+                        Console.WriteLine("-replies\t For each Note that is a REPLY, write blogname to file ");
+
+                        break;
+                    case "-parse":
+                        string blogNameToParse = args[1];
+
+                        TraverseDirectory(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains, blogNameToParse);
+
+                        break;
+                    case "-test":
+                        #region Manual test 
+
+                        string blogName = args[1];
+                        long postID = long.Parse(args[2]);
+
+                        var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
                         List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
 
                         foreach (var note in response.response.notes)
                         {
-                    note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                            note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
                             //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
                             if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
                                 break;
@@ -40,54 +84,116 @@ namespace URLNotesGrabberCORE
 
                         while (response.response._links != null)
                         {
-                    response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
+                            response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
 
                             foreach (var note in response.response.notes)
                             {
-                        note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                                note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
                                 //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
                                 if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
                                     break;
                             }
                         }
-                //var sortedList = myList.OrderBy(tuple => tuple.Item2).ToList();
+                        #endregion
-                //notes = notes.OrderBy(tuple => tuple.Item2).ThenBy(tuple => tuple.Item1).ToList();
 
-                //using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
+                        break;
-                //{
-                //    foreach (var note in notes)
-                //    {
-                //        using (StreamWriter sw = new StreamWriter(System.Configuration.ConfigurationSettings.AppSettings["PathOutput"], true))
-                //    }
-                //}
 
-            }
+                    case "-post":
-            else if (args[0] == "-p") //write post's blogs to file
+                        TraverseDirectoryForCorruption(settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
-            {
+                        break;
+
+                    case "-posts": //write post's blogs to file
                         WritePostBlogsToFile(settings.GetValue<string>("PathOutputPosts"));
-            }
+                        break;
-            else if (args[0] == "-b") //write blogs to file
+
-            {
+                    case "-blogs": //write blogs to file
                         WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"));
-            }
+                        break;
-            else if (args[0] == "-pn") //collect notes from all posts
+
+                    case "-collect": //collect notes from all posts
+                        bool withoutNotesOnly = true;
+
+                        if(args.Length != 2)
                         {
-                CollectNotes(settings.GetValue<string>("PathOutput"));
+                            Console.WriteLine("--Expected WITHOUTNOTESONLY (0, 1)--");
+                            break;
                         }
-            else if (args[0] == "-br") //collect notes from all posts
+
+                        if (args.Length > 1 && args[1] is not null)
                         {
+                            if (args[1] == "1")
+                            {
+                                withoutNotesOnly = true;
+                                Console.WriteLine("Parsed");
+                            }
+                            else
+                            {
+                                withoutNotesOnly = false;
+                                Console.WriteLine("--NOT Parsed");
+                            }
+
+                            Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
+                        }
+                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
+                        break;
+
+                    case "-blogsR": //collect notes from all posts
                         WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
+                        break;
+
+                    case "-blogsO": //collect notes from all posts
+                        int from = 1, to = 999999, top = 100;
+
+                        if (args[1] is not null && args[2] is not null && args[3] is not null)
+                        {
+                            from = int.Parse(args[1]);
+                            to = int.Parse(args[2]);
+                            top = int.Parse(args[3]);
+                        }
+                        else
+                        {
+                            Console.WriteLine("--Expected FROM TO--");
+                        }
+                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
+                        break;
+
+                    case "-bop": //collect notes from all posts
+                        from = 1; to = 999999; top = 100;
+
+                        if (args[1] is not null && args[2] is not null && args[3] is not null)
+                        {
+                            from = int.Parse(args[1]);
+                            to = int.Parse(args[2]);
+                            top = int.Parse(args[3]);
+                        }
+                        else
+                        {
+                            Console.WriteLine("--Expected FROM TO--");
+                        }
+                        WriteBlogsToFileAll(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
+                        break;
+
+                    case "-replies": //colection posts with replies
+                        try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
+                        WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
+
+                        break;
+
+                    default:
+                        Console.WriteLine("** Unknown Command ** " + args[0]);
+                        break;
+                }
             }
 
 
 
             System.Console.WriteLine("<fin>:/");
-            System.Console.ReadKey();
+            //System.Console.ReadKey();
         }
 
         static void WritePostBlogsToFile(string outPath)
         {
-            List<Tuple<string, long>> posts = DataAccess.GetPosts(); 
+            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(); 
 
             using (StreamWriter sw = new StreamWriter(outPath, true))
             {
@@ -98,6 +204,7 @@ namespace URLNotesGrabberCORE
                 }
                 blogs = blogs.Distinct().ToList();
                 blogs.Sort();
+                blogs.Reverse();
 
                 foreach (var blog in blogs)
                 {
@@ -107,10 +214,11 @@ namespace URLNotesGrabberCORE
             }
         }
 
-        static void WriteBlogsToFile(string outPath, bool reblogsOnly = false)
+        static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
         {
-            List<string> blogs = DataAccess.GetBlogs(reblogsOnly);
+            List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
             blogs.Sort();
+            blogs.Reverse();
 
             using (StreamWriter sw = new StreamWriter(outPath, true))
             {
@@ -118,23 +226,48 @@ namespace URLNotesGrabberCORE
                 {
                     Console.WriteLine(blog);
                     sw.WriteLine(blog + ".tumblr.com");
+                    DataAccess.UpdateBlogOutput(blog);
                 }
             }
         }
 
-        protected static bool ContainsAny(string input)
+        static void WriteBlogsToFileAll(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
+        {
+            List<string> blogs = DataAccess.GetBlogsAll(reblogsOnly, from, to, top);
+            blogs.Sort();
+            blogs.Reverse();
+
+            using (StreamWriter sw = new StreamWriter(outPath, true))
+            {
+                foreach (var blog in blogs)
+                {
+                    Console.WriteLine(blog);
+                    sw.WriteLine(blog + ".tumblr.com");
+                    DataAccess.UpdateBlogOutput(blog);
+                }
+            }
+        }
+
+        static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
+        {
+            List<Tuple<string, long>> posts = DataAccess.GetReplies();
+
+            using (StreamWriter sw = new StreamWriter(outPath, true))
+            {
+                foreach (var post in posts)
+                {
+                    Console.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
+                    sw.WriteLine(@"https://tumblr.com/{0}/{1}", post.Item1, post.Item2);
+                }
+            }
+        }
+
+        protected static bool ContainsAny(string input, List<string> contains)
         {
             if (string.IsNullOrEmpty(input)) return false;
-            var items = new List<string>();
-            items.Add("zombaee");
-            items.Add("zomb-eh");
-            items.Add("ahzombae");
-            items.Add("thebugandme");
-            items.Add("lovingbabybug");
-            items.Add("swarthyvillain");
 
 
-            foreach (string item in items)
+            foreach (string item in contains)
             {
                 if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
                 {
@@ -144,22 +277,33 @@ namespace URLNotesGrabberCORE
             return false;
         }
 
-        static async Task GrabNotes(Tuple<string, long> post)
+        static async Task<string> GrabNotes(Tuple<string, long, long, long> post)
         {
             try
             {
                 int APICount = DataAccess.GetAPICount();
 
-                Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now);
+                Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
-                var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult();
+                var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3.ToString()).GetAwaiter().GetResult();
                 List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
 
                 if (response.statusCode == "NotFound")
-                    return;
+                {
+                    Thread.Sleep(1000);
+                    DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
+                    return response.statusCode;
+                }
                 if (response.statusCode == "TooManyRequests")
-                    return;
+                {
+                    for (int s = 0; s <= response.retryInSeconds; s += 15)
+                    {
+                        Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
+                        Thread.Sleep(15000);
+                    }
+                    return response.statusCode;
+                }
 
-                if (response.response != null)
+                if (response.response != null && response.response.notes != null)
                 {
                     Console.WriteLine("Notes\t" + response.response.notes.Count);
                     foreach (var note in response.response.notes)
@@ -170,12 +314,34 @@ namespace URLNotesGrabberCORE
                             break;
                     }
                 }
+                else if (response.response == null)
+                {
+                    Console.WriteLine("##### Response Null - API Failure? ###");
+                    return "FAILURE";
+                }
+                else if (response.response.notes == null)
+                {
+                    Console.WriteLine("##### Notes Null - WHY? ###");
+                    return "FAILURE";
+                }
+                else
+                { 
                     if (response.response == null)
-                    throw new Exception("Response is null");
+                    {
+                        Console.WriteLine("response.response == null");
+                        return "FAILURE";
+                    }
 
-                while (response.response != null && response.response._links != null)
+                    while (response.response != null
+                            && response.response._links != null
+                            && long.Parse(response.response._links.next.query_params.before_timestamp) >= post.Item3)
                     {
                         response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
+                        if (response.response.notes is null)
+                        {
+                            Console.WriteLine("response.response.notes == null");
+                            return "NULL NOTES";
+                        }
 
                         Console.WriteLine("Notes\t" + response.response.notes.Count);
                         foreach (var note in response.response.notes)
@@ -187,12 +353,18 @@ namespace URLNotesGrabberCORE
                         }
                     }
                 }
+
+                return "Success";
+            }
             catch(Exception ex) { Console.WriteLine(ex.ToString()); }
+
+
+            return "UNKNOWN";
         }
 
-        static async void CollectNotes(string outPath)
+        static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
         {
-            List<Tuple<string, long>> posts = DataAccess.GetPosts(true);
+            List<Tuple<string, long, long, long>> posts = DataAccess.GetPosts(withoutNotesOnly);
 
             RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
                                                                     PermitLimit = 300,
@@ -208,17 +380,24 @@ namespace URLNotesGrabberCORE
                 {
                     foreach (var post in posts)
                     {
+                        string status;
+
                         using RateLimitLease lease = limiter.AttemptAcquire(1);
                         if (lease.IsAcquired)
                         {
-                            await GrabNotes(post);
+                            Console.WriteLine("{0,32} - {1,15} - {2}", post.Item1, post.Item2, DateTimeOffset.FromUnixTimeSeconds(post.Item4).ToString());
+                            Thread.Sleep(1000);
+                            status = await GrabNotes(post);
                         }
                         else
                         {
                             Console.WriteLine("!@@@@@@@ - Rate Limited Exceeded: No Lease Available");
                             return;
                         }
+                        if(status == "Success")
                             DataAccess.UpdatePostMarkNotesCollected(post.Item1, post.Item2);
+                        else
+                            Console.WriteLine("GrabNotes Result: " + status);
                     }
                 }
             }
@@ -229,20 +408,22 @@ namespace URLNotesGrabberCORE
         }
 
 
-        static void TraverseDirectory(string path, string outPath)
+        static void TraverseDirectory(string path, string outPath, List<string> contains, string blogName = "")
         {
             // Get all directories in the current directory and sort them alphabetically
             var directories = Directory.GetDirectories(path);
             Array.Sort(directories, StringComparer.InvariantCulture);
 
-            using (StreamWriter sw = new StreamWriter(outPath, true))
+            //using (StreamWriter sw = new StreamWriter(outPath, true))
-            {
+            //{
             //sw.WriteLine("=====" + path);
-            }
+            //}
+
             foreach (var directory in directories)
             {
+
                 Console.WriteLine("Directory: " + directory);
-                TraverseDirectory(directory, outPath); // Recursively traverse subdirectories
+                TraverseDirectory(directory, outPath, contains, blogName); // Recursively traverse subdirectories
             }
             try
             {
@@ -250,14 +431,14 @@ namespace URLNotesGrabberCORE
                 // Process all files in the current directory
                 foreach (var file in Directory.GetFiles(path))
                 {
-                    if (file.EndsWith(".txt"))
+                    if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
                     {
-                        Console.WriteLine($"=====File: {file}");
+                        //Console.WriteLine($"=====File: {file}");
 
-                        using (StreamWriter sw = new StreamWriter(outPath, true))
+                        //using (StreamWriter sw = new StreamWriter(outPath, true))
-                        {
+                        //{
                         //sw.WriteLine("---" + file);
-                        }
+                        //}
 
                         try
                         {
@@ -268,7 +449,40 @@ namespace URLNotesGrabberCORE
                             {
                                 //if (line.StartsWith(@"Reblog url: https://") && !line.Contains(@"zombaee") && !line.Contains(@"zomb-eh") && !line.Contains(@"deactivated"))
                                 if (line.StartsWith(@"Post id:"))
+                                {   //  if there were no files, let's still collect post info
+                                    if (reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != "." && reblog.downloadedFiles == ".")
                                     {
+                                        if (!reblog.reblogURL.Contains(@"deactivated")
+                                            && reblog.reblogURL.Length != 0
+                                            && ContainsAny(reblog.reblogURL, contains))
+                                        {
+                                            DirectoryInfo currentDir = new DirectoryInfo(path);
+                                            if (!headerWasWritten)
+                                            {
+                                                headerWasWritten = true;
+                                            }
+
+                                            //if (reblog.reblogURL.Contains("/blog/private")
+                                            //    || reblog.body.Contains("/blog/private"))
+                                            //{
+                                            //    Console.WriteLine(reblog.postID);
+                                            //    Console.WriteLine(reblog.postURL);
+                                            //    Console.WriteLine(reblog.date);
+                                            //    Console.WriteLine(reblog.body);
+                                            //    Console.WriteLine(reblog.reblogKey);
+                                            //    Console.WriteLine(reblog.reblogURL);
+                                            //    Console.WriteLine(reblog.reblogName);
+                                            //    Console.WriteLine(reblog.downloadedFiles);
+                                            //}
+                                            string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
+                                            urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
+                                            DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
+                                                                reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
+                                                                reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
+                                                                reblog.title, false);
+                                        }
+                                    }
+
                                     reblog = new ReblogRecord();
                                     reblog.postID = line.Substring(9).Trim();
                                 }
@@ -294,38 +508,109 @@ namespace URLNotesGrabberCORE
                                 {
                                     reblog.date = line.Substring(6).Trim();
                                 }
-
+                                if (line.StartsWith(@"Body:"))
-                                if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
                                 {
-                                    if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated"))
+                                    reblog.body = line.Substring(6).Trim();
+                                }
+                                if (line.StartsWith(@"Post url:"))
+                                {
+                                    reblog.postURL = line.Substring(10).Trim();
+                                }
+                                if (line.StartsWith(@"Answer:"))
+                                {
+                                    reblog.answer = line.Substring(8).Trim();
+                                }
+                                if (line.StartsWith(@"Audio Caption:"))
+                                {
+                                    reblog.audioCaption = line.Substring(15).Trim();
+                                }
+                                if (line.StartsWith(@"Blog Name:"))
+                                {
+                                    reblog.blogName = line.Substring(11).Trim();
+                                }
+                                if (line.StartsWith(@"Link:"))
+                                {
+                                    reblog.link = line.Substring(6).Trim();
+                                }
+                                if (line.StartsWith(@"Photo Caption:"))
+                                {
+                                    reblog.photoCaption = line.Substring(15).Trim();
+                                }
+                                if (line.StartsWith(@"Photo url:"))
+                                {
+                                    reblog.photoURL = line.Substring(11).Trim();
+                                }
+                                if (line.StartsWith(@"Question:"))
+                                {
+                                    reblog.question = line.Substring(10).Trim();
+                                }
+                                if (line.StartsWith(@"Quote:"))
+                                {
+                                    reblog.quote = line.Substring(7).Trim();
+                                }
+                                if (line.StartsWith(@"Slug:"))
+                                {
+                                    reblog.slug = line.Substring(6).Trim();
+                                }
+                                if (line.StartsWith(@"Summary:"))
+                                {
+                                    reblog.summary = line.Substring(9).Trim();
+                                }
+                                if (line.StartsWith(@"Tags:"))
+                                {
+                                    reblog.tags = line.Substring(6).Trim();
+                                }
+                                if (line.StartsWith(@"Title:"))
+                                {
+                                    reblog.title = line.Substring(7).Trim();
+                                }
+
+                                if ((reblog.downloadedFiles != "."
+                                        || reblog.reblogURL.Contains("/blog/private")
+                                        || reblog.body.Contains("/blog/private")) && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
+                                {
+                                    if ((ContainsAny(reblog.downloadedFiles, contains)
+                                        || reblog.reblogURL.Contains("/blog/private")
+                                        || reblog.body.Contains("/blog/private"))
+                                        // && !ContainsAny(reblog.reblogURL, contains)
+                                        && !reblog.reblogURL.Contains(@"deactivated"))
                                     {
                                         DirectoryInfo currentDir = new DirectoryInfo(path);
                                         if (!headerWasWritten)
                                         {
                                             headerWasWritten = true;
                                         }
-                                        Console.WriteLine(reblog.postID);
+                                        //if (reblog.reblogURL.Contains("/blog/private")
-                                        Console.WriteLine(reblog.date);
+                                        //    || reblog.body.Contains("/blog/private"))
-                                        Console.WriteLine(reblog.reblogKey);
+                                        //{
-                                        Console.WriteLine(reblog.reblogURL);
+                                        //    Console.WriteLine(reblog.postID);
-                                        Console.WriteLine(reblog.reblogName);
+                                        //    Console.WriteLine(reblog.postURL);
-                                        Console.WriteLine(reblog.downloadedFiles);
+                                        //    Console.WriteLine(reblog.date);
+                                        //    Console.WriteLine(reblog.body);
+                                        //    Console.WriteLine(reblog.reblogKey);
+                                        //    Console.WriteLine(reblog.reblogURL);
+                                        //    Console.WriteLine(reblog.reblogName);
+                                        //    Console.WriteLine(reblog.downloadedFiles);
+                                        //}
                                         string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
                                         urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
-                                        DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL );
+                                        DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date, reblog.postURL, reblog.slug, reblog.reblogKey,
+                                                                reblog.reblogName, reblog.summary, reblog.quote, reblog.body, reblog.tags, reblog.link, reblog.photoURL,
+                                                                reblog.photoCaption, reblog.downloadedFiles, reblog.audioCaption, reblog.question, reblog.answer,
+                                                                reblog.title, true);
                                     }
                                 }
                             }
                             urls.Sort();
 
 
-                            using (StreamWriter sw = new StreamWriter(outPath, true))
+                            //using (StreamWriter sw = new StreamWriter(outPath, true))
-                            {
+                            //{
-                                foreach (string line in urls.Distinct())
+                            //    foreach (string line in urls.Distinct())
-                                {
+                            //    {
-                                    sw.WriteLine(line);
+                            //        sw.WriteLine(line);
-                                }
+                            //    }
-                            }
+                            //}
                         }
                         catch (Exception e)
                         {
@@ -341,5 +626,50 @@ namespace URLNotesGrabberCORE
                 Console.WriteLine($"An error occurred: {ex.Message}");
             }
         }
+
+
+        static void TraverseDirectoryForCorruption(string path, string outPath, List<string> contains, string blogName = "")
+        {
+            var directories = Directory.GetDirectories(path);
+            Array.Sort(directories, StringComparer.InvariantCulture);
+
+            foreach (var directory in directories)
+            {
+                //Console.WriteLine("Directory: " + directory);
+                TraverseDirectoryForCorruption(directory, outPath, contains, blogName); // Recursively traverse subdirectories
+            }
+            try
+            {
+                bool headerWasWritten = false;
+                foreach (var file in Directory.GetFiles(path))
+                {
+                    if (file.EndsWith(".txt") && (path.Contains(blogName) || blogName == ""))
+                    {
+                        try
+                        {
+                            var urls = new List<string>();
+                            var reblog = new ReblogRecord();
+
+                            foreach (string line in File.ReadLines(file))
+                            {
+                                if (!line.StartsWith(@"Post id:") && line.Contains(@"id:"))
+                                {
+                                    Console.WriteLine(file);
+                                }
+                            }
+                        }
+                        catch (Exception e)
+                        {
+                            Console.WriteLine("The file could not be read:");
+                            Console.WriteLine(e.Message);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine($"An error occurred: {ex.Message}");
+            }
+        }
     }
 }

URLNotesGrabberCORE/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "URLNotesGrabberCORE": {
       "commandName": "Project",
-      "commandLineArgs": "-br"
+      "commandLineArgs": "-collect 1"
     }
   }
 }

URLNotesGrabberCORE/ResponseNotes.cs

@@ -74,8 +74,11 @@ namespace URLNotesGrabberCORE
     public class Root
     {
         public Meta meta { get; set; }
+        [JsonConverter(typeof(EmptyArrayOrObjectConverter<Response>))]
         public Response response { get; set; }
 
         public string statusCode { get; set; }
+
+        public int retryInSeconds { get; set; }
     }
 }

URLNotesGrabberCORE/URLNotesGrabberCORE.csproj

@@ -7,6 +7,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'" />
+
   <ItemGroup>
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />

URLNotesGrabberCORE/URLNotesGrabberCORE.sln

@@ -0,0 +1,24 @@
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.5.2.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "URLNotesGrabberCORE", "URLNotesGrabberCORE.csproj", "{008F0DF4-6E08-F5FD-D644-B9B999F533E7}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{008F0DF4-6E08-F5FD-D644-B9B999F533E7}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+	GlobalSection(ExtensibilityGlobals) = postSolution
+		SolutionGuid = {C1F768CD-5EA9-444F-87CF-923EE287ACDB}
+	EndGlobalSection
+EndGlobal

URLNotesGrabberCORE/appsettings.json

@@ -5,6 +5,14 @@
     "PathOutputJSON": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotesJSON.txt",
     "PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt",
     "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
-    "PathDB":          "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db"
+    "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
+    "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
+    "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
+  },
+  "TumblrApi": {
+    "ConsumerKey": "PtsBCGumcsgihyynxUh8b47Jfmi7uXhEIOU46bmhdsUSJ5mEP3",
+    "ConsumerSecret": "sA8BwNVTVKqBRRJmHbAD6NuyKPJ3bb9cei2bYhMgqT8cLX8tSG",
+    "OAuthToken": "HPJI6IijHoKN6WzBumG7KjS7g01iCu07jQsulpueKbWN1ZJ35J",
+    "OAuthTokenSecret": "ajkN0Z1kKrsJaIXZEDI8zLsjRSQxFkfgO1i5k78FuQYCQjSY7w"
   }
 }