.

URLNotesGrabberCORE/DataAccess.cs

@@ -8,6 +8,7 @@ using RestSharp.Authenticators.OAuth;
 using RestSharp.Authenticators;
 using RestSharp;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 
 namespace URLNotesGrabberCORE
 {
@@ -34,6 +35,14 @@ namespace URLNotesGrabberCORE
 
     internal class DataAccess
     {
+        public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
+        {
+            // Unix timestamp is seconds past epoch
+            DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
+            dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
+            return dateTime;
+        }
+
         #region Adds
         public static void AddBlog(string blogName, string DBPath = @"TL.db")
         {
@@ -50,6 +59,7 @@ namespace URLNotesGrabberCORE
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
+                Console.WriteLine("+ " + blogName);
             }
             catch (Exception ex)
             {
@@ -63,9 +73,10 @@ namespace URLNotesGrabberCORE
         }
 
 
-        public static void AddPost(string blogName, long postID, string reblogURL, string DBPath = @"TL.db")
+        public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string DBPath = @"TL.db")
         {
             try { AddBlog(blogName, DBPath); } catch { }
+            try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
 
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
@@ -73,7 +84,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL) values('" + blogName + "', " + postID + ", '" + reblogURL + "')";
+                string sql = "INSERT INTO Posts (BlogName, PostID, reblogURL, postDate) values('" + blogName + "', " + postID + ", '" + reblogURL + "', '" + postDate + "')";
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
@@ -81,8 +92,10 @@ namespace URLNotesGrabberCORE
             catch (Exception ex)
             {
                 if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                {
                     Console.WriteLine(ex.Message);
                 }
+            }
             finally
             {
                 connection.Close();
@@ -119,6 +132,8 @@ namespace URLNotesGrabberCORE
             //try { AddPost(rootBlogName, postID, DBPath); } catch { }
             try { AddBlog(noteBlogName, DBPath); } catch { }
 
+            Console.WriteLine("{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName);
+
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
             try
@@ -148,7 +163,64 @@ namespace URLNotesGrabberCORE
         #endregion Adds
 
         #region Gets
-        public static List<Tuple<string, long>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
+        public static List<Tuple<string, long, string>> GetPosts(bool withoutNotesOnly = false, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+            List<Tuple<string, long, string>> posts = new List<Tuple<string, long, string>>();
+
+            try
+            {
+                connection.Open();
+
+                string sql = "SELECT " +
+                             "  Posts.BlogName, " +
+                             "  Posts.PostID, " +
+                             "  Max(Notes.timestamp) as LatestNoteTimestamp " +
+                             "FROM " +
+                             "   Posts INNER JOIN " +
+                             "   Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " +
+                             "WHERE NotFound = 0 ";
+
+                if (withoutNotesOnly)
+                    sql +=  " and HasNotesGathered = 0 "; 
+                
+                sql +=     "GROUP BY " +
+                           "   Posts.BlogName, Posts.PostID " +
+                           "ORDER BY " +
+                           "   Posts.NotesGatheredDatetime, Max(Notes.timestamp), BlogName, Posts.PostID";
+
+                using (SQLiteCommand command = new SQLiteCommand(sql, connection))
+                {
+                    using (SQLiteDataReader reader = command.ExecuteReader())
+                    {
+                        while (reader.Read())
+                        {
+                            Tuple<string, long, string> post = default;
+                            string blog = null;
+                            long id = 0;
+                            long timestamp;
+                            timestamp = reader.GetInt64(2); // Assuming Id is the first column
+                            id = reader.GetInt64(1); // Assuming Id is the first column
+                            blog = reader.GetString(0); // Assuming Title is the second column
+
+                            post = new Tuple<string, long, string>(blog, id, timestamp.ToString());
+                            posts.Add(post);
+                        }
+                    }
+                }
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+            return posts;
+        }
+        public static List<Tuple<string, long>> GetReplies(string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
@@ -157,16 +229,8 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "SELECT " +
+                string sql = "SELECT	distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
-                             " MIN(blogName) as blogName, " +
-                             " postID " +
-                             "FROM" +
-                             " Posts ";
 
-                if (withoutNotesOnly)
-                    sql += " WHERE HasNotesGathered = 0 and NotFound = 0";
-
-                sql += " GROUP BY  postID ORDER BY reblogURL, PostID";
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     using (SQLiteDataReader reader = command.ExecuteReader())
@@ -176,8 +240,9 @@ namespace URLNotesGrabberCORE
                             Tuple<string, long> post = default;
                             string blog = null;
                             long id = 0;
-                            id = reader.GetInt64(1); // Assuming Id is the first column
+
-                            blog = reader.GetString(0); // Assuming Title is the second column
+                            id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
+                            blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
 
                             post = new Tuple<string, long>(blog, id);
                             posts.Add(post);
@@ -240,7 +305,7 @@ namespace URLNotesGrabberCORE
         }
 
 
-        public static List<string> GetBlogs(bool reblogsOnly, string DBPath = @"TL.db")
+        public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
             List<string> blogs = new List<string>();
@@ -250,9 +315,9 @@ namespace URLNotesGrabberCORE
                 connection.Open();
                 string sql = "";
                 if (reblogsOnly)
-                    sql = "SELECT distinct blogName from blogs B inner join notes N on N.NoteBlogName = B.BlogName where N.type = 'reblog' and B.HasBeenOutput = 0  ORDER BY BlogName";
+                    sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName  ORDER BY count(*) desc, BlogName LIMIT " + top;
                 else
-                    sql = "SELECT * FROM Blogs ORDER BY BlogName";
+                    sql = "SELECT NoteBlogName as blogName, count(*) from notes inner join blogs on blogs.BlogName = notes.NoteBlogName where type NOT IN ('reblog', 'reply', 'posted') and HasBeenOutput = 0 group by NoteBlogName  ORDER BY count(*) desc, BlogName LIMIT " + top;
                 using (SQLiteCommand command = new SQLiteCommand(sql, connection))
                 {
                     using (SQLiteDataReader reader = command.ExecuteReader())
@@ -291,7 +356,7 @@ namespace URLNotesGrabberCORE
             {
                 connection.Open();
 
-                string sql = "UPDATE Posts SET HasNotesGathered = 1 WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = " + DateTimeOffset.UtcNow.ToUnixTimeSeconds() + " WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
                 SQLiteCommand command = new SQLiteCommand(sql, connection);
 
                 command.ExecuteNonQuery();
@@ -333,7 +398,32 @@ namespace URLNotesGrabberCORE
         }
 
 
-        public static void UpdatePostBlogOutput(string blogName, string DBPath = @"TL.db")
+        public static void UpdatePostSetDate(string blogName, long postID, string postDate, string DBPath = @"TL.db")
+        {
+            SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
+
+            try
+            {
+                connection.Open();
+
+                string sql = "UPDATE Posts SET postDate = '" + postDate + "' WHERE BlogName = '" + blogName + "' AND PostID = " + postID;
+                SQLiteCommand command = new SQLiteCommand(sql, connection);
+
+                command.ExecuteNonQuery();
+            }
+            catch (Exception ex)
+            {
+                if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
+                    Console.WriteLine(ex.Message);
+            }
+            finally
+            {
+                connection.Close();
+            }
+        }
+
+
+        public static void UpdateBlogOutput(string blogName, string DBPath = @"TL.db")
         {
             SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
 
@@ -407,7 +497,7 @@ namespace URLNotesGrabberCORE
             if (timestamp != null)
             {
                 URL += "&before_timestamp=" + timestamp;
-                Thread.Sleep(500);
+                Thread.Sleep(1000);
             }
             var client = new RestClient(URL);
             var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: CONSUMER_KEY,
@@ -425,7 +515,7 @@ namespace URLNotesGrabberCORE
             var response = client.Execute(request);
 
             var myJsonResponse = response.Content;
-            Console.WriteLine(timestamp + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
+            Console.WriteLine(timestamp.ToString() + '\t' + DateTime.Now + '\t' + DataAccess.UpdateAPICount());
             Root myDeserializedClass = new Root();
 
             try

URLNotesGrabberCORE/Program.cs

@@ -18,21 +18,27 @@ namespace URLNotesGrabberCORE
                                     .Build();
             var settings = config.GetSection("appSettings");
 
+            List<string> contains = settings.GetValue<string>("ContainsList").Split(',').ToList();
+
             if (args.Length == 0) //Traverse folder structure to add posts and thus blogs to DB
             {
-                TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"));
+                TraverseDirectory( settings.GetValue<string>("PathInput"), settings.GetValue<string>("PathOutputBlogs"), contains);
             }
             else
-                switch(args[0])
+                switch (args[0])
                 {
-                    case "-n": //Manual test 
+                    case "-n":
+                        #region Manual test 
 
-                                var response = APIAccess.GrabNotes("mangsbraaap", 185490841455).GetAwaiter().GetResult();
+                        string blogName = args[1];
+                        long postID = long.Parse(args[2]);
+
+                        var response = APIAccess.GrabNotes(blogName, postID).GetAwaiter().GetResult();
                         List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
 
                         foreach (var note in response.response.notes)
                         {
-                                    note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                            note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString();
                             //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
                             if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
                                 break;
@@ -40,16 +46,17 @@ namespace URLNotesGrabberCORE
 
                         while (response.response._links != null)
                         {
-                                    response = APIAccess.GrabNotes("mangsbraaap", 185490841455, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
+                            response = APIAccess.GrabNotes(blogName, postID, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
 
                             foreach (var note in response.response.notes)
                             {
-                                        note.reblog_parent_blog_name = "mangsbraaap"; note.post_id = "185490841455";
+                                note.reblog_parent_blog_name = blogName; note.post_id = postID.ToString(); ;
                                 //notes.Add(new Tuple<string, string>(note.blog_url, note.type));
                                 if (DataAccess.AddNote(note.reblog_parent_blog_name, note.blog_name, long.Parse(note.post_id), note.timestamp, note.type))
                                     break;
                             }
                         }
+                        #endregion
 
                         break;
 
@@ -62,23 +69,57 @@ namespace URLNotesGrabberCORE
                         break;
 
                     case "-pn": //collect notes from all posts
-                        CollectNotes(settings.GetValue<string>("PathOutput"));
+                        bool withoutNotesOnly = true;
+
+                        if (args[1] is not null)
+                        {
+                            bool.TryParse(args[1], out withoutNotesOnly);
+
+                            Console.WriteLine("Without Notes Only: {0}\t{1}", withoutNotesOnly, args[1]);
+                        }
+                        CollectNotes(settings.GetValue<string>("PathOutput"), withoutNotesOnly);
                         break;
 
                     case "-br": //collect notes from all posts
                         WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), true);
                         break;
+
+                    case "-bo": //collect notes from all posts
+                        int from = 1, to = 999999, top = 100;
+
+                        if (args[1] is not null && args[2] is not null && args[3] is not null)
+                        {
+                            from = int.Parse(args[1]);
+                            to = int.Parse(args[2]);
+                            top = int.Parse(args[3]);
+                        }
+                        else
+                        {
+                            Console.WriteLine("--Expected FROM TO--");
+                        }
+                        WriteBlogsToFile(settings.GetValue<string>("PathOutputBlogs"), false, from, to, top);
+                        break;
+
+                    case "-pr": //colection posts with replies
+                        try { System.IO.File.Delete(settings.GetValue<string>("PathOutputReplies")); } catch { }
+                        WriteRepliesToFile(settings.GetValue<string>("PathOutputBlogs"), false);
+
+                        break;
+
+                    default:
+                        Console.WriteLine("** Unknown Command ** " + args[0]);
+                        break;
                 }
 
 
 
             System.Console.WriteLine("<fin>:/");
-            System.Console.ReadKey();
+            //System.Console.ReadKey();
         }
 
         static void WritePostBlogsToFile(string outPath)
         {
-            List<Tuple<string, long>> posts = DataAccess.GetPosts(); 
+            List<Tuple<string, long, string>> posts = DataAccess.GetPosts(); 
 
             using (StreamWriter sw = new StreamWriter(outPath, true))
             {
@@ -98,9 +139,9 @@ namespace URLNotesGrabberCORE
             }
         }
 
-        static void WriteBlogsToFile(string outPath, bool reblogsOnly = false)
+        static void WriteBlogsToFile(string outPath, bool reblogsOnly = false, int from = 0, int to = 999999, int top = 100)
         {
-            List<string> blogs = DataAccess.GetBlogs(reblogsOnly);
+            List<string> blogs = DataAccess.GetBlogs(reblogsOnly, from, to, top);
             blogs.Sort();
 
             using (StreamWriter sw = new StreamWriter(outPath, true))
@@ -109,24 +150,31 @@ namespace URLNotesGrabberCORE
                 {
                     Console.WriteLine(blog);
                     sw.WriteLine(blog + ".tumblr.com");
-                    DataAccess.UpdatePostBlogOutput(blog);
+                    DataAccess.UpdateBlogOutput(blog);
                 }
             }
         }
 
-        protected static bool ContainsAny(string input)
+        static void WriteRepliesToFile(string outPath, bool reblogsOnly = false)
+        {
+            List<Tuple<string, long>> posts = DataAccess.GetReplies();
+
+            using (StreamWriter sw = new StreamWriter(outPath, true))
+            {
+                foreach (var post in posts)
+                {
+                    Console.WriteLine("{0}\t{1}", post.Item1, post.Item2);
+                    sw.WriteLine("{0}\t{1}", post.Item1 + ".tumblr.com", post.Item2);
+                }
+            }
+        }
+
+        protected static bool ContainsAny(string input, List<string> contains)
         {
             if (string.IsNullOrEmpty(input)) return false;
-            var items = new List<string>();
-            items.Add("zombaee");
-            items.Add("zomb-eh");
-            items.Add("ahzombae");
-            items.Add("thebugandme");
-            items.Add("lovingbabybug");
-            items.Add("swarthyvillain");
 
 
-            foreach (string item in items)
+            foreach (string item in contains)
             {
                 if (input.IndexOf(item, StringComparison.OrdinalIgnoreCase) >= 0)
                 {
@@ -136,28 +184,28 @@ namespace URLNotesGrabberCORE
             return false;
         }
 
-        static async Task<string> GrabNotes(Tuple<string, long> post)
+        static async Task<string> GrabNotes(Tuple<string, long, string> post)
         {
             try
             {
                 int APICount = DataAccess.GetAPICount();
 
-                Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now);
+                Console.WriteLine(post.Item1 + '\t' + post.Item2 + '\t' + DateTime.Now + "\t" + APICount);
-                var response = APIAccess.GrabNotes(post.Item1, post.Item2).GetAwaiter().GetResult();
+                var response = APIAccess.GrabNotes(post.Item1, post.Item2, post.Item3).GetAwaiter().GetResult();
                 List<Tuple<string, string>> notes = new List<Tuple<string, string>>();
 
                 if (response.statusCode == "NotFound")
                 {
-                    Thread.Sleep(1000);
+                    Thread.Sleep(2000);
                     DataAccess.UpdatePostMarkNotFound(post.Item1, post.Item2);
                     return response.statusCode;
                 }
                 if (response.statusCode == "TooManyRequests")
                 {
-                    for( int s  = 0; s <= response.retryInSeconds; s++)
+                    for( int s  = 0; s <= response.retryInSeconds; s+=15)
                     {
-                        Console.WriteLine("Sleeping for {0} more seconds", response.retryInSeconds - s);
+                        Console.WriteLine("Sleeping for {0} more seconds, until {1}", response.retryInSeconds - s, DateTime.Now.AddSeconds(response.retryInSeconds - s).ToShortTimeString());
-                        Thread.Sleep(1000);
+                        Thread.Sleep(15000);
                     }
                     return response.statusCode;
                 }
@@ -174,13 +222,21 @@ namespace URLNotesGrabberCORE
                     }
                 }
                 if (response.response == null)
+                {
+                    Console.WriteLine("response.response == null");
                     return "FAILURE";
+                }
 
-                while (response.response != null && response.response._links != null)
+                while (     response.response != null 
+                        &&  response.response._links != null 
+                        &&  long.Parse(response.response._links.next.query_params.before_timestamp) >= long.Parse(post.Item3)) 
                 {
                     response = APIAccess.GrabNotes(post.Item1, post.Item2, response.response._links.next.query_params.before_timestamp).GetAwaiter().GetResult();
                     if (response.response.notes is null)
+                    {
+                        Console.WriteLine("response.response.notes == null");
                         return "NULL NOTES";
+                    }
 
                     Console.WriteLine("Notes\t" + response.response.notes.Count);
                     foreach (var note in response.response.notes)
@@ -200,9 +256,9 @@ namespace URLNotesGrabberCORE
             return "UNKNOWN";
         }
 
-        static async void CollectNotes(string outPath)
+        static async void CollectNotes(string outPath, bool withoutNotesOnly = true)
         {
-            List<Tuple<string, long>> posts = DataAccess.GetPosts(true);
+            List<Tuple<string, long, string>> posts = DataAccess.GetPosts(withoutNotesOnly);
 
             RateLimiter limiter = new SlidingWindowRateLimiter(new SlidingWindowRateLimiterOptions {
                                                                     PermitLimit = 300,
@@ -244,20 +300,21 @@ namespace URLNotesGrabberCORE
         }
 
 
-        static void TraverseDirectory(string path, string outPath)
+        static void TraverseDirectory(string path, string outPath, List<string> contains)
         {
             // Get all directories in the current directory and sort them alphabetically
             var directories = Directory.GetDirectories(path);
             Array.Sort(directories, StringComparer.InvariantCulture);
 
-            using (StreamWriter sw = new StreamWriter(outPath, true))
+            //using (StreamWriter sw = new StreamWriter(outPath, true))
-            {
+            //{
                 //sw.WriteLine("=====" + path);
-            }
+            //}
+
             foreach (var directory in directories)
             {
                 Console.WriteLine("Directory: " + directory);
-                TraverseDirectory(directory, outPath); // Recursively traverse subdirectories
+                TraverseDirectory(directory, outPath, contains); // Recursively traverse subdirectories
             }
             try
             {
@@ -267,12 +324,12 @@ namespace URLNotesGrabberCORE
                 {
                     if (file.EndsWith(".txt"))
                     {
-                        Console.WriteLine($"=====File: {file}");
+                        //Console.WriteLine($"=====File: {file}");
 
-                        using (StreamWriter sw = new StreamWriter(outPath, true))
+                        //using (StreamWriter sw = new StreamWriter(outPath, true))
-                        {
+                        //{
                             //sw.WriteLine("---" + file);
-                        }
+                        //}
 
                         try
                         {
@@ -312,35 +369,35 @@ namespace URLNotesGrabberCORE
 
                                 if (reblog.downloadedFiles != "." && reblog.reblogName != "." && reblog.postID != "." && reblog.date != "." && reblog.reblogURL != ".")
                                 {
-                                    if (ContainsAny(reblog.downloadedFiles) && !ContainsAny(reblog.reblogURL) && !reblog.reblogURL.Contains(@"deactivated"))
+                                    if (ContainsAny(reblog.downloadedFiles, contains) && !ContainsAny(reblog.reblogURL, contains) && !reblog.reblogURL.Contains(@"deactivated"))
                                     {
                                         DirectoryInfo currentDir = new DirectoryInfo(path);
                                         if (!headerWasWritten)
                                         {
                                             headerWasWritten = true;
                                         }
-                                        Console.WriteLine(reblog.postID);
+                                        //Console.WriteLine(reblog.postID);
-                                        Console.WriteLine(reblog.date);
+                                        //Console.WriteLine(reblog.date);
-                                        Console.WriteLine(reblog.reblogKey);
+                                        //Console.WriteLine(reblog.reblogKey);
-                                        Console.WriteLine(reblog.reblogURL);
+                                        //Console.WriteLine(reblog.reblogURL);
-                                        Console.WriteLine(reblog.reblogName);
+                                        //Console.WriteLine(reblog.reblogName);
-                                        Console.WriteLine(reblog.downloadedFiles);
+                                        //Console.WriteLine(reblog.downloadedFiles);
                                         string curDir = currentDir.Name.Replace("_1", "").Replace("_2", "").Replace("_3", "").Replace("_4", "").Replace("_5", "").Replace("_6", "").Replace("_7", "").Replace("_8", "").Replace("_9", "");
                                         urls.Add(string.Format("{0}\t{1}", curDir, reblog.postID));
-                                        DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL );
+                                        DataAccess.AddPost(curDir, long.Parse(reblog.postID), reblog.reblogURL, reblog.date );
                                     }
                                 }
                             }
                             urls.Sort();
 
 
-                            using (StreamWriter sw = new StreamWriter(outPath, true))
+                            //using (StreamWriter sw = new StreamWriter(outPath, true))
-                            {
+                            //{
-                                foreach (string line in urls.Distinct())
+                            //    foreach (string line in urls.Distinct())
-                                {
+                            //    {
-                                    sw.WriteLine(line);
+                            //        sw.WriteLine(line);
-                                }
+                            //    }
-                            }
+                            //}
                         }
                         catch (Exception e)
                         {

URLNotesGrabberCORE/Properties/launchSettings.json

@@ -2,7 +2,7 @@
   "profiles": {
     "URLNotesGrabberCORE": {
       "commandName": "Project",
-      "commandLineArgs": "-pn"
+      "commandLineArgs": "-pn true 31"
     }
   }
 }

URLNotesGrabberCORE/appsettings.json

@@ -5,6 +5,8 @@
     "PathOutputJSON": "u:\\jim\\Documents\\Web Copies\\blogs\\GetNotesJSON.txt",
     "PathOutputPosts": "u:\\jim\\Documents\\Web Copies\\blogs\\GetPosts.txt",
     "PathOutputBlogs": "u:\\jim\\Documents\\Web Copies\\blogs\\GetBlogs.txt",
-    "PathDB":          "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db"
+    "PathOutputReplies": "u:\\jim\\Documents\\Web Copies\\blogs\\GetReplies.txt",
+    "PathDB": "u:\\jim\\Documents\\Web Copies\\blogs\\TL.db",
+    "ContainsList": "zombaee,zomb-eh,ahzombae,thebugandme,lovingbabybug,swarthyvillain,h4rdspot"
   }
 }