1990 lines
95 KiB
C#
1990 lines
95 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using System.Data.SQLite;
|
|
using RestSharp.Authenticators.OAuth;
|
|
using RestSharp.Authenticators;
|
|
using RestSharp;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Converters;
|
|
using Microsoft.Extensions.Diagnostics.Latency;
|
|
using System.Collections;
|
|
using Microsoft.Extensions.Configuration;
|
|
using System.IO;
|
|
using System.Data;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
|
|
class ReblogRecord
|
|
{
|
|
public string answer;
|
|
public string audioCaption;
|
|
public string blogName;
|
|
public string body;
|
|
public string date;
|
|
public string downloadedFiles;
|
|
public string link;
|
|
public string photoCaption;
|
|
public string photoURL;
|
|
public string postID;
|
|
public string postURL;
|
|
public string question;
|
|
public string quote;
|
|
public string reblogKey;
|
|
public string reblogName;
|
|
public string reblogURL;
|
|
public string slug;
|
|
public string summary;
|
|
public string tags;
|
|
public string title;
|
|
|
|
public ReblogRecord()
|
|
{
|
|
answer = ".";
|
|
audioCaption = ".";
|
|
blogName = ".";
|
|
body = ".";
|
|
date = ".";
|
|
downloadedFiles = ".";
|
|
link = ".";
|
|
photoCaption = ".";
|
|
photoURL = ".";
|
|
postID = ".";
|
|
postURL = ".";
|
|
question = ".";
|
|
quote = ".";
|
|
reblogKey = ".";
|
|
reblogName = ".";
|
|
reblogURL = ".";
|
|
slug = ".";
|
|
summary = ".";
|
|
tags = ".";
|
|
title = ".";
|
|
}
|
|
|
|
}
|
|
|
|
internal class DataAccess
|
|
{
|
|
private static IConfiguration? _configuration;
|
|
private static HashSet<long>? _postIdsToExclude;
|
|
private static readonly object _importPragmaLock = new object();
|
|
private static Tuple<string, string>? _savedImportPragmas;
|
|
private static string? _cachedDbPath;
|
|
|
|
static DataAccess()
|
|
{
|
|
_configuration = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
|
|
.Build();
|
|
var excludeSetting = _configuration["appSettings:PostIDToExclude"];
|
|
if (!string.IsNullOrWhiteSpace(excludeSetting))
|
|
{
|
|
_postIdsToExclude = excludeSetting.Split(',')
|
|
.Select(s => long.TryParse(s.Trim(), out var id) ? id : (long?)null)
|
|
.Where(id => id.HasValue)
|
|
.Select(id => id.Value)
|
|
.ToHashSet();
|
|
}
|
|
else
|
|
{
|
|
_postIdsToExclude = new HashSet<long>();
|
|
}
|
|
}
|
|
|
|
private static string GetDefaultDbPath()
|
|
{
|
|
if (_cachedDbPath != null)
|
|
return _cachedDbPath;
|
|
|
|
string? configPath = _configuration?["appSettings:PathDB"];
|
|
if (!string.IsNullOrWhiteSpace(configPath))
|
|
{
|
|
_cachedDbPath = configPath;
|
|
}
|
|
else
|
|
{
|
|
// Fallback to 3 levels up from the executable directory
|
|
_cachedDbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "TL.db");
|
|
}
|
|
return _cachedDbPath;
|
|
}
|
|
|
|
public static string Q(string input)
|
|
{
|
|
return "'" + input.Replace("'", "''") + "'";
|
|
}
|
|
|
|
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
|
|
{
|
|
// Unix timestamp is seconds past epoch
|
|
DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
|
dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
|
|
return dateTime;
|
|
}
|
|
|
|
public static void EnableImportModePragmas(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
lock (_importPragmaLock)
|
|
{
|
|
if (_savedImportPragmas != null)
|
|
return;
|
|
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string currentJournalMode = "delete";
|
|
string currentSynchronous = "2";
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA journal_mode;", connection))
|
|
{
|
|
currentJournalMode = Convert.ToString(cmd.ExecuteScalar()) ?? "delete";
|
|
}
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA synchronous;", connection))
|
|
{
|
|
currentSynchronous = Convert.ToString(cmd.ExecuteScalar()) ?? "2";
|
|
}
|
|
|
|
_savedImportPragmas = new Tuple<string, string>(currentJournalMode, currentSynchronous);
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA journal_mode=WAL;", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA synchronous=NORMAL;", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
Console.WriteLine($"[SQLite Import Mode] Enabled | journal_mode=WAL | synchronous=NORMAL (previous: journal_mode={currentJournalMode}, synchronous={currentSynchronous})");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
}
|
|
|
|
public static void RestoreImportModePragmas(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
lock (_importPragmaLock)
|
|
{
|
|
if (_savedImportPragmas == null)
|
|
return;
|
|
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string priorJournalMode = _savedImportPragmas.Item1;
|
|
string priorSynchronous = _savedImportPragmas.Item2;
|
|
|
|
using (var cmd = new SQLiteCommand($"PRAGMA journal_mode={priorJournalMode};", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
using (var cmd = new SQLiteCommand($"PRAGMA synchronous={priorSynchronous};", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
Console.WriteLine($"[SQLite Import Mode] Restored | journal_mode={priorJournalMode} | synchronous={priorSynchronous}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[SQLite Import Mode] Failed to restore pragmas: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
_savedImportPragmas = null;
|
|
connection.Close();
|
|
}
|
|
}
|
|
}
|
|
|
|
public static void EnsureReplyTextColumnExists(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Check if replyText column exists
|
|
string checkSql = "PRAGMA table_info(Notes);";
|
|
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
bool columnExists = false;
|
|
while (reader.Read())
|
|
{
|
|
string columnName = reader.GetString(1);
|
|
if (columnName.Equals("replyText", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
columnExists = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!columnExists)
|
|
{
|
|
// Add the column if it doesn't exist
|
|
connection.Close();
|
|
connection.Open();
|
|
|
|
string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT DEFAULT '.';";
|
|
using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection))
|
|
{
|
|
addCommand.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added replyText column to Notes table");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string checkSql = "PRAGMA table_info(Blogs);";
|
|
bool likesPulledExists = false;
|
|
bool likesCursorExists = false;
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string columnName = reader.GetString(1);
|
|
if (columnName.Equals("LikesPulled", StringComparison.OrdinalIgnoreCase)) likesPulledExists = true;
|
|
if (columnName.Equals("LikesCursor", StringComparison.OrdinalIgnoreCase)) likesCursorExists = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!likesPulledExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesPulled column to Blogs table");
|
|
}
|
|
|
|
if (!likesCursorExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesCursor column to Blogs table");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
#region Adds
|
|
public static void AddBlog(string blogName, bool byLikes = false, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
if (blogName.Contains("deact"))
|
|
return;
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Insert BlogName and DateAdded (current UTC datetime)
|
|
string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, DateCreated, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @DateCreated, @ByLikes)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@ByLikes", byLikes ? 1 : 0);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
// Console.WriteLine("+ " + blogName);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
try { AddBlog(blogName, byLikes, DBPath); } catch { }
|
|
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
|
|
try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL); } catch { }
|
|
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = @"INSERT INTO Posts (
|
|
BlogName,
|
|
PostID,
|
|
reblogURL,
|
|
PostDate,
|
|
PostURL,
|
|
Slug,
|
|
ReblogKey,
|
|
ReblogName,
|
|
Summary,
|
|
Quote,
|
|
Body,
|
|
Tags,
|
|
Link,
|
|
PhotoURL,
|
|
PhotoCaption,
|
|
DownloadedFiles,
|
|
AudioCaption,
|
|
Question,
|
|
Answer,
|
|
Title,
|
|
DateModified,
|
|
DateCreated,
|
|
RootBlogName,
|
|
RootURL,
|
|
HasImage,
|
|
ByLikes
|
|
) VALUES (" +
|
|
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
int rowsInserted = 0;
|
|
try
|
|
{
|
|
rowsInserted = command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
///
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string updateSql = "UPDATE Posts SET hasImage = " + (hasImage ? 1 : 0) + ", DateModified = '" + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss") + "' WHERE blogName = '" + blogName + "' AND postID = '" + postID + "'";
|
|
SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
|
|
|
updateCommand.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex2)
|
|
{
|
|
// Breakpoint here
|
|
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
{
|
|
Console.WriteLine(ex2.Message);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
///
|
|
}
|
|
}
|
|
|
|
// Only update HasBeenOutput and DateAdded if a new post was inserted
|
|
if (rowsInserted == 1)
|
|
{
|
|
try
|
|
{
|
|
string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateAdded = @DateAdded, DateModified = @DateModified WHERE BlogName = @BlogName";
|
|
using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
|
|
{
|
|
updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
|
|
updateBlogCommand.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
updateBlogCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
updateBlogCommand.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void AddAPICount(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Use INSERT OR IGNORE to avoid UNIQUE constraint errors when the date row already exists.
|
|
// Also explicitly initialize APICount to 0 in case the table has no default.
|
|
string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount, DateCreated) values(@date, 0, @DateCreated)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
|
|
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
//Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
|
|
try { AddBlog(noteBlogName, false, DBPath); } catch { }
|
|
|
|
SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection2.Open();
|
|
|
|
string sql = "INSERT OR IGNORE INTO Notes (rootBlogName, noteBlogName, PostID, TimeStamp, Type, DatetimeCrawled, DateModified, DateCreated) values(@rootBlogName, @noteBlogName, @PostID, @TimeStamp, @Type, @DatetimeCrawled, @DateModified, @DateCreated)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection2))
|
|
{
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.Parameters.AddWithValue("@TimeStamp", timestamp);
|
|
command.Parameters.AddWithValue("@Type", type ?? string.Empty);
|
|
command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
|
|
int rowsInserted = command.ExecuteNonQuery();
|
|
|
|
if (rowsInserted == 1)
|
|
{
|
|
ConsoleColor previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
|
|
Console.ForegroundColor = previousColor;
|
|
//Thread.Sleep(1000); // Brief pause to make new notes more noticeable in the console output
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
|
|
}
|
|
|
|
// Only update HasBeenOutput if a new note was inserted
|
|
if (rowsInserted == 1)
|
|
{
|
|
try
|
|
{
|
|
string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName";
|
|
using (var updateCommand = new SQLiteCommand(updateSql, connection2))
|
|
{
|
|
updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
|
|
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
updateCommand.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
Console.WriteLine("^^^^^ - SHORTCUT");
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection2.Close();
|
|
}
|
|
return false;
|
|
}
|
|
#endregion Adds
|
|
|
|
#region Gets
|
|
|
|
/// <summary>
|
|
///
|
|
/// </summary>
|
|
/// <param name="withoutNotesOnly"></param>
|
|
/// <param name="DBPath"></param>
|
|
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
|
|
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql;
|
|
|
|
if (withoutNotesOnly)
|
|
{
|
|
string beforeDateFilter = string.Empty;
|
|
if (beforeDate.HasValue)
|
|
{
|
|
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
|
beforeDateFilter = $"WHERE (U.NotesGatheredDateTime < {unixTimestamp} OR U.NotesGatheredDateTime IS NULL)" + Environment.NewLine;
|
|
}
|
|
|
|
sql = "WITH PostsWithCount AS" + Environment.NewLine +
|
|
"(" + Environment.NewLine +
|
|
" SELECT " + Environment.NewLine +
|
|
" P.BlogName," + Environment.NewLine +
|
|
" P.PostID," + Environment.NewLine +
|
|
" 1925013599 AS LatestNoteTimestamp," + Environment.NewLine +
|
|
" P.NotesGatheredDateTime," + Environment.NewLine +
|
|
" COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT," + Environment.NewLine +
|
|
" P.HasNotesGathered," + Environment.NewLine +
|
|
" P.NotFound," + Environment.NewLine +
|
|
" P.PostDate" + Environment.NewLine +
|
|
" FROM Posts P" + Environment.NewLine +
|
|
")," + Environment.NewLine +
|
|
"Unioned AS" + Environment.NewLine +
|
|
"(" + Environment.NewLine +
|
|
" SELECT " + Environment.NewLine +
|
|
" BlogName," + Environment.NewLine +
|
|
" PostID," + Environment.NewLine +
|
|
" LatestNoteTimestamp," + Environment.NewLine +
|
|
" NotesGatheredDateTime," + Environment.NewLine +
|
|
" CNT," + Environment.NewLine +
|
|
" PostDate" + Environment.NewLine +
|
|
" FROM PostsWithCount" + Environment.NewLine +
|
|
" WHERE NotFound = 0" + Environment.NewLine +
|
|
" AND HasNotesGathered = 0" + Environment.NewLine +
|
|
"" + Environment.NewLine +
|
|
" UNION " + Environment.NewLine +
|
|
"" + Environment.NewLine +
|
|
" SELECT " + Environment.NewLine +
|
|
" BlogName," + Environment.NewLine +
|
|
" PostID," + Environment.NewLine +
|
|
" LatestNoteTimestamp," + Environment.NewLine +
|
|
" NotesGatheredDateTime," + Environment.NewLine +
|
|
" CNT," + Environment.NewLine +
|
|
" PostDate" + Environment.NewLine +
|
|
" FROM PostsWithCount" + Environment.NewLine +
|
|
" WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
|
|
" AND NotFound = 0" + Environment.NewLine +
|
|
" AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine +
|
|
")" + Environment.NewLine +
|
|
"SELECT" + Environment.NewLine +
|
|
" U.BlogName," + Environment.NewLine +
|
|
" U.PostID," + Environment.NewLine +
|
|
" U.LatestNoteTimestamp," + Environment.NewLine +
|
|
" U.NotesGatheredDateTime," + Environment.NewLine +
|
|
" U.CNT" + Environment.NewLine +
|
|
"FROM Unioned U" + Environment.NewLine +
|
|
beforeDateFilter +
|
|
"ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;" + Environment.NewLine;
|
|
}
|
|
else
|
|
{
|
|
sql = "SELECT " +
|
|
" MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
|
|
" Posts.PostID, " + Environment.NewLine +
|
|
" 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
|
|
" Posts.NotesGatheredDatetime, " + Environment.NewLine +
|
|
" CNT.CNT " + Environment.NewLine +
|
|
"FROM " + Environment.NewLine +
|
|
" Posts " + Environment.NewLine +
|
|
" LEFT OUTER JOIN " + Environment.NewLine +
|
|
" Notes ON Notes.RootBlogName = Posts.BlogName AND Notes.PostID = Posts.PostID " + Environment.NewLine +
|
|
" LEFT OUTER JOIN " + Environment.NewLine +
|
|
" ( select BlogName, count(PostID) as CNT from Posts group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
|
|
"WHERE NotFound = 0 " + Environment.NewLine;
|
|
|
|
if (beforeDate.HasValue)
|
|
{
|
|
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
|
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime IS NULL) " + Environment.NewLine;
|
|
}
|
|
|
|
sql += "GROUP BY " + Environment.NewLine +
|
|
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
|
|
"ORDER BY " + Environment.NewLine +
|
|
" notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine;
|
|
}
|
|
|
|
Console.WriteLine(withoutNotesOnly);
|
|
Console.WriteLine(sql);
|
|
Console.Write(">" ); //Console.ReadKey();
|
|
//Thread.Sleep(250);
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long, long, long> post = default;
|
|
string blog;
|
|
long postID = 0;
|
|
long lastNoteTimestamp;
|
|
long notesGatheredTimestamp;
|
|
|
|
// Diagnostic logging before each cast
|
|
//for (int i = 0; i < reader.FieldCount; i++)
|
|
//{
|
|
// var value = reader.GetValue(i);
|
|
// var type = reader.GetFieldType(i);
|
|
// Console.WriteLine($"[GetPosts] Column {i}: Name={reader.GetName(i)}, Value={value}, Type={type}");
|
|
//}
|
|
|
|
try
|
|
{
|
|
blog = Convert.ToString(reader.GetValue(0));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 0 to string: {ex.Message}");
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
postID = Convert.ToInt64(reader.GetValue(1));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 1 to Int64: {ex.Message}");
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
lastNoteTimestamp = Convert.ToInt64(reader.GetValue(2));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 2 to Int64: {ex.Message}");
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
notesGatheredTimestamp = Convert.ToInt64(reader.GetValue(3));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 3 to Int64: {ex.Message}");
|
|
throw;
|
|
}
|
|
|
|
// Exclude if postID is in the exclusion list
|
|
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
|
|
continue;
|
|
|
|
post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT distinct RootBlogName as blogName, postID FROM Notes WHERE Notes.type = 'reply' order by RootBlogName, PostID";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
|
|
id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
|
|
blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
|
|
|
|
post = new Tuple<string, long>(blog, id);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
|
|
MAX(Notes.timestamp) as LatestTimestamp
|
|
FROM Notes
|
|
WHERE Notes.type = 'reply'
|
|
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
|
|
GROUP BY Notes.RootBlogName, Notes.PostID
|
|
ORDER BY LatestTimestamp ASC
|
|
LIMIT @limit";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@limit", limit);
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
|
|
blog = reader.GetString(reader.GetOrdinal("blogName"));
|
|
id = reader.GetInt64(reader.GetOrdinal("PostID"));
|
|
|
|
post = new Tuple<string, long>(blog, id);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int limit = 1)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = @"SELECT DISTINCT Notes.RootBlogName as blogName, Notes.PostID,
|
|
MAX(Notes.timestamp) as LatestTimestamp
|
|
FROM Notes
|
|
WHERE Notes.type = 'reply'
|
|
AND (Notes.replyText IS NULL OR Notes.replyText = '' OR Notes.replyText = '.')
|
|
GROUP BY Notes.RootBlogName, Notes.PostID
|
|
ORDER BY LatestTimestamp ASC
|
|
LIMIT @limit";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@limit", limit);
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
long timestamp = 0;
|
|
|
|
blog = reader.GetString(reader.GetOrdinal("blogName"));
|
|
id = reader.GetInt64(reader.GetOrdinal("PostID"));
|
|
timestamp = reader.GetInt64(reader.GetOrdinal("LatestTimestamp"));
|
|
timestamp++;
|
|
|
|
post = new Tuple<string, long, long>(blog, id, timestamp);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static int GetAPICount(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
int count = 0;
|
|
|
|
try { AddAPICount(); } catch { }
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT APICount FROM DailyAPICount WHERE [Date] = @date";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
count = reader.GetInt32(0); // Assuming Id is the first column
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return count;
|
|
}
|
|
|
|
public static List<Tuple<string, int, long>> GetBlogsForLikes(string specificBlog = null, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, int, long>> blogs = new List<Tuple<string, int, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql;
|
|
if (!string.IsNullOrEmpty(specificBlog))
|
|
sql = "SELECT BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs WHERE BlogName = @blog";
|
|
else
|
|
sql = "SELECT B.BlogName, COALESCE(LikesPulled, 0), COALESCE(LikesCursor, 0) FROM Blogs B INNER JOIN Notes N ON N.NoteBlogName = B.BlogName WHERE B.LikesPulled = 0 AND N.TimeStamp >= 1535778000 AND N.rootBlogName = B.BlogName AND EXISTS (SELECT 1 FROM Posts P WHERE P.BlogName = B.BlogName) GROUP BY B.BlogName ORDER BY MIN(N.Timestamp);";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
if (!string.IsNullOrEmpty(specificBlog))
|
|
command.Parameters.AddWithValue("@blog", specificBlog);
|
|
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
blogs.Add(new Tuple<string, int, long>(
|
|
reader.GetString(0),
|
|
reader.GetInt32(1),
|
|
reader.GetInt64(2)
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error fetching blogs for likes: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return blogs;
|
|
}
|
|
|
|
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<string> blogs = new List<string>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = "";
|
|
if (reblogsOnly)
|
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
|
else
|
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type NOT IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@isActive", 1);
|
|
command.Parameters.AddWithValue("@top", top);
|
|
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string blog = null;
|
|
blog = reader.GetString(0); // Assuming Title is the second column
|
|
|
|
blogs.Add(blog);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return blogs;
|
|
}
|
|
|
|
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<string> blogs = new List<string>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = "";
|
|
if (reblogsOnly)
|
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND type IN ('reblog', 'reply', 'posted') AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
|
else
|
|
sql = "SELECT NoteBlogName as blogName, count(*) FROM notes INNER JOIN blogs ON blogs.BlogName = notes.NoteBlogName WHERE blogs.IsActive = @isActive AND HasBeenOutput = 0 GROUP BY NoteBlogName ORDER BY count(*) DESC, BlogName LIMIT @top";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@isActive", 1);
|
|
command.Parameters.AddWithValue("@top", top);
|
|
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string blog = null;
|
|
blog = reader.GetString(0); // Assuming Title is the second column
|
|
|
|
blogs.Add(blog);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return blogs;
|
|
}
|
|
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
try
|
|
{
|
|
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
List<string> rowTexts = new List<string>();
|
|
for (int i = 0; i < reader.FieldCount; i++)
|
|
{
|
|
if (!reader.IsDBNull(i))
|
|
{
|
|
var val = reader.GetValue(i);
|
|
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
|
|
{
|
|
rowTexts.Add(str);
|
|
}
|
|
}
|
|
}
|
|
if (rowTexts.Count > 0)
|
|
{
|
|
yield return rowTexts;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
connection.Dispose();
|
|
}
|
|
}
|
|
#endregion Gets
|
|
|
|
#region Updates
|
|
|
|
|
|
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
//string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
|
|
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = @dateModified WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
// Informative output when marking a post as NotFound
|
|
Console.WriteLine($"Marking post NotFound: {blogName}/{postID}");
|
|
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(NotFound, 0) <> 1";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(postDate, '') <> @postDate";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty);
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
|
|
try { AddBlog(noteBlogName, false, DBPath); } catch { }
|
|
|
|
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
|
|
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Notes SET timestamp = @timestamp, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND noteBlogName = @noteBlogName AND PostID = @postID AND IFNULL(timestamp, 0) <> @timestamp";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@timestamp", timestamp);
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
|
|
command.Parameters.AddWithValue("@postID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Notes.RootBlogName, Notes.PostID, Notes.TimeStamp, Notes.Type, Notes.NoteBlogName")
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
Console.WriteLine("^^^^^ - SHORTCUT");
|
|
return true;
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
return false;
|
|
}
|
|
|
|
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET ";
|
|
sql += "postDate = @postDate, ";
|
|
sql += "reblogURL = @reblogURL, ";
|
|
sql += "postURL = @postURL, ";
|
|
sql += "slug = @slug, ";
|
|
sql += "reblogKey = @reblogKey, ";
|
|
sql += "reblogName = @reblogName, ";
|
|
sql += "summary = @summary, ";
|
|
sql += "quote = @quote, ";
|
|
sql += "body = @body, ";
|
|
sql += "tags = @tags, ";
|
|
sql += "link = @link, ";
|
|
sql += "photoURL = @photoURL, ";
|
|
sql += "photoCaption = @photoCaption, ";
|
|
sql += "downloadedFiles = @downloadedFiles, ";
|
|
sql += "audioCaption = @audioCaption, ";
|
|
sql += "question = @question, ";
|
|
sql += "answer = @answer, ";
|
|
sql += "title = @title, ";
|
|
sql += "DateModified = @dateModified, ";
|
|
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
|
|
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
|
|
sql += "hasImage = @hasImage, ";
|
|
sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) ";
|
|
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
|
|
sql += "IFNULL(postDate, '') <> @postDate OR ";
|
|
sql += "IFNULL(reblogURL, '') <> @reblogURL OR ";
|
|
sql += "IFNULL(postURL, '') <> @postURL OR ";
|
|
sql += "IFNULL(slug, '') <> @slug OR ";
|
|
sql += "IFNULL(reblogKey, '') <> @reblogKey OR ";
|
|
sql += "IFNULL(reblogName, '') <> @reblogName OR ";
|
|
sql += "IFNULL(summary, '') <> @summary OR ";
|
|
sql += "IFNULL(quote, '') <> @quote OR ";
|
|
sql += "IFNULL(body, '') <> @body OR ";
|
|
sql += "IFNULL(tags, '') <> @tags OR ";
|
|
sql += "IFNULL(link, '') <> @link OR ";
|
|
sql += "IFNULL(photoURL, '') <> @photoURL OR ";
|
|
sql += "IFNULL(photoCaption, '') <> @photoCaption OR ";
|
|
sql += "IFNULL(downloadedFiles, '') <> @downloadedFiles OR ";
|
|
sql += "IFNULL(audioCaption, '') <> @audioCaption OR ";
|
|
sql += "IFNULL(question, '') <> @question OR ";
|
|
sql += "IFNULL(answer, '') <> @answer OR ";
|
|
sql += "IFNULL(title, '') <> @title OR ";
|
|
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
|
|
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
|
|
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
|
|
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)";
|
|
sql += ")";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty);
|
|
command.Parameters.AddWithValue("@reblogURL", reblogURL ?? string.Empty);
|
|
command.Parameters.AddWithValue("@postURL", postURL ?? string.Empty);
|
|
command.Parameters.AddWithValue("@slug", slug ?? string.Empty);
|
|
command.Parameters.AddWithValue("@reblogKey", reblogKey ?? string.Empty);
|
|
command.Parameters.AddWithValue("@reblogName", reblogName ?? string.Empty);
|
|
command.Parameters.AddWithValue("@summary", summary ?? string.Empty);
|
|
command.Parameters.AddWithValue("@quote", quote ?? string.Empty);
|
|
command.Parameters.AddWithValue("@body", body ?? string.Empty);
|
|
command.Parameters.AddWithValue("@tags", tags ?? string.Empty);
|
|
command.Parameters.AddWithValue("@link", link ?? string.Empty);
|
|
command.Parameters.AddWithValue("@photoURL", photoURL ?? string.Empty);
|
|
command.Parameters.AddWithValue("@photoCaption", photoCaption ?? string.Empty);
|
|
command.Parameters.AddWithValue("@downloadedFiles", downloadedFiles ?? string.Empty);
|
|
command.Parameters.AddWithValue("@audioCaption", audioCaption ?? string.Empty);
|
|
command.Parameters.AddWithValue("@question", question ?? string.Empty);
|
|
command.Parameters.AddWithValue("@answer", answer ?? string.Empty);
|
|
command.Parameters.AddWithValue("@title", title ?? string.Empty);
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", string.IsNullOrWhiteSpace(rootBlogName) ? (object)DBNull.Value : rootBlogName);
|
|
command.Parameters.AddWithValue("@rootURL", string.IsNullOrWhiteSpace(rootURL) ? (object)DBNull.Value : rootURL);
|
|
command.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
|
command.Parameters.AddWithValue("@byLikes", byLikes ? 1 : 0);
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void UpdateBlogOutput(string blogName, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName AND IFNULL(HasBeenOutput, 0) <> 1";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name AND (IFNULL(LikesPulled, 0) <> @pulled OR IFNULL(LikesCursor, 0) <> @cursor)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@pulled", likesPulled);
|
|
command.Parameters.AddWithValue("@cursor", likesCursor);
|
|
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@name", blogName);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error updating blog likes status: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static int UpdateAPICount(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
int APICount = DataAccess.GetAPICount();
|
|
APICount++;
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE DailyAPICount SET APICount = @APICount WHERE [Date] = @date";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@APICount", APICount);
|
|
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
|
|
return APICount;
|
|
}
|
|
|
|
public static void UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
//string sql = "UPDATE Notes SET replyText = @replyText WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply'";
|
|
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE PostID = @PostID AND noteBlogName = @noteBlogName AND TimeStamp = @TimeStamp AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
|
|
command.Parameters.AddWithValue("@TimeStamp", timestamp);
|
|
int rowsAffected = command.ExecuteNonQuery();
|
|
|
|
if (rowsAffected == 0)
|
|
{
|
|
Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
|
|
Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
|
|
Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} from {noteBlogName}");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
|
|
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified WHERE rootBlogName = @rootBlogName AND PostID = @PostID AND Type = 'reply' AND IFNULL(replyText, '.') <> @replyText";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
int rowsAffected = command.ExecuteNonQuery();
|
|
|
|
if (rowsAffected == 0)
|
|
{
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}/{postID} (rows not found or values unchanged)");
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}/{postID} with '{replyText}'");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
|
|
}
|
|
finally
|
|
{
|
|
connection.Close();
|
|
}
|
|
}
|
|
#endregion Updates
|
|
}
|
|
|
|
internal class APIAccess
|
|
{
|
|
public static string Blog { get; set; } = string.Empty;
|
|
private static IConfiguration Configuration { get; set; } = null!;
|
|
private static string ApiConfigSection { get; set; } = "TumblrApi";
|
|
|
|
static APIAccess()
|
|
{
|
|
Configuration = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
|
|
.Build();
|
|
|
|
ValidateApiSection(ApiConfigSection);
|
|
}
|
|
|
|
public static void SetApiConfigSection(string sectionName)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(sectionName))
|
|
sectionName = "TumblrApi";
|
|
|
|
ValidateApiSection(sectionName);
|
|
ApiConfigSection = sectionName;
|
|
Console.WriteLine($"Using API settings section: {ApiConfigSection}");
|
|
}
|
|
|
|
private static void ValidateApiSection(string sectionName)
|
|
{
|
|
if (Configuration[$"{sectionName}:ConsumerKey"] == null ||
|
|
Configuration[$"{sectionName}:ConsumerSecret"] == null ||
|
|
Configuration[$"{sectionName}:OAuthToken"] == null ||
|
|
Configuration[$"{sectionName}:OAuthTokenSecret"] == null)
|
|
{
|
|
throw new InvalidOperationException($"{sectionName} configuration is missing required keys in appsettings.json");
|
|
}
|
|
}
|
|
|
|
private static string ConsumerKey => Configuration[$"{ApiConfigSection}:ConsumerKey"] ??
|
|
throw new InvalidOperationException("ConsumerKey is not configured");
|
|
private static string ConsumerSecret => Configuration[$"{ApiConfigSection}:ConsumerSecret"] ??
|
|
throw new InvalidOperationException("ConsumerSecret is not configured");
|
|
private static string OAuthToken => Configuration[$"{ApiConfigSection}:OAuthToken"] ??
|
|
throw new InvalidOperationException("OAuthToken is not configured");
|
|
private static string OAuthTokenSecret => Configuration[$"{ApiConfigSection}:OAuthTokenSecret"] ??
|
|
throw new InvalidOperationException("OAuthTokenSecret is not configured");
|
|
|
|
private static int GetRetryDelaySecondsFromHeaders(IEnumerable<HeaderParameter>? headers)
|
|
{
|
|
if (headers == null)
|
|
return 60;
|
|
|
|
int retryInSeconds = 0;
|
|
bool remainingIsZero = false;
|
|
|
|
foreach (var header in headers)
|
|
{
|
|
string? headerName = header?.Name;
|
|
string? headerValue = header?.Value?.ToString();
|
|
if (string.IsNullOrWhiteSpace(headerName) || string.IsNullOrWhiteSpace(headerValue))
|
|
continue;
|
|
|
|
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int retrySecs))
|
|
{
|
|
retryInSeconds = Math.Max(retryInSeconds, retrySecs);
|
|
}
|
|
else if (DateTimeOffset.TryParse(headerValue, out var retryAt))
|
|
{
|
|
int secs = (int)Math.Max(0, (retryAt - DateTimeOffset.UtcNow).TotalSeconds);
|
|
retryInSeconds = Math.Max(retryInSeconds, secs);
|
|
}
|
|
}
|
|
|
|
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (long.TryParse(headerValue, out long epoch))
|
|
{
|
|
int secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
retryInSeconds = Math.Max(retryInSeconds, secs);
|
|
}
|
|
}
|
|
|
|
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
|
{
|
|
remainingIsZero = true;
|
|
}
|
|
|
|
if (remainingIsZero && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (int.TryParse(headerValue, out int resetValue))
|
|
{
|
|
retryInSeconds = Math.Max(retryInSeconds, resetValue);
|
|
}
|
|
else if (long.TryParse(headerValue, out long resetEpoch))
|
|
{
|
|
int secs = (int)Math.Max(0, resetEpoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
retryInSeconds = Math.Max(retryInSeconds, secs);
|
|
}
|
|
}
|
|
}
|
|
|
|
return retryInSeconds > 0 ? retryInSeconds : 60;
|
|
}
|
|
|
|
public static async Task<Root> GrabNotes(string blog, long ID, string? timestamp = null)
|
|
{
|
|
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all";
|
|
URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
|
|
if (!string.IsNullOrEmpty(timestamp))
|
|
{
|
|
URL += "&before_timestamp=" + timestamp;
|
|
await Task.Delay(100);
|
|
}
|
|
|
|
// Create a new RestClient for each request to ensure fresh OAuth signatures
|
|
using (var client = new RestClient(URL))
|
|
{
|
|
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
|
consumerSecret: ConsumerSecret,
|
|
token: OAuthToken,
|
|
tokenSecret: OAuthTokenSecret,
|
|
OAuthSignatureMethod.HmacSha1
|
|
);
|
|
|
|
client.Authenticator = oAuth1;
|
|
var request = new RestRequest(URL, Method.Get);
|
|
var response = await client.ExecuteAsync(request);
|
|
|
|
var myJsonResponse = response.Content ?? string.Empty;
|
|
Console.WriteLine($"{timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
|
var myDeserializedClass = new Root();
|
|
|
|
try
|
|
{
|
|
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
|
if (deserializedResult != null)
|
|
{
|
|
myDeserializedClass = deserializedResult;
|
|
myDeserializedClass.rawJson = myJsonResponse;
|
|
|
|
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
|
|
{
|
|
myDeserializedClass.statusCode = "NotFound";
|
|
}
|
|
|
|
// If the response JSON indicates a 429 (Too Many Requests) via meta.status or message,
|
|
// treat it like a rate-limited response and attempt to read Retry headers.
|
|
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
|
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
|
|
|
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
|
{
|
|
// populate retryInSeconds from headers if possible
|
|
if (response?.Headers != null)
|
|
{
|
|
bool checkResetLocal = false;
|
|
foreach (var header in response.Headers)
|
|
{
|
|
string? headerName = header?.Name;
|
|
string? headerValue = header?.Value?.ToString();
|
|
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
|
continue;
|
|
|
|
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int retrySecs))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
|
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
|
}
|
|
|
|
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
|
{
|
|
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
|
}
|
|
|
|
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
|
checkResetLocal = true;
|
|
|
|
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (int.TryParse(headerValue, out int resetValue))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
|
else if (long.TryParse(headerValue, out long epochVal))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
|
}
|
|
}
|
|
}
|
|
|
|
// surface rate-limit status back to caller
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Failed JSON: {myJsonResponse}");
|
|
Console.WriteLine(ex.ToString());
|
|
|
|
if (!response.IsSuccessful)
|
|
{
|
|
// Response.StatusCode may be null with some RestSharp responses.
|
|
// Guard it and still attempt to extract retry time from headers (Retry-After, Remaining/Reset, X-RateLimit-Reset)
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
|
// Only set the status if we actually have one; otherwise leave existing value alone (may be null)
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
|
|
bool checkReset = false;
|
|
|
|
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
|
{
|
|
bool foundRateLimitHeader = false;
|
|
foreach (var header in response.Headers)
|
|
{
|
|
if (header.Name != null && header.Value != null)
|
|
{
|
|
Console.WriteLine($"{header.Name} - {header.Value}");
|
|
// Common header patterns used by APIs to indicate retry times:
|
|
// - Retry-After: either seconds or HTTP date
|
|
// - X-RateLimit-Reset: often seconds since epoch
|
|
// - <something>Reset (after Remaining==0): seconds
|
|
var headerValue = header.Value?.ToString();
|
|
if (!string.IsNullOrEmpty(headerValue))
|
|
{
|
|
// 1) Retry-After header (seconds or date)
|
|
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int retrySecs))
|
|
{
|
|
if (myDeserializedClass.retryInSeconds < retrySecs)
|
|
myDeserializedClass.retryInSeconds = retrySecs;
|
|
}
|
|
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
|
{
|
|
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
|
if (myDeserializedClass.retryInSeconds < secs)
|
|
myDeserializedClass.retryInSeconds = secs;
|
|
}
|
|
}
|
|
|
|
// 2) Common 'Reset' header: numeric seconds or epoch seconds
|
|
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int resetValue))
|
|
{
|
|
if (myDeserializedClass.retryInSeconds < resetValue)
|
|
myDeserializedClass.retryInSeconds = resetValue;
|
|
}
|
|
else if (long.TryParse(headerValue, out long epochVal))
|
|
{
|
|
// treat as epoch seconds -> compute secs until that epoch
|
|
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
if (myDeserializedClass.retryInSeconds < secs)
|
|
myDeserializedClass.retryInSeconds = secs;
|
|
}
|
|
}
|
|
|
|
// 3) X-RateLimit-Reset header: often epoch seconds
|
|
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (long.TryParse(headerValue, out long epoch))
|
|
{
|
|
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
if (myDeserializedClass.retryInSeconds < secs)
|
|
myDeserializedClass.retryInSeconds = secs;
|
|
}
|
|
foundRateLimitHeader = true;
|
|
}
|
|
}
|
|
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
|
checkReset = true;
|
|
else
|
|
checkReset = false;
|
|
}
|
|
|
|
// If we detected rate-limit related headers (Retry-After / X-RateLimit-Reset / Remaining/Reset)
|
|
// or the HTTP status indicates 429, surface it.
|
|
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
|
|
public static async Task<PostsRoot> GrabPostWithReplies(string blog, long postID, long timestamp)
|
|
{
|
|
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/posts?id={postID}¬es_info=true&before_timestamp={timestamp}";
|
|
|
|
// Create a new RestClient for each request to ensure fresh OAuth signatures
|
|
using (var client = new RestClient(URL))
|
|
{
|
|
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
|
consumerSecret: ConsumerSecret,
|
|
token: OAuthToken,
|
|
tokenSecret: OAuthTokenSecret,
|
|
OAuthSignatureMethod.HmacSha1
|
|
);
|
|
|
|
client.Authenticator = oAuth1;
|
|
var request = new RestRequest(URL, Method.Get);
|
|
var response = await client.ExecuteAsync(request);
|
|
|
|
var myJsonResponse = response.Content ?? string.Empty;
|
|
Console.WriteLine($"[Reply API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
|
var myDeserializedClass = new PostsRoot();
|
|
|
|
// Check if response is successful and contains JSON
|
|
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
|
|
{
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
Console.WriteLine($"[Reply API] {statusStr}\t{response?.StatusDescription}");
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
// Check if response is HTML (error page) instead of JSON
|
|
if (myJsonResponse.TrimStart().StartsWith("<"))
|
|
{
|
|
Console.WriteLine($"[Reply API] Received HTML response (likely error page): {response?.StatusCode}");
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
try
|
|
{
|
|
var deserializedResult = JsonConvert.DeserializeObject<PostsRoot>(myJsonResponse);
|
|
if (deserializedResult != null)
|
|
{
|
|
myDeserializedClass = deserializedResult;
|
|
myDeserializedClass.rawJson = myJsonResponse;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
|
|
Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
|
|
public static async Task<LikesRoot> GrabLikes(string blog, long beforeTimestamp = 0)
|
|
{
|
|
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/likes?npf=false&reblog_info=true";
|
|
if (beforeTimestamp > 0)
|
|
{
|
|
URL += $"&before={beforeTimestamp}";
|
|
}
|
|
|
|
using (var client = new RestClient(URL))
|
|
{
|
|
var oAuth1 = OAuth1Authenticator.ForAccessToken(consumerKey: ConsumerKey,
|
|
consumerSecret: ConsumerSecret,
|
|
token: OAuthToken,
|
|
tokenSecret: OAuthTokenSecret,
|
|
OAuthSignatureMethod.HmacSha1
|
|
);
|
|
|
|
client.Authenticator = oAuth1;
|
|
var request = new RestRequest(URL, Method.Get);
|
|
var response = await client.ExecuteAsync(request);
|
|
|
|
var myJsonResponse = response.Content ?? string.Empty;
|
|
Console.WriteLine($"[Likes API] {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
|
var myDeserializedClass = new LikesRoot();
|
|
|
|
// Check if response is successful and contains JSON
|
|
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
|
|
{
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
Console.WriteLine($"[Likes API] {statusStr}\t{response?.StatusDescription}");
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
|
|
if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
|
|
}
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
if (myJsonResponse.TrimStart().StartsWith("<"))
|
|
{
|
|
Console.WriteLine($"[Likes API] Received HTML response (likely error page): {response?.StatusCode}");
|
|
myDeserializedClass.statusCode = response?.StatusCode.ToString();
|
|
if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
|
|
}
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
try
|
|
{
|
|
var deserializedResult = JsonConvert.DeserializeObject<LikesRoot>(myJsonResponse);
|
|
if (deserializedResult != null)
|
|
{
|
|
myDeserializedClass = deserializedResult;
|
|
myDeserializedClass.rawJson = myJsonResponse;
|
|
|
|
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
|
bool statusIndicatesRateLimit = response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests;
|
|
|
|
if (metaIndicatesRateLimit || statusIndicatesRateLimit)
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response?.Headers);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Likes API] Failed to parse JSON response: {ex.Message}");
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
}
|
|
}
|