jimandClaude Opus 5
b31d5842cc
Notes.RootBlogName/NoteBlogName/Type became RootBlogId/NoteBlogId/TypeId on 2026-08-07, resolved through the new BlogNames and NoteTypes tables. There is no compatibility view, so every affected statement is a hard cut. All 14 call sites in DataAccess.cs are ported: - Notes->Blogs joins go through Blogs.BlogId in one integer hop; the Notes->Posts join in GetRepliesWithFilledText is the only one that must route through BlogNames, since Posts carries no BlogId - AddNote registers both blog names and the note type with INSERT OR IGNORE before inserting, in one transaction committed before the console sleep. Registering the type matters: an unseen type would resolve to NULL and fail NOT NULL on TypeId, silently losing the note - The LEFT JOIN Notes in GetPosts is dropped rather than translated. It selected nothing, could not remove a row, and its duplicates were collapsed by the query's own GROUP BY - Duplicate-key detection moves to IsNotesDuplicateKey, matching the constraint and table instead of an exact column list. The old literal string is what broke on this rename - EnsureReplyTextColumnExists drops DEFAULT '.', matching the migrated schema: new rows get NULL, not a placeholder nobody wrote verify-db-schema.sql gains BlogNames, NoteTypes, Blogs.BlogId and the new Notes columns, plus query 1d naming a pre-migration file and pointing at normalize-notes.sql. Blogs.BlogId is deliberately not auto-fixable -- an added-but-empty column makes engagement joins return zero rows silently. Verified against the live 148 MB file: query plans hit the intended indexes, and the BlogId join matches an independent name-resolved formulation exactly on all 4,267 GetBlogs and 2,637 GetBlogsForLikes rows. RolodexRepository.cs (16 sites) lives in the Rolodex repo and is not covered here. Co-Authored-By: Claude Opus 5 <[email protected]>
3461 lines
175 KiB
C#
3461 lines
175 KiB
C#
using System;
|
|
using System.Collections.Generic;
|
|
using System.Linq;
|
|
using System.Text;
|
|
using System.Threading.Tasks;
|
|
using System.Data.SQLite;
|
|
using RestSharp.Authenticators.OAuth;
|
|
using RestSharp.Authenticators;
|
|
using RestSharp;
|
|
using Newtonsoft.Json;
|
|
using Newtonsoft.Json.Converters;
|
|
using Microsoft.Extensions.Diagnostics.Latency;
|
|
using System.Collections;
|
|
using Microsoft.Extensions.Configuration;
|
|
using System.IO;
|
|
using System.Data;
|
|
using System.Text.Json;
|
|
|
|
namespace URLNotesGrabberCORE
|
|
{
|
|
|
|
class ReblogRecord
|
|
{
|
|
public string answer;
|
|
public string audioCaption;
|
|
public string blogName;
|
|
public string body;
|
|
public string date;
|
|
public string downloadedFiles;
|
|
public string link;
|
|
public string photoCaption;
|
|
public string photoURL;
|
|
public string postID;
|
|
public string postURL;
|
|
public string question;
|
|
public string quote;
|
|
public string reblogKey;
|
|
public string reblogName;
|
|
public string reblogURL;
|
|
public string rootURL;
|
|
public string slug;
|
|
public string summary;
|
|
public string tags;
|
|
public string title;
|
|
|
|
public ReblogRecord()
|
|
{
|
|
answer = ".";
|
|
audioCaption = ".";
|
|
blogName = ".";
|
|
body = ".";
|
|
date = ".";
|
|
downloadedFiles = ".";
|
|
link = ".";
|
|
photoCaption = ".";
|
|
photoURL = ".";
|
|
postID = ".";
|
|
postURL = ".";
|
|
question = ".";
|
|
quote = ".";
|
|
reblogKey = ".";
|
|
reblogName = ".";
|
|
reblogURL = ".";
|
|
rootURL = ".";
|
|
slug = ".";
|
|
summary = ".";
|
|
tags = ".";
|
|
title = ".";
|
|
}
|
|
|
|
}
|
|
|
|
internal class DataAccess
|
|
{
|
|
private static IConfiguration? _configuration;
|
|
private static HashSet<long>? _postIdsToExclude;
|
|
private static readonly object _importPragmaLock = new object();
|
|
private static Tuple<string, string>? _savedImportPragmas;
|
|
private static string? _cachedDbPath;
|
|
private static SQLiteConnection? _importConnection;
|
|
private static HashSet<string>? _importBlogCache;
|
|
private static readonly object _importSessionLock = new object();
|
|
// AddAPICount/UpdateAPICount run once per API call. A schema-level failure there repeats
|
|
// identically every time, so log each distinct message once instead of per call.
|
|
private static readonly HashSet<string> _apiCountFailuresLogged = new HashSet<string>();
|
|
private static readonly object _apiCountFailureLock = new object();
|
|
|
|
static DataAccess()
|
|
{
|
|
_configuration = new ConfigurationBuilder()
|
|
.SetBasePath(Directory.GetCurrentDirectory())
|
|
.AddJsonFile("appsettings.json", optional: false, reloadOnChange: true)
|
|
.Build();
|
|
var excludeSetting = _configuration["appSettings:PostIDToExclude"];
|
|
if (!string.IsNullOrWhiteSpace(excludeSetting))
|
|
{
|
|
_postIdsToExclude = excludeSetting.Split(',')
|
|
.Select(s => long.TryParse(s.Trim(), out var id) ? id : (long?)null)
|
|
.Where(id => id.HasValue)
|
|
.Select(id => id.Value)
|
|
.ToHashSet();
|
|
}
|
|
else
|
|
{
|
|
_postIdsToExclude = new HashSet<long>();
|
|
}
|
|
}
|
|
|
|
// The database every DataAccess call defaults to, exposed so modes can report
|
|
// which file they actually read when their results are surprising.
|
|
public static string GetActiveDbPath() => GetDefaultDbPath();
|
|
|
|
private static string GetDefaultDbPath()
|
|
{
|
|
if (_cachedDbPath != null)
|
|
return _cachedDbPath;
|
|
|
|
string? configPath = _configuration?["appSettings:PathDB"];
|
|
if (!string.IsNullOrWhiteSpace(configPath))
|
|
{
|
|
_cachedDbPath = configPath;
|
|
}
|
|
else
|
|
{
|
|
// Fallback to 3 levels up from the executable directory
|
|
_cachedDbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "TL.db");
|
|
}
|
|
return _cachedDbPath;
|
|
}
|
|
|
|
#region IsActive
|
|
|
|
// Posts.IsActive and Notes.IsActive mean the same thing Blogs.IsActive does:
|
|
// 0 = removed elsewhere (Rolodex), anything else (including NULL) = live.
|
|
//
|
|
// This crawler is a reader of all three. It never writes any IsActive column --
|
|
// no INSERT lists it, no UPDATE sets it, and MapPrefixToColumn cannot map to it --
|
|
// so a row removed in Rolodex is never resurrected by a re-crawl.
|
|
//
|
|
// Unlike Blogs.IsActive, the Posts and Notes columns are optional: they are added
|
|
// from outside this app and are absent from databases that predate them. Naming a
|
|
// missing column is a hard SQLite error ("no such column"), so every read asks the
|
|
// schema first and simply drops the filter when the column is not there. The answer
|
|
// is cached per database path, so adding the columns to a live database takes effect
|
|
// on the next run.
|
|
private static readonly Dictionary<string, bool> _isActiveColumnCache =
|
|
new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
|
|
private static readonly object _isActiveColumnLock = new object();
|
|
|
|
private static bool HasIsActiveColumn(string table, string? DBPath)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
string cacheKey = DBPath + "|" + table;
|
|
|
|
lock (_isActiveColumnLock)
|
|
{
|
|
if (_isActiveColumnCache.TryGetValue(cacheKey, out bool cached))
|
|
return cached;
|
|
}
|
|
|
|
bool exists = false;
|
|
try
|
|
{
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
|
|
using SQLiteCommand command = new SQLiteCommand($"PRAGMA table_info({table});", connection);
|
|
using SQLiteDataReader reader = command.ExecuteReader();
|
|
while (reader.Read())
|
|
{
|
|
if (reader.GetString(1).Equals("IsActive", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
exists = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
// An unreadable schema is treated as "no column" so the caller's query still runs.
|
|
Console.WriteLine($"Error checking {table}.IsActive column: {ex.Message}");
|
|
}
|
|
|
|
lock (_isActiveColumnLock)
|
|
{
|
|
_isActiveColumnCache[cacheKey] = exists;
|
|
}
|
|
|
|
return exists;
|
|
}
|
|
|
|
/// <summary>
|
|
/// " AND COALESCE(alias.IsActive, 1) = 1" when the table carries the column, "" when it
|
|
/// does not. NULL is read as live, the same way Rolodex reads Blogs.IsActive.
|
|
/// </summary>
|
|
private static string AndIsActive(string table, string alias = "", string? DBPath = null)
|
|
{
|
|
if (!HasIsActiveColumn(table, DBPath)) return string.Empty;
|
|
|
|
string qualifier = string.IsNullOrEmpty(alias) ? string.Empty : alias + ".";
|
|
return $" AND COALESCE({qualifier}IsActive, 1) = 1";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Same filter as <see cref="AndIsActive"/>, for a query that has no WHERE clause yet.
|
|
/// </summary>
|
|
private static string WhereIsActive(string table, string alias = "", string? DBPath = null)
|
|
{
|
|
string clause = AndIsActive(table, alias, DBPath);
|
|
return clause.Length == 0 ? string.Empty : " WHERE" + clause.Substring(" AND".Length);
|
|
}
|
|
|
|
#endregion IsActive
|
|
|
|
#region Notes integer schema
|
|
|
|
// Notes stopped storing names on 2026-08-07: RootBlogName/NoteBlogName/Type became
|
|
// RootBlogId/NoteBlogId/TypeId, resolved through BlogNames and NoteTypes. There is no
|
|
// compatibility view -- a query naming an old column fails outright, so this is a hard
|
|
// cut rather than an optional column like IsActive. See TL.db.md.
|
|
//
|
|
// Two shapes recur below and are spelled out inline rather than hidden behind a helper,
|
|
// so that every statement reads as the SQL it actually runs:
|
|
// (SELECT BlogId FROM BlogNames WHERE BlogName = @name) -- unique-index probe, 20k rows
|
|
// (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') -- 5 rows, effectively free
|
|
// Joining Notes to Blogs is the one case that must NOT route through BlogNames: Blogs
|
|
// carries its own BlogId, so N.NoteBlogId = B.BlogId is a single integer hop. Joining
|
|
// Notes to Posts is the opposite case -- Posts has only BlogName, so it has to go
|
|
// through BlogNames.
|
|
|
|
/// <summary>
|
|
/// True when the exception is a duplicate-key collision on Notes. The message embeds the
|
|
/// primary key's column names, which the integer migration renamed, so this matches on the
|
|
/// constraint and the table instead of on an exact column list -- a literal comparison
|
|
/// silently inverts into "log every error" the next time a column is renamed.
|
|
/// </summary>
|
|
private static bool IsNotesDuplicateKey(Exception ex)
|
|
{
|
|
return ex.Message.Contains("UNIQUE constraint failed", StringComparison.OrdinalIgnoreCase)
|
|
&& ex.Message.Contains("Notes.", StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Gives a blog name an ID if it does not have one. No read-back and no round trip -- a
|
|
/// name that is already registered keeps the ID that 1.18M Notes rows point at.
|
|
/// </summary>
|
|
private static void RegisterBlogName(SQLiteConnection connection, SQLiteTransaction? transaction, string blogName)
|
|
{
|
|
using SQLiteCommand command = new SQLiteCommand("INSERT OR IGNORE INTO BlogNames (BlogName) VALUES (@BlogName)", connection, transaction);
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
/// <summary>
|
|
/// Same, for a note type. NoteTypes is a table rather than a CHECK constraint precisely so
|
|
/// that a type this crawler has not seen before is an INSERT and not a schema migration --
|
|
/// without this the type would resolve to NULL and fail the NOT NULL on Notes.TypeId.
|
|
/// </summary>
|
|
private static void RegisterNoteType(SQLiteConnection connection, SQLiteTransaction? transaction, string type)
|
|
{
|
|
using SQLiteCommand command = new SQLiteCommand("INSERT OR IGNORE INTO NoteTypes (Type) VALUES (@Type)", connection, transaction);
|
|
command.Parameters.AddWithValue("@Type", type);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
#endregion Notes integer schema
|
|
|
|
public static string Q(string input)
|
|
{
|
|
return "'" + input.Replace("'", "''") + "'";
|
|
}
|
|
|
|
public static DateTime UnixTimeStampToDateTime(double unixTimeStamp)
|
|
{
|
|
// Unix timestamp is seconds past epoch
|
|
DateTime dateTime = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
|
dateTime = dateTime.AddSeconds(unixTimeStamp).ToLocalTime();
|
|
return dateTime;
|
|
}
|
|
|
|
public static void EnableImportModePragmas(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
lock (_importPragmaLock)
|
|
{
|
|
if (_savedImportPragmas != null)
|
|
return;
|
|
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string currentJournalMode = "delete";
|
|
string currentSynchronous = "2";
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA journal_mode;", connection))
|
|
{
|
|
currentJournalMode = Convert.ToString(cmd.ExecuteScalar()) ?? "delete";
|
|
}
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA synchronous;", connection))
|
|
{
|
|
currentSynchronous = Convert.ToString(cmd.ExecuteScalar()) ?? "2";
|
|
}
|
|
|
|
_savedImportPragmas = new Tuple<string, string>(currentJournalMode, currentSynchronous);
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA journal_mode=WAL;", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
using (var cmd = new SQLiteCommand("PRAGMA synchronous=NORMAL;", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
Console.WriteLine($"[SQLite Import Mode] Enabled | journal_mode=WAL | synchronous=NORMAL (previous: journal_mode={currentJournalMode}, synchronous={currentSynchronous})");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[SQLite Import Mode] Failed to enable import pragmas: {ex.Message}");
|
|
}
|
|
}
|
|
}
|
|
|
|
public static void RestoreImportModePragmas(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
lock (_importPragmaLock)
|
|
{
|
|
if (_savedImportPragmas == null)
|
|
return;
|
|
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string priorJournalMode = _savedImportPragmas.Item1;
|
|
string priorSynchronous = _savedImportPragmas.Item2;
|
|
|
|
using (var cmd = new SQLiteCommand($"PRAGMA journal_mode={priorJournalMode};", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
using (var cmd = new SQLiteCommand($"PRAGMA synchronous={priorSynchronous};", connection))
|
|
{
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
|
|
Console.WriteLine($"[SQLite Import Mode] Restored | journal_mode={priorJournalMode} | synchronous={priorSynchronous}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[SQLite Import Mode] Failed to restore pragmas: {ex.Message}");
|
|
}
|
|
finally
|
|
{
|
|
_savedImportPragmas = null;
|
|
}
|
|
}
|
|
}
|
|
|
|
public static void BeginImportSession(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
lock (_importSessionLock)
|
|
{
|
|
if (_importConnection != null) return;
|
|
|
|
var conn = new SQLiteConnection("Data Source=" + DBPath);
|
|
conn.Open();
|
|
_importConnection = conn;
|
|
|
|
_importBlogCache = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
|
Console.WriteLine("[Import Session] Opened single connection | blog cache initialized (lazy)");
|
|
}
|
|
}
|
|
|
|
public static void EndImportSession()
|
|
{
|
|
lock (_importSessionLock)
|
|
{
|
|
try { _importConnection?.Close(); _importConnection?.Dispose(); }
|
|
catch { }
|
|
_importConnection = null;
|
|
_importBlogCache = null;
|
|
}
|
|
}
|
|
|
|
public static void EnsureReplyTextColumnExists(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Check if replyText column exists
|
|
string checkSql = "PRAGMA table_info(Notes);";
|
|
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
bool columnExists = false;
|
|
while (reader.Read())
|
|
{
|
|
string columnName = reader.GetString(1);
|
|
if (columnName.Equals("replyText", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
columnExists = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!columnExists)
|
|
{
|
|
// Add the column if it doesn't exist
|
|
connection.Close();
|
|
connection.Open();
|
|
|
|
// No column default: the migrated schema dropped the DEFAULT '.' that
|
|
// is how 1.1M rows acquired a placeholder nobody wrote. New rows get
|
|
// NULL, which every reader here already treats as "no reply text".
|
|
string addColumnSql = "ALTER TABLE Notes ADD COLUMN replyText TEXT;";
|
|
using (SQLiteCommand addCommand = new SQLiteCommand(addColumnSql, connection))
|
|
{
|
|
addCommand.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added replyText column to Notes table");
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"Error checking/creating replyText column: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
public static void EnsureBlogsLikesColumnsExist(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string checkSql = "PRAGMA table_info(Blogs);";
|
|
bool likesPulledExists = false;
|
|
bool likesCursorExists = false;
|
|
bool likesNewestTimestampExists = false;
|
|
bool likesLastRefreshedExists = false;
|
|
bool likesLastNewCountExists = false;
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(checkSql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string columnName = reader.GetString(1);
|
|
if (columnName.Equals("LikesPulled", StringComparison.OrdinalIgnoreCase)) likesPulledExists = true;
|
|
if (columnName.Equals("LikesCursor", StringComparison.OrdinalIgnoreCase)) likesCursorExists = true;
|
|
if (columnName.Equals("LikesNewestTimestamp", StringComparison.OrdinalIgnoreCase)) likesNewestTimestampExists = true;
|
|
if (columnName.Equals("LikesLastRefreshed", StringComparison.OrdinalIgnoreCase)) likesLastRefreshedExists = true;
|
|
if (columnName.Equals("LikesLastNewCount", StringComparison.OrdinalIgnoreCase)) likesLastNewCountExists = true;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!likesPulledExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesPulled INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesPulled column to Blogs table");
|
|
}
|
|
|
|
if (!likesCursorExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesCursor INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesCursor column to Blogs table");
|
|
}
|
|
|
|
if (!likesNewestTimestampExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesNewestTimestamp INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesNewestTimestamp column to Blogs table");
|
|
}
|
|
|
|
if (!likesLastRefreshedExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastRefreshed INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesLastRefreshed column to Blogs table");
|
|
}
|
|
|
|
if (!likesLastNewCountExists)
|
|
{
|
|
string addCol = "ALTER TABLE Blogs ADD COLUMN LikesLastNewCount INTEGER DEFAULT 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(addCol, connection)) cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added LikesLastNewCount column to Blogs table");
|
|
}
|
|
|
|
// One-time reset: if any of the new high-water-mark columns were just added,
|
|
// wipe all likes tracking state so the system starts from a known-good baseline.
|
|
// Existing Posts rows with ByLikes=1 remain — UNIQUE(BlogName, PostID) absorbs re-inserts.
|
|
if (!likesNewestTimestampExists || !likesLastRefreshedExists || !likesLastNewCountExists)
|
|
{
|
|
string resetSql = @"UPDATE Blogs
|
|
SET LikesPulled = 0,
|
|
LikesCursor = 0,
|
|
LikesNewestTimestamp = 0,
|
|
LikesLastRefreshed = 0,
|
|
LikesLastNewCount = 0;";
|
|
using (SQLiteCommand cmd = new SQLiteCommand(resetSql, connection))
|
|
{
|
|
int affected = cmd.ExecuteNonQuery();
|
|
Console.WriteLine($"[Migration] Reset likes tracking on {affected} blog rows for clean baseline");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error mapping Blogs likes columns: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
#region Adds
|
|
public static void AddBlog(string blogName, bool byLikes = false, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
if (blogName.Contains("deact"))
|
|
return;
|
|
|
|
SQLiteConnection connection;
|
|
bool ownsConnection;
|
|
lock (_importSessionLock)
|
|
{
|
|
if (_importConnection != null)
|
|
{
|
|
if (_importBlogCache != null && _importBlogCache.Contains(blogName))
|
|
return;
|
|
connection = _importConnection;
|
|
ownsConnection = false;
|
|
}
|
|
else
|
|
{
|
|
connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
ownsConnection = true;
|
|
}
|
|
}
|
|
|
|
try
|
|
{
|
|
// Insert BlogName and DateAdded (current UTC datetime)
|
|
string sql = "INSERT OR IGNORE INTO Blogs (BlogName, DateAdded, DateModified, DateCreated, ByLikes) VALUES (@BlogName, @DateAdded, @DateModified, @DateCreated, @ByLikes)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@DateAdded", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@ByLikes", byLikes ? 1 : 0);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
_importBlogCache?.Add(blogName);
|
|
// Console.WriteLine("+ " + blogName);
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Blogs.BlogName")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
if (ownsConnection) connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void AddPost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
try { AddBlog(blogName, byLikes, DBPath); } catch { }
|
|
try { UpdatePostSetDate(blogName, postID, postDate, DBPath); } catch { }
|
|
try { UpdatePost(blogName, postID, reblogURL, postDate, postURL, slug, reblogKey, reblogName, summary, quote, body, tags, link, photoURL, photoCaption, downloadedFiles, audioCaption, question, answer, title, hasImage, byLikes, DBPath, rootBlogName, rootURL); } catch { }
|
|
|
|
SQLiteConnection connection;
|
|
bool ownsConnection;
|
|
lock (_importSessionLock)
|
|
{
|
|
if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
|
|
else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
|
|
}
|
|
|
|
try
|
|
{
|
|
if (ownsConnection) connection.Open();
|
|
|
|
// IsActive is deliberately absent from this column list: a post removed
|
|
// elsewhere must stay removed, so the crawler never writes that flag.
|
|
string sql = @"INSERT INTO Posts (
|
|
BlogName,
|
|
PostID,
|
|
reblogURL,
|
|
PostDate,
|
|
PostURL,
|
|
Slug,
|
|
ReblogKey,
|
|
ReblogName,
|
|
Summary,
|
|
Quote,
|
|
Body,
|
|
Tags,
|
|
Link,
|
|
PhotoURL,
|
|
PhotoCaption,
|
|
DownloadedFiles,
|
|
AudioCaption,
|
|
Question,
|
|
Answer,
|
|
Title,
|
|
DateModified,
|
|
DateCreated,
|
|
RootBlogName,
|
|
RootURL,
|
|
HasImage,
|
|
ByLikes
|
|
) VALUES (" +
|
|
Q(blogName) + ", " + postID + ", " + Q(reblogURL) + ", " + Q(postDate) + ", " + Q(postURL) + ", " + Q(slug) + ", " + Q(reblogKey) + ", " + Q(reblogName) + ", " + Q(summary) + ", " + Q(quote) + ", " + Q(body) + ", " + Q(tags) + ", " + Q(link) + ", " + Q(photoURL) + ", " + Q(photoCaption) + ", " + Q(downloadedFiles) + ", " + Q(audioCaption) + ", " + Q(question) + ", " + Q(answer) + ", " + Q(title) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")) + ", " + Q(rootBlogName ?? ".") + ", " + Q(rootURL ?? ".") + ", " + (hasImage ? 1 : 0) + ", " + (byLikes ? 1 : 0) + ")";
|
|
SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
|
|
int rowsInserted = 0;
|
|
try
|
|
{
|
|
rowsInserted = command.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
///
|
|
try
|
|
{
|
|
if (ownsConnection) connection.Open();
|
|
|
|
string updateSql = "UPDATE Posts SET hasImage = @hasImage, DateModified = @DateModified WHERE blogName = @blogName AND postID = @postID AND IFNULL(hasImage, 0) <> @hasImage";
|
|
using SQLiteCommand updateCommand = new SQLiteCommand(updateSql, connection);
|
|
updateCommand.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
|
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
updateCommand.Parameters.AddWithValue("@blogName", blogName);
|
|
updateCommand.Parameters.AddWithValue("@postID", postID);
|
|
|
|
updateCommand.ExecuteNonQuery();
|
|
}
|
|
catch (Exception ex2)
|
|
{
|
|
// Breakpoint here
|
|
if (ex2.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
{
|
|
Console.WriteLine(ex2.Message);
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (ownsConnection) connection.Close();
|
|
}
|
|
///
|
|
}
|
|
}
|
|
|
|
// Only reopen the blog for output if a new post was inserted. DateAdded records
|
|
// when the blog first entered the registry and is never rewritten here -- a new
|
|
// post is not a new blog.
|
|
if (rowsInserted == 1)
|
|
{
|
|
try
|
|
{
|
|
string updateBlogSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)";
|
|
using (var updateBlogCommand = new SQLiteCommand(updateBlogSql, connection))
|
|
{
|
|
updateBlogCommand.Parameters.AddWithValue("@BlogName", blogName);
|
|
updateBlogCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
updateBlogCommand.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (ownsConnection) connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void AddAPICount(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Use INSERT OR IGNORE to avoid UNIQUE constraint errors when the date row already exists.
|
|
// Also explicitly initialize APICount to 0 in case the table has no default.
|
|
//
|
|
// DailyAPICount is (Date TEXT PK, APICount INTEGER) — the crawler never creates or
|
|
// migrates this table, and no code reads a creation timestamp off it, so the insert
|
|
// names only those two columns. Naming a DateCreated column here used to throw
|
|
// "no such column: DateCreated" into a silent catch, which meant the day's row was
|
|
// never created and the tally sat at 0 for months.
|
|
string sql = "INSERT OR IGNORE INTO DailyAPICount (Date, APICount) values(@date, 0)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
ReportAPICountFailure($"Error creating the row for {DateTime.Today.ToShortDateString()}: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// Bookkeeping writes that fail identically on every API call would flood the console, but
|
|
// swallowing them entirely is what hid the DateCreated bug. Log each distinct message once.
|
|
// Messages embed today's date, so a date rollover reports afresh.
|
|
private static void ReportAPICountFailure(string message)
|
|
{
|
|
lock (_apiCountFailureLock)
|
|
{
|
|
if (!_apiCountFailuresLogged.Add(message))
|
|
return;
|
|
}
|
|
|
|
Console.WriteLine($"[DailyAPICount] {message}");
|
|
}
|
|
|
|
public static bool AddNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
|
|
try { AddBlog(noteBlogName, false, DBPath); } catch { }
|
|
|
|
using SQLiteConnection connection2 = new SQLiteConnection("Data Source=" + DBPath);
|
|
int rowsInserted = 0;
|
|
|
|
try
|
|
{
|
|
connection2.Open();
|
|
|
|
// Notes stores integer IDs, so both participants and the type have to exist in
|
|
// their lookup table before the note can point at them.
|
|
//
|
|
// All four statements run in one transaction so a crash cannot leave a name or a
|
|
// type registered with no note. The transaction is committed before the console
|
|
// output below, which sleeps -- a write lock must not be held across that.
|
|
using (SQLiteTransaction transaction = connection2.BeginTransaction())
|
|
{
|
|
RegisterBlogName(connection2, transaction, rootBlogName);
|
|
RegisterBlogName(connection2, transaction, noteBlogName);
|
|
RegisterNoteType(connection2, transaction, type ?? string.Empty);
|
|
|
|
// INSERT OR IGNORE, and no IsActive in the column list: re-crawling a note
|
|
// that was removed elsewhere leaves the existing row -- and its flag -- alone.
|
|
string sql = "INSERT OR IGNORE INTO Notes (RootBlogId, NoteBlogId, PostID, TimeStamp, TypeId, DatetimeCrawled, DateModified, DateCreated) " +
|
|
"SELECT (SELECT BlogId FROM BlogNames WHERE BlogName = @rootBlogName), " +
|
|
" (SELECT BlogId FROM BlogNames WHERE BlogName = @noteBlogName), " +
|
|
" @PostID, @TimeStamp, " +
|
|
" (SELECT TypeId FROM NoteTypes WHERE Type = @Type), " +
|
|
" @DatetimeCrawled, @DateModified, @DateCreated";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection2, transaction))
|
|
{
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.Parameters.AddWithValue("@TimeStamp", timestamp);
|
|
command.Parameters.AddWithValue("@Type", type ?? string.Empty);
|
|
command.Parameters.AddWithValue("@DatetimeCrawled", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@DateCreated", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
|
|
rowsInserted = command.ExecuteNonQuery();
|
|
}
|
|
|
|
transaction.Commit();
|
|
}
|
|
|
|
if (rowsInserted == 1)
|
|
{
|
|
ConsoleColor previousColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = ConsoleColor.Green;
|
|
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
|
|
Console.ForegroundColor = previousColor;
|
|
Thread.Sleep(250); // Brief pause to make new notes more noticeable in the console output
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
|
|
}
|
|
|
|
// Only update HasBeenOutput if a new note was inserted
|
|
if (rowsInserted == 1)
|
|
{
|
|
try
|
|
{
|
|
// HasBeenOutput IS NULL still counts as a change: the selection queries
|
|
// test HasBeenOutput = 0, which a NULL would never match.
|
|
string updateSql = "UPDATE Blogs SET HasBeenOutput = 0, DateModified = @DateModified WHERE BlogName = @BlogName AND (HasBeenOutput IS NULL OR HasBeenOutput <> 0)";
|
|
using (var updateCommand = new SQLiteCommand(updateSql, connection2))
|
|
{
|
|
updateCommand.Parameters.AddWithValue("@BlogName", noteBlogName);
|
|
updateCommand.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
updateCommand.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch { }
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (!IsNotesDuplicateKey(ex))
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
Console.WriteLine("^^^^^ - SHORTCUT");
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
#endregion Adds
|
|
|
|
#region Gets
|
|
|
|
/// <summary>
|
|
///
|
|
/// </summary>
|
|
/// <param name="withoutNotesOnly"></param>
|
|
/// <param name="DBPath"></param>
|
|
/// <returns>blogName, postID, lastNoteTimestamp, notesGatheredTimestamp</returns>
|
|
public static List<Tuple<string, long, long, long>> GetPosts(bool withoutNotesOnly = false, DateTime? beforeDate = null, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long, long, long>> posts = new List<Tuple<string, long, long, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql;
|
|
|
|
if (withoutNotesOnly)
|
|
{
|
|
string beforeDateFilter = string.Empty;
|
|
if (beforeDate.HasValue)
|
|
{
|
|
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
|
beforeDateFilter = $"WHERE (U.NotesGatheredDateTime < {unixTimestamp} OR U.NotesGatheredDateTime IS NULL)" + Environment.NewLine;
|
|
}
|
|
|
|
sql = "WITH PostsWithCount AS" + Environment.NewLine +
|
|
"(" + Environment.NewLine +
|
|
" SELECT " + Environment.NewLine +
|
|
" P.BlogName," + Environment.NewLine +
|
|
" P.PostID," + Environment.NewLine +
|
|
" 1925013599 AS LatestNoteTimestamp," + Environment.NewLine +
|
|
" P.NotesGatheredDateTime," + Environment.NewLine +
|
|
" COUNT(P.PostID) OVER(PARTITION BY P.BlogName) AS CNT," + Environment.NewLine +
|
|
" P.HasNotesGathered," + Environment.NewLine +
|
|
" P.NotFound," + Environment.NewLine +
|
|
" P.PostDate" + Environment.NewLine +
|
|
" FROM Posts P" + WhereIsActive("Posts", "P", DBPath) + Environment.NewLine +
|
|
")," + Environment.NewLine +
|
|
"Unioned AS" + Environment.NewLine +
|
|
"(" + Environment.NewLine +
|
|
" SELECT " + Environment.NewLine +
|
|
" BlogName," + Environment.NewLine +
|
|
" PostID," + Environment.NewLine +
|
|
" LatestNoteTimestamp," + Environment.NewLine +
|
|
" NotesGatheredDateTime," + Environment.NewLine +
|
|
" CNT," + Environment.NewLine +
|
|
" PostDate" + Environment.NewLine +
|
|
" FROM PostsWithCount" + Environment.NewLine +
|
|
" WHERE NotFound = 0" + Environment.NewLine +
|
|
" AND HasNotesGathered = 0" + Environment.NewLine +
|
|
"" + Environment.NewLine +
|
|
" UNION " + Environment.NewLine +
|
|
"" + Environment.NewLine +
|
|
" SELECT " + Environment.NewLine +
|
|
" BlogName," + Environment.NewLine +
|
|
" PostID," + Environment.NewLine +
|
|
" LatestNoteTimestamp," + Environment.NewLine +
|
|
" NotesGatheredDateTime," + Environment.NewLine +
|
|
" CNT," + Environment.NewLine +
|
|
" PostDate" + Environment.NewLine +
|
|
" FROM PostsWithCount" + Environment.NewLine +
|
|
" WHERE BlogName = 'zomb-eh'" + Environment.NewLine +
|
|
" AND NotFound = 0" + Environment.NewLine +
|
|
" AND NotesGatheredDateTime < unixepoch('now', 'localtime', '-3 days')" + Environment.NewLine +
|
|
")" + Environment.NewLine +
|
|
"SELECT" + Environment.NewLine +
|
|
" U.BlogName," + Environment.NewLine +
|
|
" U.PostID," + Environment.NewLine +
|
|
" U.LatestNoteTimestamp," + Environment.NewLine +
|
|
" U.NotesGatheredDateTime," + Environment.NewLine +
|
|
" U.CNT" + Environment.NewLine +
|
|
"FROM Unioned U" + Environment.NewLine +
|
|
beforeDateFilter +
|
|
"ORDER BY U.NotesGatheredDateTime, U.PostDate DESC, U.BlogName, U.PostID;" + Environment.NewLine;
|
|
}
|
|
else
|
|
{
|
|
// The LEFT OUTER JOIN to Notes that used to sit here has been dropped rather
|
|
// than ported. Nothing was selected from it, a LEFT JOIN cannot remove a row,
|
|
// and the GROUP BY below collapsed the rows it duplicated -- so it could not
|
|
// affect the result, and it cost a join against 1.18M rows on every pass.
|
|
sql = "SELECT " +
|
|
" MAX(Posts.BlogName) as BlogName, " + Environment.NewLine +
|
|
" Posts.PostID, " + Environment.NewLine +
|
|
" 1925013599 as LatestNoteTimestamp, " + Environment.NewLine +
|
|
" Posts.NotesGatheredDatetime, " + Environment.NewLine +
|
|
" CNT.CNT " + Environment.NewLine +
|
|
"FROM " + Environment.NewLine +
|
|
" Posts " + Environment.NewLine +
|
|
" LEFT OUTER JOIN " + Environment.NewLine +
|
|
" ( select BlogName, count(PostID) as CNT from Posts" + WhereIsActive("Posts", "", DBPath) + " group by BlogName) CNT on CNT.blogName = Posts.BlogName " +
|
|
"WHERE NotFound = 0 " + AndIsActive("Posts", "Posts", DBPath) + Environment.NewLine;
|
|
|
|
if (beforeDate.HasValue)
|
|
{
|
|
long unixTimestamp = new DateTimeOffset(beforeDate.Value).ToUnixTimeSeconds();
|
|
sql += $" AND (NotesGatheredDateTime < {unixTimestamp} OR NotesGatheredDateTime= 0) " + Environment.NewLine;
|
|
}
|
|
|
|
sql += "GROUP BY " + Environment.NewLine +
|
|
" Posts.BlogName, Posts.PostID " + Environment.NewLine +
|
|
"ORDER BY " + Environment.NewLine +
|
|
" notesgathereddatetime, Posts.PostDate desc, Posts.BlogName, Posts.PostID" + Environment.NewLine;
|
|
}
|
|
|
|
Console.WriteLine(withoutNotesOnly);
|
|
Console.WriteLine(System.Text.RegularExpressions.Regex.Replace(sql, @"\s+", " ").Trim());
|
|
Console.Write(">" ); //Console.ReadKey();
|
|
//Thread.Sleep(250);
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long, long, long> post = default;
|
|
string blog;
|
|
long postID = 0;
|
|
long lastNoteTimestamp;
|
|
long notesGatheredTimestamp;
|
|
|
|
// Diagnostic logging before each cast
|
|
//for (int i = 0; i < reader.FieldCount; i++)
|
|
//{
|
|
// var value = reader.GetValue(i);
|
|
// var type = reader.GetFieldType(i);
|
|
// Console.WriteLine($"[GetPosts] Column {i}: Name={reader.GetName(i)}, Value={value}, Type={type}");
|
|
//}
|
|
|
|
try
|
|
{
|
|
blog = Convert.ToString(reader.GetValue(0));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 0 to string: {ex.Message}");
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
postID = Convert.ToInt64(reader.GetValue(1));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 1 to Int64: {ex.Message}");
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
lastNoteTimestamp = Convert.ToInt64(reader.GetValue(2));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 2 to Int64: {ex.Message}");
|
|
throw;
|
|
}
|
|
try
|
|
{
|
|
notesGatheredTimestamp = Convert.ToInt64(reader.GetValue(3));
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[GetPosts] Error casting column 3 to Int64: {ex.Message}");
|
|
throw;
|
|
}
|
|
|
|
// Exclude if postID is in the exclusion list
|
|
if (_postIdsToExclude != null && _postIdsToExclude.Contains(postID))
|
|
continue;
|
|
|
|
post = new Tuple<string, long, long, long>(blog, postID, lastNoteTimestamp, notesGatheredTimestamp);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static List<Tuple<string, long>> GetReplies(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT DISTINCT BN.BlogName as blogName, N.PostID" +
|
|
" FROM Notes N" +
|
|
" INNER JOIN BlogNames BN ON BN.BlogId = N.RootBlogId" +
|
|
" WHERE N.TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply')" + AndIsActive("Notes", "N", DBPath) +
|
|
" ORDER BY BN.BlogName, N.PostID";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
|
|
id = reader.GetInt64(reader.GetOrdinal("postID")); // Assuming Id is the first column
|
|
blog = reader.GetString(reader.GetOrdinal("blogName")); // Assuming Title is the second column
|
|
|
|
post = new Tuple<string, long>(blog, id);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static List<Tuple<string, long>> GetRepliesWithMissingText(string? DBPath = null, int limit = 50)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long>> posts = new List<Tuple<string, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Grouped on the integer rather than the name: the group key is what gets sorted,
|
|
// and BN.BlogName comes along for free off the join.
|
|
string sql = @"SELECT BN.BlogName as blogName, N.PostID,
|
|
MAX(N.TimeStamp) as LatestTimestamp
|
|
FROM Notes N
|
|
INNER JOIN BlogNames BN ON BN.BlogId = N.RootBlogId
|
|
WHERE N.TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply')
|
|
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Notes", "N", DBPath) + @"
|
|
GROUP BY N.RootBlogId, N.PostID
|
|
ORDER BY LatestTimestamp ASC
|
|
LIMIT @limit";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@limit", limit);
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
|
|
blog = reader.GetString(reader.GetOrdinal("blogName"));
|
|
id = reader.GetInt64(reader.GetOrdinal("PostID"));
|
|
|
|
post = new Tuple<string, long>(blog, id);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"Error getting replies with missing text: {ex.Message}");
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static List<Tuple<string, long, long>> GetRepliesWithFilledText(string? DBPath = null, int? limit = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, long, long>> posts = new List<Tuple<string, long, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Posts carries only BlogName, so this is the one join to Notes that has to go
|
|
// through BlogNames -- there is no Posts.BlogId to hop on. The name predicate is
|
|
// pushed into the 20k-row lookup, which then feeds integers to the Notes key.
|
|
string sql = @"SELECT DISTINCT P.BlogName, P.PostID, MAX(N.TimeStamp) as LatestTimestamp
|
|
FROM Posts P
|
|
INNER JOIN BlogNames RBN ON RBN.BlogName = P.BlogName
|
|
INNER JOIN Notes N ON N.RootBlogId = RBN.BlogId AND N.PostID = P.PostID
|
|
WHERE P.NotFound = 0
|
|
AND N.TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply')
|
|
AND (N.replyText IS NULL OR N.replyText = '' OR N.replyText = '.')" + AndIsActive("Posts", "P", DBPath) + AndIsActive("Notes", "N", DBPath) + @"
|
|
GROUP BY P.BlogName, P.PostID
|
|
ORDER BY LatestTimestamp ASC";
|
|
|
|
if (limit.HasValue)
|
|
{
|
|
sql += " LIMIT @limit";
|
|
}
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
if (limit.HasValue)
|
|
{
|
|
command.Parameters.AddWithValue("@limit", limit.Value);
|
|
}
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
Tuple<string, long, long> post = default;
|
|
string blog = null;
|
|
long id = 0;
|
|
long timestamp = 0;
|
|
|
|
blog = reader.GetString(reader.GetOrdinal("blogName"));
|
|
id = reader.GetInt64(reader.GetOrdinal("PostID"));
|
|
timestamp = reader.GetInt64(reader.GetOrdinal("LatestTimestamp"));
|
|
timestamp++;
|
|
|
|
post = new Tuple<string, long, long>(blog, id, timestamp);
|
|
posts.Add(post);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"Error getting replies with filled text: {ex.Message}");
|
|
}
|
|
return posts;
|
|
}
|
|
|
|
public static int GetAPICount(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
int count = 0;
|
|
|
|
// AddAPICount reports its own failures; this guard only stops a connection-level
|
|
// problem from taking down the read below.
|
|
try { AddAPICount(); } catch (Exception ex) { ReportAPICountFailure($"AddAPICount failed: {ex.Message}"); }
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "SELECT APICount FROM DailyAPICount WHERE [Date] = @date";
|
|
|
|
bool rowFound = false;
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
rowFound = true;
|
|
count = reader.GetInt32(0); // Assuming Id is the first column
|
|
}
|
|
}
|
|
}
|
|
|
|
// A missing row means AddAPICount did not take. Returning a silent 0 here is what
|
|
// made the tally look merely idle rather than broken.
|
|
if (!rowFound)
|
|
ReportAPICountFailure($"No row for {DateTime.Today.ToShortDateString()} after AddAPICount - reported count of 0 is not a real tally.");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
return count;
|
|
}
|
|
|
|
public static List<Tuple<string, int, long, long>> GetBlogsForLikes(string specificBlog = null, int cooldownDays = 7, bool ignoreCooldown = false, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<Tuple<string, int, long, long>> blogs = new List<Tuple<string, int, long, long>>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql;
|
|
|
|
// The two Notes branches below join on Blogs.BlogId, which is NULL for the 168k
|
|
// registry rows that have never appeared in a note. The inner join drops them,
|
|
// which is correct here -- both branches already require a note to exist -- but
|
|
// it is the wrong shape for anything that lists the registry.
|
|
if (!string.IsNullOrEmpty(specificBlog))
|
|
{
|
|
// Specific blog: always process, bypass cooldown
|
|
sql = @"SELECT BlogName,
|
|
COALESCE(LikesPulled, 0),
|
|
COALESCE(LikesCursor, 0),
|
|
COALESCE(LikesNewestTimestamp, 0)
|
|
FROM Blogs
|
|
WHERE BlogName = @blog
|
|
AND IsActive = 1";
|
|
}
|
|
else if (ignoreCooldown)
|
|
{
|
|
// All blogs with notes: backfill-pending OR any refresh-eligible blog, ignore cooldown
|
|
sql = @"SELECT B.BlogName,
|
|
COALESCE(B.LikesPulled, 0),
|
|
COALESCE(B.LikesCursor, 0),
|
|
COALESCE(B.LikesNewestTimestamp, 0)
|
|
FROM Blogs B
|
|
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
|
|
WHERE N.TimeStamp >= 1535778000
|
|
AND N.RootBlogId = B.BlogId
|
|
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
|
|
GROUP BY B.BlogName
|
|
ORDER BY MIN(N.TimeStamp);";
|
|
}
|
|
else
|
|
{
|
|
// Backfill-pending OR refresh-due (past cooldown window)
|
|
sql = @"SELECT B.BlogName,
|
|
COALESCE(B.LikesPulled, 0),
|
|
COALESCE(B.LikesCursor, 0),
|
|
COALESCE(B.LikesNewestTimestamp, 0)
|
|
FROM Blogs B
|
|
INNER JOIN Notes N ON N.NoteBlogId = B.BlogId
|
|
WHERE N.TimeStamp >= 1535778000
|
|
AND N.RootBlogId = B.BlogId
|
|
AND B.IsActive = 1" + AndIsActive("Notes", "N", DBPath) + @"
|
|
AND (
|
|
B.LikesPulled = 0
|
|
OR COALESCE(B.LikesLastRefreshed, 0)
|
|
< (CAST(strftime('%s','now') AS INTEGER) - (@cooldownDays * 86400))
|
|
)
|
|
GROUP BY B.BlogName
|
|
ORDER BY MIN(N.TimeStamp);";
|
|
}
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
if (!string.IsNullOrEmpty(specificBlog))
|
|
command.Parameters.AddWithValue("@blog", specificBlog);
|
|
else if (!ignoreCooldown)
|
|
command.Parameters.AddWithValue("@cooldownDays", cooldownDays);
|
|
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
blogs.Add(new Tuple<string, int, long, long>(
|
|
reader.GetString(0),
|
|
reader.GetInt32(1),
|
|
reader.GetInt64(2),
|
|
reader.GetInt64(3)
|
|
));
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error fetching blogs for likes: {ex.Message}");
|
|
}
|
|
return blogs;
|
|
}
|
|
|
|
public static List<string> GetBlogs(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<string> blogs = new List<string>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
// Blogs is reached in one integer hop off Blogs.BlogId, not through BlogNames --
|
|
// that would add a hop and end in the text comparison the migration removed.
|
|
// The negated form is only correct because Notes.TypeId is NOT NULL.
|
|
string sql = "";
|
|
if (reblogsOnly)
|
|
sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND N.TypeId IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply', 'posted')) AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top";
|
|
else
|
|
sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND N.TypeId NOT IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply', 'posted')) AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@isActive", 1);
|
|
command.Parameters.AddWithValue("@top", top);
|
|
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string blog = null;
|
|
blog = reader.GetString(0); // Assuming Title is the second column
|
|
|
|
blogs.Add(blog);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
return blogs;
|
|
}
|
|
|
|
public static List<string> GetBlogsAll(bool reblogsOnly, int from, int to, int top, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
List<string> blogs = new List<string>();
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = "";
|
|
if (reblogsOnly)
|
|
sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND N.TypeId IN (SELECT TypeId FROM NoteTypes WHERE Type IN ('reblog', 'reply', 'posted')) AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top";
|
|
else
|
|
sql = "SELECT B.BlogName as blogName, count(*) FROM Notes N INNER JOIN Blogs B ON B.BlogId = N.NoteBlogId WHERE B.IsActive = @isActive" + AndIsActive("Notes", "N", DBPath) + " AND B.HasBeenOutput = 0 GROUP BY N.NoteBlogId ORDER BY count(*) DESC, B.BlogName LIMIT @top";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@isActive", 1);
|
|
command.Parameters.AddWithValue("@top", top);
|
|
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string blog = null;
|
|
blog = reader.GetString(0); // Assuming Title is the second column
|
|
|
|
blogs.Add(blog);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
return blogs;
|
|
}
|
|
public static IEnumerable<List<string>> GetAllPostTextColumns(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
string sql = "SELECT BlogName, reblogURL, PostURL, Slug, ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link, PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption, Question, Answer, Title, RootBlogName, RootURL FROM Posts WHERE IFNULL(DownloadedFiles, '.') = '.'" + AndIsActive("Posts", "", DBPath);
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
List<string> rowTexts = new List<string>();
|
|
for (int i = 0; i < reader.FieldCount; i++)
|
|
{
|
|
if (!reader.IsDBNull(i))
|
|
{
|
|
var val = reader.GetValue(i);
|
|
if (val is string str && !string.IsNullOrWhiteSpace(str) && str != ".")
|
|
{
|
|
rowTexts.Add(str);
|
|
}
|
|
}
|
|
}
|
|
if (rowTexts.Count > 0)
|
|
{
|
|
yield return rowTexts;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
#endregion Gets
|
|
|
|
#region Updates
|
|
|
|
|
|
public static void UpdatePostMarkNotesCollected(string blogName, long postID, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
//string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered WHERE BlogName = @BlogName AND PostID = @PostID";
|
|
// NotesGatheredDateTime is crawl bookkeeping -- it moves on every pass and says
|
|
// nothing about the post itself, so only the HasNotesGathered flag flipping
|
|
// counts as a modification. The CASE reads the pre-UPDATE value of the flag.
|
|
string sql = "UPDATE Posts SET HasNotesGathered = 1, NotesGatheredDateTime = @notesGathered, DateModified = CASE WHEN IFNULL(HasNotesGathered, 0) <> 1 THEN @dateModified ELSE DateModified END WHERE PostID = @PostID AND (IFNULL(HasNotesGathered, 0) <> 1 OR IFNULL(NotesGatheredDateTime, 0) <> @notesGathered)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@notesGathered", DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
}
|
|
|
|
public static void UpdatePostMarkNotFound(string blogName, long postID, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
// Informative output when marking a post as NotFound
|
|
Console.WriteLine($"Marking post NotFound: {blogName}.tumblr.com/post/{postID}");
|
|
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET NotFound = 1, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(NotFound, 0) <> 1";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
}
|
|
|
|
// ----- CollectRunState: tracks the frozen cutoff + completion flag for a managed "-collect 0" full re-check run -----
|
|
// Single-row table (Id = 1), mirroring the ApiKeyPoolMeta pattern. Lets an interrupted run resume against the
|
|
// same cutoff and lets a completed run stop instead of restarting on the next launch.
|
|
|
|
public static void EnsureCollectRunStateTableExists(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
string sql = "CREATE TABLE IF NOT EXISTS CollectRunState (" +
|
|
"Id INTEGER PRIMARY KEY CHECK (Id = 1), " +
|
|
"RunCutoff INTEGER DEFAULT 0, " +
|
|
"RunComplete INTEGER DEFAULT 1, " +
|
|
"RunStarted TEXT, " +
|
|
"RunCompletedAt TEXT)";
|
|
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
// Returns (RunCutoff unix seconds, RunComplete) for the single run-state row, or null if no row exists yet.
|
|
public static (long cutoff, bool complete)? GetCollectRunState(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
using SQLiteCommand command = new SQLiteCommand("SELECT RunCutoff, RunComplete FROM CollectRunState WHERE Id = 1", connection);
|
|
using SQLiteDataReader reader = command.ExecuteReader();
|
|
if (reader.Read())
|
|
{
|
|
long cutoff = Convert.ToInt64(reader.GetValue(0));
|
|
bool complete = Convert.ToInt64(reader.GetValue(1)) != 0;
|
|
return (cutoff, complete);
|
|
}
|
|
return null;
|
|
}
|
|
|
|
// Start (or restart) a managed run: freeze the cutoff and mark the run in progress.
|
|
public static void BeginCollectRun(long cutoffUnixSeconds, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
string sql = "INSERT INTO CollectRunState (Id, RunCutoff, RunComplete, RunStarted, RunCompletedAt) " +
|
|
"VALUES (1, @cutoff, 0, @started, NULL) " +
|
|
"ON CONFLICT(Id) DO UPDATE SET RunCutoff = @cutoff, RunComplete = 0, RunStarted = @started, RunCompletedAt = NULL";
|
|
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
command.Parameters.AddWithValue("@cutoff", cutoffUnixSeconds);
|
|
command.Parameters.AddWithValue("@started", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
// Mark the active managed run complete so the next launch starts fresh instead of resuming.
|
|
public static void CompleteCollectRun(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
string sql = "UPDATE CollectRunState SET RunComplete = 1, RunCompletedAt = @completedAt WHERE Id = 1";
|
|
using SQLiteCommand command = new SQLiteCommand(sql, connection);
|
|
command.Parameters.AddWithValue("@completedAt", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
public static void UpdatePostSetDate(string blogName, long postID, string postDate, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
SQLiteConnection connection;
|
|
bool ownsConnection;
|
|
lock (_importSessionLock)
|
|
{
|
|
if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
|
|
else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
|
|
}
|
|
|
|
try
|
|
{
|
|
if (ownsConnection) connection.Open();
|
|
|
|
string sql = "UPDATE Posts SET postDate = @postDate, DateModified = @dateModified WHERE BlogName = @BlogName AND PostID = @PostID AND IFNULL(postDate, '') <> @postDate";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty);
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
if (ownsConnection) connection.Close();
|
|
}
|
|
}
|
|
|
|
public static bool UpdateNote(string rootBlogName, string noteBlogName, long postID, long timestamp, string type, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
//try { AddPost(rootBlogName, postID, DBPath); } catch { }
|
|
try { AddBlog(noteBlogName, false, DBPath); } catch { }
|
|
|
|
Console.WriteLine("{2}\t{0}\t{1}", UnixTimeStampToDateTime(timestamp), noteBlogName, type);
|
|
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Notes SET TimeStamp = @timestamp, DateModified = @dateModified " +
|
|
"WHERE RootBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @rootBlogName) " +
|
|
"AND NoteBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @noteBlogName) " +
|
|
"AND PostID = @postID AND IFNULL(TimeStamp, 0) <> @timestamp";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@timestamp", timestamp);
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
|
|
command.Parameters.AddWithValue("@postID", postID);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (!IsNotesDuplicateKey(ex))
|
|
{
|
|
Console.WriteLine(ex.Message);
|
|
Console.WriteLine("^^^^^ - SHORTCUT");
|
|
return true;
|
|
}
|
|
}
|
|
return false;
|
|
}
|
|
|
|
public static void UpdatePost(string blogName, long postID, string reblogURL, string postDate, string postURL, string slug, string reblogKey, string reblogName, string summary, string quote, string body, string tags, string link, string photoURL, string photoCaption, string downloadedFiles, string audioCaption, string question, string answer, string title, bool hasImage, bool byLikes = false, string? DBPath = null, string? rootBlogName = null, string? rootURL = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
SQLiteConnection connection;
|
|
bool ownsConnection;
|
|
lock (_importSessionLock)
|
|
{
|
|
if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
|
|
else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
|
|
}
|
|
|
|
try
|
|
{
|
|
if (ownsConnection) connection.Open();
|
|
|
|
// "." is TraverseDirectory/ReblogRecord's sentinel for "this field had no
|
|
// matching line in this particular export file" -- not an empty value. A blog
|
|
// with two export files in different formats (e.g. an "_2" duplicate folder, or
|
|
// a Tumblr export whose field set changed over time) sends one record with a real
|
|
// Title and another with Title = "." for the same PostID, and re-importing both
|
|
// on every run must not let the "not supplied" record blank out what the other
|
|
// one has. Every content field below is CASE-guarded the same way RootBlogName/
|
|
// RootURL already were, and the change-detection ignores "." too so a "."-only
|
|
// difference doesn't fire the UPDATE (and bump DateModified) on its own. Only "."
|
|
// is treated as the sentinel -- an explicit empty string from a real field still
|
|
// overwrites, same as before.
|
|
string sql = "UPDATE Posts SET ";
|
|
sql += "postDate = CASE WHEN @postDate = '.' THEN postDate ELSE @postDate END, ";
|
|
sql += "reblogURL = CASE WHEN @reblogURL = '.' THEN reblogURL ELSE @reblogURL END, ";
|
|
sql += "postURL = CASE WHEN @postURL = '.' THEN postURL ELSE @postURL END, ";
|
|
sql += "slug = CASE WHEN @slug = '.' THEN slug ELSE @slug END, ";
|
|
sql += "reblogKey = CASE WHEN @reblogKey = '.' THEN reblogKey ELSE @reblogKey END, ";
|
|
sql += "reblogName = CASE WHEN @reblogName = '.' THEN reblogName ELSE @reblogName END, ";
|
|
sql += "summary = CASE WHEN @summary = '.' THEN summary ELSE @summary END, ";
|
|
sql += "quote = CASE WHEN @quote = '.' THEN quote ELSE @quote END, ";
|
|
sql += "body = CASE WHEN @body = '.' THEN body ELSE @body END, ";
|
|
sql += "tags = CASE WHEN @tags = '.' THEN tags ELSE @tags END, ";
|
|
sql += "link = CASE WHEN @link = '.' THEN link ELSE @link END, ";
|
|
sql += "photoURL = CASE WHEN @photoURL = '.' THEN photoURL ELSE @photoURL END, ";
|
|
sql += "photoCaption = CASE WHEN @photoCaption = '.' THEN photoCaption ELSE @photoCaption END, ";
|
|
sql += "downloadedFiles = CASE WHEN @downloadedFiles = '.' THEN downloadedFiles ELSE @downloadedFiles END, ";
|
|
sql += "audioCaption = CASE WHEN @audioCaption = '.' THEN audioCaption ELSE @audioCaption END, ";
|
|
sql += "question = CASE WHEN @question = '.' THEN question ELSE @question END, ";
|
|
sql += "answer = CASE WHEN @answer = '.' THEN answer ELSE @answer END, ";
|
|
sql += "title = CASE WHEN @title = '.' THEN title ELSE @title END, ";
|
|
sql += "DateModified = @dateModified, ";
|
|
sql += "RootBlogName = CASE WHEN @rootBlogName IS NULL OR @rootBlogName = '' OR @rootBlogName = '.' THEN RootBlogName ELSE @rootBlogName END, ";
|
|
sql += "RootURL = CASE WHEN @rootURL IS NULL OR @rootURL = '' OR @rootURL = '.' THEN RootURL ELSE @rootURL END, ";
|
|
sql += "hasImage = @hasImage, ";
|
|
sql += "ByLikes = MAX(IFNULL(ByLikes, 0), @byLikes) ";
|
|
sql += " WHERE BlogName = @BlogName AND PostID = @PostID AND (";
|
|
sql += "(@postDate <> '.' AND IFNULL(postDate, '') <> @postDate) OR ";
|
|
sql += "(@reblogURL <> '.' AND IFNULL(reblogURL, '') <> @reblogURL) OR ";
|
|
sql += "(@postURL <> '.' AND IFNULL(postURL, '') <> @postURL) OR ";
|
|
sql += "(@slug <> '.' AND IFNULL(slug, '') <> @slug) OR ";
|
|
sql += "(@reblogKey <> '.' AND IFNULL(reblogKey, '') <> @reblogKey) OR ";
|
|
sql += "(@reblogName <> '.' AND IFNULL(reblogName, '') <> @reblogName) OR ";
|
|
sql += "(@summary <> '.' AND IFNULL(summary, '') <> @summary) OR ";
|
|
sql += "(@quote <> '.' AND IFNULL(quote, '') <> @quote) OR ";
|
|
sql += "(@body <> '.' AND IFNULL(body, '') <> @body) OR ";
|
|
sql += "(@tags <> '.' AND IFNULL(tags, '') <> @tags) OR ";
|
|
sql += "(@link <> '.' AND IFNULL(link, '') <> @link) OR ";
|
|
sql += "(@photoURL <> '.' AND IFNULL(photoURL, '') <> @photoURL) OR ";
|
|
sql += "(@photoCaption <> '.' AND IFNULL(photoCaption, '') <> @photoCaption) OR ";
|
|
sql += "(@downloadedFiles <> '.' AND IFNULL(downloadedFiles, '') <> @downloadedFiles) OR ";
|
|
sql += "(@audioCaption <> '.' AND IFNULL(audioCaption, '') <> @audioCaption) OR ";
|
|
sql += "(@question <> '.' AND IFNULL(question, '') <> @question) OR ";
|
|
sql += "(@answer <> '.' AND IFNULL(answer, '') <> @answer) OR ";
|
|
sql += "(@title <> '.' AND IFNULL(title, '') <> @title) OR ";
|
|
sql += "IFNULL(hasImage, 0) <> @hasImage OR ";
|
|
sql += "(@byLikes = 1 AND IFNULL(ByLikes, 0) = 0) OR ";
|
|
sql += "((@rootBlogName IS NOT NULL AND @rootBlogName <> '' AND @rootBlogName <> '.') AND IFNULL(RootBlogName, '') <> @rootBlogName) OR ";
|
|
sql += "((@rootURL IS NOT NULL AND @rootURL <> '' AND @rootURL <> '.') AND IFNULL(RootURL, '') <> @rootURL)";
|
|
sql += ")";
|
|
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@postDate", postDate ?? string.Empty);
|
|
command.Parameters.AddWithValue("@reblogURL", reblogURL ?? string.Empty);
|
|
command.Parameters.AddWithValue("@postURL", postURL ?? string.Empty);
|
|
command.Parameters.AddWithValue("@slug", slug ?? string.Empty);
|
|
command.Parameters.AddWithValue("@reblogKey", reblogKey ?? string.Empty);
|
|
command.Parameters.AddWithValue("@reblogName", reblogName ?? string.Empty);
|
|
command.Parameters.AddWithValue("@summary", summary ?? string.Empty);
|
|
command.Parameters.AddWithValue("@quote", quote ?? string.Empty);
|
|
command.Parameters.AddWithValue("@body", body ?? string.Empty);
|
|
command.Parameters.AddWithValue("@tags", tags ?? string.Empty);
|
|
command.Parameters.AddWithValue("@link", link ?? string.Empty);
|
|
command.Parameters.AddWithValue("@photoURL", photoURL ?? string.Empty);
|
|
command.Parameters.AddWithValue("@photoCaption", photoCaption ?? string.Empty);
|
|
command.Parameters.AddWithValue("@downloadedFiles", downloadedFiles ?? string.Empty);
|
|
command.Parameters.AddWithValue("@audioCaption", audioCaption ?? string.Empty);
|
|
command.Parameters.AddWithValue("@question", question ?? string.Empty);
|
|
command.Parameters.AddWithValue("@answer", answer ?? string.Empty);
|
|
command.Parameters.AddWithValue("@title", title ?? string.Empty);
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", string.IsNullOrWhiteSpace(rootBlogName) ? (object)DBNull.Value : rootBlogName);
|
|
command.Parameters.AddWithValue("@rootURL", string.IsNullOrWhiteSpace(rootURL) ? (object)DBNull.Value : rootURL);
|
|
command.Parameters.AddWithValue("@hasImage", hasImage ? 1 : 0);
|
|
command.Parameters.AddWithValue("@byLikes", byLikes ? 1 : 0);
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
|
|
command.ExecuteNonQuery();
|
|
}
|
|
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
finally
|
|
{
|
|
if (ownsConnection) connection.Close();
|
|
}
|
|
}
|
|
|
|
public static void UpdateBlogOutput(string blogName, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Blogs SET HasBeenOutput = 1, DateModified = @DateModified WHERE BlogName = @BlogName AND IFNULL(HasBeenOutput, 0) <> 1";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@BlogName", blogName);
|
|
command.Parameters.AddWithValue("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
}
|
|
|
|
public static void UpdateBlogLikesStatus(string blogName, int likesPulled, long likesCursor, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = "UPDATE Blogs SET LikesPulled = @pulled, LikesCursor = @cursor, DateModified = @modified WHERE BlogName = @name AND (IFNULL(LikesPulled, 0) <> @pulled OR IFNULL(LikesCursor, 0) <> @cursor)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@pulled", likesPulled);
|
|
command.Parameters.AddWithValue("@cursor", likesCursor);
|
|
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@name", blogName);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error updating blog likes status: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// Bumps the high-water mark for a blog. Used during Branch A (initial backfill) when we
|
|
// capture the newest liked_timestamp on the first page so subsequent refresh runs have a
|
|
// stopping point. MAX(...) protects against out-of-order updates.
|
|
public static void UpdateBlogLikesNewestTimestamp(string blogName, long newestTimestamp, string? DBPath = null)
|
|
{
|
|
if (newestTimestamp <= 0) return;
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = @"UPDATE Blogs
|
|
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
|
|
DateModified = @modified
|
|
WHERE BlogName = @name
|
|
AND COALESCE(LikesNewestTimestamp, 0) < @newest";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@newest", newestTimestamp);
|
|
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@name", blogName);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error updating blog likes newest timestamp: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// Called at the end of a refresh pass (Branch B). Bumps the high-water mark, stamps the
|
|
// last-refreshed time so cooldown takes effect, and records how many new likes were found.
|
|
public static void UpdateBlogLikesRefreshStatus(string blogName, long newestTimestamp, int newCount, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
try
|
|
{
|
|
connection.Open();
|
|
string sql = @"UPDATE Blogs
|
|
SET LikesNewestTimestamp = MAX(COALESCE(LikesNewestTimestamp, 0), @newest),
|
|
LikesLastRefreshed = CAST(strftime('%s','now') AS INTEGER),
|
|
LikesLastNewCount = @count,
|
|
DateModified = @modified
|
|
WHERE BlogName = @name";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@newest", newestTimestamp);
|
|
command.Parameters.AddWithValue("@count", newCount);
|
|
command.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@name", blogName);
|
|
command.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error updating blog likes refresh status: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
public static int UpdateAPICount(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
int APICount = DataAccess.GetAPICount();
|
|
APICount++;
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE DailyAPICount SET APICount = @APICount WHERE [Date] = @date";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@APICount", APICount);
|
|
command.Parameters.AddWithValue("@date", DateTime.Today.ToShortDateString());
|
|
|
|
// No row for today means this UPDATE matched nothing and the increment was
|
|
// thrown away, while the value returned below still looks like a real count.
|
|
if (command.ExecuteNonQuery() == 0)
|
|
ReportAPICountFailure($"UPDATE matched no row for {DateTime.Today.ToShortDateString()} - the count of {APICount} was not persisted.");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
if (ex.Message != "constraint failed\r\nUNIQUE constraint failed: Posts.BlogName, Posts.PostID")
|
|
Console.WriteLine(ex.Message);
|
|
}
|
|
|
|
return APICount;
|
|
}
|
|
|
|
public static int UpdateNoteReplyText(string rootBlogName, long postID, string noteBlogName, long timestamp, string replyText, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
int rowsAffected = 0;
|
|
|
|
// A bare "." collides with the "needs processing" sentinel in GetRepliesWithFilledText, which would loop the post forever. Store as ". " so the data is preserved but no longer matches the sentinel.
|
|
if (replyText == ".")
|
|
replyText = ". ";
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
// Match on (noteBlogName, TimeStamp ±5s) only - a reply by a given blog at a given timestamp is the same reply across the original post and every reblog of it, so this fans out across reblog chains in one shot. Tolerance absorbs the ~1s drift between what -collect stored and what mode=conversation returns now.
|
|
// Only fan out to rows that match the SELECT criteria in GetRepliesWithFilledText (NULL/empty/legacy-'.'). Never overwrite '?' (confirmed-empty) or already-fetched text.
|
|
// The ABS() term cannot use an index on TimeStamp, before or after the integer schema; the NoteBlogId probe is what keeps this off a full scan.
|
|
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified " +
|
|
"WHERE NoteBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @noteBlogName) " +
|
|
"AND ABS(TimeStamp - @TimeStamp) <= 5 " +
|
|
"AND TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') " +
|
|
"AND (replyText IS NULL OR replyText = '' OR replyText = '.') " +
|
|
"AND (replyText IS NULL OR replyText <> @replyText)";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@replyText", replyText ?? "?");
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
command.Parameters.AddWithValue("@noteBlogName", noteBlogName);
|
|
command.Parameters.AddWithValue("@TimeStamp", timestamp);
|
|
rowsAffected = command.ExecuteNonQuery();
|
|
|
|
if (rowsAffected == 0)
|
|
{
|
|
Console.WriteLine($"[UpdateNoteReplyText] INFO: No rows updated for {rootBlogName}.tumblr.com/post/{postID} from {noteBlogName} at {UnixTimeStampToDateTime(timestamp)} (row not found or value unchanged)");
|
|
Console.WriteLine($"[UpdateNoteReplyText] Query: {sql}");
|
|
Console.WriteLine($"[UpdateNoteReplyText] Params: rootBlogName={rootBlogName}, PostID={postID}, noteBlogName={noteBlogName}, TimeStamp={timestamp}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"[UpdateNoteReplyText] Successfully updated {rowsAffected} row(s) for {rootBlogName}.tumblr.com/post/{postID} from {noteBlogName}");
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[UpdateNoteReplyText] Error updating reply text: {ex.Message}");
|
|
Console.WriteLine($"[UpdateNoteReplyText] StackTrace: {ex.StackTrace}");
|
|
}
|
|
return rowsAffected;
|
|
}
|
|
|
|
public static int UpdateAllNoteReplyTextForPost(string rootBlogName, long postID, string replyText, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
string sql = "UPDATE Notes SET replyText = @replyText, DateModified = @dateModified " +
|
|
"WHERE RootBlogId = (SELECT BlogId FROM BlogNames WHERE BlogName = @rootBlogName) " +
|
|
"AND PostID = @PostID " +
|
|
"AND TypeId = (SELECT TypeId FROM NoteTypes WHERE Type = 'reply') " +
|
|
"AND IFNULL(replyText, '.') <> @replyText";
|
|
using (SQLiteCommand command = new SQLiteCommand(sql, connection))
|
|
{
|
|
command.Parameters.AddWithValue("@replyText", replyText ?? ".");
|
|
command.Parameters.AddWithValue("@dateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
command.Parameters.AddWithValue("@rootBlogName", rootBlogName);
|
|
command.Parameters.AddWithValue("@PostID", postID);
|
|
int rowsAffected = command.ExecuteNonQuery();
|
|
|
|
if (rowsAffected == 0)
|
|
{
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] INFO: No rows updated for {rootBlogName}.tumblr.com/post/{postID} (rows not found or values unchanged)");
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Query: {sql}");
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Params: rootBlogName={rootBlogName}, PostID={postID}, replyText={replyText}");
|
|
}
|
|
else
|
|
{
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Successfully updated {rowsAffected} row(s) for {rootBlogName}.tumblr.com/post/{postID} with '{replyText}'");
|
|
}
|
|
|
|
return rowsAffected;
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// Breakpoint here
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] Error updating reply text: {ex.Message}");
|
|
Console.WriteLine($"[UpdateAllNoteReplyTextForPost] StackTrace: {ex.StackTrace}");
|
|
return 0;
|
|
}
|
|
}
|
|
#endregion Updates
|
|
|
|
#region TTFileHelper
|
|
|
|
// Idempotent migration: adds Blogs.TTFolderPath and Posts.PostType if missing.
|
|
// Mirrors the EnsureBlogsLikesColumnsExist pattern (PRAGMA table_info + ALTER TABLE ADD COLUMN).
|
|
public static void EnsureTTFileHelperColumnsExist(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
using SQLiteConnection connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
|
|
try
|
|
{
|
|
connection.Open();
|
|
|
|
bool ttFolderPathExists = false;
|
|
using (SQLiteCommand command = new SQLiteCommand("PRAGMA table_info(Blogs);", connection))
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string columnName = reader.GetString(1);
|
|
if (columnName.Equals("TTFolderPath", StringComparison.OrdinalIgnoreCase)) ttFolderPathExists = true;
|
|
}
|
|
}
|
|
|
|
if (!ttFolderPathExists)
|
|
{
|
|
using (SQLiteCommand cmd = new SQLiteCommand("ALTER TABLE Blogs ADD COLUMN TTFolderPath TEXT;", connection))
|
|
cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added TTFolderPath column to Blogs table");
|
|
}
|
|
|
|
bool postTypeExists = false;
|
|
using (SQLiteCommand command = new SQLiteCommand("PRAGMA table_info(Posts);", connection))
|
|
using (SQLiteDataReader reader = command.ExecuteReader())
|
|
{
|
|
while (reader.Read())
|
|
{
|
|
string columnName = reader.GetString(1);
|
|
if (columnName.Equals("PostType", StringComparison.OrdinalIgnoreCase)) postTypeExists = true;
|
|
}
|
|
}
|
|
|
|
if (!postTypeExists)
|
|
{
|
|
using (SQLiteCommand cmd = new SQLiteCommand("ALTER TABLE Posts ADD COLUMN PostType TEXT;", connection))
|
|
cmd.ExecuteNonQuery();
|
|
Console.WriteLine("[Migration] Added PostType column to Posts table");
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"Error ensuring TTFileHelper columns: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
// INSERT-or-UPDATE for a post arriving from a Tumblr text-file export.
|
|
// On collision, only content columns + PostType + DateModified are updated;
|
|
// engagement columns (ByLikes, RootBlogName, RootURL, HasNotesGathered, NotFound,
|
|
// NotesGatheredDateTime) are preserved.
|
|
public static void UpsertPostFromTextFile(
|
|
string blogName,
|
|
string postID,
|
|
string? reblogURL,
|
|
string? postDate,
|
|
string? postURL,
|
|
string? slug,
|
|
string? reblogKey,
|
|
string? reblogName,
|
|
string? summary,
|
|
string? quote,
|
|
string? body,
|
|
string? tags,
|
|
string? link,
|
|
string? photoURL,
|
|
string? photoCaption,
|
|
string? downloadedFiles,
|
|
string? audioCaption,
|
|
string? question,
|
|
string? answer,
|
|
string? title,
|
|
string? postType,
|
|
bool hasImage,
|
|
string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
try { AddBlog(blogName, false, DBPath); } catch { }
|
|
|
|
SQLiteConnection connection;
|
|
bool ownsConnection;
|
|
lock (_importSessionLock)
|
|
{
|
|
if (_importConnection != null) { connection = _importConnection; ownsConnection = false; }
|
|
else { connection = new SQLiteConnection("Data Source=" + DBPath); ownsConnection = true; }
|
|
}
|
|
|
|
string now = DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss");
|
|
|
|
try
|
|
{
|
|
if (ownsConnection) connection.Open();
|
|
|
|
// As in AddPost, IsActive is never written -- neither here nor in the
|
|
// UPDATE below, which is why an ingest cannot un-remove a post.
|
|
string insertSql = @"INSERT INTO Posts (
|
|
BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
|
|
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
|
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
|
Question, Answer, Title, PostType,
|
|
DateModified, DateCreated, RootBlogName, RootURL,
|
|
HasImage, ByLikes
|
|
) VALUES (
|
|
@BlogName, @PostID, @reblogURL, @PostDate, @PostURL, @Slug,
|
|
@ReblogKey, @ReblogName, @Summary, @Quote, @Body, @Tags, @Link,
|
|
@PhotoURL, @PhotoCaption, @DownloadedFiles, @AudioCaption,
|
|
@Question, @Answer, @Title, @PostType,
|
|
@DateModified, @DateCreated, '.', '.', @HasImage, 0)";
|
|
|
|
int rowsInserted = 0;
|
|
try
|
|
{
|
|
using (var cmd = new SQLiteCommand(insertSql, connection))
|
|
{
|
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
|
cmd.Parameters.AddWithValue("@PostID", postID);
|
|
cmd.Parameters.AddWithValue("@reblogURL", (object?)reblogURL ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PostDate", (object?)postDate ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PostURL", (object?)postURL ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Slug", (object?)slug ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@ReblogKey", (object?)reblogKey ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@ReblogName", (object?)reblogName ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Summary", (object?)summary ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Quote", (object?)quote ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Body", (object?)body ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Tags", (object?)tags ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Link", (object?)link ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PhotoURL", (object?)photoURL ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PhotoCaption", (object?)photoCaption ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@DownloadedFiles", (object?)downloadedFiles ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@AudioCaption", (object?)audioCaption ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Question", (object?)question ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Answer", (object?)answer ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Title", (object?)title ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PostType", (object?)postType ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@DateModified", now);
|
|
cmd.Parameters.AddWithValue("@DateCreated", now);
|
|
cmd.Parameters.AddWithValue("@HasImage", hasImage ? 1 : 0);
|
|
rowsInserted = cmd.ExecuteNonQuery();
|
|
}
|
|
}
|
|
catch (Exception insertEx)
|
|
{
|
|
string msg = insertEx.Message ?? string.Empty;
|
|
bool isUniqueViolation = msg.Contains("UNIQUE constraint failed");
|
|
if (!isUniqueViolation)
|
|
{
|
|
Console.WriteLine($"[UpsertPostFromTextFile] INSERT failed for {blogName}/{postID}: {msg}");
|
|
}
|
|
}
|
|
|
|
if (rowsInserted == 0)
|
|
{
|
|
// NULL is this function's sentinel for "this file's record had no line for
|
|
// that field" (IngestMode's G(key) misses return null; LegacyPostsDbImporter
|
|
// passes null straight from a NULL source column) -- it does not mean "clear
|
|
// this field". --ingest's entire reason to exist is reconciling multiple
|
|
// export files for the same (BlogName, PostID) -- IngestMode normalizes a
|
|
// "_2"-suffixed duplicate folder onto the same blog name specifically so a
|
|
// second, differently-formatted file for a post it already has gets merged in.
|
|
// Files are walked in filesystem enumeration order, not sorted, so which
|
|
// file's UpsertPostFromTextFile call runs last for a given PostID is
|
|
// effectively arbitrary. An unconditional SET here would let whichever file
|
|
// processed last silently null out every column its own record didn't carry,
|
|
// erasing real content the other file had -- the opposite of "clean up". Each
|
|
// column is CASE-guarded to keep the existing value when this call's parameter
|
|
// is NULL, and the change-detection ignores a NULL-vs-real mismatch the same
|
|
// way, so a partial record converges into the row instead of overwriting it.
|
|
string updateSql = @"UPDATE Posts SET
|
|
reblogURL = CASE WHEN @reblogURL IS NULL THEN reblogURL ELSE @reblogURL END,
|
|
PostDate = CASE WHEN @PostDate IS NULL THEN PostDate ELSE @PostDate END,
|
|
PostURL = CASE WHEN @PostURL IS NULL THEN PostURL ELSE @PostURL END,
|
|
Slug = CASE WHEN @Slug IS NULL THEN Slug ELSE @Slug END,
|
|
ReblogKey = CASE WHEN @ReblogKey IS NULL THEN ReblogKey ELSE @ReblogKey END,
|
|
ReblogName = CASE WHEN @ReblogName IS NULL THEN ReblogName ELSE @ReblogName END,
|
|
Summary = CASE WHEN @Summary IS NULL THEN Summary ELSE @Summary END,
|
|
Quote = CASE WHEN @Quote IS NULL THEN Quote ELSE @Quote END,
|
|
Body = CASE WHEN @Body IS NULL THEN Body ELSE @Body END,
|
|
Tags = CASE WHEN @Tags IS NULL THEN Tags ELSE @Tags END,
|
|
Link = CASE WHEN @Link IS NULL THEN Link ELSE @Link END,
|
|
PhotoURL = CASE WHEN @PhotoURL IS NULL THEN PhotoURL ELSE @PhotoURL END,
|
|
PhotoCaption = CASE WHEN @PhotoCaption IS NULL THEN PhotoCaption ELSE @PhotoCaption END,
|
|
DownloadedFiles = CASE WHEN @DownloadedFiles IS NULL THEN DownloadedFiles ELSE @DownloadedFiles END,
|
|
AudioCaption = CASE WHEN @AudioCaption IS NULL THEN AudioCaption ELSE @AudioCaption END,
|
|
Question = CASE WHEN @Question IS NULL THEN Question ELSE @Question END,
|
|
Answer = CASE WHEN @Answer IS NULL THEN Answer ELSE @Answer END,
|
|
Title = CASE WHEN @Title IS NULL THEN Title ELSE @Title END,
|
|
PostType = CASE WHEN @PostType IS NULL THEN PostType ELSE @PostType END,
|
|
HasImage = @HasImage,
|
|
DateModified = @DateModified
|
|
WHERE BlogName = @BlogName AND PostID = @PostID AND (
|
|
(@reblogURL IS NOT NULL AND IFNULL(reblogURL, '') <> @reblogURL) OR
|
|
(@PostDate IS NOT NULL AND IFNULL(PostDate, '') <> @PostDate) OR
|
|
(@PostURL IS NOT NULL AND IFNULL(PostURL, '') <> @PostURL) OR
|
|
(@Slug IS NOT NULL AND IFNULL(Slug, '') <> @Slug) OR
|
|
(@ReblogKey IS NOT NULL AND IFNULL(ReblogKey, '') <> @ReblogKey) OR
|
|
(@ReblogName IS NOT NULL AND IFNULL(ReblogName, '') <> @ReblogName) OR
|
|
(@Summary IS NOT NULL AND IFNULL(Summary, '') <> @Summary) OR
|
|
(@Quote IS NOT NULL AND IFNULL(Quote, '') <> @Quote) OR
|
|
(@Body IS NOT NULL AND IFNULL(Body, '') <> @Body) OR
|
|
(@Tags IS NOT NULL AND IFNULL(Tags, '') <> @Tags) OR
|
|
(@Link IS NOT NULL AND IFNULL(Link, '') <> @Link) OR
|
|
(@PhotoURL IS NOT NULL AND IFNULL(PhotoURL, '') <> @PhotoURL) OR
|
|
(@PhotoCaption IS NOT NULL AND IFNULL(PhotoCaption, '') <> @PhotoCaption) OR
|
|
(@DownloadedFiles IS NOT NULL AND IFNULL(DownloadedFiles, '') <> @DownloadedFiles) OR
|
|
(@AudioCaption IS NOT NULL AND IFNULL(AudioCaption, '') <> @AudioCaption) OR
|
|
(@Question IS NOT NULL AND IFNULL(Question, '') <> @Question) OR
|
|
(@Answer IS NOT NULL AND IFNULL(Answer, '') <> @Answer) OR
|
|
(@Title IS NOT NULL AND IFNULL(Title, '') <> @Title) OR
|
|
(@PostType IS NOT NULL AND IFNULL(PostType, '') <> @PostType) OR
|
|
IFNULL(HasImage, 0) <> @HasImage
|
|
)";
|
|
|
|
using (var cmd = new SQLiteCommand(updateSql, connection))
|
|
{
|
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
|
cmd.Parameters.AddWithValue("@PostID", postID);
|
|
cmd.Parameters.AddWithValue("@reblogURL", (object?)reblogURL ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PostDate", (object?)postDate ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PostURL", (object?)postURL ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Slug", (object?)slug ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@ReblogKey", (object?)reblogKey ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@ReblogName", (object?)reblogName ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Summary", (object?)summary ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Quote", (object?)quote ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Body", (object?)body ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Tags", (object?)tags ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Link", (object?)link ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PhotoURL", (object?)photoURL ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PhotoCaption", (object?)photoCaption ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@DownloadedFiles", (object?)downloadedFiles ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@AudioCaption", (object?)audioCaption ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Question", (object?)question ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Answer", (object?)answer ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@Title", (object?)title ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@PostType", (object?)postType ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@HasImage", hasImage ? 1 : 0);
|
|
cmd.Parameters.AddWithValue("@DateModified", now);
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
}
|
|
}
|
|
finally
|
|
{
|
|
if (ownsConnection) connection.Close();
|
|
}
|
|
}
|
|
|
|
public static List<TTPostRecord> GetAllPostsForBlog(string blogName, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
var results = new List<TTPostRecord>();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
|
|
string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
|
|
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
|
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
|
Question, Answer, Title, PostType,
|
|
HasImage, DateCreated, DateModified
|
|
FROM Posts WHERE BlogName = @BlogName" + AndIsActive("Posts", "", DBPath);
|
|
using var cmd = new SQLiteCommand(sql, connection);
|
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
|
using var reader = cmd.ExecuteReader();
|
|
while (reader.Read())
|
|
{
|
|
results.Add(new TTPostRecord
|
|
{
|
|
BlogName = SafeStr(reader, 0),
|
|
PostId = SafeStr(reader, 1),
|
|
ReblogUrl = SafeNullableStr(reader, 2),
|
|
Date = SafeNullableStr(reader, 3),
|
|
PostUrl = SafeNullableStr(reader, 4),
|
|
Slug = SafeNullableStr(reader, 5),
|
|
ReblogKey = SafeNullableStr(reader, 6),
|
|
ReblogName = SafeNullableStr(reader, 7),
|
|
Summary = SafeNullableStr(reader, 8),
|
|
Quote = SafeNullableStr(reader, 9),
|
|
Body = SafeNullableStr(reader, 10),
|
|
Tags = SafeNullableStr(reader, 11),
|
|
Link = SafeNullableStr(reader, 12),
|
|
PhotoUrl = SafeNullableStr(reader, 13),
|
|
PhotoCaption = SafeNullableStr(reader, 14),
|
|
DownloadedFiles = SafeNullableStr(reader, 15),
|
|
AudioCaption = SafeNullableStr(reader, 16),
|
|
Question = SafeNullableStr(reader, 17),
|
|
Answer = SafeNullableStr(reader, 18),
|
|
Title = SafeNullableStr(reader, 19),
|
|
PostType = SafeNullableStr(reader, 20),
|
|
HasImage = SafeNullableStr(reader, 21),
|
|
CreatedDate = SafeNullableStr(reader, 22),
|
|
ModifiedDate = SafeNullableStr(reader, 23)
|
|
});
|
|
}
|
|
return results;
|
|
}
|
|
|
|
public static TTPostRecord? GetPost(string blogName, string postId, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
|
|
string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
|
|
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
|
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
|
Question, Answer, Title, PostType,
|
|
HasImage, DateCreated, DateModified
|
|
FROM Posts WHERE BlogName = @BlogName AND PostID = @PostID" + AndIsActive("Posts", "", DBPath);
|
|
using var cmd = new SQLiteCommand(sql, connection);
|
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
|
cmd.Parameters.AddWithValue("@PostID", postId);
|
|
using var reader = cmd.ExecuteReader();
|
|
if (!reader.Read()) return null;
|
|
|
|
return new TTPostRecord
|
|
{
|
|
BlogName = SafeStr(reader, 0),
|
|
PostId = SafeStr(reader, 1),
|
|
ReblogUrl = SafeNullableStr(reader, 2),
|
|
Date = SafeNullableStr(reader, 3),
|
|
PostUrl = SafeNullableStr(reader, 4),
|
|
Slug = SafeNullableStr(reader, 5),
|
|
ReblogKey = SafeNullableStr(reader, 6),
|
|
ReblogName = SafeNullableStr(reader, 7),
|
|
Summary = SafeNullableStr(reader, 8),
|
|
Quote = SafeNullableStr(reader, 9),
|
|
Body = SafeNullableStr(reader, 10),
|
|
Tags = SafeNullableStr(reader, 11),
|
|
Link = SafeNullableStr(reader, 12),
|
|
PhotoUrl = SafeNullableStr(reader, 13),
|
|
PhotoCaption = SafeNullableStr(reader, 14),
|
|
DownloadedFiles = SafeNullableStr(reader, 15),
|
|
AudioCaption = SafeNullableStr(reader, 16),
|
|
Question = SafeNullableStr(reader, 17),
|
|
Answer = SafeNullableStr(reader, 18),
|
|
Title = SafeNullableStr(reader, 19),
|
|
PostType = SafeNullableStr(reader, 20),
|
|
HasImage = SafeNullableStr(reader, 21),
|
|
CreatedDate = SafeNullableStr(reader, 22),
|
|
ModifiedDate = SafeNullableStr(reader, 23)
|
|
};
|
|
}
|
|
|
|
// Find a post by PostID alone (matches ThreeTxtFileHelper's correction-mode
|
|
// lookup, which does not scope by blog name). Returns the first match if any.
|
|
public static TTPostRecord? GetPostByIdAnyBlog(string postId, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
|
|
string sql = @"SELECT BlogName, PostID, reblogURL, PostDate, PostURL, Slug,
|
|
ReblogKey, ReblogName, Summary, Quote, Body, Tags, Link,
|
|
PhotoURL, PhotoCaption, DownloadedFiles, AudioCaption,
|
|
Question, Answer, Title, PostType,
|
|
HasImage, DateCreated, DateModified
|
|
FROM Posts WHERE PostID = @PostID" + AndIsActive("Posts", "", DBPath) + @" LIMIT 1";
|
|
using var cmd = new SQLiteCommand(sql, connection);
|
|
cmd.Parameters.AddWithValue("@PostID", postId);
|
|
using var reader = cmd.ExecuteReader();
|
|
if (!reader.Read()) return null;
|
|
|
|
return new TTPostRecord
|
|
{
|
|
BlogName = SafeStr(reader, 0),
|
|
PostId = SafeStr(reader, 1),
|
|
ReblogUrl = SafeNullableStr(reader, 2),
|
|
Date = SafeNullableStr(reader, 3),
|
|
PostUrl = SafeNullableStr(reader, 4),
|
|
Slug = SafeNullableStr(reader, 5),
|
|
ReblogKey = SafeNullableStr(reader, 6),
|
|
ReblogName = SafeNullableStr(reader, 7),
|
|
Summary = SafeNullableStr(reader, 8),
|
|
Quote = SafeNullableStr(reader, 9),
|
|
Body = SafeNullableStr(reader, 10),
|
|
Tags = SafeNullableStr(reader, 11),
|
|
Link = SafeNullableStr(reader, 12),
|
|
PhotoUrl = SafeNullableStr(reader, 13),
|
|
PhotoCaption = SafeNullableStr(reader, 14),
|
|
DownloadedFiles = SafeNullableStr(reader, 15),
|
|
AudioCaption = SafeNullableStr(reader, 16),
|
|
Question = SafeNullableStr(reader, 17),
|
|
Answer = SafeNullableStr(reader, 18),
|
|
Title = SafeNullableStr(reader, 19),
|
|
PostType = SafeNullableStr(reader, 20),
|
|
HasImage = SafeNullableStr(reader, 21),
|
|
CreatedDate = SafeNullableStr(reader, 22),
|
|
ModifiedDate = SafeNullableStr(reader, 23)
|
|
};
|
|
}
|
|
|
|
// Returns true only when a row's TTFolderPath actually changed. A false means either
|
|
// the row already held this value or no row matched the name -- callers must not
|
|
// report a write they did not get, which is how a --updatepaths run could once print
|
|
// "Updated <blog>" for every metadata file while leaving the column entirely NULL.
|
|
public static bool SetBlogTTFolderPath(string blogName, string? path, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
try { AddBlog(blogName, false, DBPath); } catch { }
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
using var cmd = new SQLiteCommand(
|
|
"UPDATE Blogs SET TTFolderPath = @path, DateModified = @modified WHERE BlogName = @name AND IFNULL(TTFolderPath, '') <> IFNULL(@path, '')",
|
|
connection);
|
|
cmd.Parameters.AddWithValue("@path", (object?)path ?? DBNull.Value);
|
|
cmd.Parameters.AddWithValue("@modified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"));
|
|
cmd.Parameters.AddWithValue("@name", blogName);
|
|
return cmd.ExecuteNonQuery() > 0;
|
|
}
|
|
|
|
// Whether a Blogs row exists under this exact name. BlogName is a BINARY-collated
|
|
// primary key, so a metadata filename that differs only in case is a different blog
|
|
// as far as the UPDATE above is concerned -- worth telling the user about.
|
|
public static bool BlogExists(string blogName, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
using var cmd = new SQLiteCommand("SELECT 1 FROM Blogs WHERE BlogName = @name", connection);
|
|
cmd.Parameters.AddWithValue("@name", blogName);
|
|
return cmd.ExecuteScalar() != null;
|
|
}
|
|
|
|
// Partial UPDATE used by the correct-apply path. fieldsToUpdate maps
|
|
// ThreeTxtFileHelper prefix names ("Reblog URL", "Body", etc.) to non-empty
|
|
// values pulled from a BAK file. Only those columns + DateModified are written;
|
|
// other content columns and all engagement columns are left intact.
|
|
// Returns true if a row was actually changed. A row whose columns already hold
|
|
// the incoming values is left alone, DateModified included.
|
|
public static bool UpdatePostContentFields(string blogName, string postId, IDictionary<string, string> fieldsToUpdate, string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
var setClauses = new List<string>();
|
|
var changedClauses = new List<string>();
|
|
var parameters = new List<(string Name, object Value)>();
|
|
|
|
foreach (var kvp in fieldsToUpdate)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(kvp.Value)) continue;
|
|
string? column = MapPrefixToColumn(kvp.Key);
|
|
if (column == null) continue;
|
|
string paramName = "@p" + parameters.Count;
|
|
setClauses.Add($"{column} = {paramName}");
|
|
changedClauses.Add($"IFNULL({column}, '') <> {paramName}");
|
|
parameters.Add((paramName, kvp.Value));
|
|
}
|
|
|
|
if (setClauses.Count == 0) return false;
|
|
|
|
setClauses.Add("DateModified = @DateModified");
|
|
parameters.Add(("@DateModified", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss")));
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
|
|
string sql = $"UPDATE Posts SET {string.Join(", ", setClauses)} WHERE BlogName = @BlogName AND PostID = @PostID AND ({string.Join(" OR ", changedClauses)})";
|
|
using var cmd = new SQLiteCommand(sql, connection);
|
|
foreach (var (name, value) in parameters)
|
|
cmd.Parameters.AddWithValue(name, value);
|
|
cmd.Parameters.AddWithValue("@BlogName", blogName);
|
|
cmd.Parameters.AddWithValue("@PostID", postId);
|
|
|
|
return cmd.ExecuteNonQuery() > 0;
|
|
}
|
|
|
|
private static string? MapPrefixToColumn(string prefix)
|
|
{
|
|
return prefix.ToLowerInvariant() switch
|
|
{
|
|
"reblog url" => "reblogURL",
|
|
"date" => "PostDate",
|
|
"has image" => "HasImage",
|
|
"post url" => "PostURL",
|
|
"slug" => "Slug",
|
|
"reblog key" => "ReblogKey",
|
|
"reblog name" => "ReblogName",
|
|
"summary" => "Summary",
|
|
"quote" => "Quote",
|
|
"body" => "Body",
|
|
"tags" => "Tags",
|
|
"link" => "Link",
|
|
"photo url" => "PhotoURL",
|
|
"photo caption" => "PhotoCaption",
|
|
"downloaded files" => "DownloadedFiles",
|
|
"audio caption" => "AudioCaption",
|
|
"question" => "Question",
|
|
"answer" => "Answer",
|
|
"title" => "Title",
|
|
"post id" => null,
|
|
_ => null
|
|
};
|
|
}
|
|
|
|
// Export targets only: active blogs that actually carry a TTFolderPath.
|
|
// Blogs is a 144k-row crawl registry and only the few hundred blogs downloaded
|
|
// locally have a folder, so returning the unset rows made --output print a skip
|
|
// line for every blog Tumblr has ever handed us.
|
|
public static List<(string BlogName, string TTFolderPath)> GetAllBlogsWithTTFolderPath(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
var results = new List<(string, string)>();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
using var cmd = new SQLiteCommand(
|
|
"SELECT BlogName, TRIM(TTFolderPath) FROM Blogs WHERE IsActive = 1 AND IFNULL(TRIM(TTFolderPath), '') <> '' ORDER BY BlogName",
|
|
connection);
|
|
using var reader = cmd.ExecuteReader();
|
|
while (reader.Read())
|
|
results.Add((reader.GetString(0), reader.GetString(1)));
|
|
return results;
|
|
}
|
|
|
|
// Companion counts for the messages --output and --updatepaths print about coverage.
|
|
public static int CountActiveBlogs(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
using var cmd = new SQLiteCommand("SELECT COUNT(*) FROM Blogs WHERE IsActive = 1", connection);
|
|
return Convert.ToInt32(cmd.ExecuteScalar());
|
|
}
|
|
|
|
public static int CountBlogsWithTTFolderPath(string? DBPath = null)
|
|
{
|
|
DBPath ??= GetDefaultDbPath();
|
|
|
|
using var connection = new SQLiteConnection("Data Source=" + DBPath);
|
|
connection.Open();
|
|
using var cmd = new SQLiteCommand(
|
|
"SELECT COUNT(*) FROM Blogs WHERE IFNULL(TRIM(TTFolderPath), '') <> ''", connection);
|
|
return Convert.ToInt32(cmd.ExecuteScalar());
|
|
}
|
|
|
|
private static string SafeStr(SQLiteDataReader reader, int ordinal)
|
|
{
|
|
return reader.IsDBNull(ordinal) ? string.Empty : reader.GetValue(ordinal)?.ToString() ?? string.Empty;
|
|
}
|
|
|
|
private static string? SafeNullableStr(SQLiteDataReader reader, int ordinal)
|
|
{
|
|
return reader.IsDBNull(ordinal) ? null : reader.GetValue(ordinal)?.ToString();
|
|
}
|
|
|
|
#endregion TTFileHelper
|
|
}
|
|
|
|
// Mirrors ThreeTxtFileHelper.PostData so the ported ingest/output/correct logic
|
|
// can read/write rows without an ORM. All content fields are nullable except keys.
|
|
public class TTPostRecord
|
|
{
|
|
public string BlogName { get; set; } = "";
|
|
public string PostId { get; set; } = "";
|
|
public string? ReblogUrl { get; set; }
|
|
public string? Date { get; set; }
|
|
public string? HasImage { get; set; }
|
|
public string? PostUrl { get; set; }
|
|
public string? Slug { get; set; }
|
|
public string? ReblogKey { get; set; }
|
|
public string? ReblogName { get; set; }
|
|
public string? Summary { get; set; }
|
|
public string? Quote { get; set; }
|
|
public string? Body { get; set; }
|
|
public string? Tags { get; set; }
|
|
public string? Link { get; set; }
|
|
public string? PhotoUrl { get; set; }
|
|
public string? PhotoCaption { get; set; }
|
|
public string? DownloadedFiles { get; set; }
|
|
public string? AudioCaption { get; set; }
|
|
public string? Question { get; set; }
|
|
public string? Answer { get; set; }
|
|
public string? Title { get; set; }
|
|
public string? PostType { get; set; }
|
|
public string? CreatedDate { get; set; }
|
|
public string? ModifiedDate { get; set; }
|
|
}
|
|
|
|
public class ApiKeyConfig
|
|
{
|
|
public string SectionName { get; set; } = string.Empty;
|
|
public int KeyNumber { get; set; }
|
|
public string ConsumerKey { get; set; } = string.Empty;
|
|
public string ConsumerSecret { get; set; } = string.Empty;
|
|
public string OAuthToken { get; set; } = string.Empty;
|
|
public string OAuthTokenSecret { get; set; } = string.Empty;
|
|
public bool PoolEnabled { get; set; }
|
|
public string ColorName { get; set; } = string.Empty;
|
|
public ConsoleColor ParsedColor { get; set; } = ConsoleColor.White;
|
|
}
|
|
|
|
internal class ApiKeyPool
|
|
{
|
|
private static List<ApiKeyConfig> _keys = new();
|
|
private static ApiKeyConfig? _overrideKey = null;
|
|
private static bool _usePool = false;
|
|
private static int _currentIndex = 0;
|
|
private static string _dbPath = string.Empty;
|
|
private static ApiKeyConfig? _activeKey = null;
|
|
private static readonly string[] DefaultAutoColors =
|
|
{
|
|
"Cyan", "Yellow", "Green", "Magenta", "Blue", "Red",
|
|
"DarkCyan", "DarkYellow", "DarkGreen", "DarkMagenta"
|
|
};
|
|
private static int _nextAutoColorIndex = 0;
|
|
private static string _configFilePath = string.Empty;
|
|
|
|
public static bool IsPoolActive => _usePool;
|
|
public static List<ApiKeyConfig> Keys => _keys;
|
|
|
|
public static void Initialize(IConfiguration config, string? dbPath, string configFilePath, string? overrideSection = null)
|
|
{
|
|
_dbPath = dbPath ?? throw new ArgumentNullException(nameof(dbPath));
|
|
_configFilePath = configFilePath;
|
|
EnsureStateTableExists();
|
|
|
|
if (!string.IsNullOrEmpty(overrideSection))
|
|
{
|
|
var section = config.GetSection(overrideSection);
|
|
if (section["ConsumerKey"] == null)
|
|
throw new InvalidOperationException($"API section '{overrideSection}' not found in appsettings.json");
|
|
|
|
var colorName = section["Color"] ?? string.Empty;
|
|
var parsedColor = ParseColor(colorName);
|
|
|
|
_overrideKey = new ApiKeyConfig
|
|
{
|
|
SectionName = overrideSection,
|
|
KeyNumber = 1,
|
|
ConsumerKey = section["ConsumerKey"]!,
|
|
ConsumerSecret = section["ConsumerSecret"]!,
|
|
OAuthToken = section["OAuthToken"]!,
|
|
OAuthTokenSecret = section["OAuthTokenSecret"]!,
|
|
PoolEnabled = true,
|
|
ColorName = colorName,
|
|
ParsedColor = parsedColor
|
|
};
|
|
_usePool = false;
|
|
Console.WriteLine($"[Pool] Single-key mode: {overrideSection} (Color: {_overrideKey.ParsedColor})");
|
|
return;
|
|
}
|
|
|
|
var root = config.AsEnumerable()
|
|
.Where(kv => kv.Key.Contains(":ConsumerKey"))
|
|
.Select(kv => kv.Key.Split(':')[0])
|
|
.Distinct()
|
|
.ToList();
|
|
|
|
var autoAssignedColors = new List<(string sectionName, string color)>();
|
|
|
|
int keyNum = 1;
|
|
foreach (var sectionName in root.OrderBy(s => s))
|
|
{
|
|
var section = config.GetSection(sectionName);
|
|
var poolEnabledStr = section["PoolEnabled"];
|
|
bool poolEnabled = !string.IsNullOrEmpty(poolEnabledStr) && bool.TryParse(poolEnabledStr, out var parsed) && parsed;
|
|
|
|
if (poolEnabled)
|
|
{
|
|
var colorName = section["Color"] ?? string.Empty;
|
|
var parsedColor = ParseColor(colorName);
|
|
|
|
if (string.IsNullOrEmpty(colorName))
|
|
{
|
|
colorName = GetNextAutoColor();
|
|
parsedColor = ParseColor(colorName);
|
|
autoAssignedColors.Add((sectionName, colorName));
|
|
}
|
|
|
|
_keys.Add(new ApiKeyConfig
|
|
{
|
|
SectionName = sectionName,
|
|
KeyNumber = keyNum,
|
|
ConsumerKey = section["ConsumerKey"]!,
|
|
ConsumerSecret = section["ConsumerSecret"]!,
|
|
OAuthToken = section["OAuthToken"]!,
|
|
OAuthTokenSecret = section["OAuthTokenSecret"]!,
|
|
PoolEnabled = true,
|
|
ColorName = colorName,
|
|
ParsedColor = parsedColor
|
|
});
|
|
keyNum++;
|
|
}
|
|
}
|
|
|
|
if (_keys.Count == 0)
|
|
{
|
|
var fallback = config.GetSection("TumblrApi");
|
|
if (fallback["ConsumerKey"] != null)
|
|
{
|
|
var colorName = fallback["Color"] ?? string.Empty;
|
|
var parsedColor = ParseColor(colorName);
|
|
|
|
if (string.IsNullOrEmpty(colorName))
|
|
{
|
|
colorName = GetNextAutoColor();
|
|
parsedColor = ParseColor(colorName);
|
|
autoAssignedColors.Add(("TumblrApi", colorName));
|
|
}
|
|
|
|
_keys.Add(new ApiKeyConfig
|
|
{
|
|
SectionName = "TumblrApi",
|
|
KeyNumber = 1,
|
|
ConsumerKey = fallback["ConsumerKey"]!,
|
|
ConsumerSecret = fallback["ConsumerSecret"]!,
|
|
OAuthToken = fallback["OAuthToken"]!,
|
|
OAuthTokenSecret = fallback["OAuthTokenSecret"]!,
|
|
PoolEnabled = true,
|
|
ColorName = colorName,
|
|
ParsedColor = parsedColor
|
|
});
|
|
Console.WriteLine("[Pool] No keys with PoolEnabled, falling back to TumblrApi");
|
|
}
|
|
}
|
|
|
|
if (autoAssignedColors.Count > 0)
|
|
{
|
|
PersistAutoAssignedColors(autoAssignedColors);
|
|
}
|
|
|
|
_usePool = true;
|
|
LoadState();
|
|
|
|
var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName} (Color: {k.ParsedColor})"));
|
|
Console.WriteLine($"[Pool] Active keys: {mapping}");
|
|
}
|
|
|
|
private static string GetNextAutoColor()
|
|
{
|
|
if (_nextAutoColorIndex < DefaultAutoColors.Length)
|
|
return DefaultAutoColors[_nextAutoColorIndex++];
|
|
return "White";
|
|
}
|
|
|
|
private static ConsoleColor ParseColor(string colorName)
|
|
{
|
|
if (string.IsNullOrEmpty(colorName))
|
|
return ConsoleColor.White;
|
|
if (Enum.TryParse<ConsoleColor>(colorName, out var color))
|
|
return color;
|
|
Console.WriteLine($"[Pool] Warning: Invalid color '{colorName}', using White");
|
|
return ConsoleColor.White;
|
|
}
|
|
|
|
private static void PersistAutoAssignedColors(List<(string sectionName, string color)> assignments)
|
|
{
|
|
try
|
|
{
|
|
var json = File.ReadAllText(_configFilePath);
|
|
using var doc = JsonDocument.Parse(json);
|
|
var root = doc.RootElement;
|
|
|
|
using var stream = new MemoryStream();
|
|
using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true });
|
|
|
|
writer.WriteStartObject();
|
|
foreach (var prop in root.EnumerateObject())
|
|
{
|
|
writer.WritePropertyName(prop.Name);
|
|
|
|
if (prop.Value.ValueKind == JsonValueKind.Object)
|
|
{
|
|
writer.WriteStartObject();
|
|
foreach (var subProp in prop.Value.EnumerateObject())
|
|
{
|
|
writer.WritePropertyName(subProp.Name);
|
|
subProp.Value.WriteTo(writer);
|
|
}
|
|
|
|
var match = assignments.FirstOrDefault(a => a.sectionName == prop.Name);
|
|
if (!string.IsNullOrEmpty(match.sectionName))
|
|
{
|
|
writer.WriteString("Color", match.color);
|
|
}
|
|
writer.WriteEndObject();
|
|
}
|
|
else
|
|
{
|
|
prop.Value.WriteTo(writer);
|
|
}
|
|
}
|
|
writer.WriteEndObject();
|
|
writer.Flush();
|
|
|
|
File.WriteAllText(_configFilePath, Encoding.UTF8.GetString(stream.ToArray()));
|
|
Console.WriteLine($"[Pool] Auto-assigned colors persisted to config: {string.Join(", ", assignments.Select(a => $"{a.sectionName}={a.color}"))}");
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Pool] Warning: Could not persist auto-assigned colors to config: {ex.Message}");
|
|
}
|
|
}
|
|
|
|
private static void EnsureStateTableExists()
|
|
{
|
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
|
conn.Open();
|
|
using var cmd1 = new System.Data.SQLite.SQLiteCommand(
|
|
"CREATE TABLE IF NOT EXISTS ApiKeyPoolState (KeyName TEXT PRIMARY KEY, RetryUntil INTEGER DEFAULT 0)", conn);
|
|
cmd1.ExecuteNonQuery();
|
|
using var cmd2 = new System.Data.SQLite.SQLiteCommand(
|
|
"CREATE TABLE IF NOT EXISTS ApiKeyPoolMeta (Id INTEGER PRIMARY KEY CHECK (Id = 1), LastIndex INTEGER DEFAULT 0)", conn);
|
|
cmd2.ExecuteNonQuery();
|
|
}
|
|
|
|
private static void LoadState()
|
|
{
|
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
|
conn.Open();
|
|
|
|
using var cmd = new System.Data.SQLite.SQLiteCommand("SELECT LastIndex FROM ApiKeyPoolMeta WHERE Id = 1", conn);
|
|
var idx = cmd.ExecuteScalar();
|
|
if (idx != null)
|
|
_currentIndex = Convert.ToInt32(idx);
|
|
|
|
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
|
foreach (var key in _keys)
|
|
{
|
|
using var readCmd = new System.Data.SQLite.SQLiteCommand(
|
|
"SELECT RetryUntil FROM ApiKeyPoolState WHERE KeyName = @name", conn);
|
|
readCmd.Parameters.AddWithValue("@name", key.SectionName);
|
|
var retryUntil = readCmd.ExecuteScalar();
|
|
if (retryUntil != null)
|
|
{
|
|
var retryTs = Convert.ToInt64(retryUntil);
|
|
if (retryTs > now)
|
|
{
|
|
var remaining = retryTs - now;
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = key.ParsedColor;
|
|
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, clears in {remaining}s");
|
|
Console.ForegroundColor = prevColor;
|
|
}
|
|
else if (retryTs > 0)
|
|
{
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = key.ParsedColor;
|
|
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared on startup");
|
|
Console.ForegroundColor = prevColor;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public static ApiKeyConfig GetCurrentKey()
|
|
{
|
|
ApiKeyConfig key;
|
|
|
|
if (!_usePool || _overrideKey != null)
|
|
key = _overrideKey!;
|
|
else if (_keys.Count == 1)
|
|
key = _keys[0];
|
|
else
|
|
{
|
|
int attempts = 0;
|
|
|
|
while (attempts < _keys.Count)
|
|
{
|
|
key = _keys[_currentIndex % _keys.Count];
|
|
_currentIndex = (_currentIndex + 1) % _keys.Count;
|
|
|
|
if (IsKeyAvailable(key))
|
|
{
|
|
SaveState();
|
|
_activeKey = key;
|
|
return key;
|
|
}
|
|
|
|
attempts++;
|
|
}
|
|
|
|
var earliest = _keys
|
|
.Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) })
|
|
.OrderBy(x => x.RetryUntil)
|
|
.First();
|
|
|
|
Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)");
|
|
|
|
key = earliest.Key;
|
|
_currentIndex = (_keys.IndexOf(key) + 1) % _keys.Count;
|
|
SaveState();
|
|
}
|
|
|
|
_activeKey = key;
|
|
return key;
|
|
}
|
|
|
|
private static bool IsKeyAvailable(ApiKeyConfig key)
|
|
{
|
|
var retryUntil = GetRetryUntil(key);
|
|
return retryUntil <= DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
|
}
|
|
|
|
private static long GetRetryUntil(ApiKeyConfig key)
|
|
{
|
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
|
conn.Open();
|
|
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
|
"SELECT RetryUntil FROM ApiKeyPoolState WHERE KeyName = @name", conn);
|
|
cmd.Parameters.AddWithValue("@name", key.SectionName);
|
|
var result = cmd.ExecuteScalar();
|
|
return result != null ? Convert.ToInt64(result) : 0;
|
|
}
|
|
|
|
public static void MarkRateLimited(ApiKeyConfig key, int retryInSeconds)
|
|
{
|
|
var retryUntil = DateTimeOffset.UtcNow.ToUnixTimeSeconds() + retryInSeconds;
|
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
|
conn.Open();
|
|
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
|
"INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, @until)", conn);
|
|
cmd.Parameters.AddWithValue("@name", key.SectionName);
|
|
cmd.Parameters.AddWithValue("@until", retryUntil);
|
|
cmd.ExecuteNonQuery();
|
|
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = key.ParsedColor;
|
|
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, retry in {retryInSeconds}s (until {DateTimeOffset.FromUnixTimeSeconds(retryUntil).LocalDateTime:HH:mm:ss})");
|
|
Console.ForegroundColor = prevColor;
|
|
}
|
|
|
|
public static void MarkAvailable(ApiKeyConfig key)
|
|
{
|
|
// Called after every successful call; skip the write and the log line when nothing was flagged.
|
|
if (GetRetryUntil(key) == 0)
|
|
return;
|
|
|
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
|
conn.Open();
|
|
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
|
"INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, 0)", conn);
|
|
cmd.Parameters.AddWithValue("@name", key.SectionName);
|
|
cmd.ExecuteNonQuery();
|
|
|
|
var prevColor = Console.ForegroundColor;
|
|
Console.ForegroundColor = key.ParsedColor;
|
|
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared");
|
|
Console.ForegroundColor = prevColor;
|
|
}
|
|
|
|
public static bool IsAllRateLimited(out int minRetrySeconds)
|
|
{
|
|
minRetrySeconds = 0;
|
|
if (!_usePool || _overrideKey != null) return false;
|
|
if (_keys.Count == 0) return false;
|
|
|
|
var now = DateTimeOffset.UtcNow.ToUnixTimeSeconds();
|
|
bool all = true;
|
|
int min = int.MaxValue;
|
|
|
|
foreach (var key in _keys)
|
|
{
|
|
var retryUntil = GetRetryUntil(key);
|
|
if (retryUntil <= now) { all = false; break; }
|
|
int remaining = (int)(retryUntil - now);
|
|
if (remaining < min) min = remaining;
|
|
}
|
|
|
|
minRetrySeconds = all ? min : 0;
|
|
return all;
|
|
}
|
|
|
|
public static void SleepUntilAnyAvailable(int refreshSeconds = 30)
|
|
{
|
|
if (refreshSeconds <= 0) refreshSeconds = 30;
|
|
if (!IsAllRateLimited(out int minRetry) || minRetry <= 0) return;
|
|
|
|
DateTime retryAt = DateTime.Now.AddSeconds(minRetry);
|
|
int remaining = minRetry;
|
|
while (remaining > 0)
|
|
{
|
|
Console.WriteLine("[Pool] All API keys rate-limited. Sleeping {0}s, until {1}", remaining, retryAt.ToString("T"));
|
|
int sleepSeconds = Math.Min(refreshSeconds, remaining);
|
|
Thread.Sleep(sleepSeconds * 1000);
|
|
remaining -= sleepSeconds;
|
|
|
|
if (remaining > 0 && IsAllRateLimited(out int refreshed) && refreshed > 0 && refreshed < remaining)
|
|
{
|
|
remaining = refreshed;
|
|
retryAt = DateTime.Now.AddSeconds(remaining);
|
|
}
|
|
}
|
|
}
|
|
|
|
private static void SaveState()
|
|
{
|
|
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
|
|
conn.Open();
|
|
using var cmd = new System.Data.SQLite.SQLiteCommand(
|
|
"INSERT OR REPLACE INTO ApiKeyPoolMeta (Id, LastIndex) VALUES (1, @idx)", conn);
|
|
cmd.Parameters.AddWithValue("@idx", _currentIndex);
|
|
cmd.ExecuteNonQuery();
|
|
}
|
|
}
|
|
|
|
internal class APIAccess
|
|
{
|
|
public static string Blog { get; set; } = string.Empty;
|
|
|
|
public static void SetApiConfigSection(string sectionName)
|
|
{
|
|
if (!ApiKeyPool.IsPoolActive)
|
|
Console.WriteLine($"Using API settings section: {sectionName}");
|
|
}
|
|
|
|
private static RestClient BuildClient(ApiKeyConfig key, string url)
|
|
{
|
|
var client = new RestClient(url);
|
|
var oAuth1 = OAuth1Authenticator.ForAccessToken(
|
|
consumerKey: key.ConsumerKey,
|
|
consumerSecret: key.ConsumerSecret,
|
|
token: key.OAuthToken,
|
|
tokenSecret: key.OAuthTokenSecret,
|
|
OAuthSignatureMethod.HmacSha1);
|
|
client.Authenticator = oAuth1;
|
|
return client;
|
|
}
|
|
|
|
private static string FormatKeyLabel(ApiKeyConfig key) => $"[Key#{key.KeyNumber}]";
|
|
|
|
private static string SummarizeBody(string body)
|
|
{
|
|
if (string.IsNullOrWhiteSpace(body))
|
|
return "(empty)";
|
|
|
|
var flat = System.Text.RegularExpressions.Regex.Replace(body, @"<[^>]+>|\s+", " ").Trim();
|
|
return flat.Length <= 80 ? flat : flat.Substring(0, 80) + "...";
|
|
}
|
|
|
|
private static int GetRetryDelaySecondsFromHeaders(IEnumerable<HeaderParameter>? headers)
|
|
{
|
|
if (headers == null)
|
|
return 60;
|
|
|
|
int retryInSeconds = 0;
|
|
bool remainingIsZero = false;
|
|
|
|
foreach (var header in headers)
|
|
{
|
|
string? headerName = header?.Name;
|
|
string? headerValue = header?.Value?.ToString();
|
|
if (string.IsNullOrWhiteSpace(headerName) || string.IsNullOrWhiteSpace(headerValue))
|
|
continue;
|
|
|
|
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int retrySecs))
|
|
{
|
|
retryInSeconds = Math.Max(retryInSeconds, retrySecs);
|
|
}
|
|
else if (DateTimeOffset.TryParse(headerValue, out var retryAt))
|
|
{
|
|
int secs = (int)Math.Max(0, (retryAt - DateTimeOffset.UtcNow).TotalSeconds);
|
|
retryInSeconds = Math.Max(retryInSeconds, secs);
|
|
}
|
|
}
|
|
|
|
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (long.TryParse(headerValue, out long epoch))
|
|
{
|
|
int secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
retryInSeconds = Math.Max(retryInSeconds, secs);
|
|
}
|
|
}
|
|
|
|
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
|
{
|
|
remainingIsZero = true;
|
|
}
|
|
|
|
if (remainingIsZero && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (int.TryParse(headerValue, out int resetValue))
|
|
{
|
|
retryInSeconds = Math.Max(retryInSeconds, resetValue);
|
|
}
|
|
else if (long.TryParse(headerValue, out long resetEpoch))
|
|
{
|
|
int secs = (int)Math.Max(0, resetEpoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
retryInSeconds = Math.Max(retryInSeconds, secs);
|
|
}
|
|
}
|
|
}
|
|
|
|
return retryInSeconds > 0 ? retryInSeconds : 60;
|
|
}
|
|
|
|
public static async Task<Root> GrabNotes(ApiKeyConfig key, string blog, long ID, string? timestamp = null)
|
|
{
|
|
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=all";
|
|
URL = URL.Replace("[0]", blog).Replace("[1]", ID.ToString());
|
|
if (!string.IsNullOrEmpty(timestamp))
|
|
{
|
|
URL += "&before_timestamp=" + timestamp;
|
|
await Task.Delay(100);
|
|
}
|
|
|
|
using (var client = BuildClient(key, URL))
|
|
{
|
|
var request = new RestRequest(URL, Method.Get);
|
|
var response = await client.ExecuteAsync(request);
|
|
|
|
var myJsonResponse = response.Content ?? string.Empty;
|
|
Console.WriteLine($"{FormatKeyLabel(key)} {timestamp}\t{DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
|
var myDeserializedClass = new Root();
|
|
|
|
// Never reached the API: there is no body to interpret, so the post's state is still unknown.
|
|
if (response.ResponseStatus != ResponseStatus.Completed)
|
|
{
|
|
myDeserializedClass.statusCode = response.ResponseStatus.ToString();
|
|
myDeserializedClass.transientFailure = true;
|
|
Console.WriteLine($"[Transient] {FormatKeyLabel(key)} transport {response.ResponseStatus}: {response.ErrorException?.Message}");
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
try
|
|
{
|
|
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
|
if (deserializedResult == null)
|
|
{
|
|
// Empty body behind an HTTP status: an edge/proxy response, not the API.
|
|
myDeserializedClass.statusCode = response.StatusCode.ToString();
|
|
myDeserializedClass.transientFailure = true;
|
|
Console.WriteLine($"[Transient] {FormatKeyLabel(key)} HTTP {(int)response.StatusCode} {response.StatusDescription} — empty body");
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
myDeserializedClass = deserializedResult;
|
|
myDeserializedClass.rawJson = myJsonResponse;
|
|
|
|
if (myDeserializedClass.meta != null && myDeserializedClass.meta.status == 404)
|
|
{
|
|
myDeserializedClass.statusCode = "NotFound";
|
|
}
|
|
|
|
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
|
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
|
|
|
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
|
{
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, GetRetryDelaySecondsFromHeaders(response.Headers));
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
// A body that will not parse came from infrastructure (CDN/proxy/WAF), not the Tumblr
|
|
// API, so it says nothing about this post. Retryable, not a failure of the post itself.
|
|
myDeserializedClass.statusCode = response.StatusCode.ToString();
|
|
|
|
if (response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
|
{
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
}
|
|
else
|
|
{
|
|
myDeserializedClass.transientFailure = true;
|
|
Console.WriteLine($"[Transient] {FormatKeyLabel(key)} HTTP {(int)response.StatusCode} {response.StatusDescription} — unparseable body: {SummarizeBody(myJsonResponse)}");
|
|
|
|
// A 2xx that will not parse is a genuine surprise; keep the detail for that case only.
|
|
if (response.IsSuccessful)
|
|
Console.WriteLine(ex.ToString());
|
|
}
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
|
|
public static async Task<Root> GrabPostWithReplies(ApiKeyConfig key, string blog, long postID, long timestamp)
|
|
{
|
|
var URL = "https://api.tumblr.com/v2/blog/[0].tumblr.com/notes?id=[1]&mode=conversation";
|
|
URL = URL.Replace("[0]", blog).Replace("[1]", postID.ToString());
|
|
if (timestamp > 0)
|
|
{
|
|
URL += $"&before_timestamp={timestamp}";
|
|
await Task.Delay(100);
|
|
}
|
|
|
|
using (var client = BuildClient(key, URL))
|
|
{
|
|
var request = new RestRequest(URL, Method.Get);
|
|
var response = await client.ExecuteAsync(request);
|
|
|
|
var myJsonResponse = response.Content ?? string.Empty;
|
|
Console.WriteLine($"[Reply API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
|
var myDeserializedClass = new Root();
|
|
|
|
// Check if response is successful and contains JSON
|
|
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
|
|
{
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
Console.WriteLine($"[Reply API] {statusStr}\t{response?.StatusDescription}");
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
// Check if response is HTML (error page) instead of JSON
|
|
if (myJsonResponse.TrimStart().StartsWith("<"))
|
|
{
|
|
Console.WriteLine($"[Reply API] Received HTML response (likely error page): {response?.StatusCode}");
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
try
|
|
{
|
|
var deserializedResult = JsonConvert.DeserializeObject<Root>(myJsonResponse);
|
|
if (deserializedResult != null)
|
|
{
|
|
myDeserializedClass = deserializedResult;
|
|
myDeserializedClass.rawJson = myJsonResponse;
|
|
|
|
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
|
bool metaMsgIndicatesRateLimit = myDeserializedClass.meta != null && !string.IsNullOrEmpty(myDeserializedClass.meta.msg) && myDeserializedClass.meta.msg.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0;
|
|
|
|
if (metaIndicatesRateLimit || metaMsgIndicatesRateLimit || (response != null && (response.StatusDescription?.IndexOf("Too Many", StringComparison.OrdinalIgnoreCase) >= 0 || response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)))
|
|
{
|
|
if (response?.Headers != null)
|
|
{
|
|
bool checkResetLocal = false;
|
|
foreach (var header in response.Headers)
|
|
{
|
|
string? headerName = header?.Name;
|
|
string? headerValue = header?.Value?.ToString();
|
|
if (string.IsNullOrEmpty(headerName) || string.IsNullOrEmpty(headerValue))
|
|
continue;
|
|
|
|
if (string.Equals(headerName, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int retrySecs))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, retrySecs);
|
|
else if (DateTimeOffset.TryParse(headerValue, out var dto))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds));
|
|
}
|
|
|
|
if (headerName.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0 && long.TryParse(headerValue, out long epoch))
|
|
{
|
|
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, secs);
|
|
}
|
|
|
|
if (headerName.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && headerValue == "0")
|
|
checkResetLocal = true;
|
|
|
|
if (checkResetLocal && headerName.IndexOf("Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (int.TryParse(headerValue, out int resetValue))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, resetValue);
|
|
else if (long.TryParse(headerValue, out long epochVal))
|
|
myDeserializedClass.retryInSeconds = Math.Max(myDeserializedClass.retryInSeconds, (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds()));
|
|
}
|
|
}
|
|
}
|
|
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Reply API] Failed to parse JSON response: {ex.Message}");
|
|
Console.WriteLine($"[Reply API] Response content: {myJsonResponse.Substring(0, Math.Min(200, myJsonResponse.Length))}");
|
|
|
|
if (!response.IsSuccessful)
|
|
{
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
Console.WriteLine($"{statusStr}\t{response?.StatusDescription}");
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
|
|
bool checkReset = false;
|
|
|
|
if ((myDeserializedClass.statusCode != "NotFound" || myDeserializedClass.retryInSeconds > 0) && response.Headers != null)
|
|
{
|
|
bool foundRateLimitHeader = false;
|
|
foreach (var header in response.Headers)
|
|
{
|
|
if (header.Name != null && header.Value != null)
|
|
{
|
|
Console.WriteLine($"{header.Name} - {header.Value}");
|
|
var headerValue = header.Value?.ToString();
|
|
if (!string.IsNullOrEmpty(headerValue))
|
|
{
|
|
if (string.Equals(header.Name, "Retry-After", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int retrySecs))
|
|
{
|
|
if (myDeserializedClass.retryInSeconds < retrySecs)
|
|
myDeserializedClass.retryInSeconds = retrySecs;
|
|
}
|
|
else if (DateTimeOffset.TryParse(headerValue, out DateTimeOffset dto))
|
|
{
|
|
var secs = (int)Math.Max(0, (dto - DateTimeOffset.UtcNow).TotalSeconds);
|
|
if (myDeserializedClass.retryInSeconds < secs)
|
|
myDeserializedClass.retryInSeconds = secs;
|
|
}
|
|
}
|
|
|
|
if (checkReset && header.Name.Contains("Reset", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
if (int.TryParse(headerValue, out int resetValue))
|
|
{
|
|
if (myDeserializedClass.retryInSeconds < resetValue)
|
|
myDeserializedClass.retryInSeconds = resetValue;
|
|
}
|
|
else if (long.TryParse(headerValue, out long epochVal))
|
|
{
|
|
var secs = (int)Math.Max(0, epochVal - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
if (myDeserializedClass.retryInSeconds < secs)
|
|
myDeserializedClass.retryInSeconds = secs;
|
|
}
|
|
}
|
|
|
|
if (header.Name.IndexOf("X-RateLimit-Reset", StringComparison.OrdinalIgnoreCase) >= 0)
|
|
{
|
|
if (long.TryParse(headerValue, out long epoch))
|
|
{
|
|
var secs = (int)Math.Max(0, epoch - DateTimeOffset.UtcNow.ToUnixTimeSeconds());
|
|
if (myDeserializedClass.retryInSeconds < secs)
|
|
myDeserializedClass.retryInSeconds = secs;
|
|
}
|
|
foundRateLimitHeader = true;
|
|
}
|
|
}
|
|
if (header.Name.Contains("Remaining", StringComparison.OrdinalIgnoreCase) && header.Value.ToString() == "0")
|
|
checkReset = true;
|
|
else
|
|
checkReset = false;
|
|
}
|
|
|
|
if (foundRateLimitHeader || (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests))
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
|
|
public static async Task<LikesRoot> GrabLikes(ApiKeyConfig key, string blog, long beforeTimestamp = 0)
|
|
{
|
|
var URL = $"https://api.tumblr.com/v2/blog/{Uri.EscapeDataString(blog)}.tumblr.com/likes?npf=false&reblog_info=true";
|
|
if (beforeTimestamp > 0)
|
|
{
|
|
URL += $"&before={beforeTimestamp}";
|
|
}
|
|
|
|
using (var client = BuildClient(key, URL))
|
|
{
|
|
var request = new RestRequest(URL, Method.Get);
|
|
var response = await client.ExecuteAsync(request);
|
|
|
|
var myJsonResponse = response.Content ?? string.Empty;
|
|
Console.WriteLine($"[Likes API] {FormatKeyLabel(key)} {DateTime.Now}\t{DataAccess.UpdateAPICount()}");
|
|
var myDeserializedClass = new LikesRoot();
|
|
|
|
// Check if response is successful and contains JSON
|
|
if (!response.IsSuccessful || string.IsNullOrEmpty(myJsonResponse))
|
|
{
|
|
string? statusStr = null;
|
|
try { statusStr = response != null ? response.StatusCode.ToString() : null; } catch { statusStr = null; }
|
|
Console.WriteLine($"[Likes API] {statusStr}\t{response?.StatusDescription}");
|
|
if (!string.IsNullOrEmpty(statusStr))
|
|
myDeserializedClass.statusCode = statusStr;
|
|
|
|
if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
|
|
}
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
if (myJsonResponse.TrimStart().StartsWith("<"))
|
|
{
|
|
Console.WriteLine($"[Likes API] Received HTML response (likely error page): {response?.StatusCode}");
|
|
myDeserializedClass.statusCode = response?.StatusCode.ToString();
|
|
if (response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests)
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response.Headers);
|
|
}
|
|
return myDeserializedClass;
|
|
}
|
|
|
|
try
|
|
{
|
|
var deserializedResult = JsonConvert.DeserializeObject<LikesRoot>(myJsonResponse);
|
|
if (deserializedResult != null)
|
|
{
|
|
myDeserializedClass = deserializedResult;
|
|
myDeserializedClass.rawJson = myJsonResponse;
|
|
|
|
bool metaIndicatesRateLimit = myDeserializedClass.meta != null && myDeserializedClass.meta.status == 429;
|
|
bool statusIndicatesRateLimit = response != null && response.StatusCode == System.Net.HttpStatusCode.TooManyRequests;
|
|
|
|
if (metaIndicatesRateLimit || statusIndicatesRateLimit)
|
|
{
|
|
myDeserializedClass.statusCode = "TooManyRequests";
|
|
myDeserializedClass.retryInSeconds = GetRetryDelaySecondsFromHeaders(response?.Headers);
|
|
}
|
|
}
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
Console.WriteLine($"[Likes API] Failed to parse JSON response: {ex.Message}");
|
|
}
|
|
|
|
return myDeserializedClass;
|
|
}
|
|
}
|
|
}
|
|
}
|