jim/URLNotesGrabberCore
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add API key color assignment, auto-persist, AGENTS.md

Browse Source
This commit is contained in:
jim
2026-05-04 10:33:46 -05:00
parent 34e5745e99
commit 5e62b58261
3 changed files with 213 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
URLNotesGrabberCORE/DataAccess.cs
+167 -31
View File
@@ -14,6 +14,7 @@ using System.Collections;
using Microsoft.Extensions.Configuration;
using System.IO;
using System.Data;
using System.Text.Json;
namespace URLNotesGrabberCORE
{
@@ -1556,6 +1557,8 @@ namespace URLNotesGrabberCORE
public string OAuthToken { get; set; } = string.Empty;
public string OAuthTokenSecret { get; set; } = string.Empty;
public bool PoolEnabled { get; set; }
public string ColorName { get; set; } = string.Empty;
public ConsoleColor ParsedColor { get; set; } = ConsoleColor.White;
}
internal class ApiKeyPool
@@ -1565,13 +1568,22 @@ namespace URLNotesGrabberCORE
private static bool _usePool = false;
private static int _currentIndex = 0;
private static string _dbPath = string.Empty;
private static ApiKeyConfig? _activeKey = null;
private static readonly string[] DefaultAutoColors =
{
"Cyan", "Yellow", "Green", "Magenta", "Blue", "Red",
"DarkCyan", "DarkYellow", "DarkGreen", "DarkMagenta"
};
private static int _nextAutoColorIndex = 0;
private static string _configFilePath = string.Empty;
public static bool IsPoolActive => _usePool;
public static List<ApiKeyConfig> Keys => _keys;
public static void Initialize(IConfiguration config, string? dbPath, string? overrideSection = null)
public static void Initialize(IConfiguration config, string? dbPath, string configFilePath, string? overrideSection = null)
{
_dbPath = dbPath ?? "..\\..\\..\\tl.db";
_configFilePath = configFilePath;
EnsureStateTableExists();
if (!string.IsNullOrEmpty(overrideSection))
@@ -1580,6 +1592,9 @@ namespace URLNotesGrabberCORE
if (section["ConsumerKey"] == null)
throw new InvalidOperationException($"API section '{overrideSection}' not found in appsettings.json");
var colorName = section["Color"] ?? string.Empty;
var parsedColor = ParseColor(colorName);
_overrideKey = new ApiKeyConfig
{
SectionName = overrideSection,
@@ -1588,10 +1603,12 @@ namespace URLNotesGrabberCORE
ConsumerSecret = section["ConsumerSecret"]!,
OAuthToken = section["OAuthToken"]!,
OAuthTokenSecret = section["OAuthTokenSecret"]!,
PoolEnabled = true
PoolEnabled = true,
ColorName = colorName,
ParsedColor = parsedColor
};
_usePool = false;
Console.WriteLine($"[Pool] Single-key mode: {overrideSection}");
Console.WriteLine($"[Pool] Single-key mode: {overrideSection} (Color: {_overrideKey.ParsedColor})");
return;
}
@@ -1601,6 +1618,8 @@ namespace URLNotesGrabberCORE
.Distinct()
.ToList();
var autoAssignedColors = new List<(string sectionName, string color)>();
int keyNum = 1;
foreach (var sectionName in root.OrderBy(s => s))
{
@@ -1610,6 +1629,16 @@ namespace URLNotesGrabberCORE
if (poolEnabled)
{
var colorName = section["Color"] ?? string.Empty;
var parsedColor = ParseColor(colorName);
if (string.IsNullOrEmpty(colorName))
{
colorName = GetNextAutoColor();
parsedColor = ParseColor(colorName);
autoAssignedColors.Add((sectionName, colorName));
}
_keys.Add(new ApiKeyConfig
{
SectionName = sectionName,
@@ -1618,7 +1647,9 @@ namespace URLNotesGrabberCORE
ConsumerSecret = section["ConsumerSecret"]!,
OAuthToken = section["OAuthToken"]!,
OAuthTokenSecret = section["OAuthTokenSecret"]!,
PoolEnabled = true
PoolEnabled = true,
ColorName = colorName,
ParsedColor = parsedColor
});
keyNum++;
}
@@ -1629,6 +1660,16 @@ namespace URLNotesGrabberCORE
var fallback = config.GetSection("TumblrApi");
if (fallback["ConsumerKey"] != null)
{
var colorName = fallback["Color"] ?? string.Empty;
var parsedColor = ParseColor(colorName);
if (string.IsNullOrEmpty(colorName))
{
colorName = GetNextAutoColor();
parsedColor = ParseColor(colorName);
autoAssignedColors.Add(("TumblrApi", colorName));
}
_keys.Add(new ApiKeyConfig
{
SectionName = "TumblrApi",
@@ -1637,19 +1678,92 @@ namespace URLNotesGrabberCORE
ConsumerSecret = fallback["ConsumerSecret"]!,
OAuthToken = fallback["OAuthToken"]!,
OAuthTokenSecret = fallback["OAuthTokenSecret"]!,
PoolEnabled = true
PoolEnabled = true,
ColorName = colorName,
ParsedColor = parsedColor
});
Console.WriteLine("[Pool] No keys with PoolEnabled, falling back to TumblrApi");
}
}
if (autoAssignedColors.Count > 0)
{
PersistAutoAssignedColors(autoAssignedColors);
}
_usePool = true;
LoadState();
var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName}"));
var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName} (Color: {k.ParsedColor})"));
Console.WriteLine($"[Pool] Active keys: {mapping}");
}
private static string GetNextAutoColor()
{
if (_nextAutoColorIndex < DefaultAutoColors.Length)
return DefaultAutoColors[_nextAutoColorIndex++];
return "White";
}
private static ConsoleColor ParseColor(string colorName)
{
if (string.IsNullOrEmpty(colorName))
return ConsoleColor.White;
if (Enum.TryParse<ConsoleColor>(colorName, out var color))
return color;
Console.WriteLine($"[Pool] Warning: Invalid color '{colorName}', using White");
return ConsoleColor.White;
}
private static void PersistAutoAssignedColors(List<(string sectionName, string color)> assignments)
{
try
{
var json = File.ReadAllText(_configFilePath);
using var doc = JsonDocument.Parse(json);
var root = doc.RootElement;
using var stream = new MemoryStream();
using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true });
writer.WriteStartObject();
foreach (var prop in root.EnumerateObject())
{
writer.WritePropertyName(prop.Name);
if (prop.Value.ValueKind == JsonValueKind.Object)
{
writer.WriteStartObject();
foreach (var subProp in prop.Value.EnumerateObject())
{
writer.WritePropertyName(subProp.Name);
subProp.Value.WriteTo(writer);
}
var match = assignments.FirstOrDefault(a => a.sectionName == prop.Name);
if (!string.IsNullOrEmpty(match.sectionName))
{
writer.WriteString("Color", match.color);
}
writer.WriteEndObject();
}
else
{
prop.Value.WriteTo(writer);
}
}
writer.WriteEndObject();
writer.Flush();
File.WriteAllText(_configFilePath, Encoding.UTF8.GetString(stream.ToArray()));
Console.WriteLine($"[Pool] Auto-assigned colors persisted to config: {string.Join(", ", assignments.Select(a => $"{a.sectionName}={a.color}"))}");
}
catch (Exception ex)
{
Console.WriteLine($"[Pool] Warning: Could not persist auto-assigned colors to config: {ex.Message}");
}
}
private static void EnsureStateTableExists()
{
using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath);
@@ -1685,11 +1799,17 @@ namespace URLNotesGrabberCORE
if (retryTs > now)
{
var remaining = retryTs - now;
var prevColor = Console.ForegroundColor;
Console.ForegroundColor = key.ParsedColor;
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, clears in {remaining}s");
Console.ForegroundColor = prevColor;
}
else if (retryTs > 0)
{
var prevColor = Console.ForegroundColor;
Console.ForegroundColor = key.ParsedColor;
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared on startup");
Console.ForegroundColor = prevColor;
}
}
}
@@ -1697,39 +1817,47 @@ namespace URLNotesGrabberCORE
public static ApiKeyConfig GetCurrentKey()
{
ApiKeyConfig key;
if (!_usePool || _overrideKey != null)
return _overrideKey!;
if (_keys.Count == 1)
return _keys[0];
int attempts = 0;
while (attempts < _keys.Count)
key = _overrideKey!;
else if (_keys.Count == 1)
key = _keys[0];
else
{
var key = _keys[_currentIndex % _keys.Count];
_currentIndex = (_currentIndex + 1) % _keys.Count;
int attempts = 0;
if (IsKeyAvailable(key))
while (attempts < _keys.Count)
{
SaveState();
return key;
key = _keys[_currentIndex % _keys.Count];
_currentIndex = (_currentIndex + 1) % _keys.Count;
if (IsKeyAvailable(key))
{
SaveState();
_activeKey = key;
Console.ForegroundColor = key.ParsedColor;
return key;
}
attempts++;
}
attempts++;
var earliest = _keys
.Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) })
.OrderBy(x => x.RetryUntil)
.First();
Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)");
key = earliest.Key;
_currentIndex = (_keys.IndexOf(key) + 1) % _keys.Count;
SaveState();
}
var earliest = _keys
.Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) })
.OrderBy(x => x.RetryUntil)
.First();
Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)");
var chosen = earliest.Key;
_currentIndex = (_keys.IndexOf(chosen) + 1) % _keys.Count;
SaveState();
return chosen;
_activeKey = key;
Console.ForegroundColor = key.ParsedColor;
return key;
}
private static bool IsKeyAvailable(ApiKeyConfig key)
@@ -1760,7 +1888,10 @@ namespace URLNotesGrabberCORE
cmd.Parameters.AddWithValue("@until", retryUntil);
cmd.ExecuteNonQuery();
var prevColor = Console.ForegroundColor;
Console.ForegroundColor = key.ParsedColor;
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, retry in {retryInSeconds}s (until {DateTimeOffset.FromUnixTimeSeconds(retryUntil).LocalDateTime:HH:mm:ss})");
Console.ForegroundColor = prevColor;
}
public static void MarkAvailable(ApiKeyConfig key)
@@ -1771,6 +1902,11 @@ namespace URLNotesGrabberCORE
"INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, 0)", conn);
cmd.Parameters.AddWithValue("@name", key.SectionName);
cmd.ExecuteNonQuery();
var prevColor = Console.ForegroundColor;
Console.ForegroundColor = key.ParsedColor;
Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared");
Console.ForegroundColor = prevColor;
}
public static bool IsAllRateLimited(out int minRetrySeconds)
URLNotesGrabberCORE/Program.cs
+1 -1
View File
@@ -84,7 +84,7 @@ namespace URLNotesGrabberCORE
if (string.IsNullOrWhiteSpace(dbPath))
dbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "tl.db");
ApiKeyPool.Initialize(config, dbPath, apiExplicitlySet ? apiSectionName : null);
ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null);
// Setup Dual Logging
bool enableFileLogging = settings.GetValue("EnableFileLogging", true);