From 5e62b582617d127bb630178011bfda174bbb4b63 Mon Sep 17 00:00:00 2001 From: jim Date: Mon, 4 May 2026 10:33:46 -0500 Subject: [PATCH] feat: add API key color assignment, auto-persist, AGENTS.md --- AGENTS.md | 45 +++++++ URLNotesGrabberCORE/DataAccess.cs | 198 +++++++++++++++++++++++++----- URLNotesGrabberCORE/Program.cs | 2 +- 3 files changed, 213 insertions(+), 32 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..1ba4253 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,45 @@ +# AGENTS.md + +## Project Overview +- **Primary Language**: C# (.NET 8 Console Application) +- **Key Libraries**: RestSharp, Newtonsoft.Json, System.Data.SQLite, Microsoft.Extensions.Configuration +- **Purpose**: Tumblr API data harvester for collecting notes, posts, likes, and replies, storing results in SQLite. + +## Architectural Patterns +- CLI entry point in `Program.cs` with workflow orchestration +- `DataAccess.cs`: Database operations, `ApiKeyPool` (API key management), `APIAccess` (Tumblr client) +- `ResponseNotes.cs`: Tumblr API response models +- Round-robin API key rotation with rate-limit tracking +- Automatic console color assignment per API key for output differentiation + +## Developer Guidelines + +### Code Formatting +- 4-space indentation, no tabs, match existing C# style +- PascalCase for public members, camelCase for locals +- Minimize code comments unless explicitly requested +- Use only existing project libraries; no new dependencies without confirmation +- Match accessibility modifiers (`public` for models, `internal` for helpers) + +### Error Handling +- Wrap file/network operations in `try-catch` +- Log non-critical errors (e.g., config write failures) with `[Warning]` prefix +- Preserve console color state: use save/restore pattern for temporary color changes +- API rate limits must use `ApiKeyPool.MarkRateLimited()`/`MarkAvailable()` + +### Testing +- No existing test suite; use xUnit if adding tests +- Test critical logic: `ApiKeyPool` init, color parsing, config persistence +- Avoid testing one-off CLI workflows + +### Git Commit Messages +- Imperative mood ("Add feature" not "Added feature") +- Prefix with type: `feat:`, `fix:`, `chore:`, `docs:` +- Keep messages under 72 characters +- Never commit sensitive data (API keys/tokens) + +### API Key Color Rules +- Unconfigured keys auto-assign colors from a preset palette +- Auto-assigned colors persist to `appsettings.json` +- All output for an active key uses its assigned color +- Temporary color changes (e.g., errors) must restore the key's color afterward diff --git a/URLNotesGrabberCORE/DataAccess.cs b/URLNotesGrabberCORE/DataAccess.cs index c62baec..2900f4d 100644 --- a/URLNotesGrabberCORE/DataAccess.cs +++ b/URLNotesGrabberCORE/DataAccess.cs @@ -14,6 +14,7 @@ using System.Collections; using Microsoft.Extensions.Configuration; using System.IO; using System.Data; +using System.Text.Json; namespace URLNotesGrabberCORE { @@ -1556,6 +1557,8 @@ namespace URLNotesGrabberCORE public string OAuthToken { get; set; } = string.Empty; public string OAuthTokenSecret { get; set; } = string.Empty; public bool PoolEnabled { get; set; } + public string ColorName { get; set; } = string.Empty; + public ConsoleColor ParsedColor { get; set; } = ConsoleColor.White; } internal class ApiKeyPool @@ -1565,13 +1568,22 @@ namespace URLNotesGrabberCORE private static bool _usePool = false; private static int _currentIndex = 0; private static string _dbPath = string.Empty; + private static ApiKeyConfig? _activeKey = null; + private static readonly string[] DefaultAutoColors = + { + "Cyan", "Yellow", "Green", "Magenta", "Blue", "Red", + "DarkCyan", "DarkYellow", "DarkGreen", "DarkMagenta" + }; + private static int _nextAutoColorIndex = 0; + private static string _configFilePath = string.Empty; public static bool IsPoolActive => _usePool; public static List Keys => _keys; - public static void Initialize(IConfiguration config, string? dbPath, string? overrideSection = null) + public static void Initialize(IConfiguration config, string? dbPath, string configFilePath, string? overrideSection = null) { _dbPath = dbPath ?? "..\\..\\..\\tl.db"; + _configFilePath = configFilePath; EnsureStateTableExists(); if (!string.IsNullOrEmpty(overrideSection)) @@ -1580,6 +1592,9 @@ namespace URLNotesGrabberCORE if (section["ConsumerKey"] == null) throw new InvalidOperationException($"API section '{overrideSection}' not found in appsettings.json"); + var colorName = section["Color"] ?? string.Empty; + var parsedColor = ParseColor(colorName); + _overrideKey = new ApiKeyConfig { SectionName = overrideSection, @@ -1588,10 +1603,12 @@ namespace URLNotesGrabberCORE ConsumerSecret = section["ConsumerSecret"]!, OAuthToken = section["OAuthToken"]!, OAuthTokenSecret = section["OAuthTokenSecret"]!, - PoolEnabled = true + PoolEnabled = true, + ColorName = colorName, + ParsedColor = parsedColor }; _usePool = false; - Console.WriteLine($"[Pool] Single-key mode: {overrideSection}"); + Console.WriteLine($"[Pool] Single-key mode: {overrideSection} (Color: {_overrideKey.ParsedColor})"); return; } @@ -1601,6 +1618,8 @@ namespace URLNotesGrabberCORE .Distinct() .ToList(); + var autoAssignedColors = new List<(string sectionName, string color)>(); + int keyNum = 1; foreach (var sectionName in root.OrderBy(s => s)) { @@ -1610,6 +1629,16 @@ namespace URLNotesGrabberCORE if (poolEnabled) { + var colorName = section["Color"] ?? string.Empty; + var parsedColor = ParseColor(colorName); + + if (string.IsNullOrEmpty(colorName)) + { + colorName = GetNextAutoColor(); + parsedColor = ParseColor(colorName); + autoAssignedColors.Add((sectionName, colorName)); + } + _keys.Add(new ApiKeyConfig { SectionName = sectionName, @@ -1618,7 +1647,9 @@ namespace URLNotesGrabberCORE ConsumerSecret = section["ConsumerSecret"]!, OAuthToken = section["OAuthToken"]!, OAuthTokenSecret = section["OAuthTokenSecret"]!, - PoolEnabled = true + PoolEnabled = true, + ColorName = colorName, + ParsedColor = parsedColor }); keyNum++; } @@ -1629,6 +1660,16 @@ namespace URLNotesGrabberCORE var fallback = config.GetSection("TumblrApi"); if (fallback["ConsumerKey"] != null) { + var colorName = fallback["Color"] ?? string.Empty; + var parsedColor = ParseColor(colorName); + + if (string.IsNullOrEmpty(colorName)) + { + colorName = GetNextAutoColor(); + parsedColor = ParseColor(colorName); + autoAssignedColors.Add(("TumblrApi", colorName)); + } + _keys.Add(new ApiKeyConfig { SectionName = "TumblrApi", @@ -1637,19 +1678,92 @@ namespace URLNotesGrabberCORE ConsumerSecret = fallback["ConsumerSecret"]!, OAuthToken = fallback["OAuthToken"]!, OAuthTokenSecret = fallback["OAuthTokenSecret"]!, - PoolEnabled = true + PoolEnabled = true, + ColorName = colorName, + ParsedColor = parsedColor }); Console.WriteLine("[Pool] No keys with PoolEnabled, falling back to TumblrApi"); } } + if (autoAssignedColors.Count > 0) + { + PersistAutoAssignedColors(autoAssignedColors); + } + _usePool = true; LoadState(); - var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName}")); + var mapping = string.Join(", ", _keys.Select(k => $"Key#{k.KeyNumber}={k.SectionName} (Color: {k.ParsedColor})")); Console.WriteLine($"[Pool] Active keys: {mapping}"); } + private static string GetNextAutoColor() + { + if (_nextAutoColorIndex < DefaultAutoColors.Length) + return DefaultAutoColors[_nextAutoColorIndex++]; + return "White"; + } + + private static ConsoleColor ParseColor(string colorName) + { + if (string.IsNullOrEmpty(colorName)) + return ConsoleColor.White; + if (Enum.TryParse(colorName, out var color)) + return color; + Console.WriteLine($"[Pool] Warning: Invalid color '{colorName}', using White"); + return ConsoleColor.White; + } + + private static void PersistAutoAssignedColors(List<(string sectionName, string color)> assignments) + { + try + { + var json = File.ReadAllText(_configFilePath); + using var doc = JsonDocument.Parse(json); + var root = doc.RootElement; + + using var stream = new MemoryStream(); + using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true }); + + writer.WriteStartObject(); + foreach (var prop in root.EnumerateObject()) + { + writer.WritePropertyName(prop.Name); + + if (prop.Value.ValueKind == JsonValueKind.Object) + { + writer.WriteStartObject(); + foreach (var subProp in prop.Value.EnumerateObject()) + { + writer.WritePropertyName(subProp.Name); + subProp.Value.WriteTo(writer); + } + + var match = assignments.FirstOrDefault(a => a.sectionName == prop.Name); + if (!string.IsNullOrEmpty(match.sectionName)) + { + writer.WriteString("Color", match.color); + } + writer.WriteEndObject(); + } + else + { + prop.Value.WriteTo(writer); + } + } + writer.WriteEndObject(); + writer.Flush(); + + File.WriteAllText(_configFilePath, Encoding.UTF8.GetString(stream.ToArray())); + Console.WriteLine($"[Pool] Auto-assigned colors persisted to config: {string.Join(", ", assignments.Select(a => $"{a.sectionName}={a.color}"))}"); + } + catch (Exception ex) + { + Console.WriteLine($"[Pool] Warning: Could not persist auto-assigned colors to config: {ex.Message}"); + } + } + private static void EnsureStateTableExists() { using var conn = new System.Data.SQLite.SQLiteConnection("Data Source=" + _dbPath); @@ -1685,11 +1799,17 @@ namespace URLNotesGrabberCORE if (retryTs > now) { var remaining = retryTs - now; + var prevColor = Console.ForegroundColor; + Console.ForegroundColor = key.ParsedColor; Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, clears in {remaining}s"); + Console.ForegroundColor = prevColor; } else if (retryTs > 0) { + var prevColor = Console.ForegroundColor; + Console.ForegroundColor = key.ParsedColor; Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared on startup"); + Console.ForegroundColor = prevColor; } } } @@ -1697,39 +1817,47 @@ namespace URLNotesGrabberCORE public static ApiKeyConfig GetCurrentKey() { + ApiKeyConfig key; + if (!_usePool || _overrideKey != null) - return _overrideKey!; - - if (_keys.Count == 1) - return _keys[0]; - - int attempts = 0; - - while (attempts < _keys.Count) + key = _overrideKey!; + else if (_keys.Count == 1) + key = _keys[0]; + else { - var key = _keys[_currentIndex % _keys.Count]; - _currentIndex = (_currentIndex + 1) % _keys.Count; + int attempts = 0; - if (IsKeyAvailable(key)) + while (attempts < _keys.Count) { - SaveState(); - return key; + key = _keys[_currentIndex % _keys.Count]; + _currentIndex = (_currentIndex + 1) % _keys.Count; + + if (IsKeyAvailable(key)) + { + SaveState(); + _activeKey = key; + Console.ForegroundColor = key.ParsedColor; + return key; + } + + attempts++; } - attempts++; + var earliest = _keys + .Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) }) + .OrderBy(x => x.RetryUntil) + .First(); + + Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)"); + + key = earliest.Key; + _currentIndex = (_keys.IndexOf(key) + 1) % _keys.Count; + SaveState(); } - var earliest = _keys - .Select(k => new { Key = k, RetryUntil = GetRetryUntil(k) }) - .OrderBy(x => x.RetryUntil) - .First(); - - Console.WriteLine($"[Pool] All keys rate-limited, using earliest: Key#{earliest.Key.KeyNumber} ({earliest.Key.SectionName}, clears in {Math.Max(0, earliest.RetryUntil - DateTimeOffset.UtcNow.ToUnixTimeSeconds())}s)"); - - var chosen = earliest.Key; - _currentIndex = (_keys.IndexOf(chosen) + 1) % _keys.Count; - SaveState(); - return chosen; + _activeKey = key; + Console.ForegroundColor = key.ParsedColor; + return key; } private static bool IsKeyAvailable(ApiKeyConfig key) @@ -1760,7 +1888,10 @@ namespace URLNotesGrabberCORE cmd.Parameters.AddWithValue("@until", retryUntil); cmd.ExecuteNonQuery(); + var prevColor = Console.ForegroundColor; + Console.ForegroundColor = key.ParsedColor; Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limited, retry in {retryInSeconds}s (until {DateTimeOffset.FromUnixTimeSeconds(retryUntil).LocalDateTime:HH:mm:ss})"); + Console.ForegroundColor = prevColor; } public static void MarkAvailable(ApiKeyConfig key) @@ -1771,6 +1902,11 @@ namespace URLNotesGrabberCORE "INSERT OR REPLACE INTO ApiKeyPoolState (KeyName, RetryUntil) VALUES (@name, 0)", conn); cmd.Parameters.AddWithValue("@name", key.SectionName); cmd.ExecuteNonQuery(); + + var prevColor = Console.ForegroundColor; + Console.ForegroundColor = key.ParsedColor; + Console.WriteLine($"[Pool] Key#{key.KeyNumber} ({key.SectionName}) rate-limit cleared"); + Console.ForegroundColor = prevColor; } public static bool IsAllRateLimited(out int minRetrySeconds) diff --git a/URLNotesGrabberCORE/Program.cs b/URLNotesGrabberCORE/Program.cs index 0babc2b..6d45c80 100644 --- a/URLNotesGrabberCORE/Program.cs +++ b/URLNotesGrabberCORE/Program.cs @@ -84,7 +84,7 @@ namespace URLNotesGrabberCORE if (string.IsNullOrWhiteSpace(dbPath)) dbPath = Path.Combine(Directory.GetCurrentDirectory(), "..", "..", "..", "tl.db"); - ApiKeyPool.Initialize(config, dbPath, apiExplicitlySet ? apiSectionName : null); + ApiKeyPool.Initialize(config, dbPath, "appsettings.json", apiExplicitlySet ? apiSectionName : null); // Setup Dual Logging bool enableFileLogging = settings.GetValue("EnableFileLogging", true);