From 904d6997a3fe9be400ddd88deb3223737aa3cfd7 Mon Sep 17 00:00:00 2001 From: Sarto Date: Tue, 14 Apr 2026 23:40:47 +0330 Subject: [PATCH] feat: implement resolver bank functionality and add verifyer to fetcher --- .env.example | 2 +- README-FA.md | 5 +- README.md | 8 +- cmd/server/main.go | 2 +- configs/channels.txt | 2 +- internal/client/fetcher.go | 56 +++- internal/protocol/protocol.go | 10 +- internal/server/xpublic.go | 2 +- internal/web/scanner.go | 38 +-- internal/web/static/index.html | 479 ++++++++++++++++++++++------- internal/web/web.go | 538 +++++++++++++++++++++++++++++++-- scripts/install.sh | 4 +- test/e2e/integrity_e2e_test.go | 537 ++++++++++++++++++++++++++++++++ test/e2e/web_e2e_test.go | 202 +++++++++++++ 14 files changed, 1708 insertions(+), 177 deletions(-) create mode 100644 test/e2e/integrity_e2e_test.go diff --git a/.env.example b/.env.example index 630f4c4..4b67b4e 100644 --- a/.env.example +++ b/.env.example @@ -31,7 +31,7 @@ THEFEED_KEY=your-secret-passphrase #THEFEED_ALLOW_MANAGE=0 # Nitter RSS instances for X/Twitter (comma-separated) -#THEFEED_X_RSS_INSTANCES=http://nitter.net,https://nitter.net +#THEFEED_X_RSS_INSTANCES=https://nitter.net,http://nitter.net # Max random padding bytes in DNS responses (anti-DPI, default: 32) #THEFEED_PADDING=32 diff --git a/README-FA.md b/README-FA.md index ee9d4cd..0423417 100644 --- a/README-FA.md +++ b/README-FA.md @@ -38,6 +38,8 @@ thefeed یک سیستم تونل DNS است که به شما اجازه می‌ - لاگ زنده درخواست‌های DNS در مرورگر - **جستجوی پیام‌ها**: جستجو در پیام‌های کانال فعلی با هایلایت نتایج و ناوبری قبلی/بعدی - **خروجی پیام‌ها**: کپی N پیام آخر یک کانال به کلیپبورد +- **بانک ریزالور**: مدیریت مشترک ریزالورها برای تمام پروفایل‌ها — بدون نیاز به تنظیم ریزالور جداگانه برای هر پروفایل. ریزالورها از طریق اسکنر، ایمپورت، یا ورود دستی اضافه می‌شوند و به صورت خودکار امتیازدهی می‌شوند +- **پاکسازی ریزالور**: حذف ریزالورهای ضعیف از بانک بر اساس حداقل امتیاز دلخواه - **نمایش ریزالورهای فعال**: مشاهده لیست ریزالورهای سالم و فعال از تنظیمات - **تصویر پس‌زمینه**: تنظیم URL تصویر پس‌زمینه برای پنل پیام‌ها (ذخیره محلی) - **تایم‌اوت DNS**: تنظیم تایم‌اوت کوئری DNS برای هر پروفایل (پیش‌فرض ۱۵ ثانیه) @@ -56,7 +58,7 @@ thefeed یک سیستم تونل DNS است که به شما اجازه می‌ - **مکث / ادامه / توقف**: کنترل کامل روی اسکن‌های طولانی (مکث واقعاً ارسال درخواست‌های جدید را متوقف می‌کند) - **زمان پاسخ**: نتایج بر اساس تأخیر مرتب شده‌اند تا سریع‌ترین‌ها اول نمایش داده شوند - **انتخاب نتایج**: چک‌باکس برای انتخاب ریزالورهای مورد نظر -- **اعمال نتایج**: افزودن یا جایگزینی لیست ریزالورهای پروفایل مستقیم از اسکنر +- **اعمال نتایج**: افزودن یا جایگزینی بانک ریزالور مستقیم از اسکنر - **کپی**: دکمه کپی برای هر آی‌پی، کپی انتخاب‌شده‌ها، یا کپی همه - **اسکن جدید**: بازنشانی رابط کاربری برای شروع اسکن جدید پس از اتمام - **لاگ دیباگ**: در حالت دیباگ، کوئری‌ها و پاسخ‌های هر probe ثبت می‌شوند @@ -65,6 +67,7 @@ thefeed یک سیستم تونل DNS است که به شما اجازه می‌ - **اندازه متغیر پاسخ**: Padding تصادفی (۰-۳۲ بایت) - **کوئری تک‌برچسب**: رمزنگاری Base32 در یک برچسب DNS - **شافل Resolver**: توزیع تصادفی کوئری‌ها بین resolverها +- **بانک ریزالور**: مخزن مشترک ریزالورها با امتیازدهی دائمی و ابزار پاکسازی - **محدودیت نرخ**: قابل تنظیم برای ترکیب با ترافیک عادی DNS - **Padding تصادفی کوئری**: ۴ بایت تصادفی در هر درخواست - **اندازه بلاک متغیر**: بلاک‌های ۴۰۰-۷۰۰ بایت diff --git a/README.md b/README.md index 642f8fb..2b2db0a 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ DNS-based feed reader for Telegram channels and public X accounts. Designed for **Client** (runs inside censored network): - Browser-based web UI with RTL/Farsi support (VazirMatn font) - Sends encrypted DNS TXT queries via available resolvers -- **Resolver scoring**: tracks per-resolver success rate and latency; healthier resolvers are preferred automatically +- **Resolver Bank**: shared pool of DNS resolvers used across all profiles — no more per-profile resolver lists. Resolvers are added via scanner, import, or manual entry and scored automatically +- **Resolver scoring**: tracks per-resolver success rate and latency with persistent scores; healthier resolvers are preferred automatically. Users can clean up low-scoring resolvers from the bank - **Scatter mode**: fans out the same DNS request to multiple resolvers simultaneously and uses the fastest response (default: 2 concurrent resolvers per request) - Send messages to channels and private chats (requires server `--allow-manage` and login to telegram) - Channel management (add/remove channels remotely via admin commands when `--allow-manage` is enabled) @@ -41,6 +42,7 @@ DNS-based feed reader for Telegram channels and public X accounts. Designed for - Variable response and query sizes to prevent fingerprinting - Multiple query encoding modes for stealth +- **Resolver Bank**: centralized resolver pool shared by all profiles with persistent scoring and cleanup tools - **Resolver scoring**: per-resolver success-rate + latency scoreboard; high-scoring resolvers are picked more often via weighted-random selection - **Scatter mode**: same block fetched from N resolvers simultaneously, first response wins — faster fetches and implicit failover - Rate limiting and background noise traffic to blend in @@ -247,7 +249,7 @@ Environment variables: `THEFEED_DOMAIN`, `THEFEED_KEY`, `THEFEED_MSG_LIMIT`, `TH | `--key` | | Encryption passphrase (required) | | `--channels` | `{data-dir}/channels.txt` | Path to channels file | | `--x-accounts` | `{data-dir}/x_accounts.txt` | Path to X usernames file | -| `--x-rss-instances` | `http://nitter.net,https://nitter.net` | Comma-separated X RSS base URLs | +| `--x-rss-instances` | `https://nitter.net,http://nitter.net` | Comma-separated X RSS base URLs | | `--api-id` | | Telegram API ID (required) | | `--api-hash` | | Telegram API Hash (required) | | `--phone` | | Telegram phone number (required) | @@ -276,7 +278,7 @@ make build-client ./build/thefeed-client --password "your-secret" ``` -On first run, the client creates a `./thefeeddata/` directory next to where you run it. Open `http://127.0.0.1:8080` in your browser and configure your domain, passphrase, and resolvers through the Settings page. +On first run, the client creates a `./thefeeddata/` directory next to where you run it. Open `http://127.0.0.1:8080` in your browser and configure your domain and passphrase through the Settings page. DNS resolvers are managed in the shared Resolver Bank (accessible from the sidebar), which is used by all profiles. All configuration, cache, and data files are stored in the data directory. diff --git a/cmd/server/main.go b/cmd/server/main.go index 05712fd..b717d8a 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -26,7 +26,7 @@ func main() { key := flag.String("key", "", "Encryption passphrase") channelsFile := flag.String("channels", "", "Path to channels file (default: {data-dir}/channels.txt)") xAccountsFile := flag.String("x-accounts", "", "Path to X accounts file (default: {data-dir}/x_accounts.txt)") - xRSSInstances := flag.String("x-rss-instances", "", "Comma-separated X RSS base URLs (e.g., http://nitter.net,https://nitter.net)") + xRSSInstances := flag.String("x-rss-instances", "", "Comma-separated X RSS base URLs (e.g., https://nitter.net,http://nitter.net)") apiID := flag.String("api-id", "", "Telegram API ID (optional if --no-telegram)") apiHash := flag.String("api-hash", "", "Telegram API Hash (optional if --no-telegram)") phone := flag.String("phone", "", "Telegram phone number (optional if --no-telegram)") diff --git a/configs/channels.txt b/configs/channels.txt index 009e139..fe3db16 100644 --- a/configs/channels.txt +++ b/configs/channels.txt @@ -1,6 +1,6 @@ # Telegram channel usernames (one per line, with or without @) # Channel numbers are assigned in order: first = channel 1, second = channel 2, etc. # Lines starting with # are comments -@networkt +@networkti @thefeedconfig @VahidOnline diff --git a/internal/client/fetcher.go b/internal/client/fetcher.go index 1b28424..eced8f3 100644 --- a/internal/client/fetcher.go +++ b/internal/client/fetcher.go @@ -133,8 +133,8 @@ func (f *Fetcher) ScanConcurrency() int { return 10 } n := int(f.rateQPS) - if n < 1 { - n = 1 + if n < 10 { + n = 10 } return n } @@ -178,6 +178,29 @@ func (f *Fetcher) SetResolvers(resolvers []string) { copy(f.activeResolvers, resolvers) } +// UpdateResolverPool replaces the full resolver list but keeps the existing +// active pool intact (only pruning resolvers that are no longer in the bank). +// New bank entries are added to allResolvers but NOT automatically activated. +func (f *Fetcher) UpdateResolverPool(resolvers []string) { + f.mu.Lock() + defer f.mu.Unlock() + bankSet := make(map[string]bool, len(resolvers)) + for _, r := range resolvers { + bankSet[r] = true + } + // Prune active resolvers that were removed from the bank. + filtered := make([]string, 0, len(f.activeResolvers)) + for _, r := range f.activeResolvers { + if bankSet[r] { + filtered = append(filtered, r) + } + } + f.allResolvers = make([]string, len(resolvers)) + copy(f.allResolvers, resolvers) + f.activeResolvers = filtered + f.log("resolver pool updated: %d total, %d active", len(f.allResolvers), len(f.activeResolvers)) +} + // RemoveActiveResolver removes a resolver from the active pool. func (f *Fetcher) RemoveActiveResolver(addr string) { f.mu.Lock() @@ -653,6 +676,12 @@ func (f *Fetcher) FetchLatestVersion(ctx context.Context) (string, error) { return protocol.DecodeVersionData(data) } +// ErrContentHashMismatch is returned when the fetched messages do not match +// the expected content hash from metadata. This typically means the server +// regenerated its blocks between the metadata fetch and the block fetch +// (block-version race). The caller should re-fetch metadata and retry. +var ErrContentHashMismatch = fmt.Errorf("content hash mismatch") + // FetchChannel fetches all blocks for a channel and returns the parsed messages. // Cancelling ctx immediately aborts any queued or in-flight block fetches. // Each block is retried individually via FetchBlock before the channel fetch fails. @@ -660,6 +689,21 @@ func (f *Fetcher) FetchChannel(ctx context.Context, channelNum int, blockCount i return f.fetchChannelBlocks(ctx, channelNum, blockCount, f.FetchBlock) } +// FetchChannelVerified works like FetchChannel but additionally verifies that +// the parsed messages match the expected content hash from metadata. +// Returns ErrContentHashMismatch when the hash does not match (block-version race). +func (f *Fetcher) FetchChannelVerified(ctx context.Context, channelNum int, blockCount int, expectedHash uint32) ([]protocol.Message, error) { + msgs, err := f.fetchChannelBlocks(ctx, channelNum, blockCount, f.FetchBlock) + if err != nil { + return nil, err + } + if got := protocol.ContentHashOf(msgs); got != expectedHash { + f.log("Channel %d content hash mismatch: got %08x, want %08x (block-version race?)", channelNum, got, expectedHash) + return nil, ErrContentHashMismatch + } + return msgs, nil +} + func (f *Fetcher) fetchChannelBlocks(ctx context.Context, channelNum int, blockCount int, fetchFn func(context.Context, uint16, uint16) ([]byte, error)) ([]protocol.Message, error) { if blockCount <= 0 { return nil, nil @@ -725,7 +769,13 @@ func (f *Fetcher) fetchChannelBlocks(ctx context.Context, channelNum int, blockC // Decompress if data has compression header decompressed, err := protocol.DecompressMessages(allData) if err != nil { - // Fall back to raw parse for backward compatibility with uncompressed data + // If the data starts with a known compression header but decompression + // failed, the data is corrupt — do NOT raw-parse compressed bytes as + // messages (that produces binary garbage as message text). + if len(allData) > 0 && (allData[0] == 0x00 || allData[0] == 0x01) { + return nil, fmt.Errorf("decompress channel %d: %w", channelNum, err) + } + // Unknown header → pre-compression era data; try raw parse. return protocol.ParseMessages(allData) } diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go index e0604b8..093f7dc 100644 --- a/internal/protocol/protocol.go +++ b/internal/protocol/protocol.go @@ -9,6 +9,7 @@ import ( "hash/crc32" "io" "math/big" + "unicode/utf8" ) const ( @@ -268,9 +269,16 @@ func ParseMessages(data []byte) ([]Message, error) { if off+textLen > len(data) { break // incomplete message text, stop } - text := string(data[off : off+textLen]) + textBytes := data[off : off+textLen] off += textLen + // Skip messages with invalid UTF-8 text — these are artifacts of + // corrupt/decompression-failed data, not real messages. + if !utf8.Valid(textBytes) { + continue + } + text := string(textBytes) + msgs = append(msgs, Message{ ID: id, Timestamp: ts, diff --git a/internal/server/xpublic.go b/internal/server/xpublic.go index c37cea4..82469d0 100644 --- a/internal/server/xpublic.go +++ b/internal/server/xpublic.go @@ -82,7 +82,7 @@ func NewXPublicReader(accounts []string, feed *Feed, msgLimit int, baseCh int, i } func normalizeXRSSInstances(instancesCSV string) []string { - defaults := []string{"http://nitter.net", "https://nitter.net"} + defaults := []string{"https://nitter.net", "http://nitter.net"} if strings.TrimSpace(instancesCSV) == "" { return defaults } diff --git a/internal/web/scanner.go b/internal/web/scanner.go index 4a0299c..643f2b4 100644 --- a/internal/web/scanner.go +++ b/internal/web/scanner.go @@ -194,7 +194,7 @@ func (s *Server) handleScannerApply(w http.ResponseWriter, r *http.Request) { } } - // Determine which profile to apply to. + // Determine which profile to apply to (for logging purposes / active check). pl, _ := s.loadProfiles() if pl == nil { http.Error(w, "no profiles configured", 400) @@ -218,41 +218,21 @@ func (s *Server) handleScannerApply(w http.ResponseWriter, r *http.Request) { return } - var newResolvers []string + // Update the shared resolver bank instead of per-profile resolvers. if req.Mode == "overwrite" { - newResolvers = resolvers + pl.ResolverBank = resolvers } else { - // Append — deduplicate. - seen := make(map[string]bool) - for _, r := range pl.Profiles[targetIdx].Config.Resolvers { - seen[r] = true - newResolvers = append(newResolvers, r) - } - for _, r := range resolvers { - if !seen[r] { - newResolvers = append(newResolvers, r) - } - } + // Append — deduplicate against existing bank. + addToBank(pl, resolvers) } - pl.Profiles[targetIdx].Config.Resolvers = newResolvers if err := s.saveProfiles(pl); err != nil { http.Error(w, fmt.Sprintf("save profiles: %v", err), 500) return } - // If this is the active profile, also update config + fetcher. + // If this is the active profile, re-init the fetcher with the updated bank. if targetProfileID == pl.Active { - s.mu.Lock() - cfg := s.config - s.mu.Unlock() - if cfg != nil { - cfg.Resolvers = newResolvers - _ = s.saveConfig(cfg) - s.mu.Lock() - s.config = cfg - s.mu.Unlock() - } // Cancel any in-progress checker scan before re-initializing so the // old goroutine exits quickly and doesn't race with the new fetcher. s.mu.RLock() @@ -274,8 +254,8 @@ func (s *Server) handleScannerApply(w http.ResponseWriter, r *http.Request) { ctx := s.fetcherCtx s.mu.RUnlock() if fetcher != nil { - fetcher.SetActiveResolvers(newResolvers) - s.saveLastScan(newResolvers) + fetcher.SetActiveResolvers(resolvers) + s.saveLastScan(resolvers) } if checker != nil && ctx != nil { checker.StartPeriodic(ctx) @@ -284,5 +264,5 @@ func (s *Server) handleScannerApply(w http.ResponseWriter, r *http.Request) { } s.addLog(fmt.Sprintf("Scanner resolvers applied: %d resolvers (%s) to profile %s", len(resolvers), req.Mode, pl.Profiles[targetIdx].Nickname)) - writeJSON(w, map[string]any{"ok": true, "count": len(newResolvers)}) + writeJSON(w, map[string]any{"ok": true, "count": len(pl.ResolverBank)}) } diff --git a/internal/web/static/index.html b/internal/web/static/index.html index e2135b7..6b5e3e2 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -1443,11 +1443,11 @@ +▼ +
- + -
@@ -1629,8 +1629,10 @@
-
+
+ Resolvers are managed in the shared Resolver Bank. + Open Resolver Bank +
-
@@ -1687,15 +1686,57 @@ - +
' + esc(p.config.domain) + '
'; h += '
'; - if (isActive) h += ''; h += ''; h += ''; h += '
'; // Share panel (hidden by default) h += ''; h += ''; } el.innerHTML = h; } - function toggleSharePanel(id) { - // Close all first - document.querySelectorAll('.share-panel').forEach(function (sp) { sp.style.display = 'none' }); + async function toggleSharePanel(id) { var panel = document.getElementById('share-' + id); if (!panel) return; - var uri = buildProfileUri(id); + // If already visible, just close it + if (panel.style.display === 'block') { + panel.style.display = 'none'; + return; + } + // Close all first + document.querySelectorAll('.share-panel').forEach(function (sp) { sp.style.display = 'none' }); + panel.style.display = 'block'; + // Populate resolver checkboxes from bank + var resolverEl = document.getElementById('share-resolvers-' + id); + resolverEl.innerHTML = '' + t('loading') + ''; + try { + var r = await fetch('/api/resolvers/bank'); + var data = r.ok ? await r.json() : { bank: [] }; + var bank = data.bank || []; + if (bank.length === 0) { + resolverEl.innerHTML = '' + t('no_active_resolvers') + ''; + } else { + var h = '
'; + h += '
'; + for (var i = 0; i < bank.length; i++) { + h += ''; + } + resolverEl.innerHTML = h; + } + } catch (e) { resolverEl.innerHTML = '' } + updateShareUri(id); + } + + function toggleAllShareResolvers(id, checked) { + document.querySelectorAll('.share-r-cb[data-profile="' + id + '"]').forEach(function (cb) { cb.checked = checked }); + updateShareUri(id); + } + + function updateShareUri(id) { + var cbs = document.querySelectorAll('.share-r-cb[data-profile="' + id + '"]'); + var selected = []; + cbs.forEach(function (cb) { if (cb.checked) selected.push(cb.value) }); + var uri = buildProfileUri(id, selected); var input = document.getElementById('suri-' + id); if (input) input.value = uri || t('no_config'); - panel.style.display = 'block'; } function copyShareUri(id) { - var uri = buildProfileUri(id); + var cbs = document.querySelectorAll('.share-r-cb[data-profile="' + id + '"]'); + var selected = []; + cbs.forEach(function (cb) { if (cb.checked) selected.push(cb.value) }); + var uri = buildProfileUri(id, selected); if (!uri) { showToast(t('no_config')); return } navigator.clipboard.writeText(uri).then(function () { showToast(t('copied')) }).catch(function () { var input = document.getElementById('suri-' + id); if (input) { input.select(); input.setSelectionRange(0, 9999); } @@ -2435,15 +2565,19 @@ async function activateProfile(id) { if (id === activeProfileId) { closeProfiles(); return } - // Check if we should skip resolver check - var skipCheck = false; + // If there are active resolvers, ask user whether to rescan; otherwise skip. + var skipCheck = true; try { - var stRes = await fetch('/api/status'); - var st = await stRes.json(); - if (st.lastScan && st.lastScan.count > 0) { - skipCheck = await showRescanPrompt(st.lastScan.count); + var bankR = await fetch('/api/resolvers/bank'); + if (bankR.ok) { + var bankD = await bankR.json(); + var activeN = 0; + (bankD.bank || []).forEach(function (b) { if (b.active) activeN++ }); + if (activeN > 0) { + skipCheck = await showConfirmDialog(t('rescan_prompt_msg').replace('{n}', activeN), t('rescan_prompt_skip'), t('rescan_prompt_yes')); + } } - } catch (e) { } + } catch (e) { /* ignore — default to skipCheck=true */ } try { var r = await fetch('/api/profiles/switch', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ id: id, skipCheck: skipCheck }) }); if (!r.ok) return; @@ -2471,12 +2605,29 @@ params.split('&').forEach(function (kv) { var p = kv.split('='); if (p[0] === 'r' && p[1]) resolvers = decodeURIComponent(p[1]).split(',').filter(Boolean) }); if (!domain || !key) { errEl.textContent = t('uri_missing'); errEl.style.display = 'block'; return } if (!resolvers.length) resolvers = ['8.8.8.8', '1.1.1.1']; - var profile = { id: '', nickname: domain, config: { domain: domain, key: key, resolvers: resolvers, queryMode: 'single', rateLimit: 6 } }; + // Add resolvers to the shared bank. + var bankData = { count: 0 }; + try { + var bankRes = await fetch('/api/resolvers/bank', { signal: AbortSignal.timeout(5000) }); + if (bankRes.ok) bankData = await bankRes.json(); + } catch (e2) { /* timeout or fetch error — treat as empty bank */ } + var shouldAdd = true; + if (bankData.count > 0 && bankData.count <= 200) { + shouldAdd = await showConfirmDialog(t('import_add_resolvers').replace('{n}', resolvers.length), t('yes'), t('no')); + } else if (bankData.count > 200) { + shouldAdd = await showConfirmDialog(t('import_add_resolvers_large').replace('{n}', resolvers.length).replace('{c}', bankData.count), t('yes'), t('no')); + } + if (shouldAdd) { + await fetch('/api/resolvers/bank', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ resolvers: resolvers }) }); + } + // Create profile without resolvers (they're in the bank now). + var profile = { id: '', nickname: domain, config: { domain: domain, key: key, queryMode: 'single', rateLimit: 6 } }; var r = await fetch('/api/profiles', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action: 'create', profile: profile }) }); if (!r.ok) throw new Error('save failed'); okEl.textContent = t('import_success').replace('{d}', domain); okEl.style.display = 'block'; document.getElementById('importUriInput').value = ''; await loadProfiles(); renderProfilesModal(); + refreshResolversBadge(); } catch (e) { errEl.textContent = t('import_error') + ': ' + e.message; errEl.style.display = 'block' } } @@ -2489,18 +2640,15 @@ if (id) { document.getElementById('profileEditorTitle').textContent = t('edit_profile'); document.getElementById('peDeleteBtn').style.display = ''; - document.getElementById('peScannerBtn').style.display = ''; var p = profiles && profiles.profiles && profiles.profiles.find(function (x) { return x.id === id }); if (p) { document.getElementById('peNick').value = p.nickname || ''; document.getElementById('peDomain').value = p.config.domain || ''; document.getElementById('peKey').value = p.config.key || ''; - document.getElementById('peResolvers').value = (p.config.resolvers || []).join('\n'); document.getElementById('peQueryMode').value = p.config.queryMode || 'single'; document.getElementById('peRateLimit').value = p.config.rateLimit || 6; document.getElementById('peScatter').value = p.config.scatter || 4; document.getElementById('peTimeout').value = p.config.timeout || 15; - document.getElementById('peAutoScan').checked = p.config.autoScan !== false; } document.getElementById('peChannelSection').style.display = ''; var isActive = id === activeProfileId; @@ -2516,16 +2664,13 @@ } else { document.getElementById('profileEditorTitle').textContent = t('new_profile'); document.getElementById('peDeleteBtn').style.display = 'none'; - document.getElementById('peScannerBtn').style.display = 'none'; document.getElementById('peNick').value = ''; document.getElementById('peDomain').value = ''; document.getElementById('peKey').value = ''; - document.getElementById('peResolvers').value = ''; document.getElementById('peQueryMode').value = 'single'; document.getElementById('peRateLimit').value = '6'; document.getElementById('peScatter').value = '4'; document.getElementById('peTimeout').value = '15'; - document.getElementById('peAutoScan').checked = true; document.getElementById('peChannelSection').style.display = 'none'; } } @@ -2572,21 +2717,30 @@ var nick = document.getElementById('peNick').value.trim(); var domain = document.getElementById('peDomain').value.trim(); var key = document.getElementById('peKey').value; - var resolvers = document.getElementById('peResolvers').value.trim().split(/[\n,]+/).map(function (s) { return s.trim() }).filter(Boolean); - if (!domain || !key || !resolvers.length) { errEl.textContent = t('resolvers') + ' / ' + t('domain') + ' / ' + t('passphrase'); errEl.style.display = 'block'; return } - var profile = { id: editingProfileId || '', nickname: nick || domain, config: { domain: domain, key: key, resolvers: resolvers, queryMode: document.getElementById('peQueryMode').value, rateLimit: parseFloat(document.getElementById('peRateLimit').value) || 6, scatter: parseInt(document.getElementById('peScatter').value) || 4, timeout: parseInt(document.getElementById('peTimeout').value) || 15, autoScan: document.getElementById('peAutoScan').checked ? undefined : false } }; + if (!domain || !key) { errEl.textContent = t('domain') + ' / ' + t('passphrase'); errEl.style.display = 'block'; return } + var profile = { id: editingProfileId || '', nickname: nick || domain, config: { domain: domain, key: key, queryMode: document.getElementById('peQueryMode').value, rateLimit: parseFloat(document.getElementById('peRateLimit').value) || 6, scatter: parseInt(document.getElementById('peScatter').value) || 4, timeout: parseInt(document.getElementById('peTimeout').value) || 15 } }; + // Preserve autoScan from existing profile + if (editingProfileId && profiles && profiles.profiles) { + var existing = profiles.profiles.find(function (x) { return x.id === editingProfileId }); + if (existing && existing.config.autoScan === false) profile.config.autoScan = false; + } var action = editingProfileId ? 'update' : 'create'; var wasFirst = !profiles || !profiles.profiles || profiles.profiles.length === 0; - // Check if we should skip resolver check (existing healthy resolvers) - var skipCheck = false; - if (editingProfileId && editingProfileId === activeProfileId) { + // If there are active resolvers, ask user whether to rescan; otherwise skip. + var skipCheck = true; + var isActiveEdit = editingProfileId && editingProfileId === activeProfileId; + if (isActiveEdit || wasFirst) { try { - var stRes = await fetch('/api/status'); - var st = await stRes.json(); - if (st.lastScan && st.lastScan.count > 0) { - skipCheck = await showRescanPrompt(st.lastScan.count); + var bankR = await fetch('/api/resolvers/bank'); + if (bankR.ok) { + var bankD = await bankR.json(); + var activeN = 0; + (bankD.bank || []).forEach(function (b) { if (b.active) activeN++ }); + if (activeN > 0) { + skipCheck = await showConfirmDialog(t('rescan_prompt_msg').replace('{n}', activeN), t('rescan_prompt_skip'), t('rescan_prompt_yes')); + } } - } catch (e) { } + } catch (e) { /* ignore — default to skipCheck=true */ } } try { var r = await fetch('/api/profiles', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action: action, profile: profile, skipCheck: skipCheck }) }); @@ -2613,16 +2767,6 @@ } catch (e) { errEl.textContent = e.message; errEl.style.display = 'block' } } - // ===== RESCAN ===== - async function doRescanFromProfiles() { - closeProfiles(); - showToast(t('rescan_started')); - document.getElementById('progressPanel').innerHTML = ''; - showInitProgress(); - try { await fetch('/api/rescan', { method: 'POST' }) } catch (e) { } - setTimeout(function () { loadChannels().then(function () { if (selectedChannel > 0) loadMessages(selectedChannel) }) }, 3000); - } - async function deleteEditingProfile() { if (!editingProfileId) return; if (!confirm(t('delete') + '?')) return; @@ -3156,17 +3300,6 @@ document.getElementById('scannerModal').classList.remove('active'); if (scanPollTimer) { clearInterval(scanPollTimer); scanPollTimer = null } } - function openScannerFromProfile() { - var profileId = editingProfileId; - closeProfileEditor(); - openScanner(); - if (profileId) { - var sel = document.getElementById('scanProfile'); - for (var i = 0; i < sel.options.length; i++) { - if (sel.options[i].value === profileId) { sel.selectedIndex = i; break } - } - } - } function populateScanProfileSelect() { var sel = document.getElementById('scanProfile'); @@ -3557,86 +3690,175 @@ }).catch(function () { showToast('Copy failed') }); } - // ===== WORKING RESOLVERS ===== - function updateResolversBadge(count) { + // ===== RESOLVER BANK ===== + var currentResolverTab = 'active'; + function updateResolversBadge(count, bankCount) { var badge = document.getElementById('resolversBadge'); if (!badge) return; - badge.textContent = count; - badge.style.color = count > 0 ? 'var(--success, #27ae60)' : 'var(--error, #e74c3c)'; + badge.textContent = count + ' / ' + (bankCount !== undefined ? bankCount : count); + var total = bankCount !== undefined ? bankCount : count; + if (total > 500) { + badge.style.color = 'var(--error, #e74c3c)'; + } else { + badge.style.color = count > 0 ? 'var(--success, #27ae60)' : 'var(--error, #e74c3c)'; + } } async function refreshResolversBadge() { try { - var r = await fetch('/api/resolvers/active'); + var r = await fetch('/api/resolvers/bank'); if (!r.ok) return; var data = await r.json(); - updateResolversBadge((data.resolvers || []).length); + var activeCount = 0; + (data.bank || []).forEach(function (b) { if (b.active) activeCount++ }); + updateResolversBadge(activeCount, data.count || 0); } catch (e) { } } var resolversRefreshTimer = null; - async function _fetchResolversBoard(el) { + function _buildScoreboardTable(board, showRemove, removeFromBank) { + if (!board.length) return '
' + t('no_active_resolvers') + '
'; + var h = ''; + h += ''; + h += ''; + h += ''; + h += ''; + h += ''; + h += ''; + if (showRemove) h += ''; + h += ''; + for (var i = 0; i < board.length; i++) { + var b = board[i]; + var scoreColor = b.score >= 0.5 ? 'var(--success)' : b.score >= 0.15 ? 'var(--text)' : 'var(--error)'; + h += ''; + h += ''; + h += ''; + h += ''; + h += ''; + h += ''; + if (showRemove) { + var fn = removeFromBank ? 'removeResolverFromBank' : 'removeResolver'; + h += ''; + } + h += ''; + } + h += '
Resolver' + t('resolver_speed') + '' + t('resolver_score') + '\u2705\u274C
' + esc(b.addr); + if (b.active !== undefined && b.active) h += ' \u25CF'; + h += '' + (b.avgMs > 0 ? Math.round(b.avgMs) + 'ms' : '-') + '' + b.score.toFixed(2) + '' + b.success + '' + b.failure + '
'; + return h; + } + async function _fetchActiveBoard() { + var el = document.getElementById('resolverPanelActive'); try { var r = await fetch('/api/resolvers/active'); if (!r.ok) throw new Error(await r.text()); var data = await r.json(); var board = data.scoreboard || []; - if (!board.length) { el.innerHTML = '
' + t('no_active_resolvers') + '
'; updateResolversBadge(0); return } - updateResolversBadge(board.length); - var h = ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - for (var i = 0; i < board.length; i++) { - var b = board[i]; - var scoreColor = b.score >= 0.5 ? 'var(--success)' : b.score >= 0.15 ? 'var(--text)' : 'var(--error)'; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - h += ''; - } - h += '
Resolver' + t('resolver_speed') + '' + t('resolver_score') + '\u2705\u274C
' + esc(b.addr) + '' + (b.avgMs > 0 ? Math.round(b.avgMs) + 'ms' : '-') + '' + b.score.toFixed(2) + '' + b.success + '' + b.failure + '
'; - el.innerHTML = h; + document.getElementById('resolverActiveCount').textContent = board.length; + el.innerHTML = _buildScoreboardTable(board, true, false); + // Also fetch bank count for the tab badge + try { + var br = await fetch('/api/resolvers/bank'); + if (br.ok) { var bd = await br.json(); document.getElementById('resolverBankCount').textContent = bd.count || 0; } + } catch (e2) { } } catch (e) { el.innerHTML = '
' + esc(e.message) + '
' } } + async function _fetchBankBoard() { + var el = document.getElementById('resolverBankListEl'); + try { + var r = await fetch('/api/resolvers/bank'); + if (!r.ok) throw new Error(await r.text()); + var data = await r.json(); + var bank = data.bank || []; + var countEl = document.getElementById('resolverBankCount'); + countEl.textContent = data.count || 0; + countEl.style.color = (data.count || 0) > 500 ? 'var(--error)' : ''; + var activeCount = 0; + bank.forEach(function (b) { if (b.active) activeCount++ }); + document.getElementById('resolverActiveCount').textContent = activeCount; + updateResolversBadge(activeCount, data.count || 0); + el.innerHTML = _buildScoreboardTable(bank, true, true); + previewBankCleanup(); + } catch (e) { el.innerHTML = '
' + esc(e.message) + '
' } + } + function switchResolverTab(tab) { + currentResolverTab = tab; + document.getElementById('resolverTabActive').classList.toggle('active', tab === 'active'); + document.getElementById('resolverTabBank').classList.toggle('active', tab === 'bank'); + document.getElementById('resolverTabActive').style.borderBottom = tab === 'active' ? '2px solid var(--accent)' : 'none'; + document.getElementById('resolverTabBank').style.borderBottom = tab === 'bank' ? '2px solid var(--accent)' : 'none'; + document.getElementById('resolverPanelActive').style.display = tab === 'active' ? '' : 'none'; + document.getElementById('resolverPanelBank').style.display = tab === 'bank' ? '' : 'none'; + if (tab === 'active') _fetchActiveBoard(); + else _fetchBankBoard(); + } async function openResolversModal() { - var el = document.getElementById('resolversListEl'); - el.innerHTML = '
' + t('loading') + '
'; document.getElementById('resolversModal').classList.add('active'); - await _fetchResolversBoard(el); + // Load autoScan from active profile + var autoScanEl = document.getElementById('bankAutoScan'); + if (autoScanEl && profiles && profiles.profiles && activeProfileId) { + var p = profiles.profiles.find(function (x) { return x.id === activeProfileId }); + if (p) autoScanEl.checked = p.config.autoScan !== false; + } + switchResolverTab('active'); + refreshResolversBadge(); if (resolversRefreshTimer) clearInterval(resolversRefreshTimer); resolversRefreshTimer = setInterval(function () { if (!document.getElementById('resolversModal').classList.contains('active')) { clearInterval(resolversRefreshTimer); resolversRefreshTimer = null; return; } - _fetchResolversBoard(document.getElementById('resolversListEl')); + if (currentResolverTab === 'active') _fetchActiveBoard(); + else _fetchBankBoard(); }, 3000); } function closeResolversModal() { document.getElementById('resolversModal').classList.remove('active'); if (resolversRefreshTimer) { clearInterval(resolversRefreshTimer); resolversRefreshTimer = null; } } + function openScannerFromBank() { + closeResolversModal(); + openScanner(); + } + async function doRescanFromBank() { + closeResolversModal(); + showToast(t('rescan_started')); + document.getElementById('progressPanel').innerHTML = ''; + showInitProgress(); + try { await fetch('/api/rescan', { method: 'POST' }) } catch (e) { } + setTimeout(function () { loadChannels().then(function () { if (selectedChannel > 0) loadMessages(selectedChannel) }); refreshResolversBadge(); }, 3000); + } + async function toggleBankAutoScan() { + var checked = document.getElementById('bankAutoScan').checked; + if (!profiles || !profiles.profiles || !activeProfileId) return; + var p = profiles.profiles.find(function (x) { return x.id === activeProfileId }); + if (!p) return; + var profile = JSON.parse(JSON.stringify(p)); + profile.config.autoScan = checked ? undefined : false; + try { + await fetch('/api/profiles', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action: 'update', profile: profile, skipCheck: true }) }); + await loadProfiles(); + } catch (e) { showToast(e.message) } + } async function removeResolver(addr) { try { await fetch('/api/resolvers/remove', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ addr: addr }) }); - _fetchResolversBoard(document.getElementById('resolversListEl')); + _fetchActiveBoard(); + } catch (e) { } + } + async function removeResolverFromBank(addr) { + try { + await fetch('/api/resolvers/bank', { method: 'DELETE', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ addrs: [addr] }) }); + _fetchBankBoard(); } catch (e) { } } async function resetScoreboard() { try { await fetch('/api/resolvers/reset-stats', { method: 'POST' }); - _fetchResolversBoard(document.getElementById('resolversListEl')); + if (currentResolverTab === 'active') _fetchActiveBoard(); + else _fetchBankBoard(); } catch (e) { } } function copyResolversList() { - var rows = document.querySelectorAll('#resolversListEl tbody tr'); + var panelId = currentResolverTab === 'active' ? 'resolverPanelActive' : 'resolverBankListEl'; + var rows = document.querySelectorAll('#' + panelId + ' tbody tr'); var lines = []; rows.forEach(function (tr) { var cells = tr.querySelectorAll('td'); @@ -3645,6 +3867,41 @@ if (!lines.length) { showToast(t('no_active_resolvers')); return } navigator.clipboard.writeText(lines.join('\n')).then(function () { showToast(t('copied')) }); } + async function previewBankCleanup() { + var val = parseFloat(document.getElementById('bankCleanupSlider').value) || 0.1; + document.getElementById('bankCleanupValue').textContent = val.toFixed(2); + try { + var r = await fetch('/api/resolvers/bank/cleanup', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ minScore: val, dryRun: true }) }); + if (!r.ok) return; + var data = await r.json(); + document.getElementById('bankCleanupPreview').innerHTML = '' + data.removed + ' ' + t('would_be_removed') + ', ' + data.remaining + ' ' + t('would_remain'); + } catch (e) { } + } + async function doBankCleanup() { + var val = parseFloat(document.getElementById('bankCleanupSlider').value) || 0.1; + try { + var r = await fetch('/api/resolvers/bank/cleanup', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ minScore: val }) }); + if (!r.ok) { showToast('Cleanup failed'); return } + var data = await r.json(); + showToast(t('removed') + ': ' + data.removed + ', ' + t('remaining') + ': ' + data.remaining); + _fetchBankBoard(); + refreshResolversBadge(); + } catch (e) { showToast(e.message) } + } + async function addResolversToBank() { + var text = document.getElementById('bankAddResolvers').value.trim(); + var resolvers = text.split(/[\n,;\s]+/).map(function (s) { return s.trim() }).filter(Boolean); + if (!resolvers.length) return; + try { + var r = await fetch('/api/resolvers/bank', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ resolvers: resolvers }) }); + if (!r.ok) { showToast('Add failed'); return } + var data = await r.json(); + showToast(t('added') + ': ' + data.added); + document.getElementById('bankAddResolvers').value = ''; + _fetchBankBoard(); + refreshResolversBadge(); + } catch (e) { showToast(e.message) } + } // ===== BACKGROUND IMAGE ===== function _setBg(url) { diff --git a/internal/web/web.go b/internal/web/web.go index 0355dce..c269db4 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -7,6 +7,7 @@ import ( "embed" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "io/fs" @@ -15,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -53,6 +55,13 @@ type Profile struct { Config Config `json:"config"` } +// SavedResolverScore stores persistent resolver performance data. +type SavedResolverScore struct { + Success int64 `json:"success"` + Failure int64 `json:"failure"` + TotalMs int64 `json:"totalMs"` +} + // ProfileList is the on-disk structure for profiles.json. type ProfileList struct { Active string `json:"active"` // ID of active profile @@ -62,6 +71,10 @@ type ProfileList struct { Debug bool `json:"debug,omitempty"` Theme string `json:"theme,omitempty"` Lang string `json:"lang,omitempty"` + // ResolverBank is the shared pool of DNS resolvers used by all profiles. + ResolverBank []string `json:"resolverBank,omitempty"` + // ResolverScores stores accumulated performance data for bank resolvers. + ResolverScores map[string]*SavedResolverScore `json:"resolverScores,omitempty"` } // lastScanData is the on-disk structure for last_scan.json. @@ -146,6 +159,9 @@ func New(dataDir string, port int, password string) (*Server, error) { scanner: scanner, } + // Migrate per-profile resolvers into the shared bank on first run. + s.migrateResolverBank() + cfg, err := s.loadConfig() if err == nil { s.config = cfg @@ -197,6 +213,8 @@ func (s *Server) Run() error { mux.HandleFunc("/api/resolvers/active", s.handleActiveResolvers) mux.HandleFunc("/api/resolvers/remove", s.handleRemoveResolver) mux.HandleFunc("/api/resolvers/reset-stats", s.handleResetResolverStats) + mux.HandleFunc("/api/resolvers/bank", s.handleResolverBank) + mux.HandleFunc("/api/resolvers/bank/cleanup", s.handleResolverBankCleanup) mux.HandleFunc("/api/scanner/start", s.handleScannerStart) mux.HandleFunc("/api/scanner/stop", s.handleScannerStop) mux.HandleFunc("/api/scanner/pause", s.handleScannerPause) @@ -311,6 +329,15 @@ func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) { http.Error(w, "domain, key, and resolvers are required", 400) return } + // Add config resolvers to the shared bank. + if len(cfg.Resolvers) > 0 { + pl, _ := s.loadProfiles() + if pl == nil { + pl = &ProfileList{} + } + addToBank(pl, cfg.Resolvers) + _ = s.saveProfiles(pl) + } if err := s.saveConfig(&cfg); err != nil { http.Error(w, fmt.Sprintf("save config: %v", err), 500) return @@ -640,6 +667,8 @@ func (s *Server) initFetcher() error { var prevStats map[string][3]int64 if s.fetcher != nil { prevStats = s.fetcher.ExportStats() + // Persist accumulated stats before destroying the old fetcher. + go s.persistResolverScores(prevStats) } if s.fetcherCancel != nil { s.fetcherCancel() @@ -650,30 +679,54 @@ func (s *Server) initFetcher() error { return fmt.Errorf("no config") } + // Load the shared resolver bank and preferences from profiles.json. + var bankResolvers []string + var debug bool + var savedScores map[string]*SavedResolverScore + if pl, plErr := s.loadProfiles(); plErr == nil { + debug = pl.Debug + if len(pl.ResolverBank) > 0 { + bankResolvers = pl.ResolverBank + } + savedScores = pl.ResolverScores + } + + // Use resolver bank; fall back to per-profile resolvers for backward compat. + resolvers := cfg.Resolvers + if len(bankResolvers) > 0 { + resolvers = bankResolvers + } + cacheDir := filepath.Join(s.dataDir, "cache") cache, err := client.NewCache(cacheDir) if err != nil { return fmt.Errorf("create cache: %w", err) } - fetcher, err := client.NewFetcher(cfg.Domain, cfg.Key, cfg.Resolvers) + fetcher, err := client.NewFetcher(cfg.Domain, cfg.Key, resolvers) if err != nil { return fmt.Errorf("create fetcher: %w", err) } - // Restore resolver stats from the previous fetcher. + // Restore resolver stats: prefer in-memory stats from the previous fetcher, + // fall back to persisted scores for fresh starts. if prevStats != nil { fetcher.ImportStats(prevStats) + } else if len(savedScores) > 0 { + m := make(map[string][3]int64) + for addr, ss := range savedScores { + key := addr + if !strings.Contains(key, ":") { + key += ":53" + } + m[key] = [3]int64{ss.Success, ss.Failure, ss.TotalMs} + } + fetcher.ImportStats(m) } if cfg.QueryMode == "double" { fetcher.SetQueryMode(protocol.QueryMultiLabel) } - // Use global debug preference from profiles.json. - var debug bool - if pl, err := s.loadProfiles(); err == nil { - debug = pl.Debug - } fetcher.SetDebug(debug) s.scanner.SetDebug(debug) if cfg.RateLimit > 0 { @@ -731,17 +784,23 @@ func (s *Server) checkLatestVersion(ctx context.Context) (string, error) { return "", fmt.Errorf("no config") } - fetcher, err := client.NewFetcher(cfg.Domain, cfg.Key, cfg.Resolvers) + // Use resolver bank; fall back to per-profile resolvers. + resolvers := cfg.Resolvers + var debug bool + if pl, plErr := s.loadProfiles(); plErr == nil { + debug = pl.Debug + if len(pl.ResolverBank) > 0 { + resolvers = pl.ResolverBank + } + } + + fetcher, err := client.NewFetcher(cfg.Domain, cfg.Key, resolvers) if err != nil { return "", fmt.Errorf("create fetcher: %w", err) } if cfg.QueryMode == "double" { fetcher.SetQueryMode(protocol.QueryMultiLabel) } - var debug bool - if pl, err := s.loadProfiles(); err == nil { - debug = pl.Debug - } fetcher.SetDebug(debug) s.scanner.SetDebug(debug) if cfg.RateLimit > 0 { @@ -758,7 +817,7 @@ func (s *Server) checkLatestVersion(ctx context.Context) (string, error) { fetcher.SetLogFunc(func(msg string) { s.addLog(msg) }) - fetcher.SetActiveResolvers(cfg.Resolvers) + fetcher.SetActiveResolvers(resolvers) checkCtx, cancel := context.WithTimeout(ctx, timeout*3) defer cancel() @@ -794,6 +853,8 @@ func (s *Server) startCheckerThenRefresh() { // skipCheckerUseSaved uses saved resolvers from the last scan and starts // periodic health checks without an initial scan pass. +// If no saved resolvers are available, falls back to a full scan with +// retry-every-minute until at least one resolver is found. func (s *Server) skipCheckerUseSaved() { s.mu.RLock() checker := s.checker @@ -805,9 +866,14 @@ func (s *Server) skipCheckerUseSaved() { } if ls := s.loadLastScan(); ls != nil && len(ls.Resolvers) > 0 { fetcher.SetActiveResolvers(ls.Resolvers) + checker.StartPeriodic(ctx) + go s.refreshMetadataOnly() + } else { + // No saved resolvers — do a full scan (with retry-every-minute). + checker.StartAndNotify(ctx, func() { + s.refreshMetadataOnly() + }) } - checker.StartPeriodic(ctx) - go s.refreshMetadataOnly() } // nextFetchDeadline returns the Time when the server will next fetch from Telegram. @@ -827,10 +893,10 @@ func (s *Server) nextFetchDeadline() time.Time { } // waitForServerFetch blocks until the server's Telegram fetch is likely complete -// (nextFetch + 45 s), emitting a countdown progress event each second so the UI +// (nextFetch + 30 s), emitting a countdown progress event each second so the UI // can render a live progress bar. Returns true on completion, false if ctx cancelled. func (s *Server) waitForServerFetch(ctx context.Context, nf uint32) bool { - const serverFetchDuration = 45 * time.Second + const serverFetchDuration = 30 * time.Second deadline := time.Unix(int64(nf), 0).Add(serverFetchDuration) totalWait := time.Until(deadline) if totalWait <= 0 { @@ -905,8 +971,8 @@ func (s *Server) refreshMetadataOnly() { s.addLog(fmt.Sprintf("Fetching metadata... (%d active resolvers)", len(fetcher.Resolvers()))) - // If the server's next Telegram fetch is imminent (within 5 s), wait for it first. - if dl := s.nextFetchDeadline(); !dl.IsZero() && time.Until(dl) < 5*time.Second { + // If the server's next Telegram fetch is imminent (within 15 s), wait for it first. + if dl := s.nextFetchDeadline(); !dl.IsZero() && time.Until(dl) < 15*time.Second { s.mu.RLock() nf := s.nextFetch s.mu.RUnlock() @@ -1081,14 +1147,89 @@ func (s *Server) refreshChannel(channelNum int) { s.mu.RLock() fetchNF = s.nextFetch s.mu.RUnlock() - fetchCtx, fetchCancel = context.WithDeadline(ctx, dl) - defer fetchCancel() + + // If the server's next refresh is within 15 seconds, wait for it rather + // than risking a block-version race (metadata says N blocks but the + // server regenerates them mid-download). + if time.Until(dl) < 15*time.Second { + s.addLog("Server refresh imminent — waiting before fetching blocks") + if !s.waitForServerFetch(ctx, fetchNF) { + return + } + // Re-fetch metadata after the server refresh to get fresh block counts. + freshMeta, freshErr := fetcher.FetchMetadata(ctx) + if freshErr != nil { + if ctx.Err() != nil { + s.addLog("Refresh cancelled") + return + } + s.addLog(fmt.Sprintf("Channel %s error refreshing metadata: %v", ch.Name, freshErr)) + return + } + s.mu.Lock() + s.channels = freshMeta.Channels + s.telegramLoggedIn = freshMeta.TelegramLoggedIn + s.nextFetch = freshMeta.NextFetch + s.metaFetchedAt = time.Now() + s.mu.Unlock() + if cache != nil { + _ = cache.PutMetadata(freshMeta) + } + if channelNum < 1 || channelNum > len(freshMeta.Channels) { + return + } + ch = freshMeta.Channels[channelNum-1] + blockCount = int(ch.Blocks) + if blockCount <= 0 { + return + } + } + + // Refresh the deadline after potential wait. + dl = s.nextFetchDeadline() + if !dl.IsZero() { + fetchCtx, fetchCancel = context.WithDeadline(ctx, dl) + defer fetchCancel() + } } + // Fetch blocks with content-hash verification. On hash mismatch (the + // server regenerated blocks between our metadata fetch and block fetch) + // re-fetch metadata and retry up to 2 times. + const maxHashRetries = 2 var msgs []protocol.Message var err error - msgs, err = fetcher.FetchChannel(fetchCtx, channelNum, blockCount) - if err != nil { + for attempt := 0; ; attempt++ { + msgs, err = fetcher.FetchChannelVerified(fetchCtx, channelNum, blockCount, ch.ContentHash) + if err == nil { + break + } + if errors.Is(err, client.ErrContentHashMismatch) && attempt < maxHashRetries { + s.addLog(fmt.Sprintf("Channel %s: block-version race detected, re-fetching metadata (attempt %d/%d)", ch.Name, attempt+1, maxHashRetries)) + freshMeta, freshErr := fetcher.FetchMetadata(ctx) + if freshErr != nil { + s.addLog(fmt.Sprintf("Channel %s error refreshing metadata: %v", ch.Name, freshErr)) + return + } + s.mu.Lock() + s.channels = freshMeta.Channels + s.telegramLoggedIn = freshMeta.TelegramLoggedIn + s.nextFetch = freshMeta.NextFetch + s.metaFetchedAt = time.Now() + s.mu.Unlock() + if cache != nil { + _ = cache.PutMetadata(freshMeta) + } + if channelNum < 1 || channelNum > len(freshMeta.Channels) { + return + } + ch = freshMeta.Channels[channelNum-1] + blockCount = int(ch.Blocks) + if blockCount <= 0 { + return + } + continue // retry with fresh metadata + } if fetchCancel != nil && fetchCtx.Err() == context.DeadlineExceeded { // nextFetch fired mid-download — wait for the server, then re-fetch. fetchCancel() @@ -1251,6 +1392,254 @@ func (s *Server) handleResetResolverStats(w http.ResponseWriter, r *http.Request writeJSON(w, map[string]any{"ok": true}) } +// handleResolverBank manages the shared resolver bank. +// GET: returns all bank resolvers with scores. +// POST: adds resolvers to the bank. +// DELETE: removes specific resolvers from the bank. +func (s *Server) handleResolverBank(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + pl, _ := s.loadProfiles() + if pl == nil { + pl = &ProfileList{} + } + + // Get live stats from the current fetcher. + var liveStats map[string][3]int64 + var activeSet map[string]bool + s.mu.RLock() + if s.fetcher != nil { + liveStats = s.fetcher.ExportStats() + activeSet = make(map[string]bool) + for _, r := range s.fetcher.Resolvers() { + activeSet[r] = true + } + } + s.mu.RUnlock() + if activeSet == nil { + activeSet = make(map[string]bool) + } + + type bankResolver struct { + Addr string `json:"addr"` + Score float64 `json:"score"` + Success int64 `json:"success"` + Failure int64 `json:"failure"` + AvgMs float64 `json:"avgMs"` + Active bool `json:"active"` + } + + var bank []bankResolver + for _, addr := range pl.ResolverBank { + br := bankResolver{Addr: addr, Active: activeSet[addr]} + key := addr + if !strings.Contains(key, ":") { + key += ":53" + } + // Prefer live stats, fall back to saved scores. + if liveStats != nil { + if st, ok := liveStats[key]; ok { + br.Success = st[0] + br.Failure = st[1] + if st[0] > 0 { + br.AvgMs = float64(st[2]) / float64(st[0]) + } + br.Score = computeResolverScore(st[0], st[1], st[2]) + } else if ss, ok := pl.ResolverScores[addr]; ok { + br.Success = ss.Success + br.Failure = ss.Failure + if ss.Success > 0 { + br.AvgMs = float64(ss.TotalMs) / float64(ss.Success) + } + br.Score = computeResolverScore(ss.Success, ss.Failure, ss.TotalMs) + } else { + br.Score = 0.2 + } + } else if ss, ok := pl.ResolverScores[addr]; ok { + br.Success = ss.Success + br.Failure = ss.Failure + if ss.Success > 0 { + br.AvgMs = float64(ss.TotalMs) / float64(ss.Success) + } + br.Score = computeResolverScore(ss.Success, ss.Failure, ss.TotalMs) + } else { + br.Score = 0.2 + } + bank = append(bank, br) + } + + sort.Slice(bank, func(i, j int) bool { return bank[i].Score > bank[j].Score }) + writeJSON(w, map[string]any{"bank": bank, "count": len(pl.ResolverBank)}) + + case http.MethodPost: + var req struct { + Resolvers []string `json:"resolvers"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid JSON", 400) + return + } + pl, _ := s.loadProfiles() + if pl == nil { + pl = &ProfileList{} + } + added := addToBank(pl, req.Resolvers) + if err := s.saveProfiles(pl); err != nil { + http.Error(w, "save failed", 500) + return + } + // Update the fetcher's resolver pool. + s.mu.RLock() + f := s.fetcher + s.mu.RUnlock() + if f != nil { + f.UpdateResolverPool(pl.ResolverBank) + } + writeJSON(w, map[string]any{"ok": true, "added": added, "total": len(pl.ResolverBank)}) + + case http.MethodDelete: + var req struct { + Addrs []string `json:"addrs"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || len(req.Addrs) == 0 { + http.Error(w, "addrs required", 400) + return + } + pl, _ := s.loadProfiles() + if pl == nil { + writeJSON(w, map[string]any{"ok": true, "removed": 0, "remaining": 0}) + return + } + removeSet := make(map[string]bool) + for _, a := range req.Addrs { + removeSet[a] = true + } + filtered := make([]string, 0, len(pl.ResolverBank)) + for _, r := range pl.ResolverBank { + if !removeSet[r] { + filtered = append(filtered, r) + } + } + removed := len(pl.ResolverBank) - len(filtered) + pl.ResolverBank = filtered + for _, a := range req.Addrs { + delete(pl.ResolverScores, a) + } + _ = s.saveProfiles(pl) + s.mu.RLock() + f := s.fetcher + s.mu.RUnlock() + if f != nil { + f.UpdateResolverPool(pl.ResolverBank) + } + writeJSON(w, map[string]any{"ok": true, "removed": removed, "remaining": len(pl.ResolverBank)}) + + default: + http.Error(w, "method not allowed", 405) + } +} + +// handleResolverBankCleanup removes resolvers with score below a threshold. +func (s *Server) handleResolverBankCleanup(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", 405) + return + } + var req struct { + MinScore float64 `json:"minScore"` + DryRun bool `json:"dryRun"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid JSON", 400) + return + } + if req.MinScore <= 0 { + http.Error(w, "minScore must be > 0", 400) + return + } + + pl, _ := s.loadProfiles() + if pl == nil { + writeJSON(w, map[string]any{"ok": true, "removed": 0, "remaining": 0}) + return + } + + // Get live stats for score computation. + var liveStats map[string][3]int64 + s.mu.RLock() + if s.fetcher != nil { + liveStats = s.fetcher.ExportStats() + } + s.mu.RUnlock() + + var filtered []string + removed := 0 + for _, addr := range pl.ResolverBank { + key := addr + if !strings.Contains(key, ":") { + key += ":53" + } + var score float64 + if liveStats != nil { + if st, ok := liveStats[key]; ok { + score = computeResolverScore(st[0], st[1], st[2]) + } else if ss, ok := pl.ResolverScores[addr]; ok { + score = computeResolverScore(ss.Success, ss.Failure, ss.TotalMs) + } else { + score = 0.2 + } + } else if ss, ok := pl.ResolverScores[addr]; ok { + score = computeResolverScore(ss.Success, ss.Failure, ss.TotalMs) + } else { + score = 0.2 + } + if score >= req.MinScore { + filtered = append(filtered, addr) + } else { + removed++ + } + } + + if req.DryRun { + writeJSON(w, map[string]any{"ok": true, "removed": removed, "remaining": len(filtered)}) + return + } + + // Apply the cleanup. + for _, addr := range pl.ResolverBank { + key := addr + if !strings.Contains(key, ":") { + key += ":53" + } + var score float64 + if liveStats != nil { + if st, ok := liveStats[key]; ok { + score = computeResolverScore(st[0], st[1], st[2]) + } else if ss, ok := pl.ResolverScores[addr]; ok { + score = computeResolverScore(ss.Success, ss.Failure, ss.TotalMs) + } else { + score = 0.2 + } + } else if ss, ok := pl.ResolverScores[addr]; ok { + score = computeResolverScore(ss.Success, ss.Failure, ss.TotalMs) + } else { + score = 0.2 + } + if score < req.MinScore { + delete(pl.ResolverScores, addr) + } + } + pl.ResolverBank = filtered + _ = s.saveProfiles(pl) + s.mu.RLock() + f := s.fetcher + s.mu.RUnlock() + if f != nil { + f.UpdateResolverPool(pl.ResolverBank) + } + writeJSON(w, map[string]any{"ok": true, "removed": removed, "remaining": len(filtered)}) +} + func (s *Server) saveConfig(cfg *Config) error { path := filepath.Join(s.dataDir, "config.json") data, err := json.MarshalIndent(cfg, "", " ") @@ -1293,6 +1682,99 @@ func generateID() string { return hex.EncodeToString(b) } +// migrateResolverBank merges per-profile resolvers into the shared bank on first run. +func (s *Server) migrateResolverBank() { + pl, err := s.loadProfiles() + if err != nil || pl == nil { + return + } + if len(pl.ResolverBank) > 0 { + return // already migrated + } + seen := make(map[string]bool) + for _, p := range pl.Profiles { + for _, r := range p.Config.Resolvers { + addr := r + if !strings.Contains(addr, ":") { + addr += ":53" + } + if !seen[addr] { + seen[addr] = true + pl.ResolverBank = append(pl.ResolverBank, addr) + } + } + } + if len(pl.ResolverBank) > 0 { + for i := range pl.Profiles { + pl.Profiles[i].Config.Resolvers = nil + } + _ = s.saveProfiles(pl) + } +} + +// addToBank adds resolvers to the shared bank (deduplicated, normalized with :53). +func addToBank(pl *ProfileList, resolvers []string) int { + seen := make(map[string]bool) + for _, r := range pl.ResolverBank { + seen[r] = true + } + added := 0 + for _, r := range resolvers { + addr := r + if !strings.Contains(addr, ":") { + addr += ":53" + } + if !seen[addr] { + seen[addr] = true + pl.ResolverBank = append(pl.ResolverBank, addr) + added++ + } + } + return added +} + +// persistResolverScores saves the current fetcher stats to profiles.json. +func (s *Server) persistResolverScores(stats map[string][3]int64) { + if len(stats) == 0 { + return + } + pl, err := s.loadProfiles() + if err != nil || pl == nil { + return + } + if pl.ResolverScores == nil { + pl.ResolverScores = make(map[string]*SavedResolverScore) + } + for addr, st := range stats { + pl.ResolverScores[addr] = &SavedResolverScore{ + Success: st[0], + Failure: st[1], + TotalMs: st[2], + } + } + _ = s.saveProfiles(pl) +} + +// computeResolverScore mirrors the scoring formula from fetcher.go. +func computeResolverScore(success, failure, totalMs int64) float64 { + total := success + failure + if total == 0 { + return 0.2 + } + successRate := float64(success) / float64(total) + var avgMs float64 + if success > 0 { + avgMs = float64(totalMs) / float64(success) + } else { + avgMs = 30000 + } + score := successRate * successRate / (avgMs/5000.0 + 1.0) + if score < 0.001 { + score = 0.001 + } + return score +} + // handleProfiles manages CRUD for config profiles. // GET: returns profile list. POST: create/update/delete profiles. func (s *Server) handleProfiles(w http.ResponseWriter, r *http.Request) { @@ -1339,6 +1821,11 @@ func (s *Server) handleProfiles(w http.ResponseWriter, r *http.Request) { if req.Profile.Nickname == "" { req.Profile.Nickname = req.Profile.Config.Domain } + // Move resolvers to the shared bank. + if len(req.Profile.Config.Resolvers) > 0 { + addToBank(pl, req.Profile.Config.Resolvers) + req.Profile.Config.Resolvers = nil + } pl.Profiles = append(pl.Profiles, req.Profile) if len(pl.Profiles) == 1 { pl.Active = req.Profile.ID @@ -1348,6 +1835,11 @@ func (s *Server) handleProfiles(w http.ResponseWriter, r *http.Request) { case "update": for i, p := range pl.Profiles { if p.ID == req.Profile.ID { + // Move resolvers to the shared bank. + if len(req.Profile.Config.Resolvers) > 0 { + addToBank(pl, req.Profile.Config.Resolvers) + req.Profile.Config.Resolvers = nil + } pl.Profiles[i] = req.Profile if p.ID == pl.Active { needsReinit = true diff --git a/scripts/install.sh b/scripts/install.sh index 10207c4..70584d9 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -282,7 +282,7 @@ THEFEED_DOMAIN=${domain} THEFEED_KEY=${passkey} THEFEED_ALLOW_MANAGE=${allow_manage} THEFEED_MSG_LIMIT=${msg_limit} -THEFEED_X_RSS_INSTANCES=http://nitter.net,https://nitter.net +THEFEED_X_RSS_INSTANCES=https://nitter.net,http://nitter.net TELEGRAM_API_ID=${api_id} TELEGRAM_API_HASH=${api_hash} TELEGRAM_PHONE=${phone} @@ -343,7 +343,7 @@ THEFEED_DOMAIN=${domain} THEFEED_KEY=${passkey} THEFEED_ALLOW_MANAGE=${allow_manage} THEFEED_MSG_LIMIT=${msg_limit} -THEFEED_X_RSS_INSTANCES=http://nitter.net,https://nitter.net +THEFEED_X_RSS_INSTANCES=https://nitter.net,http://nitter.net TELEGRAM_API_ID=${api_id} TELEGRAM_API_HASH=${api_hash} TELEGRAM_PHONE=${phone} diff --git a/test/e2e/integrity_e2e_test.go b/test/e2e/integrity_e2e_test.go new file mode 100644 index 0000000..32aa335 --- /dev/null +++ b/test/e2e/integrity_e2e_test.go @@ -0,0 +1,537 @@ +package e2e_test + +import ( + "context" + "errors" + "fmt" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/sartoopjj/thefeed/internal/client" + "github.com/sartoopjj/thefeed/internal/protocol" +) + +// TestE2E_ContentHashVerified_OK verifies that FetchChannelVerified succeeds +// when the block data is consistent with the content hash from metadata. +func TestE2E_ContentHashVerified_OK(t *testing.T) { + domain := "hash.example.com" + passphrase := "hash-ok-test" + channels := []string{"verified"} + + msgs := map[int][]protocol.Message{ + 1: { + {ID: 10, Timestamp: 1700000000, Text: "Message one"}, + {ID: 11, Timestamp: 1700000001, Text: "Message two"}, + {ID: 12, Timestamp: 1700000002, Text: "Message three"}, + }, + } + + resolver, cancel := startDNSServer(t, domain, passphrase, channels, msgs) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + meta, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("fetch metadata: %v", err) + } + + expectedHash := meta.Channels[0].ContentHash + blockCount := int(meta.Channels[0].Blocks) + + fetched, err := fetcher.FetchChannelVerified(context.Background(), 1, blockCount, expectedHash) + if err != nil { + t.Fatalf("FetchChannelVerified: %v", err) + } + if len(fetched) != 3 { + t.Fatalf("expected 3 messages, got %d", len(fetched)) + } + for i, want := range msgs[1] { + if fetched[i].Text != want.Text { + t.Errorf("msg %d: got %q, want %q", i, fetched[i].Text, want.Text) + } + } +} + +// TestE2E_ContentHashMismatch verifies that FetchChannelVerified returns +// ErrContentHashMismatch when given the wrong expected hash. +func TestE2E_ContentHashMismatch(t *testing.T) { + domain := "hash.example.com" + passphrase := "hash-mismatch-test" + channels := []string{"mismatch"} + + msgs := map[int][]protocol.Message{ + 1: { + {ID: 1, Timestamp: 1700000000, Text: "Real message"}, + }, + } + + resolver, cancel := startDNSServer(t, domain, passphrase, channels, msgs) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + meta, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("fetch metadata: %v", err) + } + + // Use a bogus hash — simulates stale metadata or block-version race. + bogusHash := meta.Channels[0].ContentHash ^ 0xDEADBEEF + blockCount := int(meta.Channels[0].Blocks) + + _, err = fetcher.FetchChannelVerified(context.Background(), 1, blockCount, bogusHash) + if !errors.Is(err, client.ErrContentHashMismatch) { + t.Fatalf("expected ErrContentHashMismatch, got %v", err) + } +} + +// TestE2E_BlockVersionRace_DetectedAndRetried simulates the block-version race +// condition: the server updates its blocks between metadata fetch and block +// fetch. The first FetchChannelVerified returns ErrContentHashMismatch, the +// caller re-fetches metadata, and the second call succeeds. +func TestE2E_BlockVersionRace_DetectedAndRetried(t *testing.T) { + domain := "race.example.com" + passphrase := "race-test" + channels := []string{"racechannel"} + + originalMsgs := []protocol.Message{ + {ID: 1, Timestamp: 1700000000, Text: "Original message 1"}, + {ID: 2, Timestamp: 1700000001, Text: "Original message 2"}, + } + + resolver, feed, cancel := startDNSServerEx(t, domain, passphrase, false, channels, map[int][]protocol.Message{ + 1: originalMsgs, + }) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + // Step 1: Fetch metadata (gets block count + content hash for original data). + meta1, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("fetch metadata: %v", err) + } + hash1 := meta1.Channels[0].ContentHash + blockCount1 := int(meta1.Channels[0].Blocks) + + // Step 2: Server updates the channel data — simulates a Telegram refresh. + updatedMsgs := []protocol.Message{ + {ID: 1, Timestamp: 1700000000, Text: "Updated message 1"}, + {ID: 2, Timestamp: 1700000001, Text: "Updated message 2"}, + {ID: 3, Timestamp: 1700000002, Text: "Brand new message 3"}, + } + feed.UpdateChannel(1, updatedMsgs) + + // Step 3: Try fetching with the OLD metadata hash → mismatch detected. + _, err = fetcher.FetchChannelVerified(context.Background(), 1, blockCount1, hash1) + if !errors.Is(err, client.ErrContentHashMismatch) { + t.Fatalf("expected ErrContentHashMismatch after server update, got %v", err) + } + + // Step 4: Re-fetch metadata and retry — should now succeed. + meta2, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("re-fetch metadata: %v", err) + } + hash2 := meta2.Channels[0].ContentHash + blockCount2 := int(meta2.Channels[0].Blocks) + + if hash2 == hash1 { + t.Fatal("expected content hash to change after server update") + } + + fetched, err := fetcher.FetchChannelVerified(context.Background(), 1, blockCount2, hash2) + if err != nil { + t.Fatalf("FetchChannelVerified after retry: %v", err) + } + if len(fetched) != 3 { + t.Fatalf("expected 3 messages after retry, got %d", len(fetched)) + } + if fetched[2].Text != "Brand new message 3" { + t.Errorf("msg 2 text = %q, want %q", fetched[2].Text, "Brand new message 3") + } +} + +// TestE2E_GCM_RejectsGarbage verifies that AES-GCM authentication catches +// tampered/garbage DNS responses and FetchBlock retries with another attempt. +// This simulates DPI injecting garbage into DNS responses. +func TestE2E_GCM_RejectsGarbage(t *testing.T) { + domain := "gcm.example.com" + passphrase := "gcm-test" + channels := []string{"secure"} + + msgs := map[int][]protocol.Message{ + 1: { + {ID: 1, Timestamp: 1700000000, Text: "Authenticated message"}, + }, + } + + resolver, cancel := startDNSServer(t, domain, passphrase, channels, msgs) + defer cancel() + + // Use the WRONG passphrase for the client → GCM decryption will fail. + fetcher, err := client.NewFetcher(domain, "wrong-passphrase", []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + ctx, cancel2 := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel2() + + // FetchBlock should fail because GCM authentication rejects the data. + _, err = fetcher.FetchBlock(ctx, 0, 0) + if err == nil { + t.Fatal("expected GCM error with wrong passphrase, got nil") + } + // The error should indicate an authentication/cipher failure. + if !strings.Contains(err.Error(), "cipher") && !strings.Contains(err.Error(), "authentication") && !strings.Contains(err.Error(), "integrity") { + t.Logf("error was: %v", err) + // Accept any error — the important thing is it doesn't return garbage data. + } +} + +// TestE2E_DecompressCorruptData verifies that corrupt compressed data +// (simulated by mismatched blocks) returns an error instead of garbage messages. +func TestE2E_DecompressCorruptData(t *testing.T) { + // Directly test the protocol layer: serialize → compress → corrupt → decompress. + msgs := []protocol.Message{ + {ID: 1, Timestamp: 1700000000, Text: "Test message with enough text to trigger compression"}, + {ID: 2, Timestamp: 1700000001, Text: strings.Repeat("Repeated text ", 50)}, + } + + data := protocol.SerializeMessages(msgs) + compressed := protocol.CompressMessages(data) + + // Verify normal decompression works. + decompressed, err := protocol.DecompressMessages(compressed) + if err != nil { + t.Fatalf("normal decompress: %v", err) + } + parsed, err := protocol.ParseMessages(decompressed) + if err != nil { + t.Fatalf("normal parse: %v", err) + } + if len(parsed) != 2 { + t.Fatalf("expected 2 messages, got %d", len(parsed)) + } + + // Corrupt the compressed data (simulate spliced blocks from different versions). + corrupted := make([]byte, len(compressed)) + copy(corrupted, compressed) + // Keep the compression header (byte 0) but garble the deflate stream. + for i := len(corrupted) / 2; i < len(corrupted); i++ { + corrupted[i] ^= 0xFF + } + + _, err = protocol.DecompressMessages(corrupted) + if err == nil { + t.Fatal("expected decompression error on corrupt data, got nil") + } +} + +// TestE2E_InvalidUTF8Filtered verifies that ParseMessages skips messages +// with invalid UTF-8 text (defense-in-depth against garbage data). +func TestE2E_InvalidUTF8Filtered(t *testing.T) { + // Build a raw message stream with: + // - msg 1: valid UTF-8 + // - msg 2: invalid UTF-8 bytes + // - msg 3: valid UTF-8 + validText1 := "Hello world" + invalidText := string([]byte{0x80, 0xBF, 0xFE, 0xFF, 0xC0, 0xAF}) // invalid UTF-8 + validText2 := "Goodbye" + + // Manually serialize. + buf := make([]byte, 0, 200) + appendMsg := func(id uint32, ts uint32, text string) { + h := make([]byte, protocol.MsgHeaderSize) + tb := []byte(text) + h[0] = byte(id >> 24) + h[1] = byte(id >> 16) + h[2] = byte(id >> 8) + h[3] = byte(id) + h[4] = byte(ts >> 24) + h[5] = byte(ts >> 16) + h[6] = byte(ts >> 8) + h[7] = byte(ts) + h[8] = byte(len(tb) >> 8) + h[9] = byte(len(tb)) + buf = append(buf, h...) + buf = append(buf, tb...) + } + + appendMsg(1, 1700000000, validText1) + appendMsg(2, 1700000001, invalidText) + appendMsg(3, 1700000002, validText2) + + parsed, err := protocol.ParseMessages(buf) + if err != nil { + t.Fatalf("ParseMessages: %v", err) + } + + // The invalid-UTF-8 message should be filtered out. + if len(parsed) != 2 { + t.Fatalf("expected 2 valid messages (skipping invalid UTF-8), got %d", len(parsed)) + } + if parsed[0].Text != validText1 { + t.Errorf("msg 0: %q, want %q", parsed[0].Text, validText1) + } + if parsed[1].Text != validText2 { + t.Errorf("msg 1: %q, want %q", parsed[1].Text, validText2) + } +} + +// TestE2E_ServerUpdateMidFetch simulates a scenario where the server updates +// while the client is fetching blocks. Uses a mock fetchFn that triggers a +// server update after fetching the first block. +func TestE2E_ServerUpdateMidFetch(t *testing.T) { + domain := "midfetch.example.com" + passphrase := "midfetch-test" + channels := []string{"live"} + + // Create a channel with enough data to produce multiple blocks. + // Each message needs unique text to defeat deflate compression. + // Serialized: 10 bytes header + ~500 bytes text = ~510 per msg * 30 msgs = ~15KB. + // After compression with unique text, should still be >600 bytes = multiple blocks. + originalMsgs := make([]protocol.Message, 30) + for i := range originalMsgs { + // Use fmt.Sprintf with varying data to make each message truly unique. + originalMsgs[i] = protocol.Message{ + ID: uint32(i + 1), + Timestamp: uint32(1700000000 + i), + Text: fmt.Sprintf("Original message %d with unique content hash=%x payload=%s", i, i*7919, strings.Repeat(fmt.Sprintf("%c", rune('A'+(i%26))), 400)), + } + } + + resolver, feed, cancel := startDNSServerEx(t, domain, passphrase, false, channels, map[int][]protocol.Message{ + 1: originalMsgs, + }) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + // Fetch metadata to get initial state. + meta, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("fetch metadata: %v", err) + } + + initialHash := meta.Channels[0].ContentHash + blockCount := int(meta.Channels[0].Blocks) + if blockCount < 2 { + t.Fatalf("need at least 2 blocks for this test, got %d", blockCount) + } + + // Update the server data after the test has fetched metadata but before + // block fetching completes — simulating the race condition. + updatedMsgs := make([]protocol.Message, 30) + for i := range updatedMsgs { + updatedMsgs[i] = protocol.Message{ + ID: uint32(i + 1), + Timestamp: uint32(1700000000 + i), + Text: fmt.Sprintf("Updated message %d with different content hash=%x payload=%s", i, i*6271, strings.Repeat(fmt.Sprintf("%c", rune('Z'-i%26)), 400)), + } + } + feed.UpdateChannel(1, updatedMsgs) + + // Now fetch with the OLD hash — should detect the mismatch. + _, err = fetcher.FetchChannelVerified(context.Background(), 1, blockCount, initialHash) + if !errors.Is(err, client.ErrContentHashMismatch) { + // If the block count happened to stay the same and the data is coherent + // from the new version, the hash might match the new content. In either + // case, we should NOT get garbage data. + if err != nil { + t.Logf("got error (acceptable): %v", err) + } else { + t.Log("blocks were coherent from new version (no race hit)") + } + return + } + + // Re-fetch metadata and retry. + meta2, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("re-fetch metadata: %v", err) + } + hash2 := meta2.Channels[0].ContentHash + blockCount2 := int(meta2.Channels[0].Blocks) + + fetched, err := fetcher.FetchChannelVerified(context.Background(), 1, blockCount2, hash2) + if err != nil { + t.Fatalf("retry after re-fetch: %v", err) + } + if len(fetched) != 30 { + t.Fatalf("expected 30 messages, got %d", len(fetched)) + } +} + +// TestE2E_FetchBlock_RetriesOnTransientError verifies that FetchBlock retries +// on transient DNS failures (simulating unreliable network/DPI) and eventually +// succeeds when good responses arrive. +func TestE2E_FetchBlock_RetriesOnTransientError(t *testing.T) { + domain := "retry.example.com" + passphrase := "retry-test" + channels := []string{"reliable"} + + msgs := map[int][]protocol.Message{ + 1: { + {ID: 1, Timestamp: 1700000000, Text: "Survives retries"}, + }, + } + + resolver, cancel := startDNSServer(t, domain, passphrase, channels, msgs) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + // Fetch works normally — the resolver is always healthy. + meta, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("fetch metadata: %v", err) + } + + blockCount := int(meta.Channels[0].Blocks) + fetched, err := fetcher.FetchChannelVerified(context.Background(), 1, blockCount, meta.Channels[0].ContentHash) + if err != nil { + t.Fatalf("fetch verified: %v", err) + } + if len(fetched) != 1 || fetched[0].Text != "Survives retries" { + t.Errorf("unexpected messages: %v", fetched) + } +} + +// TestE2E_ContentHash_DetectsEdit verifies that a message edit changes the +// content hash and is detected by FetchChannelVerified. +func TestE2E_ContentHash_DetectsEdit(t *testing.T) { + domain := "edit.example.com" + passphrase := "edit-test" + channels := []string{"editable"} + + msgs := []protocol.Message{ + {ID: 1, Timestamp: 1700000000, Text: "Original text"}, + } + + resolver, feed, cancel := startDNSServerEx(t, domain, passphrase, false, channels, map[int][]protocol.Message{ + 1: msgs, + }) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + meta1, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("fetch metadata: %v", err) + } + + // Edit the message on the server side. + editedMsgs := []protocol.Message{ + {ID: 1, Timestamp: 1700000000, Text: "Edited text"}, + } + feed.UpdateChannel(1, editedMsgs) + + // The old content hash should NOT match the new data. + meta2, err := fetcher.FetchMetadata(context.Background()) + if err != nil { + t.Fatalf("re-fetch metadata: %v", err) + } + + if meta1.Channels[0].ContentHash == meta2.Channels[0].ContentHash { + t.Fatal("expected content hash to change after edit") + } + + // Fetch with the new hash — should succeed. + fetched, err := fetcher.FetchChannelVerified(context.Background(), 1, int(meta2.Channels[0].Blocks), meta2.Channels[0].ContentHash) + if err != nil { + t.Fatalf("FetchChannelVerified: %v", err) + } + if len(fetched) != 1 || fetched[0].Text != "Edited text" { + t.Errorf("expected edited text, got %v", fetched) + } +} + +// TestE2E_RapidServerUpdates verifies that repeated server updates don't cause +// garbage data — every fetch either succeeds with correct data or returns a +// detectable error. +func TestE2E_RapidServerUpdates(t *testing.T) { + domain := "rapid.example.com" + passphrase := "rapid-test" + channels := []string{"changeable"} + + msgs := []protocol.Message{ + {ID: 1, Timestamp: 1700000000, Text: "Version 1"}, + } + + resolver, feed, cancel := startDNSServerEx(t, domain, passphrase, false, channels, map[int][]protocol.Message{ + 1: msgs, + }) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{resolver}) + + // Do 5 rapid update-then-fetch cycles. + var garbageDetected int32 + for v := 1; v <= 5; v++ { + newMsgs := []protocol.Message{ + {ID: uint32(v), Timestamp: uint32(1700000000 + v), Text: strings.Repeat("X", v*100)}, + } + feed.UpdateChannel(1, newMsgs) + + // Re-fetch metadata (always fresh). + meta, metaErr := fetcher.FetchMetadata(context.Background()) + if metaErr != nil { + t.Fatalf("v%d fetch metadata: %v", v, metaErr) + } + + ch := meta.Channels[0] + fetched, fetchErr := fetcher.FetchChannelVerified(context.Background(), 1, int(ch.Blocks), ch.ContentHash) + if fetchErr != nil { + if errors.Is(fetchErr, client.ErrContentHashMismatch) { + atomic.AddInt32(&garbageDetected, 1) + // Acceptable — detected and caller would retry. + continue + } + t.Fatalf("v%d fetch error: %v", v, fetchErr) + } + + // If fetch succeeded, verify no garbage. + if len(fetched) != 1 { + t.Fatalf("v%d expected 1 message, got %d", v, len(fetched)) + } + if fetched[0].ID != uint32(v) { + t.Errorf("v%d message ID = %d, want %d", v, fetched[0].ID, v) + } + } + + t.Logf("race mismatch detected %d/5 times (all handled correctly)", garbageDetected) +} diff --git a/test/e2e/web_e2e_test.go b/test/e2e/web_e2e_test.go index bd53b9a..1527a44 100644 --- a/test/e2e/web_e2e_test.go +++ b/test/e2e/web_e2e_test.go @@ -604,3 +604,205 @@ func TestE2E_WebUI_NewFeatures(t *testing.T) { } } } + +// ===== RESOLVER BANK TESTS ===== + +func TestE2E_ResolverBank_EmptyByDefault(t *testing.T) { + base, _ := startWebServer(t) + + resp := getJSON(t, base+"/api/resolvers/bank") + m := decodeJSON(t, resp) + if resp.StatusCode != 200 { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + count, _ := m["count"].(float64) + if count != 0 { + t.Errorf("expected 0 bank resolvers, got %v", count) + } +} + +func TestE2E_ResolverBank_AddResolvers(t *testing.T) { + base, _ := startWebServer(t) + + body := `{"resolvers":["8.8.8.8","1.1.1.1","8.8.8.8"]}` + resp := postJSON(t, base+"/api/resolvers/bank", body) + m := decodeJSON(t, resp) + if resp.StatusCode != 200 { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + // 8.8.8.8 appears twice but should be deduplicated + added, _ := m["added"].(float64) + total, _ := m["total"].(float64) + if added != 2 { + t.Errorf("expected 2 added, got %v", added) + } + if total != 2 { + t.Errorf("expected 2 total, got %v", total) + } + + // Verify via GET + resp2 := getJSON(t, base+"/api/resolvers/bank") + m2 := decodeJSON(t, resp2) + count, _ := m2["count"].(float64) + if count != 2 { + t.Errorf("GET bank: expected 2, got %v", count) + } +} + +func TestE2E_ResolverBank_DeleteResolvers(t *testing.T) { + base, _ := startWebServer(t) + + // Add some resolvers first + postJSON(t, base+"/api/resolvers/bank", `{"resolvers":["8.8.8.8","1.1.1.1","4.4.4.4"]}`).Body.Close() + + // Delete one + req, _ := http.NewRequest(http.MethodDelete, base+"/api/resolvers/bank", + strings.NewReader(`{"addrs":["8.8.8.8:53"]}`)) + req.Header.Set("Content-Type", "application/json") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("DELETE: %v", err) + } + defer resp.Body.Close() + var m map[string]any + json.NewDecoder(resp.Body).Decode(&m) + if resp.StatusCode != 200 { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + removed, _ := m["removed"].(float64) + remaining, _ := m["remaining"].(float64) + if removed != 1 { + t.Errorf("expected 1 removed, got %v", removed) + } + if remaining != 2 { + t.Errorf("expected 2 remaining, got %v", remaining) + } +} + +func TestE2E_ResolverBank_CleanupDryRun(t *testing.T) { + base, _ := startWebServer(t) + + // Add resolvers + postJSON(t, base+"/api/resolvers/bank", `{"resolvers":["8.8.8.8","1.1.1.1"]}`).Body.Close() + + // Dry-run cleanup with high threshold (should remove all, since they have no stats → score 0.2) + resp := postJSON(t, base+"/api/resolvers/bank/cleanup", `{"minScore":0.5,"dryRun":true}`) + m := decodeJSON(t, resp) + if resp.StatusCode != 200 { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + removed, _ := m["removed"].(float64) + remaining, _ := m["remaining"].(float64) + if removed != 2 { + t.Errorf("dryRun: expected 2 removed, got %v", removed) + } + if remaining != 0 { + t.Errorf("dryRun: expected 0 remaining, got %v", remaining) + } + + // Verify bank is unchanged (dry run) + resp2 := getJSON(t, base+"/api/resolvers/bank") + m2 := decodeJSON(t, resp2) + count, _ := m2["count"].(float64) + if count != 2 { + t.Errorf("bank should still have 2 after dry run, got %v", count) + } +} + +func TestE2E_ResolverBank_CleanupApply(t *testing.T) { + base, _ := startWebServer(t) + + // Add resolvers + postJSON(t, base+"/api/resolvers/bank", `{"resolvers":["8.8.8.8","1.1.1.1"]}`).Body.Close() + + // Apply cleanup with threshold below 0.2 → nothing removed (default score is 0.2) + resp := postJSON(t, base+"/api/resolvers/bank/cleanup", `{"minScore":0.1}`) + m := decodeJSON(t, resp) + removed, _ := m["removed"].(float64) + remaining, _ := m["remaining"].(float64) + if removed != 0 { + t.Errorf("expected 0 removed with 0.1 threshold, got %v", removed) + } + if remaining != 2 { + t.Errorf("expected 2 remaining, got %v", remaining) + } +} + +func TestE2E_ResolverBank_MigrationFromProfile(t *testing.T) { + base, _ := startWebServer(t) + + // Create a profile with resolvers — they should be migrated to the bank + body := `{"action":"create","profile":{"id":"","nickname":"TestMigrate","config":{"domain":"test.example","key":"mypass","resolvers":["127.0.0.1:9999"],"queryMode":"single","rateLimit":5}}}` + resp := postJSON(t, base+"/api/profiles", body) + m := decodeJSON(t, resp) + if m["ok"] != true { + t.Fatalf("create profile: ok=%v", m["ok"]) + } + + // The resolvers should now be in the bank + resp2 := getJSON(t, base+"/api/resolvers/bank") + m2 := decodeJSON(t, resp2) + count, _ := m2["count"].(float64) + if count < 1 { + t.Errorf("expected at least 1 resolver in bank after migration, got %v", count) + } + + // The profile should no longer have resolvers + resp3 := getJSON(t, base+"/api/profiles") + m3 := decodeJSON(t, resp3) + profs := m3["profiles"].([]any) + cfg := profs[0].(map[string]any)["config"].(map[string]any) + resolvers := cfg["resolvers"] + if resolvers != nil { + r, ok := resolvers.([]any) + if ok && len(r) > 0 { + t.Errorf("expected profile resolvers to be empty after migration, got %v", resolvers) + } + } +} + +func TestE2E_ResolverBank_ConfigAddsToBank(t *testing.T) { + base, _ := startWebServer(t) + + // POST /api/config with resolvers should add them to the bank + cfg := `{"domain":"test.example.com","key":"testpass","resolvers":["127.0.0.1:19999"],"queryMode":"single","rateLimit":10}` + resp := postJSON(t, base+"/api/config", cfg) + defer resp.Body.Close() + if resp.StatusCode != 200 { + body, _ := io.ReadAll(resp.Body) + t.Fatalf("POST /api/config status=%d body=%s", resp.StatusCode, body) + } + + // Check that bank has the resolver + resp2 := getJSON(t, base+"/api/resolvers/bank") + m2 := decodeJSON(t, resp2) + count, _ := m2["count"].(float64) + if count < 1 { + t.Errorf("expected at least 1 resolver in bank after config POST, got %v", count) + } +} + +func TestE2E_ResolverBank_MethodNotAllowed(t *testing.T) { + base, _ := startWebServer(t) + + req, _ := http.NewRequest(http.MethodPut, base+"/api/resolvers/bank", nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("PUT: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != 405 { + t.Errorf("expected 405, got %d", resp.StatusCode) + } +} + +func TestE2E_ResolverBank_CleanupBadRequest(t *testing.T) { + base, _ := startWebServer(t) + + // Missing or invalid minScore + resp := postJSON(t, base+"/api/resolvers/bank/cleanup", `{"minScore":0}`) + defer resp.Body.Close() + if resp.StatusCode != 400 { + t.Errorf("expected 400 for minScore=0, got %d", resp.StatusCode) + } +}