diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 493ded7..ae95eb0 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,13 +36,20 @@ jobs: goarch: arm64 - goos: windows goarch: amd64 + - goos: android + goarch: arm64 steps: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: go-version: '1.26' + - name: Install UPX + if: matrix.goos == 'linux' || matrix.goos == 'windows' || matrix.goos == 'android' + run: sudo apt-get update && sudo apt-get install -y upx-ucl + - name: Build Server + if: matrix.goos != 'android' env: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} @@ -70,6 +77,15 @@ jobs: if [ "${{ matrix.goos }}" = "windows" ]; then ext=".exe"; fi go build -trimpath -ldflags="${LDFLAGS}" -o build/thefeed-client-${{ matrix.goos }}-${{ matrix.goarch }}${ext} ./cmd/client + - name: Compress with UPX + if: matrix.goos == 'linux' || matrix.goos == 'windows' || matrix.goos == 'android' + run: | + for f in build/*; do + if [ -f "$f" ]; then + upx --best --lzma "$f" || true + fi + done + - name: Upload artifacts uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index 0f3f1c1..258a212 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,5 @@ session.json .env todo + +tmp diff --git a/Makefile b/Makefile index 3ea51ee..c0970dd 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ clean: rm -rf $(BUILD_DIR) # Cross-compilation targets -build-all: build-linux-amd64 build-linux-arm64 build-darwin-amd64 build-darwin-arm64 build-windows-amd64 +build-all: build-linux-amd64 build-linux-arm64 build-darwin-amd64 build-darwin-arm64 build-windows-amd64 build-android-arm64 build-linux-amd64: @mkdir -p $(BUILD_DIR) @@ -70,3 +70,16 @@ build-windows-amd64: @mkdir -p $(BUILD_DIR) GOOS=windows GOARCH=amd64 go build $(GOFLAGS) -o $(BUILD_DIR)/$(BINARY_SERVER)-windows-amd64.exe ./cmd/server GOOS=windows GOARCH=amd64 go build $(GOFLAGS) -o $(BUILD_DIR)/$(BINARY_CLIENT)-windows-amd64.exe ./cmd/client + +build-android-arm64: + @mkdir -p $(BUILD_DIR) + GOOS=android GOARCH=arm64 go build $(GOFLAGS) -o $(BUILD_DIR)/$(BINARY_CLIENT)-android-arm64 ./cmd/client + +# UPX compression (requires upx in PATH) — only for Linux/Windows binaries +upx: + @command -v upx >/dev/null 2>&1 || { echo "upx not found, skipping compression"; exit 0; } + @for f in $(BUILD_DIR)/$(BINARY_SERVER)-linux-* $(BUILD_DIR)/$(BINARY_CLIENT)-linux-* \ + $(BUILD_DIR)/$(BINARY_SERVER)-windows-*.exe $(BUILD_DIR)/$(BINARY_CLIENT)-windows-*.exe \ + $(BUILD_DIR)/$(BINARY_CLIENT)-android-*; do \ + if [ -f "$$f" ]; then echo "UPX: $$f"; upx --best --lzma "$$f" || true; fi \ + done diff --git a/README-FA.md b/README-FA.md new file mode 100644 index 0000000..9034bdf --- /dev/null +++ b/README-FA.md @@ -0,0 +1,211 @@ +
+ +# 🌍 thefeed + +**خواندن کانال‌های تلگرام از طریق DNS — برای اینترنت سانسورشده** + +[English](README.md) | فارسی + +--- + +## thefeed چیست؟ + +thefeed یک سیستم تونل DNS است که به شما اجازه می‌دهد پیام‌های کانال‌های تلگرام را حتی وقتی تلگرام و اینترنت فیلتر شده، بخوانید. تنها چیزی که نیاز دارید **DNS** است — که تقریباً هیچ‌وقت مسدود نمی‌شود. + +``` +┌──────────────┐ درخواست DNS TXT ┌──────────────┐ MTProto ┌──────────┐ +│ کلاینت │ ─────────────▸ │ سرور │ ──────────────────────▸ │ تلگرام │ +│ (رابط وب) │ ◂────────────────────── │ (DNS) │ ◂───────────── │ API │ +└──────────────┘ پاسخ رمزنگاری‌شده └──────────────┘ └──────────┘ +``` + +## ✨ ویژگی‌ها + +### سمت سرور (خارج از ایران) +- اتصال به تلگرام و خواندن پیام‌ کانال‌ها +- سرو داده‌ها به صورت پاسخ DNS TXT رمزنگاری‌شده +- padding تصادفی برای جلوگیری از شناسایی DPI +- ذخیره‌سازی session — یک‌بار لاگین، همیشه اجرا +- پشتیبانی از حالت بدون تلگرام (`--no-telegram`) — خواندن کانال‌های عمومی بدون نیاز به ورود به تلگرام + +### سمت کلاینت (داخل ایران) +- رابط کاربری وب با پشتیبانی RTL/فارسی (فونت وزیرمتن) +- تنظیمات از طریق مرورگر — بدون نیاز به خط فرمان +- ارسال پیام به کانال‌ها و چت‌های خصوصی (نیاز به `--allow-manage` سمت سرور) +- مدیریت کانال‌ها از راه دور (افزودن/حذف کانال‌ها) +- فشرده‌سازی پیام‌ها (deflate) +- محافظت رابط وب با رمز عبور (`--password` سمت کلاینت) +- لاگ زنده درخواست‌های DNS در مرورگر +- کش آفلاین — داده‌ها بعد از قطع اتصال نگه داشته می‌شوند + +### ضد DPI +- **اندازه متغیر پاسخ**: Padding تصادفی (۰-۳۲ بایت) +- **کوئری تک‌برچسب**: رمزنگاری Base32 در یک برچسب DNS +- **شافل Resolver**: توزیع تصادفی کوئری‌ها بین resolverها +- **محدودیت نرخ**: قابل تنظیم برای ترکیب با ترافیک عادی DNS +- **Padding تصادفی کوئری**: ۴ بایت تصادفی در هر درخواست +- **اندازه بلاک متغیر**: بلاک‌های ۴۰۰-۷۰۰ بایت + +## 🔐 رمزنگاری و احراز هویت + +### مدل دو بخشی + +**بخش ۱ — رمز عبور رمزنگاری (`--key`):** روی سرور و کلاینت هر دو لازم است. هر کسی با این رمز می‌تواند همه پیام‌ها (از جمله کانال‌های خصوصی) را بخواند. می‌توانید آن را با دوستان مورد اعتماد به اشتراک بگذارید. + +**بخش ۲ — مدیریت از راه دور (`--allow-manage` سمت سرور):** وقتی فعال باشد، هر کسی با کلید رمزنگاری می‌تواند پیام ارسال کند و کانال‌ها را مدیریت کند. به صورت پیش‌فرض غیرفعال است. + +**رمز عبور وب کلاینت (`--password`):** تمام صفحات رابط وب را با HTTP Basic Auth محافظت می‌کند. این فقط محافظت محلی است. + +### ویژگی‌های امنیتی + +- **AES-256-GCM** برای تمام پاسخ‌ها و پیام‌های ارسالی +- کلیدهای مجزا از طریق HKDF برای کوئری و پاسخ +- Padding تصادفی در هر دو جهت +- بدون state — هر درخواست مستقل است +- بررسی رمز عبور ادمین سمت سرور با مقایسه زمان‌ثابت +- فایل session با مجوز ۰۶۰۰ + +> ⚠️ هرگز رمز عبور رمزنگاری (passphrase) خود را عمومی به اشتراک نگذارید — هر کسی با آن می‌تواند کلاینت خودش را اجرا و تمام پیام‌های شما را بخواند. `--password` سمت کلاینت فقط رابط وب روی دستگاه خودتان را محافظت می‌کند. + +## ⚡ نصب سریع سرور + +```bash +bash <(curl -Ls https://raw.githubusercontent.com/sartoopjj/thefeed/main/scripts/install.sh) +``` + +اسکریپت: +1. آخرین باینری را از GitHub دانلود می‌کند +2. دامنه، رمز عبور، اطلاعات تلگرام و کانال‌ها را می‌پرسد +3. به تلگرام لاگین می‌کند (یک‌بار) +4. سرویس systemd را راه‌اندازی می‌کند + +```bash +# بروزرسانی +sudo bash install.sh + +# لاگین مجدد تلگرام +sudo bash install.sh --login + +# حذف +sudo bash install.sh --uninstall +``` + +## 🖥️ نصب کلاینت + +### لینوکس / macOS / ویندوز +از صفحه [Releases](https://github.com/sartoopjj/thefeed/releases) باینری مناسب سیستم خود را دانلود کنید. + +```bash +# اجرا (مرورگر خودکار باز می‌شود) +./thefeed-client + +# با پورت و دایرکتوری سفارشی +./thefeed-client --port 9090 --data-dir ./mydata + +# با رمز عبور ادمین +./thefeed-client --password "your-password" +``` + +### اندروید (Termux) + +```bash +# نصب Termux از F-Droid +pkg update && pkg install curl + +# دانلود باینری اندروید +curl -Lo thefeed-client https://github.com/sartoopjj/thefeed/releases/latest/download/thefeed-client-android-arm64 +chmod +x thefeed-client + +# اجرا +./thefeed-client +# مرورگر را باز کنید: http://127.0.0.1:8080 +``` + +## ⚙️ تنظیمات DNS + +شما به **دو رکورد DNS** نیاز دارید. فرض کنید IP سرور شما `203.0.113.10` است: + +| نوع | نام | مقدار | +|-----|-----|-------| +| A | `ns.example.com` | `203.0.113.10` | +| NS | `t.example.com` | `ns.example.com` | + +> **توجه:** سرور باید روی پورت ۵۳ پاسخ بدهد. بهتر است روی پورت غیرمحدود (`:5300`) اجرا و با iptables فوروارد کنید: +> +> نام اینترفیس شبکه خود را با `ip a` پیدا کنید و `eth0` را جایگزین کنید: +> ```bash +> sudo iptables -I INPUT -p udp --dport 5300 -j ACCEPT +> sudo iptables -t nat -I PREROUTING -i eth0 -p udp --dport 53 -j REDIRECT --to-ports 5300 +> sudo ip6tables -I INPUT -p udp --dport 5300 -j ACCEPT +> sudo ip6tables -t nat -I PREROUTING -i eth0 -p udp --dport 53 -j REDIRECT --to-ports 5300 +> ``` +> +> برای ماندگار کردن این قوانین بعد از ریبوت: +> ```bash +> sudo apt install iptables-persistent # Debian/Ubuntu +> sudo netfilter-persistent save +> ``` + +## 🛠️ ساخت از سورس + +```bash +# پیش‌نیازها: Go 1.26+ +make build # ساخت سرور و کلاینت +make build-all # کراس‌کامپایل تمام پلتفرم‌ها +make test # اجرای تست‌ها +make upx # فشرده‌سازی باینری‌ها با UPX +``` + +## 📋 پرچم‌های سرور + +| پرچم | پیش‌فرض | توضیح | +|-------|---------|-------| +| `--data-dir` | `./data` | دایرکتوری داده‌ها | +| `--domain` | | دامنه DNS (الزامی) | +| `--key` | | رمز عبور رمزنگاری (الزامی) | +| `--channels` | `{data-dir}/channels.txt` | فایل کانال‌ها | +| `--api-id` | | شناسه API تلگرام | +| `--api-hash` | | هش API تلگرام | +| `--phone` | | شماره تلفن تلگرام | +| `--listen` | `:53` | آدرس شنود DNS | +| `--login-only` | `false` | فقط لاگین به تلگرام | +| `--no-telegram` | `false` | اجرا بدون ورود به تلگرام (فقط کانال‌های عمومی) | +| `--padding` | `32` | حداکثر padding تصادفی | +| `--msg-limit` | `15` | حداکثر تعداد پیام‌ها برای هر کانال تلگرام | +| `--allow-manage` | `false` | فعال‌سازی مدیریت از راه دور (ارسال پیام و مدیریت کانال‌ها) | + +## 📋 پرچم‌های کلاینت + +| پرچم | پیش‌فرض | توضیح | +|-------|---------|-------| +| `--data-dir` | `./thefeeddata` | دایرکتوری داده‌ها | +| `--port` | `8080` | پورت رابط وب | +| `--password` | | رمز عبور ادمین (خالی = بدون احراز هویت) | + +## 📂 فرمت channels.txt + +``` +# خطوط با # کامنت هستند +@VahidOnline +@SomeChannel +``` + +## 🤝 مشارکت + +مشارکت شما خوش‌آمد است! Issue بزنید یا Pull Request بفرستید. + +## 📄 لایسنس + +MIT + +--- + +
+ +**برای ایران آزاد 🇮🇷** + +*هر ایرانی حق دسترسی آزاد به اطلاعات را دارد* + +
+ +
diff --git a/README.md b/README.md index 486cd10..33bc594 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ DNS-based feed reader for Telegram channels. Designed for environments where only DNS queries work. +[English](README.md) | [فارسی](README-FA.md) + ## How It Works ``` @@ -16,37 +18,34 @@ DNS-based feed reader for Telegram channels. Designed for environments where onl - Serves feed data as encrypted DNS TXT responses - Random padding on responses to vary size (anti-DPI) - Session persistence — login once, run forever +- No-Telegram mode (`--no-telegram`) — reads public channels without needing Telegram credentials - All data stored in a single directory **Client** (runs inside censored network): - Browser-based web UI with RTL/Farsi support (VazirMatn font) - Configure via the web UI — no CLI flags needed - Sends encrypted DNS TXT queries via available resolvers -- Single-label base32 encoding (stealthier) or double-label hex -- Rate limiting to respect resolver limits +- Send messages to channels and private chats (requires server `--allow-manage`) +- Channel management (add/remove channels remotely via admin commands) +- Message compression (deflate) for efficient transfer +- Web UI password protection (`--password` on client) +- New message indicators and next-fetch countdown timer +- Channel type badges (Private/Public) +- Media type detection (`[IMAGE]`, `[VIDEO]`, etc.) - Live DNS query log in the browser - All data (config, cache) stored next to the binary ## Anti-DPI Features -- **Variable response size**: Random padding (0-32 bytes) on each DNS response prevents fingerprinting by fixed packet size -- **Single-label queries**: Base32 encoded subdomain in one DNS label (`abc123def.t.example.com`) instead of the more detectable two-label hex pattern -- **Resolver shuffling**: Queries are distributed across resolvers randomly -- **Rate limiting**: Configurable query rate to blend with normal DNS traffic -- **Concurrency limiting**: Max 3 concurrent block fetches to avoid DNS bursts -- **Random query padding**: 4 random bytes in each query payload +- Variable response and query sizes to prevent fingerprinting +- Multiple query encoding modes for stealth +- Resolver shuffling and rate limiting +- Background noise traffic +- Message compression to minimize query count ## Protocol -**Block size**: 180 bytes payload (fits in 512-byte UDP DNS with padding + encryption overhead) - -**Query format** (single-label, default): `[base32_encrypted].t.example.com` -**Query format** (double-label): `[hex_part1].[hex_part2].t.example.com` -- Payload: 4 random bytes + 2 channel + 2 block = 8 bytes, AES-256-GCM encrypted - -**Response**: `[2-byte length][data][random padding]` → AES-256-GCM encrypted → Base64 - -**Encryption**: AES-256-GCM with HKDF-derived keys from shared passphrase +All communication is encrypted with AES-256 and transmitted via standard DNS TXT queries and responses. Traffic is designed to blend with normal DNS activity. Message data is compressed before encryption. ## Quick Install (Server) @@ -66,21 +65,25 @@ sudo bash install.sh The script will: 1. Download the latest release binary from GitHub -2. Ask for your domain, passphrase, Telegram credentials, channels -3. Login to Telegram interactively (one-time) -4. Set up a systemd service +2. Ask for your domain, passphrase, and channels +3. Ask whether to use Telegram login (recommended: **No** — public channels work without it) +4. If Telegram mode: ask for API credentials and login +5. Set up a systemd service -Update: `sudo bash install.sh` (detects existing config, only updates binary) -Re-login: `sudo bash install.sh --login` -Uninstall: `sudo bash install.sh --uninstall` +Update: +```bash +sudo bash <(curl -Ls https://raw.githubusercontent.com/sartoopjj/thefeed/main/scripts/install.sh) +``` +Re-login: `sudo bash <(curl -Ls ...) --login` +Uninstall: `sudo bash <(curl -Ls ...) --uninstall` ## Manual Setup ### Prerequisites - Go 1.26+ -- Telegram API credentials from https://my.telegram.org - A domain with NS records pointing to your server +- Telegram API credentials from https://my.telegram.org (only if you need private channels) ### Server @@ -106,7 +109,7 @@ make build-server --api-id 12345 \ --api-hash "your-api-hash" \ --phone "+1234567890" \ - --listen ":5300" + --listen ":53" ``` All data files (session, channels) are stored in the `--data-dir` directory (default: `./data`). @@ -126,8 +129,11 @@ Environment variables: `THEFEED_DOMAIN`, `THEFEED_KEY`, `TELEGRAM_API_ID`, `TELE | `--phone` | | Telegram phone number (required) | | `--session` | `{data-dir}/session.json` | Path to Telegram session file | | `--login-only` | `false` | Authenticate to Telegram, save session, exit | +| `--no-telegram` | `false` | Run without Telegram login (public channels only) | | `--listen` | `:5300` | DNS listen address | | `--padding` | `32` | Max random padding bytes (0=disabled) | +| `--msg-limit` | `15` | Maximum messages to fetch per Telegram channel | +| `--allow-manage` | `false` | Allow remote send/channel management (default: disabled) | | `--version` | | Show version and exit | ### Client @@ -141,6 +147,9 @@ make build-client # Custom data directory and port ./build/thefeed-client --data-dir ./mydata --port 9090 + +# With remote management enabled +./build/thefeed-client --password "your-secret" ``` On first run, the client creates a `./thefeeddata/` directory next to where you run it. Open `http://127.0.0.1:8080` in your browser and configure your domain, passphrase, and resolvers through the Settings page. @@ -153,22 +162,41 @@ All configuration, cache, and data files are stored in the data directory. |------|---------|-------------| | `--data-dir` | `./thefeeddata` | Data directory for config, cache | | `--port` | `8080` | Web UI port | +| `--password` | | Password for web UI (empty = no auth) | | `--version` | | Show version and exit | +#### Android (Termux) + +```bash +# Install Termux from F-Droid +pkg update && pkg install curl + +# Download Android binary +curl -Lo thefeed-client https://github.com/sartoopjj/thefeed/releases/latest/download/thefeed-client-android-arm64 +chmod +x thefeed-client +./thefeed-client +# Open in browser: http://127.0.0.1:8080 +``` + ### Web UI The browser-based UI has: -- **Channels sidebar** (left): channel list with selection +- **Channels sidebar** (left): channel list grouped by type (Public/Private) with badges - **Messages panel** (right): messages with native RTL/Farsi rendering (VazirMatn font) +- **Send panel**: send messages to channels and private chats when Telegram is connected +- **New message badges**: visual indicators for channels with new messages +- **Next-fetch timer**: countdown to next automatic refresh +- **Media detection**: `[IMAGE]`, `[VIDEO]`, `[DOCUMENT]` tag highlighting - **Log panel** (bottom): live DNS query log -- **Settings modal**: configure domain, passphrase, resolvers, query mode, rate limit +- **Settings modal**: configure domain, passphrase, resolvers, query mode, rate limit, timeout, debug mode ## Development ```bash -make test # Run tests +make test # Run tests with race detector make build # Build both binaries -make build-all # Cross-compile all platforms +make build-all # Cross-compile all platforms (incl. Android) +make upx # Compress Linux/Windows/Android binaries with UPX make vet # Go vet make fmt # Format code make clean # Remove build artifacts @@ -219,14 +247,28 @@ This delegates all DNS queries for `t.example.com` (and its subdomains) to your ## Security -- All queries and responses are encrypted with AES-256-GCM -- Separate HKDF-derived keys for queries and responses -- Random padding in queries prevents caching and replay -- Random padding in responses prevents DPI size fingerprinting -- No session state — each query is independent +### Two-Part Access Control + +**Encryption passphrase (`--key`):** Required on both server and client. Anyone with this passphrase can read all channel messages (including private channels). You can share it with trusted friends so they can read too. + +**Remote management (`--allow-manage` on server):** When enabled, anyone with the encryption key can also send messages and manage channels. Disabled by default. Only enable on trusted servers. + +**Client web password (`--password`):** Protects all web UI endpoints with HTTP Basic Auth. This is local protection only — it does NOT affect DNS-level access. + +### Security Properties + +- All communication is end-to-end encrypted (AES-256) - Pre-shared passphrase required for both client and server -- Telegram 2FA password is prompted interactively (not stored in CLI args) -- Session file stored with 0600 permissions +- Each query is independent — no session state on the wire +- Random padding in both directions prevents traffic analysis +- Write operations gated by server-side `--allow-manage` flag +- Telegram 2FA password is prompted interactively (never stored in args) +- Session file stored with restricted permissions (0600) + +> **⚠️ Warning:** If you share your passphrase publicly, **anyone** can run their own +> client with your passphrase and read all your messages. There is no way to prevent this. +> The client `--password` flag only protects the web UI on your own machine — it does NOT stop +> others from using the passphrase. **Never share your passphrase publicly.** ## Service Management @@ -247,3 +289,13 @@ sudo bash scripts/install.sh ## License MIT + +--- + +
+ +**For FREE IRAN 🇮🇷** + +*Everyone deserves free access to information* + +
diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..c9ba275 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,158 @@ +# Release Notes + +## What's New + +### Access Control +- **Encryption passphrase** (`--key`): for reading — share with trusted friends +- **Remote management** (`--allow-manage` on server): enables send/channel management — disabled by default +- If `--allow-manage` not set on server, send and admin features are completely disabled +- Client `--password` now protects ALL web endpoints with global HTTP Basic Auth + +### Channel Management +- Add/remove Telegram channels remotely via admin commands through DNS +- Channel management UI in the web frontend (requires `--allow-manage`) +- List/refresh channel configuration from the browser + +### Send Messages +- Send messages to Telegram channels and private chats through the DNS tunnel +- Full-stack implementation: client web UI → DNS query → server → Telegram API +- GCM-encrypted message data split into DNS labels +- Telegram RandomID fix — sending to own channels now works correctly + +### Message Compression +- Deflate compression reduces the number of DNS queries needed +- Backward compatible — clients auto-detect compressed vs raw data +- 1-byte compression header (0x00=raw, 0x01=deflate) + +### Web UI Password +- Protect the web UI with `--password` flag +- HTTP Basic Auth on all endpoints (constant-time comparison) +- Empty password = no authentication (default) + +### Web UI Overhaul +- Channel type badges (Private / Public) +- New message indicator badges +- Next-fetch countdown timer +- Send message panel (when Telegram is connected) +- Media type tag highlighting (`[IMAGE]`, `[VIDEO]`, `[DOCUMENT]`) +- Channels grouped by type in sidebar +- Telegram connection warning banner +- Debug mode enabled by default +- Footer with GitHub link + +### Android Support +- `android/arm64` build target for Termux +- UPX compression for smaller binaries + +### Edit Detection +- Detects message edits even when message count stays the same +- CRC32 content hash per channel transmitted in metadata +- Client skips refresh only when both message ID and content hash match + +### No-Telegram Mode +- Server `--no-telegram` flag for users who can't or don't want to sign in to Telegram +- Reads public channels without needing Telegram API credentials or phone number +- Safer: no credentials stored on the server +- Install script supports no-Telegram setup (recommended by default) + +### Install Script Improvements +- Telegram mode selection during install (no-Telegram recommended by default) +- Update flow: option to switch between Telegram and no-Telegram modes +- Easy one-liner curl commands for update and uninstall +- Passphrase sharing warning: anyone with your passphrase can read your messages + +### Protocol Improvements +- Variable block sizes (400-700 bytes) for anti-DPI +- DNS noise queries at random intervals (10-30s) +- Metadata expansion: NextFetch, TelegramLoggedIn, ChatType, CanSend +- Block retry on transient DNS failures + +### Security Hardening +- HTTP server timeouts (read: 30s, write: 60s, idle: 120s) +- DNS query name length validation for send messages +- Generic error responses (no internal error leakage) +- Constant-time password comparison +- ⚠️ Never share your passphrase publicly — anyone with it can run their own client and read all your messages. `--password` only protects the web UI on your machine + +### Other Improvements +- Auto-open browser on client start +- Server next-fetch timer in protocol metadata +- Skip refresh when no new messages +- Prevent duplicate channel fetches +- Handle invalid passphrase errors gracefully +- Default rate limit: 5 QPS +- Configurable DNS timeout +- Persian README (README-FA.md) + +--- + +
+ +# یادداشت‌های انتشار + +## تغییرات جدید + +### کنترل دسترسی +- **رمز عبور رمزنگاری** (`--key`): برای خواندن — با دوستان مورد اعتماد به اشتراک بگذارید +- **مدیریت از راه دور** (`--allow-manage` سمت سرور): برای ارسال پیام و مدیریت کانال‌ها — به صورت پیش‌فرض غیرفعال +- اگر `--allow-manage` سمت سرور تنظیم نشده باشد، قابلیت‌های ارسال و مدیریت کاملاً غیرفعال هستند +- `--password` سمت کلاینت حالا تمام صفحات وب را با HTTP Basic Auth محافظت می‌کند + +### مدیریت کانال‌ها +- افزودن/حذف کانال‌های تلگرام از راه دور از طریق DNS +- رابط مدیریت کانال‌ها در وب (نیاز به `--allow-manage`) + +### ارسال پیام +- ارسال پیام به کانال‌ها و چت‌های خصوصی تلگرام از طریق تونل DNS +- پیاده‌سازی کامل: رابط وب → درخواست DNS → سرور → API تلگرام +- رفع باگ RandomID — ارسال به کانال‌های خودتان حالا درست کار می‌کند + +### فشرده‌سازی پیام +- فشرده‌سازی deflate تعداد درخواست‌های DNS مورد نیاز را کاهش می‌دهد +- سازگاری عقب‌گرد — کلاینت‌ها داده فشرده و خام را خودکار تشخیص می‌دهند + +### رمز عبور وب +- محافظت از رابط وب با `--password` (تمام صفحات) +- احراز هویت HTTP Basic Auth + +### بازطراحی رابط وب +- نشان‌های نوع کانال (خصوصی / عمومی) +- نشانگر پیام جدید +- تایمر شمارش معکوس دریافت بعدی +- پنل ارسال پیام +- تشخیص نوع رسانه +- دسته‌بندی کانال‌ها بر اساس نوع + +### پشتیبانی اندروید +- باینری `android/arm64` برای Termux +- فشرده‌سازی UPX + +### حالت بدون تلگرام +- پرچم `--no-telegram` برای کاربرانی که نمی‌توانند یا نمی‌خواهند وارد تلگرام شوند +- خواندن کانال‌های عمومی بدون نیاز به ورود به تلگرام +- امن‌تر: هیچ اطلاعات حساسی روی سرور ذخیره نمی‌شود + +### تشخیص ویرایش پیام +- تشخیص ویرایش پیام حتی وقتی تعداد پیام‌ها تغییر نکرده +- هش محتوا برای هر کانال در متادیتا ارسال می‌شود + +### بهبود اسکریپت نصب +- انتخاب حالت تلگرام هنگام نصب (بدون تلگرام پیشنهاد پیش‌فرض) +- امکان تغییر حالت تلگرام هنگام آپدیت +- دستورات curl ساده برای آپدیت و حذف + +### بهبود امنیت +- تایم‌اوت سرور HTTP +- اعتبارسنجی طول نام DNS +- پاسخ‌های خطای عمومی +- ⚠️ هرگز رمز عبور (passphrase) خود را عمومی به اشتراک نگذارید — هر کسی با آن می‌تواند کلاینت خودش را اجرا و تمام پیام‌های شما را بخواند. `--password` فقط رابط وب روی دستگاه خودتان را محافظت می‌کند + +### بهبودهای دیگر +- باز شدن خودکار مرورگر +- رد کردن رفرش وقتی پیام جدیدی نیست +- جلوگیری از دریالت تکراری کانال‌ها +- مدیریت خطای رمز عبور نامعتبر +- محدودیت نرخ پیش‌فرض: ۵ کوئری در ثانیه +- README فارسی + +
diff --git a/cmd/client/main.go b/cmd/client/main.go index f63a8c4..906e0f6 100644 --- a/cmd/client/main.go +++ b/cmd/client/main.go @@ -5,6 +5,8 @@ import ( "fmt" "log" "os" + "os/exec" + "runtime" "github.com/sartoopjj/thefeed/internal/version" "github.com/sartoopjj/thefeed/internal/web" @@ -13,6 +15,7 @@ import ( func main() { dataDir := flag.String("data-dir", "./thefeeddata", "Data directory for config, cache, and sessions") port := flag.Int("port", 8080, "Web UI port") + password := flag.String("password", "", "Admin password for web UI (empty = no auth)") showVersion := flag.Bool("version", false, "Show version and exit") flag.Parse() @@ -21,12 +24,31 @@ func main() { os.Exit(0) } - srv, err := web.New(*dataDir, *port) + srv, err := web.New(*dataDir, *port, *password) if err != nil { log.Fatalf("Failed to start: %v", err) } + // Try to open browser automatically + url := fmt.Sprintf("http://127.0.0.1:%d", *port) + go openBrowser(url) + if err := srv.Run(); err != nil { log.Fatalf("Server error: %v", err) } } + +func openBrowser(url string) { + var cmd *exec.Cmd + switch runtime.GOOS { + case "darwin": + cmd = exec.Command("open", url) + case "linux": + cmd = exec.Command("xdg-open", url) + case "windows": + cmd = exec.Command("rundll32", "url.dll,FileProtocolHandler", url) + default: + return + } + _ = cmd.Start() +} diff --git a/cmd/server/main.go b/cmd/server/main.go index ced94f2..8439352 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -25,13 +25,15 @@ func main() { domain := flag.String("domain", "", "DNS domain (e.g., t.example.com)") key := flag.String("key", "", "Encryption passphrase") channelsFile := flag.String("channels", "", "Path to channels file (default: {data-dir}/channels.txt)") - apiID := flag.String("api-id", "", "Telegram API ID") - apiHash := flag.String("api-hash", "", "Telegram API Hash") - phone := flag.String("phone", "", "Telegram phone number") + apiID := flag.String("api-id", "", "Telegram API ID (optional if --no-telegram)") + apiHash := flag.String("api-hash", "", "Telegram API Hash (optional if --no-telegram)") + phone := flag.String("phone", "", "Telegram phone number (optional if --no-telegram)") loginOnly := flag.Bool("login-only", false, "Authenticate to Telegram, save session, and exit") + noTelegram := flag.Bool("no-telegram", false, "Fetch public channels without Telegram login") sessionPath := flag.String("session", "", "Path to Telegram session file (default: {data-dir}/session.json)") maxPadding := flag.Int("padding", 32, "Max random padding bytes in DNS responses (anti-DPI, 0=disabled)") msgLimit := flag.Int("msg-limit", 15, "Maximum messages to fetch per Telegram channel") + allowManage := flag.Bool("allow-manage", false, "Allow remote channel management and sending via DNS") showVersion := flag.Bool("version", false, "Show version and exit") flag.Parse() @@ -59,6 +61,9 @@ func main() { if *key == "" { *key = os.Getenv("THEFEED_KEY") } + if !*allowManage && os.Getenv("THEFEED_ALLOW_MANAGE") == "1" { + *allowManage = true + } if *apiID == "" { *apiID = os.Getenv("TELEGRAM_API_ID") } @@ -74,20 +79,29 @@ func main() { flag.Usage() os.Exit(1) } - if *apiID == "" || *apiHash == "" || *phone == "" { - fmt.Fprintln(os.Stderr, "Error: --api-id, --api-hash, and --phone are required") - flag.Usage() - os.Exit(1) + + // Telegram credentials are required unless --no-telegram + needTelegram := !*noTelegram + if needTelegram { + if *apiID == "" || *apiHash == "" || *phone == "" { + fmt.Fprintln(os.Stderr, "Error: --api-id, --api-hash, and --phone are required (use --no-telegram to skip)") + flag.Usage() + os.Exit(1) + } } - id, err := strconv.Atoi(*apiID) - if err != nil { - log.Fatalf("Invalid API ID: %v", err) + var id int + if *apiID != "" { + var err error + id, err = strconv.Atoi(*apiID) + if err != nil { + log.Fatalf("Invalid API ID: %v", err) + } } - // Interactive 2FA password prompt — only when --login-only or no existing session + // Interactive 2FA password prompt — only when Telegram is enabled password := os.Getenv("TELEGRAM_PASSWORD") - if password == "" { + if password == "" && needTelegram { hasSession := false if info, statErr := os.Stat(*sessionPath); statErr == nil && info.Size() > 0 { hasSession = true @@ -109,6 +123,8 @@ func main() { ChannelsFile: *channelsFile, MaxPadding: *maxPadding, MsgLimit: *msgLimit, + NoTelegram: *noTelegram, + AllowManage: *allowManage, Telegram: server.TelegramConfig{ APIID: id, APIHash: *apiHash, diff --git a/internal/client/fetcher.go b/internal/client/fetcher.go index f5080d9..b9554d8 100644 --- a/internal/client/fetcher.go +++ b/internal/client/fetcher.go @@ -2,6 +2,8 @@ package client import ( "context" + cryptoRand "crypto/rand" + "encoding/binary" "fmt" "math/rand" "strings" @@ -93,14 +95,11 @@ func (f *Fetcher) SetQueryMode(mode protocol.QueryEncoding) { } // SetActiveResolvers updates the healthy resolver pool. Called by ResolverChecker. -// If the new list is empty, the current pool is unchanged (to avoid blackout during a bad check). func (f *Fetcher) SetActiveResolvers(resolvers []string) { f.mu.Lock() defer f.mu.Unlock() - if len(resolvers) > 0 { - f.activeResolvers = make([]string, len(resolvers)) - copy(f.activeResolvers, resolvers) - } + f.activeResolvers = make([]string, len(resolvers)) + copy(f.activeResolvers, resolvers) } // SetResolvers replaces the full resolver list and resets the active pool. @@ -189,7 +188,7 @@ func (f *Fetcher) runNoise(ctx context.Context) { m := new(dns.Msg) m.SetQuestion(dns.Fqdn(d), dns.TypeA) m.RecursionDesired = true - c.Exchange(m, r) //nolint:errcheck — fire-and-forget noise query + _, _, _ = c.Exchange(m, r) }(resolver, target) } } @@ -339,6 +338,10 @@ func (f *Fetcher) FetchMetadata(ctx context.Context) (*protocol.Metadata, error) // Cancelling ctx immediately aborts any queued or in-flight block fetches. // Each block is retried individually via FetchBlock before the channel fetch fails. func (f *Fetcher) FetchChannel(ctx context.Context, channelNum int, blockCount int) ([]protocol.Message, error) { + return f.fetchChannelBlocks(ctx, channelNum, blockCount, f.FetchBlock) +} + +func (f *Fetcher) fetchChannelBlocks(ctx context.Context, channelNum int, blockCount int, fetchFn func(context.Context, uint16, uint16) ([]byte, error)) ([]protocol.Message, error) { if blockCount <= 0 { return nil, nil } @@ -368,7 +371,7 @@ func (f *Fetcher) FetchChannel(ctx context.Context, channelNum int, blockCount i } defer func() { <-sem }() - data, err := f.FetchBlock(ctx, uint16(channelNum), uint16(idx)) + data, err := fetchFn(ctx, uint16(channelNum), uint16(idx)) results <- blockResult{idx: idx, data: data, err: err} }(i) } @@ -400,7 +403,14 @@ func (f *Fetcher) FetchChannel(ctx context.Context, channelNum int, blockCount i allData = append(allData, b...) } - return protocol.ParseMessages(allData) + // Decompress if data has compression header + decompressed, err := protocol.DecompressMessages(allData) + if err != nil { + // Fall back to raw parse for backward compatibility with uncompressed data + return protocol.ParseMessages(allData) + } + + return protocol.ParseMessages(decompressed) } func (f *Fetcher) queryResolver(ctx context.Context, resolver, qname string) ([]byte, error) { @@ -408,17 +418,9 @@ func (f *Fetcher) queryResolver(ctx context.Context, resolver, qname string) ([] resolver += ":53" } - c := new(dns.Client) - c.Timeout = f.timeout - - m := new(dns.Msg) - m.SetQuestion(dns.Fqdn(qname), dns.TypeTXT) - m.RecursionDesired = true - m.SetEdns0(4096, false) // advertise 4 KiB UDP buffer to avoid response truncation - - resp, _, err := c.ExchangeContext(ctx, m, resolver) + resp, err := f.exchangeResolver(ctx, resolver, qname) if err != nil { - return nil, fmt.Errorf("dns exchange with %s: %w", resolver, err) + return nil, err } if resp.Rcode != dns.RcodeSuccess { @@ -434,3 +436,159 @@ func (f *Fetcher) queryResolver(ctx context.Context, resolver, qname string) ([] return nil, fmt.Errorf("no TXT record in response from %s", resolver) } + +func (f *Fetcher) exchangeResolver(ctx context.Context, resolver, qname string) (*dns.Msg, error) { + resolverCtx, cancel := context.WithTimeout(ctx, f.timeout) + defer cancel() + + c := &dns.Client{Timeout: f.timeout, Net: "udp"} + + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(qname), dns.TypeTXT) + m.RecursionDesired = true + m.SetEdns0(1232, false) + + resp, _, err := c.ExchangeContext(resolverCtx, m, resolver) + if err != nil { + return nil, fmt.Errorf("dns exchange with %s: %w", resolver, err) + } + return resp, nil +} + +func (f *Fetcher) queryUpload(ctx context.Context, qname string) ([]byte, error) { + if err := f.rateWait(ctx); err != nil { + return nil, err + } + + resolvers := f.Resolvers() + if len(resolvers) == 0 { + return nil, fmt.Errorf("no active resolvers") + } + + shuffled := make([]string, len(resolvers)) + copy(shuffled, resolvers) + rand.Shuffle(len(shuffled), func(i, j int) { shuffled[i], shuffled[j] = shuffled[j], shuffled[i] }) + + var lastErr error + for _, resolver := range shuffled { + if ctx.Err() != nil { + return nil, ctx.Err() + } + data, err := f.queryResolver(ctx, resolver, qname) + if err != nil { + lastErr = err + continue + } + return data, nil + } + return nil, lastErr +} + +func splitUploadPayload(data []byte) [][]byte { + chunks := make([][]byte, 0, (len(data)+protocol.MaxUpstreamBlockPayload-1)/protocol.MaxUpstreamBlockPayload) + for len(data) > 0 { + n := protocol.MaxUpstreamBlockPayload + if n > len(data) { + n = len(data) + } + chunk := make([]byte, n) + copy(chunk, data[:n]) + chunks = append(chunks, chunk) + data = data[n:] + } + return chunks +} + +func randomSessionID() (uint16, error) { + var buf [2]byte + for { + if _, err := cryptoRand.Read(buf[:]); err != nil { + return 0, err + } + sessionID := binary.BigEndian.Uint16(buf[:]) + if sessionID != 0 { + return sessionID, nil + } + } +} + +func (f *Fetcher) sendUpstream(ctx context.Context, kind protocol.UpstreamKind, targetChannel uint16, payload []byte) ([]byte, error) { + chunks := splitUploadPayload(payload) + if len(chunks) == 0 { + return nil, fmt.Errorf("empty payload") + } + if len(chunks) > protocol.MaxUpstreamBlocks { + return nil, fmt.Errorf("payload requires too many DNS blocks: %d > %d", len(chunks), protocol.MaxUpstreamBlocks) + } + + sessionID, err := randomSessionID() + if err != nil { + return nil, fmt.Errorf("generate session id: %w", err) + } + + initQname, err := protocol.EncodeUpstreamInitQuery(f.queryKey, protocol.UpstreamInit{ + SessionID: sessionID, + TotalBlocks: uint8(len(chunks)), + Kind: kind, + TargetChannel: uint8(targetChannel), + }, f.domain, f.queryMode) + if err != nil { + return nil, fmt.Errorf("encode upstream init: %w", err) + } + if f.debug { + f.log("[debug] upstream init kind=%d blocks=%d qname=%s", kind, len(chunks), initQname) + } + + data, err := f.queryUpload(ctx, initQname) + if err != nil { + return nil, fmt.Errorf("start upstream session: %w", err) + } + if string(data) != "READY" { + return nil, fmt.Errorf("unexpected upstream init response: %s", string(data)) + } + + for idx, chunk := range chunks { + blockQname, err := protocol.EncodeUpstreamBlockQuery(f.queryKey, sessionID, uint8(idx), chunk, f.domain, f.queryMode) + if err != nil { + return nil, fmt.Errorf("encode upstream block %d: %w", idx, err) + } + if f.debug { + f.log("[debug] upstream block kind=%d idx=%d len=%d qname=%s", kind, idx, len(chunk), blockQname) + } + + data, err = f.queryUpload(ctx, blockQname) + if err != nil { + return nil, fmt.Errorf("upload block %d: %w", idx, err) + } + + if idx+1 < len(chunks) && string(data) != "CONTINUE" { + return nil, fmt.Errorf("unexpected upstream block response: %s", string(data)) + } + } + + return data, nil +} + +// SendMessage sends a text message to the given channel via chunked upstream DNS queries. +// Returns an error if the message is too long or sending fails. +func (f *Fetcher) SendMessage(ctx context.Context, channelNum int, text string) error { + data, err := f.sendUpstream(ctx, protocol.UpstreamKindSend, uint16(channelNum), []byte(text)) + if err != nil { + return fmt.Errorf("send failed: %w", err) + } + if string(data) != "OK" { + return fmt.Errorf("unexpected response: %s", string(data)) + } + return nil +} + +// SendAdminCommand sends an admin command to the server via chunked upstream DNS queries. +// The payload is a single AdminCmd byte followed by the argument string. +func (f *Fetcher) SendAdminCommand(ctx context.Context, cmd protocol.AdminCmd, arg string) (string, error) { + payload := append([]byte{byte(cmd)}, []byte(arg)...) + data, err := f.sendUpstream(ctx, protocol.UpstreamKindAdmin, 0, payload) + if err != nil { + return "", fmt.Errorf("admin command failed: %w", err) + } + return string(data), nil +} diff --git a/internal/client/fetcher_test.go b/internal/client/fetcher_test.go new file mode 100644 index 0000000..d8864ad --- /dev/null +++ b/internal/client/fetcher_test.go @@ -0,0 +1,26 @@ +package client + +import "testing" + +func TestSetActiveResolversAllowsEmpty(t *testing.T) { + fetcher, err := NewFetcher("t.example.com", "test-passphrase", []string{"1.1.1.1:53", "8.8.8.8:53"}) + if err != nil { + t.Fatalf("NewFetcher: %v", err) + } + fetcher.SetActiveResolvers(nil) + if got := fetcher.Resolvers(); len(got) != 0 { + t.Fatalf("len(Resolvers()) = %d, want 0", len(got)) + } +} + +func TestSetActiveResolversReplacesPool(t *testing.T) { + fetcher, err := NewFetcher("t.example.com", "test-passphrase", []string{"1.1.1.1:53", "8.8.8.8:53"}) + if err != nil { + t.Fatalf("NewFetcher: %v", err) + } + fetcher.SetActiveResolvers([]string{"9.9.9.9:53"}) + got := fetcher.Resolvers() + if len(got) != 1 || got[0] != "9.9.9.9:53" { + t.Fatalf("Resolvers() = %v, want [9.9.9.9:53]", got) + } +} diff --git a/internal/client/resolver.go b/internal/client/resolver.go index 5b42234..4f204a8 100644 --- a/internal/client/resolver.go +++ b/internal/client/resolver.go @@ -43,7 +43,7 @@ func (rc *ResolverChecker) SetLogFunc(fn LogFunc) { // ctx controls the lifetime — cancel it to stop the checker. func (rc *ResolverChecker) Start(ctx context.Context) { go func() { - rc.runCheck() + rc.CheckNow() ticker := time.NewTicker(10 * time.Minute) defer ticker.Stop() for { @@ -51,13 +51,14 @@ func (rc *ResolverChecker) Start(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: - rc.runCheck() + rc.CheckNow() } } }() } -func (rc *ResolverChecker) runCheck() { +// CheckNow runs a single resolver health-check pass immediately. +func (rc *ResolverChecker) CheckNow() { resolvers := rc.fetcher.AllResolvers() if len(resolvers) == 0 { return @@ -90,6 +91,10 @@ func (rc *ResolverChecker) runCheck() { wg.Wait() rc.fetcher.SetActiveResolvers(healthy) + if len(healthy) == 0 { + rc.log("Resolver check done: 0/%d healthy", len(resolvers)) + return + } rc.log("Resolver check done: %d/%d healthy", len(healthy), len(resolvers)) } @@ -115,6 +120,7 @@ func (rc *ResolverChecker) checkOne(resolver string) bool { m := new(dns.Msg) m.SetQuestion(dns.Fqdn(qname), dns.TypeTXT) m.RecursionDesired = true + m.SetEdns0(1232, false) resp, _, err := c.Exchange(m, resolver) // We consider the resolver healthy if we get any DNS response back diff --git a/internal/protocol/dns.go b/internal/protocol/dns.go index 193ae96..a5932e1 100644 --- a/internal/protocol/dns.go +++ b/internal/protocol/dns.go @@ -8,15 +8,60 @@ import ( "encoding/hex" "fmt" "math/big" - "strconv" "strings" ) const ( maxDNSLabelLen = 63 maxDNSNameLen = 253 // without trailing dot + + // SendChannel is the special channel number used for upstream message sending. + // When a query has channel == SendChannel, the block field encodes the target + // channel number, and additional data labels carry the encrypted message text. + SendChannel uint16 = 0xFFFE + + // AdminChannel is the special channel number for admin commands (add/remove + // channels, hard refresh). The encrypted payload is "password\ncmd\narg". + AdminChannel uint16 = 0xFFFD + + // UpstreamInitChannel starts a chunked upstream session for admin/send payloads. + UpstreamInitChannel uint16 = 0xFFFC + // UpstreamDataChannel carries one chunk of a chunked upstream session. + UpstreamDataChannel uint16 = 0xFFFB + + // MaxUpstreamBlockPayload keeps uploaded query chunks comfortably below DNS + // name limits across typical domains and resolver paths. + MaxUpstreamBlockPayload = 8 + // MaxUpstreamBlocks bounds the amount of server-side session state. + MaxUpstreamBlocks = 128 ) +// UpstreamKind identifies the payload carried by a chunked upstream session. +type UpstreamKind byte + +const ( + UpstreamKindSend UpstreamKind = 1 + UpstreamKindAdmin UpstreamKind = 2 +) + +// AdminCmd identifies admin commands carried in upstream admin payloads. +type AdminCmd byte + +const ( + AdminCmdAddChannel AdminCmd = 1 + AdminCmdRemoveChannel AdminCmd = 2 + AdminCmdListChannels AdminCmd = 3 + AdminCmdRefresh AdminCmd = 4 +) + +// UpstreamInit describes a chunked upstream session. +type UpstreamInit struct { + SessionID uint16 + TotalBlocks uint8 + Kind UpstreamKind + TargetChannel uint8 +} + // QueryEncoding controls how DNS query subdomains are encoded. type QueryEncoding int @@ -25,9 +70,6 @@ const ( QuerySingleLabel QueryEncoding = iota // QueryMultiLabel uses hex split across multiple DNS labels. QueryMultiLabel - // QueryPlainLabel encodes channel and block as plain decimal text (no query encryption). - // Responses are always encrypted regardless of this setting. - QueryPlainLabel ) var b32 = base32.StdEncoding.WithPadding(base32.NoPadding) @@ -35,20 +77,13 @@ var b32 = base32.StdEncoding.WithPadding(base32.NoPadding) // EncodeQuery creates a DNS query subdomain for the given channel and block. // Single-label (default): [base32_encrypted].domain // Multi-label: [hex_part1].[hex_part2].domain -// Plain-label: cb.domain (no query encryption) -// Responses are always encrypted regardless of mode. +// All queries are encrypted to prevent DPI detection. func EncodeQuery(queryKey [KeySize]byte, channel, block uint16, domain string, mode QueryEncoding) (string, error) { domain = strings.TrimSuffix(domain, ".") if domain == "" { return "", fmt.Errorf("empty domain") } - // Plain text mode: no encryption, just human-readable label. - if mode == QueryPlainLabel { - label := fmt.Sprintf("c%db%d", channel, block) - return joinQName([]string{label}, domain) - } - payload := make([]byte, QueryPayloadSize) if _, err := rand.Read(payload[:QueryPaddingSize]); err != nil { @@ -132,11 +167,6 @@ func DecodeQuery(queryKey [KeySize]byte, qname, domain string) (channel, block u encoded := qname[:len(qname)-len(suffix)] - // Try plain-label first: cb (short, no dots, all decimal) - if ch, blk, ok := parsePlainLabel(encoded); ok { - return ch, blk, nil - } - // Try base32 (single-label: no dots or dots stripped) b32str := strings.ReplaceAll(encoded, ".", "") if ct, e := b32.DecodeString(strings.ToUpper(b32str)); e == nil { @@ -145,34 +175,23 @@ func DecodeQuery(queryKey [KeySize]byte, qname, domain string) (channel, block u // Fall back to hex (multi-label: dots stripped) hexStr := strings.ReplaceAll(encoded, ".", "") - ct, e := hex.DecodeString(hexStr) - if e != nil { - return 0, 0, fmt.Errorf("decode query: invalid encoding") + if ct, e := hex.DecodeString(hexStr); e == nil { + return parseQueryCiphertext(queryKey, ct) } - return parseQueryCiphertext(queryKey, ct) -} -// parsePlainLabel parses the plain-text query format "cb". -// Returns ok=false if the string does not match this pattern. -func parsePlainLabel(s string) (channel, block uint16, ok bool) { - if len(s) < 3 || s[0] != 'c' { - return 0, 0, false + // For multi-label data queries (header_b32.data_hex...), the concatenated + // string is neither valid base32 nor valid hex. Try the first label alone + // — it contains the AES-ECB encrypted header with channel and block. + if parts := strings.SplitN(encoded, ".", 2); len(parts) == 2 { + if ct, e := b32.DecodeString(strings.ToUpper(parts[0])); e == nil { + return parseQueryCiphertext(queryKey, ct) + } + if ct, e := hex.DecodeString(parts[0]); e == nil { + return parseQueryCiphertext(queryKey, ct) + } } - bi := strings.IndexByte(s[1:], 'b') - if bi < 0 { - return 0, 0, false - } - bi++ // adjust for the slice offset - chStr, bStr := s[1:bi], s[bi+1:] - if len(chStr) == 0 || len(bStr) == 0 { - return 0, 0, false - } - ch, err1 := strconv.ParseUint(chStr, 10, 16) - blk, err2 := strconv.ParseUint(bStr, 10, 16) - if err1 != nil || err2 != nil { - return 0, 0, false - } - return uint16(ch), uint16(blk), true + + return 0, 0, fmt.Errorf("decode query: invalid encoding") } func parseQueryCiphertext(queryKey [KeySize]byte, ciphertext []byte) (channel, block uint16, err error) { @@ -228,3 +247,391 @@ func DecodeResponse(responseKey [KeySize]byte, encoded string) ([]byte, error) { } return padded[PadLengthSize : PadLengthSize+dataLen], nil } + +// EncodeSendQuery creates a DNS query that carries an upstream message. +// Format: [header_b32].[data_b32].domain +// The header is a normal encrypted 8-byte query with channel=SendChannel and +// block=targetChannel. The data label contains GCM-encrypted message text. +// Returns an error if the message is too long for a single DNS query. +func EncodeSendQuery(queryKey [KeySize]byte, targetChannel uint16, message []byte, domain string, mode QueryEncoding) (string, error) { + return encodeDataQuery(queryKey, SendChannel, targetChannel, message, domain, mode) +} + +// EncodeAdminQuery creates a DNS query that carries an admin command to the server. +// The payload is a single AdminCmd byte followed by optional argument bytes, +// GCM-encrypted and split across DNS labels. +func EncodeAdminQuery(queryKey [KeySize]byte, cmd AdminCmd, arg []byte, domain string, mode QueryEncoding) (string, error) { + payload := append([]byte{byte(cmd)}, arg...) + return encodeDataQuery(queryKey, AdminChannel, 0, payload, domain, mode) +} + +// encodeDataQuery builds a DNS query carrying encrypted data in additional labels. +func encodeDataQuery(queryKey [KeySize]byte, specialCh, block uint16, data []byte, domain string, mode QueryEncoding) (string, error) { + domain = strings.TrimSuffix(domain, ".") + if domain == "" { + return "", fmt.Errorf("empty domain") + } + if len(data) == 0 { + return "", fmt.Errorf("empty payload") + } + + // Build header + header := make([]byte, QueryPayloadSize) + if _, err := rand.Read(header[:QueryPaddingSize]); err != nil { + return "", fmt.Errorf("random padding: %w", err) + } + binary.BigEndian.PutUint16(header[QueryPaddingSize:], specialCh) + binary.BigEndian.PutUint16(header[QueryPaddingSize+QueryChannelSize:], block) + + encHeader, err := encryptQueryBlock(queryKey, header) + if err != nil { + return "", fmt.Errorf("encrypt header: %w", err) + } + + // Encrypt data with GCM + encData, err := Encrypt(queryKey, data) + if err != nil { + return "", fmt.Errorf("encrypt message: %w", err) + } + + // Encode header and data + headerStr := strings.ToLower(b32.EncodeToString(encHeader)) + dataStr := strings.ToLower(hex.EncodeToString(encData)) + + // Validate total query name fits in DNS limits (253 chars max) + // Each data label adds len+1 (for dot), header adds len+1, domain adds len+1 + totalLen := len(headerStr) + 1 + len(dataStr) + (len(dataStr) / maxDNSLabelLen) + 1 + len(domain) + if totalLen > 253 { + return "", fmt.Errorf("message too large for DNS query (%d chars, max 253)", totalLen) + } + + // Split data into DNS labels (max 63 chars each) + var dataLabels []string + for len(dataStr) > maxDNSLabelLen { + dataLabels = append(dataLabels, dataStr[:maxDNSLabelLen]) + dataStr = dataStr[maxDNSLabelLen:] + } + if len(dataStr) > 0 { + dataLabels = append(dataLabels, dataStr) + } + + // Build query name: header.data1.data2...dataN.domain + allLabels := append([]string{headerStr}, dataLabels...) + return joinQName(allLabels, domain) +} + +// DecodeSendQuery decodes a send-message DNS query. Returns the target channel +// number and decrypted message text. +func DecodeSendQuery(queryKey [KeySize]byte, qname, domain string) (targetChannel uint16, message []byte, err error) { + qname = strings.TrimSuffix(qname, ".") + domain = strings.TrimSuffix(domain, ".") + + suffix := "." + domain + if !strings.HasSuffix(strings.ToLower(qname), strings.ToLower(suffix)) { + return 0, nil, fmt.Errorf("domain mismatch") + } + + encoded := qname[:len(qname)-len(suffix)] + parts := strings.Split(encoded, ".") + if len(parts) < 2 { + return 0, nil, fmt.Errorf("send query needs at least header + data labels") + } + + // Decode header (first label) + headerLabel := parts[0] + headerCT, err := b32.DecodeString(strings.ToUpper(headerLabel)) + if err != nil { + // Try hex fallback + headerCT, err = hex.DecodeString(headerLabel) + if err != nil { + return 0, nil, fmt.Errorf("decode header: %w", err) + } + } + + plaintext, err := decryptQueryBlock(queryKey, headerCT) + if err != nil { + return 0, nil, fmt.Errorf("decrypt header: %w", err) + } + + ch := binary.BigEndian.Uint16(plaintext[QueryPaddingSize:]) + if ch != SendChannel { + return 0, nil, fmt.Errorf("not a send query (channel=%d)", ch) + } + targetChannel = binary.BigEndian.Uint16(plaintext[QueryPaddingSize+QueryChannelSize:]) + + // Decode data labels (remaining labels, concatenated hex) + dataHex := strings.Join(parts[1:], "") + dataCT, err := hex.DecodeString(dataHex) + if err != nil { + return 0, nil, fmt.Errorf("decode data: %w", err) + } + + message, err = Decrypt(queryKey, dataCT) + if err != nil { + return 0, nil, fmt.Errorf("decrypt message: %w", err) + } + + return targetChannel, message, nil +} + +// DecodeAdminQuery decodes an admin command DNS query and returns the command and argument. +func DecodeAdminQuery(queryKey [KeySize]byte, qname, domain string) (cmd AdminCmd, arg []byte, err error) { + qname = strings.TrimSuffix(qname, ".") + domain = strings.TrimSuffix(domain, ".") + + suffix := "." + domain + if !strings.HasSuffix(strings.ToLower(qname), strings.ToLower(suffix)) { + return 0, nil, fmt.Errorf("domain mismatch") + } + + encoded := qname[:len(qname)-len(suffix)] + parts := strings.Split(encoded, ".") + if len(parts) < 2 { + return 0, nil, fmt.Errorf("admin query needs at least header + data labels") + } + + headerLabel := parts[0] + headerCT, err := b32.DecodeString(strings.ToUpper(headerLabel)) + if err != nil { + headerCT, err = hex.DecodeString(headerLabel) + if err != nil { + return 0, nil, fmt.Errorf("decode header: %w", err) + } + } + + plaintext, err := decryptQueryBlock(queryKey, headerCT) + if err != nil { + return 0, nil, fmt.Errorf("decrypt header: %w", err) + } + + ch := binary.BigEndian.Uint16(plaintext[QueryPaddingSize:]) + if ch != AdminChannel { + return 0, nil, fmt.Errorf("not an admin query (channel=%d)", ch) + } + + dataHex := strings.Join(parts[1:], "") + dataCT, err := hex.DecodeString(dataHex) + if err != nil { + return 0, nil, fmt.Errorf("decode data: %w", err) + } + + payload, err := Decrypt(queryKey, dataCT) + if err != nil { + return 0, nil, fmt.Errorf("decrypt payload: %w", err) + } + + if len(payload) == 0 { + return 0, nil, fmt.Errorf("empty admin payload") + } + cmd = AdminCmd(payload[0]) + if len(payload) > 1 { + arg = payload[1:] + } + return cmd, arg, nil +} + +// EncodeUpstreamInitQuery creates a compact single-label query that registers +// a chunked upstream session. All init data is packed into the AES-ECB header: +// +// [0:2] session_id, [2] total_blocks, [3] kind, +// [4:6] channel=UpstreamInitChannel, [6] target_channel, [7] 0 +// +// No GCM data labels — just one 26-char base32 label + domain. +func EncodeUpstreamInitQuery(queryKey [KeySize]byte, init UpstreamInit, domain string, mode QueryEncoding) (string, error) { + if init.SessionID == 0 { + return "", fmt.Errorf("session id is required") + } + if init.TotalBlocks == 0 || int(init.TotalBlocks) > MaxUpstreamBlocks { + return "", fmt.Errorf("invalid block count: %d", init.TotalBlocks) + } + domain = strings.TrimSuffix(domain, ".") + if domain == "" { + return "", fmt.Errorf("empty domain") + } + + payload := make([]byte, QueryPayloadSize) + binary.BigEndian.PutUint16(payload[0:], init.SessionID) + payload[2] = init.TotalBlocks + payload[3] = byte(init.Kind) + binary.BigEndian.PutUint16(payload[QueryPaddingSize:], UpstreamInitChannel) + payload[6] = init.TargetChannel + // payload[7] = 0 (zero-padded) + + encrypted, err := encryptQueryBlock(queryKey, payload) + if err != nil { + return "", fmt.Errorf("encrypt init: %w", err) + } + + encoded := strings.ToLower(b32.EncodeToString(encrypted)) + return joinQName([]string{encoded}, domain) +} + +// DecodeUpstreamInitQuery parses a compact single-label upstream init query. +func DecodeUpstreamInitQuery(queryKey [KeySize]byte, qname, domain string) (*UpstreamInit, error) { + qname = strings.TrimSuffix(qname, ".") + domain = strings.TrimSuffix(domain, ".") + + suffix := "." + domain + if !strings.HasSuffix(strings.ToLower(qname), strings.ToLower(suffix)) { + return nil, fmt.Errorf("domain mismatch") + } + + encoded := qname[:len(qname)-len(suffix)] + label := strings.ReplaceAll(encoded, ".", "") + + ct, err := b32.DecodeString(strings.ToUpper(label)) + if err != nil { + ct, err = hex.DecodeString(label) + if err != nil { + return nil, fmt.Errorf("decode init: %w", err) + } + } + + plaintext, err := decryptQueryBlock(queryKey, ct) + if err != nil { + return nil, fmt.Errorf("decrypt init: %w", err) + } + + ch := binary.BigEndian.Uint16(plaintext[QueryPaddingSize:]) + if ch != UpstreamInitChannel { + return nil, fmt.Errorf("not an upstream init query (channel=%d)", ch) + } + + init := &UpstreamInit{ + SessionID: binary.BigEndian.Uint16(plaintext[0:2]), + TotalBlocks: plaintext[2], + Kind: UpstreamKind(plaintext[3]), + TargetChannel: plaintext[6], + } + if init.SessionID == 0 { + return nil, fmt.Errorf("invalid upstream session id") + } + if init.TotalBlocks == 0 || int(init.TotalBlocks) > MaxUpstreamBlocks { + return nil, fmt.Errorf("invalid upstream block count: %d", init.TotalBlocks) + } + if init.Kind != UpstreamKindSend && init.Kind != UpstreamKindAdmin { + return nil, fmt.Errorf("invalid upstream kind: %d", init.Kind) + } + return init, nil +} + +// EncodeUpstreamBlockQuery encodes one chunk of a chunked upstream payload +// into a single DNS label. The first min(2, len(chunk)) bytes are embedded in +// the AES-ECB header at [6:8] (not covered by integrity check); any remaining +// bytes are appended raw after the 16-byte ciphertext. The upstream payload is +// already GCM-encrypted, so confidentiality is preserved; tampering is caught +// by GCM on reassembly. +// +// Header: [0:2] session_id, [2] index, [3] chunk_len, +// [4:6] channel=UpstreamDataChannel, [6:8] chunk prefix +// Suffix: chunk[2:] (raw, up to 6 bytes) +// +// Max label: 16 + 6 = 22 bytes → 36 base32 chars (fits in 63-char DNS label). +func EncodeUpstreamBlockQuery(queryKey [KeySize]byte, sessionID uint16, index uint8, chunk []byte, domain string, mode QueryEncoding) (string, error) { + if sessionID == 0 { + return "", fmt.Errorf("session id is required") + } + if len(chunk) == 0 { + return "", fmt.Errorf("empty upstream block") + } + if len(chunk) > MaxUpstreamBlockPayload { + return "", fmt.Errorf("upstream block too large: %d > %d", len(chunk), MaxUpstreamBlockPayload) + } + domain = strings.TrimSuffix(domain, ".") + if domain == "" { + return "", fmt.Errorf("empty domain") + } + + header := make([]byte, QueryPayloadSize) + binary.BigEndian.PutUint16(header[0:], sessionID) + header[2] = index + header[3] = byte(len(chunk)) + binary.BigEndian.PutUint16(header[QueryPaddingSize:], UpstreamDataChannel) + + // Embed first 2 bytes of chunk in header[6:8]. + inHeader := len(chunk) + if inHeader > 2 { + inHeader = 2 + } + copy(header[6:], chunk[:inHeader]) + + encHeader, err := encryptQueryBlock(queryKey, header) + if err != nil { + return "", fmt.Errorf("encrypt header: %w", err) + } + + // Append remaining chunk bytes (raw) after the encrypted header. + combined := encHeader + if len(chunk) > 2 { + combined = append(combined, chunk[2:]...) + } + + label := strings.ToLower(b32.EncodeToString(combined)) + + return joinQName([]string{label}, domain) +} + +// DecodeUpstreamBlockQuery decodes one chunk of a chunked upstream payload. +// The first 2 bytes of chunk data live in the encrypted header[6:8]; any +// remaining bytes follow the 16-byte ciphertext as raw bytes. +func DecodeUpstreamBlockQuery(queryKey [KeySize]byte, qname, domain string) (sessionID uint16, index uint8, chunk []byte, err error) { + qname = strings.TrimSuffix(qname, ".") + domain = strings.TrimSuffix(domain, ".") + + suffix := "." + domain + if !strings.HasSuffix(strings.ToLower(qname), strings.ToLower(suffix)) { + return 0, 0, nil, fmt.Errorf("domain mismatch") + } + + encoded := qname[:len(qname)-len(suffix)] + // Single label — no dots expected + label := strings.ReplaceAll(encoded, ".", "") + + raw, err := b32.DecodeString(strings.ToUpper(label)) + if err != nil { + return 0, 0, nil, fmt.Errorf("decode label: %w", err) + } + if len(raw) < 16 { + return 0, 0, nil, fmt.Errorf("block query too short: %d bytes", len(raw)) + } + + plaintext, err := decryptQueryBlock(queryKey, raw[:16]) + if err != nil { + return 0, 0, nil, fmt.Errorf("decrypt header: %w", err) + } + + ch := binary.BigEndian.Uint16(plaintext[QueryPaddingSize:]) + if ch != UpstreamDataChannel { + return 0, 0, nil, fmt.Errorf("not an upstream data query (channel=%d)", ch) + } + + sessionID = binary.BigEndian.Uint16(plaintext[0:2]) + if sessionID == 0 { + return 0, 0, nil, fmt.Errorf("invalid upstream session id") + } + index = plaintext[2] + chunkLen := int(plaintext[3]) + if chunkLen == 0 || chunkLen > MaxUpstreamBlockPayload { + return 0, 0, nil, fmt.Errorf("invalid chunk length: %d", chunkLen) + } + + chunk = make([]byte, chunkLen) + // First min(2, chunkLen) bytes from header[6:8]. + inHeader := chunkLen + if inHeader > 2 { + inHeader = 2 + } + copy(chunk[:inHeader], plaintext[6:6+inHeader]) + + // Remaining bytes from raw suffix after the 16-byte ciphertext. + if chunkLen > 2 { + extra := raw[16:] + need := chunkLen - 2 + if len(extra) < need { + return 0, 0, nil, fmt.Errorf("insufficient data bytes: have %d, need %d", len(extra), need) + } + copy(chunk[2:], extra[:need]) + } + + return sessionID, index, chunk, nil +} diff --git a/internal/protocol/dns_test.go b/internal/protocol/dns_test.go index 92f6b5e..2856671 100644 --- a/internal/protocol/dns_test.go +++ b/internal/protocol/dns_test.go @@ -1,7 +1,6 @@ package protocol import ( - "fmt" "strings" "testing" ) @@ -88,49 +87,12 @@ func TestEncodeQueryTooLongDomain(t *testing.T) { } } -func TestEncodeDecodeQueryPlainLabel(t *testing.T) { +func TestSingleLabelNotConfusedWithHex(t *testing.T) { qk, _, err := DeriveKeys("test-key") if err != nil { t.Fatalf("DeriveKeys: %v", err) } domain := "t.example.com" - tests := []struct { - channel uint16 - block uint16 - }{ - {0, 0}, - {1, 42}, - {255, 65535}, - {3, 100}, - } - for _, tt := range tests { - qname, err := EncodeQuery(qk, tt.channel, tt.block, domain, QueryPlainLabel) - if err != nil { - t.Fatalf("EncodeQuery(%d, %d): %v", tt.channel, tt.block, err) - } - // Label should be "cb" — human readable, no padding hex. - want := fmt.Sprintf("c%db%d.%s", tt.channel, tt.block, domain) - if qname != want { - t.Errorf("got %q, want %q", qname, want) - } - // DecodeQuery must recover channel and block regardless of key. - ch, blk, err := DecodeQuery(qk, qname, domain) - if err != nil { - t.Fatalf("DecodeQuery: %v", err) - } - if ch != tt.channel || blk != tt.block { - t.Errorf("got ch=%d blk=%d, want ch=%d blk=%d", ch, blk, tt.channel, tt.block) - } - } -} - -func TestPlainLabelNotConfusedWithEncrypted(t *testing.T) { - qk, _, err := DeriveKeys("test-key") - if err != nil { - t.Fatalf("DeriveKeys: %v", err) - } - domain := "t.example.com" - // Encode with single-label then check that DecodeQuery does NOT treat it as plain. qname, _ := EncodeQuery(qk, 5, 10, domain, QuerySingleLabel) ch, blk, err := DecodeQuery(qk, qname, domain) if err != nil { @@ -235,3 +197,187 @@ func TestQueryDomainWithTrailingDot(t *testing.T) { t.Errorf("got ch=%d blk=%d, want ch=1 blk=0", ch, blk) } } + +func TestEncodeDecodeSendQuery(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + domain := "t.example.com" + msg := []byte("Hello!") + + qname, err := EncodeSendQuery(qk, 3, msg, domain, QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeSendQuery: %v", err) + } + + targetCh, gotMsg, err := DecodeSendQuery(qk, qname, domain) + if err != nil { + t.Fatalf("DecodeSendQuery: %v", err) + } + if targetCh != 3 { + t.Errorf("targetChannel = %d, want 3", targetCh) + } + if string(gotMsg) != string(msg) { + t.Errorf("message = %q, want %q", string(gotMsg), string(msg)) + } +} + +func TestEncodeDecodeSendQueryNoPassword(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + domain := "t.example.com" + msg := []byte("No password") + + qname, err := EncodeSendQuery(qk, 1, msg, domain, QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeSendQuery: %v", err) + } + + targetCh, gotMsg, err := DecodeSendQuery(qk, qname, domain) + if err != nil { + t.Fatalf("DecodeSendQuery: %v", err) + } + if targetCh != 1 { + t.Errorf("targetChannel = %d, want 1", targetCh) + } + if string(gotMsg) != string(msg) { + t.Errorf("message = %q, want %q", string(gotMsg), string(msg)) + } +} + +func TestEncodeDecodeAdminQuery(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + domain := "t.example.com" + + qname, err := EncodeAdminQuery(qk, AdminCmdAddChannel, []byte("testchan"), domain, QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeAdminQuery: %v", err) + } + + gotCmd, gotArg, err := DecodeAdminQuery(qk, qname, domain) + if err != nil { + t.Fatalf("DecodeAdminQuery: %v", err) + } + if gotCmd != AdminCmdAddChannel { + t.Errorf("cmd = %d, want %d", gotCmd, AdminCmdAddChannel) + } + if string(gotArg) != "testchan" { + t.Errorf("arg = %q, want %q", string(gotArg), "testchan") + } +} + +func TestEncodeDecodeUpstreamInitQuery(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + init := UpstreamInit{ + SessionID: 0x1122, + TotalBlocks: 7, + Kind: UpstreamKindSend, + TargetChannel: 15, + } + qname, err := EncodeUpstreamInitQuery(qk, init, "t.example.com", QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeUpstreamInitQuery: %v", err) + } + // Init query should be a single compact label — no data labels + if strings.Count(strings.TrimSuffix(qname, ".t.example.com"), ".") != 0 { + t.Fatalf("init query should be a single label, got: %s", qname) + } + got, err := DecodeUpstreamInitQuery(qk, qname, "t.example.com") + if err != nil { + t.Fatalf("DecodeUpstreamInitQuery: %v", err) + } + if *got != init { + t.Fatalf("got %+v, want %+v", *got, init) + } +} + +func TestEncodeDecodeUpstreamBlockQuery(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + chunk := strings.Repeat("x", MaxUpstreamBlockPayload) + qname, err := EncodeUpstreamBlockQuery(qk, 0xAABB, 3, []byte(chunk), "t.example.com", QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeUpstreamBlockQuery: %v", err) + } + if len(qname) > 253 { + t.Fatalf("upstream block query too long: %d", len(qname)) + } + sessionID, index, gotChunk, err := DecodeUpstreamBlockQuery(qk, qname, "t.example.com") + if err != nil { + t.Fatalf("DecodeUpstreamBlockQuery: %v", err) + } + if sessionID != 0xAABB { + t.Fatalf("sessionID = %#x, want %#x", sessionID, 0xAABB) + } + if index != 3 { + t.Fatalf("index = %d, want 3", index) + } + if string(gotChunk) != chunk { + t.Fatalf("chunk = %q, want %q", string(gotChunk), chunk) + } +} + +func TestEncodeUpstreamInitQueryRejectsInvalidBlockCount(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + _, err = EncodeUpstreamInitQuery(qk, UpstreamInit{SessionID: 1, TotalBlocks: MaxUpstreamBlocks + 1, Kind: UpstreamKindAdmin}, "t.example.com", QuerySingleLabel) + if err == nil { + t.Fatal("expected invalid block count error") + } +} + +func TestDecodeQueryRoutesUpstreamInitQuery(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + init := UpstreamInit{ + SessionID: 0xBEEF, + TotalBlocks: 1, + Kind: UpstreamKindSend, + TargetChannel: 5, + } + qname, err := EncodeUpstreamInitQuery(qk, init, "t.example.com", QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeUpstreamInitQuery: %v", err) + } + // DecodeQuery must extract the channel from multi-label upstream queries + ch, _, err := DecodeQuery(qk, qname, "t.example.com") + if err != nil { + t.Fatalf("DecodeQuery failed on upstream init query: %v", err) + } + if ch != UpstreamInitChannel { + t.Fatalf("channel = %#x, want %#x (UpstreamInitChannel)", ch, UpstreamInitChannel) + } +} + +func TestDecodeQueryRoutesUpstreamBlockQuery(t *testing.T) { + qk, _, err := DeriveKeys("test-key") + if err != nil { + t.Fatalf("DeriveKeys: %v", err) + } + qname, err := EncodeUpstreamBlockQuery(qk, 0xAABB, 0, []byte("HI"), "t.example.com", QuerySingleLabel) + if err != nil { + t.Fatalf("EncodeUpstreamBlockQuery: %v", err) + } + ch, _, err := DecodeQuery(qk, qname, "t.example.com") + if err != nil { + t.Fatalf("DecodeQuery failed on upstream block query: %v", err) + } + if ch != UpstreamDataChannel { + t.Fatalf("channel = %#x, want %#x (UpstreamDataChannel)", ch, UpstreamDataChannel) + } +} diff --git a/internal/protocol/protocol.go b/internal/protocol/protocol.go index 4c74893..9f42854 100644 --- a/internal/protocol/protocol.go +++ b/internal/protocol/protocol.go @@ -1,9 +1,13 @@ package protocol import ( + "bytes" + "compress/flate" "crypto/rand" "encoding/binary" "fmt" + "hash/crc32" + "io" "math/big" ) @@ -11,8 +15,8 @@ const ( // MinBlockPayload is the minimum decrypted payload per DNS TXT block. MinBlockPayload = 400 // MaxBlockPayload is the maximum decrypted payload per DNS TXT block. - // 700 bytes data + 28 GCM overhead + 2 prefix + 32 padding → ~996 base64 chars. - // Well within the 4096-byte EDNS0 UDP buffer the client advertises. + // 700 bytes data + 28 GCM overhead + 2 prefix + 32 padding → ~1016 base64 chars. + // Fits within the standard 1232-byte EDNS0 UDP buffer (DNS Flag Day 2020). MaxBlockPayload = 700 // DefaultBlockPayload is kept for compatibility; equals MaxBlockPayload. DefaultBlockPayload = MaxBlockPayload @@ -36,10 +40,11 @@ const ( QueryPayloadSize = QueryPaddingSize + QueryChannelSize + QueryBlockSize // 8 // Message header sizes (in the serialized message stream). - MsgIDSize = 4 - MsgTimestampSize = 4 - MsgLengthSize = 2 - MsgHeaderSize = MsgIDSize + MsgTimestampSize + MsgLengthSize // 10 + MsgIDSize = 4 + MsgTimestampSize = 4 + MsgLengthSize = 2 + MsgHeaderSize = MsgIDSize + MsgTimestampSize + MsgLengthSize // 10 + MsgContentHashSize = 4 ) // Media placeholder strings for non-text content. @@ -55,18 +60,31 @@ const ( MediaLocation = "[LOCATION]" ) +// ChatType distinguishes channel types in metadata. +type ChatType uint8 + +const ( + ChatTypeChannel ChatType = 0 // public Telegram channel + ChatTypePrivate ChatType = 1 // private chat / bot +) + // Metadata holds channel 0 data: server info + channel list. type Metadata struct { - Marker [MarkerSize]byte - Timestamp uint32 - Channels []ChannelInfo + Marker [MarkerSize]byte + Timestamp uint32 + NextFetch uint32 // unix timestamp of next server-side fetch (0 = unknown) + TelegramLoggedIn bool // true if server has an active Telegram session + Channels []ChannelInfo } // ChannelInfo describes a single feed channel. type ChannelInfo struct { - Name string - Blocks uint16 - LastMsgID uint32 + Name string + Blocks uint16 + LastMsgID uint32 + ContentHash uint32 // CRC32 of serialized message data; changes on edits + ChatType ChatType // 0=channel, 1=private + CanSend bool // true if server allows sending messages to this chat } // Message represents a single feed message in a channel. @@ -76,12 +94,21 @@ type Message struct { Text string } +// ContentHashOf computes a CRC32 hash of serialized message data. +// This changes when any message is edited, even if IDs stay the same. +func ContentHashOf(msgs []Message) uint32 { + data := SerializeMessages(msgs) + return crc32.ChecksumIEEE(data) +} + // SerializeMetadata encodes metadata into bytes for channel 0 blocks. +// Format: marker(3) + timestamp(4) + nextFetch(4) + flags(1) + channelCount(2) + per-channel data +// Per-channel: nameLen(1) + name + blocks(2) + lastMsgID(4) + contentHash(4) + chatType(1) + flags(1) func SerializeMetadata(m *Metadata) []byte { - // 3 marker + 4 timestamp + 2 channel count + per-channel data - size := MarkerSize + 4 + 2 + // 3 marker + 4 timestamp + 4 nextFetch + 1 flags + 2 channel count + per-channel data + size := MarkerSize + 4 + 4 + 1 + 2 for _, ch := range m.Channels { - size += 1 + len(ch.Name) + 2 + 4 + size += 1 + len(ch.Name) + 2 + 4 + 4 + 1 + 1 // +4 for contentHash } buf := make([]byte, size) off := 0 @@ -92,6 +119,16 @@ func SerializeMetadata(m *Metadata) []byte { binary.BigEndian.PutUint32(buf[off:], m.Timestamp) off += 4 + binary.BigEndian.PutUint32(buf[off:], m.NextFetch) + off += 4 + + var flags byte + if m.TelegramLoggedIn { + flags |= 0x01 + } + buf[off] = flags + off++ + binary.BigEndian.PutUint16(buf[off:], uint16(len(m.Channels))) off += 2 @@ -108,6 +145,16 @@ func SerializeMetadata(m *Metadata) []byte { off += 2 binary.BigEndian.PutUint32(buf[off:], ch.LastMsgID) off += 4 + binary.BigEndian.PutUint32(buf[off:], ch.ContentHash) + off += 4 + buf[off] = byte(ch.ChatType) + off++ + var chFlags byte + if ch.CanSend { + chFlags |= 0x01 + } + buf[off] = chFlags + off++ } return buf @@ -115,7 +162,8 @@ func SerializeMetadata(m *Metadata) []byte { // ParseMetadata decodes metadata from concatenated channel 0 block data. func ParseMetadata(data []byte) (*Metadata, error) { - if len(data) < MarkerSize+4+2 { + // Minimum: marker(3) + timestamp(4) + nextFetch(4) + flags(1) + count(2) = 14 + if len(data) < MarkerSize+4+4+1+2 { return nil, fmt.Errorf("metadata too short: %d bytes", len(data)) } m := &Metadata{} @@ -127,6 +175,13 @@ func ParseMetadata(data []byte) (*Metadata, error) { m.Timestamp = binary.BigEndian.Uint32(data[off:]) off += 4 + m.NextFetch = binary.BigEndian.Uint32(data[off:]) + off += 4 + + flags := data[off] + off++ + m.TelegramLoggedIn = flags&0x01 != 0 + count := binary.BigEndian.Uint16(data[off:]) off += 2 @@ -143,18 +198,27 @@ func ParseMetadata(data []byte) (*Metadata, error) { name := string(data[off : off+nameLen]) off += nameLen - if off+6 > len(data) { + if off+12 > len(data) { return nil, fmt.Errorf("truncated channel info at %d", i) } blocks := binary.BigEndian.Uint16(data[off:]) off += 2 lastID := binary.BigEndian.Uint32(data[off:]) off += 4 + contentHash := binary.BigEndian.Uint32(data[off:]) + off += 4 + chatType := ChatType(data[off]) + off++ + chFlags := data[off] + off++ m.Channels = append(m.Channels, ChannelInfo{ - Name: name, - Blocks: blocks, - LastMsgID: lastID, + Name: name, + Blocks: blocks, + LastMsgID: lastID, + ContentHash: contentHash, + ChatType: chatType, + CanSend: chFlags&0x01 != 0, }) } @@ -245,3 +309,58 @@ func randBlockSize() int { } return MinBlockPayload + int(n.Int64()) } + +const ( + // compressionNone means no compression applied (raw serialized messages). + compressionNone byte = 0x00 + // compressionDeflate means data is deflate-compressed. + compressionDeflate byte = 0x01 +) + +// CompressMessages compresses serialized message data using deflate. +// The output has a 1-byte header (compression type) followed by the payload. +// If compression doesn't reduce size, the raw data is stored instead. +func CompressMessages(data []byte) []byte { + if len(data) == 0 { + return append([]byte{compressionNone}, data...) + } + + var buf bytes.Buffer + w, err := flate.NewWriter(&buf, flate.BestCompression) + if err != nil { + return append([]byte{compressionNone}, data...) + } + w.Write(data) + w.Close() + + compressed := buf.Bytes() + if len(compressed) >= len(data) { + // Compression didn't help — store raw + return append([]byte{compressionNone}, data...) + } + + return append([]byte{compressionDeflate}, compressed...) +} + +// DecompressMessages decompresses data produced by CompressMessages. +// Reads the 1-byte header to determine the compression type. +func DecompressMessages(data []byte) ([]byte, error) { + if len(data) == 0 { + return nil, fmt.Errorf("empty compressed data") + } + + switch data[0] { + case compressionNone: + return data[1:], nil + case compressionDeflate: + r := flate.NewReader(bytes.NewReader(data[1:])) + defer r.Close() + out, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("deflate decompress: %w", err) + } + return out, nil + default: + return nil, fmt.Errorf("unknown compression type: 0x%02x", data[0]) + } +} diff --git a/internal/server/dns.go b/internal/server/dns.go index 17e2d91..9d73501 100644 --- a/internal/server/dns.go +++ b/internal/server/dns.go @@ -1,9 +1,14 @@ package server import ( + "bufio" "context" + "fmt" "log" + "os" "strings" + "sync" + "time" "github.com/miekg/dns" @@ -12,24 +17,44 @@ import ( // DNSServer serves feed data over DNS TXT queries. type DNSServer struct { - domain string - feed *Feed - queryKey [protocol.KeySize]byte - responseKey [protocol.KeySize]byte - listenAddr string - maxPadding int + domain string + feed *Feed + reader *TelegramReader // nil when --no-telegram + queryKey [protocol.KeySize]byte + responseKey [protocol.KeySize]byte + listenAddr string + maxPadding int + allowManage bool // if true, admin/send commands are accepted + channelsFile string // path to channels.txt for admin commands + + sessionsMu sync.Mutex + sessions map[uint16]*uploadSession +} + +type uploadSession struct { + kind protocol.UpstreamKind + targetChannel uint8 + totalBlocks uint8 + blocks [][]byte + received []bool + expiresAt time.Time } // NewDNSServer creates a DNS server for the given domain. -func NewDNSServer(listenAddr, domain string, feed *Feed, queryKey, responseKey [protocol.KeySize]byte, maxPadding int) *DNSServer { - return &DNSServer{ - domain: strings.TrimSuffix(domain, "."), - feed: feed, - queryKey: queryKey, - responseKey: responseKey, - listenAddr: listenAddr, - maxPadding: maxPadding, +func NewDNSServer(listenAddr, domain string, feed *Feed, queryKey, responseKey [protocol.KeySize]byte, maxPadding int, reader *TelegramReader, allowManage bool, channelsFile string) *DNSServer { + s := &DNSServer{ + domain: strings.TrimSuffix(domain, "."), + feed: feed, + reader: reader, + queryKey: queryKey, + responseKey: responseKey, + listenAddr: listenAddr, + maxPadding: maxPadding, + allowManage: allowManage, + channelsFile: channelsFile, + sessions: make(map[uint16]*uploadSession), } + return s } // ListenAndServe starts the DNS server on UDP, shutting down when ctx is cancelled. @@ -78,6 +103,28 @@ func (s *DNSServer) handleQuery(w dns.ResponseWriter, r *dns.Msg) { return } + // Handle upstream init/data queries + switch channel { + case protocol.UpstreamInitChannel: + s.handleUpstreamInitQuery(w, m, q) + return + case protocol.UpstreamDataChannel: + s.handleUpstreamDataQuery(w, m, q) + return + } + + // Handle send-message queries + if channel == protocol.SendChannel { + s.handleSendQuery(w, m, q) + return + } + + // Handle admin command queries + if channel == protocol.AdminChannel { + s.handleAdminQuery(w, m, q) + return + } + data, err := s.feed.GetBlock(int(channel), int(block)) if err != nil { log.Printf("[dns] get block ch=%d blk=%d: %v", channel, block, err) @@ -122,3 +169,381 @@ func splitTXT(s string) []string { } return parts } + +func (s *DNSServer) writeEncodedResponse(w dns.ResponseWriter, m *dns.Msg, name string, data []byte) { + encoded, err := protocol.EncodeResponse(s.responseKey, data, s.maxPadding) + if err != nil { + m.Rcode = dns.RcodeServerFailure + w.WriteMsg(m) + return + } + m.Answer = append(m.Answer, &dns.TXT{ + Hdr: dns.RR_Header{ + Name: name, + Rrtype: dns.TypeTXT, + Class: dns.ClassINET, + Ttl: 1, + }, + Txt: splitTXT(encoded), + }) + w.WriteMsg(m) +} + +func (s *DNSServer) cleanupExpiredSessions(now time.Time) { + for id, sess := range s.sessions { + if now.After(sess.expiresAt) { + delete(s.sessions, id) + } + } +} + +func (s *DNSServer) handleUpstreamInitQuery(w dns.ResponseWriter, m *dns.Msg, q dns.Question) { + if !s.allowManage { + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + + init, err := protocol.DecodeUpstreamInitQuery(s.queryKey, q.Name, s.domain) + if err != nil { + log.Printf("[dns] decode upstream init: %v", err) + m.Rcode = dns.RcodeNameError + w.WriteMsg(m) + return + } + + if init.Kind == protocol.UpstreamKindSend { + if s.reader == nil { + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + } + + now := time.Now() + s.sessionsMu.Lock() + s.cleanupExpiredSessions(now) + s.sessions[init.SessionID] = &uploadSession{ + kind: init.Kind, + targetChannel: init.TargetChannel, + totalBlocks: init.TotalBlocks, + blocks: make([][]byte, init.TotalBlocks), + received: make([]bool, init.TotalBlocks), + expiresAt: now.Add(5 * time.Minute), + } + s.sessionsMu.Unlock() + + s.writeEncodedResponse(w, m, q.Name, []byte("READY")) +} + +func (s *DNSServer) handleUpstreamDataQuery(w dns.ResponseWriter, m *dns.Msg, q dns.Question) { + if !s.allowManage { + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + + sessionID, index, chunk, err := protocol.DecodeUpstreamBlockQuery(s.queryKey, q.Name, s.domain) + if err != nil { + log.Printf("[dns] decode upstream block: %v", err) + m.Rcode = dns.RcodeNameError + w.WriteMsg(m) + return + } + + now := time.Now() + s.sessionsMu.Lock() + s.cleanupExpiredSessions(now) + sess, ok := s.sessions[sessionID] + if !ok || now.After(sess.expiresAt) { + if ok { + delete(s.sessions, sessionID) + } + s.sessionsMu.Unlock() + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + if int(index) >= len(sess.blocks) { + s.sessionsMu.Unlock() + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + if !sess.received[index] { + copied := make([]byte, len(chunk)) + copy(copied, chunk) + sess.blocks[index] = copied + sess.received[index] = true + } + sess.expiresAt = now.Add(5 * time.Minute) + complete := true + for _, received := range sess.received { + if !received { + complete = false + break + } + } + if !complete { + s.sessionsMu.Unlock() + s.writeEncodedResponse(w, m, q.Name, []byte("CONTINUE")) + return + } + + payload := make([]byte, 0) + for _, block := range sess.blocks { + payload = append(payload, block...) + } + delete(s.sessions, sessionID) + s.sessionsMu.Unlock() + + result, err := s.executeUpstreamPayload(sess, payload) + if err != nil { + log.Printf("[dns] upstream execute: %v", err) + m.Rcode = dns.RcodeServerFailure + w.WriteMsg(m) + return + } + + s.writeEncodedResponse(w, m, q.Name, result) +} + +func (s *DNSServer) executeUpstreamPayload(sess *uploadSession, payload []byte) ([]byte, error) { + switch sess.kind { + case protocol.UpstreamKindSend: + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + if err := s.reader.SendMessage(ctx, int(sess.targetChannel), string(payload)); err != nil { + return nil, err + } + return []byte("OK"), nil + case protocol.UpstreamKindAdmin: + if len(payload) == 0 { + return nil, fmt.Errorf("empty admin payload") + } + cmd := protocol.AdminCmd(payload[0]) + arg := "" + if len(payload) > 1 { + arg = string(payload[1:]) + } + + var result string + var err error + switch cmd { + case protocol.AdminCmdAddChannel: + result, err = s.adminAddChannel(arg) + case protocol.AdminCmdRemoveChannel: + result, err = s.adminRemoveChannel(arg) + case protocol.AdminCmdListChannels: + result, err = s.adminListChannels() + case protocol.AdminCmdRefresh: + result, err = s.adminRefresh() + default: + err = fmt.Errorf("unknown command: %d", cmd) + } + if err != nil { + return nil, err + } + return []byte(result), nil + default: + return nil, fmt.Errorf("unknown upstream kind: %d", sess.kind) + } +} + +func (s *DNSServer) handleSendQuery(w dns.ResponseWriter, m *dns.Msg, q dns.Question) { + if !s.allowManage { + log.Printf("[dns] send query rejected: --allow-manage not set") + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + + if s.reader == nil { + log.Printf("[dns] send query rejected: no Telegram reader") + m.Rcode = dns.RcodeServerFailure + w.WriteMsg(m) + return + } + + targetCh, message, err := protocol.DecodeSendQuery(s.queryKey, q.Name, s.domain) + if err != nil { + log.Printf("[dns] decode send query: %v", err) + m.Rcode = dns.RcodeNameError + w.WriteMsg(m) + return + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + if err := s.reader.SendMessage(ctx, int(targetCh), string(message)); err != nil { + log.Printf("[dns] send message to ch %d: %v", targetCh, err) + m.Rcode = dns.RcodeServerFailure + w.WriteMsg(m) + return + } + + // Respond with an ACK TXT record + s.writeEncodedResponse(w, m, q.Name, []byte("OK")) +} + +func (s *DNSServer) handleAdminQuery(w dns.ResponseWriter, m *dns.Msg, q dns.Question) { + if !s.allowManage { + log.Printf("[dns] admin query rejected: --allow-manage not set") + m.Rcode = dns.RcodeRefused + w.WriteMsg(m) + return + } + + cmd, arg, err := protocol.DecodeAdminQuery(s.queryKey, q.Name, s.domain) + if err != nil { + log.Printf("[dns] decode admin query: %v", err) + m.Rcode = dns.RcodeNameError + w.WriteMsg(m) + return + } + + var result string + switch cmd { + case protocol.AdminCmdAddChannel: + result, err = s.adminAddChannel(string(arg)) + case protocol.AdminCmdRemoveChannel: + result, err = s.adminRemoveChannel(string(arg)) + case protocol.AdminCmdListChannels: + result, err = s.adminListChannels() + case protocol.AdminCmdRefresh: + result, err = s.adminRefresh() + default: + err = fmt.Errorf("unknown command: %d", cmd) + } + + if err != nil { + log.Printf("[dns] admin cmd=%d: %v", cmd, err) + m.Rcode = dns.RcodeServerFailure + w.WriteMsg(m) + return + } + + s.writeEncodedResponse(w, m, q.Name, []byte(result)) +} + +func (s *DNSServer) adminAddChannel(username string) (string, error) { + username = strings.TrimSpace(username) + if username == "" { + return "", fmt.Errorf("empty channel name") + } + username = strings.TrimPrefix(username, "@") + + // Check if already exists + existing, err := loadChannelsFromFile(s.channelsFile) + if err != nil { + return "", fmt.Errorf("read channels: %w", err) + } + for _, ch := range existing { + if strings.EqualFold(ch, username) { + return "already exists", nil + } + } + + // Append to file + f, err := os.OpenFile(s.channelsFile, os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return "", fmt.Errorf("open channels file: %w", err) + } + defer f.Close() + if _, err := fmt.Fprintf(f, "\n@%s\n", username); err != nil { + return "", fmt.Errorf("write channel: %w", err) + } + + log.Printf("[admin] added channel @%s", username) + + // Update the live reader and trigger immediate fetch. + if s.reader != nil { + all, _ := loadChannelsFromFile(s.channelsFile) + s.reader.UpdateChannels(all) + s.reader.RequestRefresh() + } + + return "OK", nil +} + +func (s *DNSServer) adminRemoveChannel(username string) (string, error) { + username = strings.TrimSpace(username) + if username == "" { + return "", fmt.Errorf("empty channel name") + } + username = strings.TrimPrefix(username, "@") + + existing, err := loadChannelsFromFile(s.channelsFile) + if err != nil { + return "", fmt.Errorf("read channels: %w", err) + } + + found := false + var remaining []string + for _, ch := range existing { + if strings.EqualFold(ch, username) { + found = true + continue + } + remaining = append(remaining, ch) + } + if !found { + return "not found", nil + } + + // Rewrite file + content := "# Telegram channel usernames (one per line)\n" + for _, ch := range remaining { + content += "@" + ch + "\n" + } + if err := os.WriteFile(s.channelsFile, []byte(content), 0600); err != nil { + return "", fmt.Errorf("write channels: %w", err) + } + + log.Printf("[admin] removed channel @%s", username) + + // Update the live reader and trigger immediate fetch. + if s.reader != nil { + s.reader.UpdateChannels(remaining) + s.reader.RequestRefresh() + } + + return "OK", nil +} + +func (s *DNSServer) adminListChannels() (string, error) { + channels, err := loadChannelsFromFile(s.channelsFile) + if err != nil { + return "", err + } + return strings.Join(channels, "\n"), nil +} + +func (s *DNSServer) adminRefresh() (string, error) { + if s.reader == nil { + return "", fmt.Errorf("no Telegram reader") + } + s.reader.RequestRefresh() + log.Printf("[admin] hard refresh requested") + return "OK", nil +} + +func loadChannelsFromFile(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + + var channels []string + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + channels = append(channels, strings.TrimPrefix(line, "@")) + } + return channels, scanner.Err() +} diff --git a/internal/server/feed.go b/internal/server/feed.go index 152a1fb..a5b7fbe 100644 --- a/internal/server/feed.go +++ b/internal/server/feed.go @@ -11,21 +11,29 @@ import ( // Feed manages the block data for all channels. type Feed struct { - mu sync.RWMutex - marker [protocol.MarkerSize]byte - channels []string - blocks map[int][][]byte - lastIDs map[int]uint32 - metaBlocks [][]byte // cached metadata split into blocks - updated time.Time + mu sync.RWMutex + marker [protocol.MarkerSize]byte + channels []string + blocks map[int][][]byte + lastIDs map[int]uint32 + contentHashes map[int]uint32 + chatTypes map[int]protocol.ChatType + canSend map[int]bool + metaBlocks [][]byte // metadata for all channels + updated time.Time + telegramLoggedIn bool + nextFetch uint32 } // NewFeed creates a new Feed with the given channel names. func NewFeed(channels []string) *Feed { f := &Feed{ - channels: channels, - blocks: make(map[int][][]byte), - lastIDs: make(map[int]uint32), + channels: channels, + blocks: make(map[int][][]byte), + lastIDs: make(map[int]uint32), + contentHashes: make(map[int]uint32), + chatTypes: make(map[int]protocol.ChatType), + canSend: make(map[int]bool), } f.rotateMarker() f.rebuildMetaBlocks() @@ -39,18 +47,21 @@ func (f *Feed) rotateMarker() { // UpdateChannel replaces the messages for a channel, re-serializing into blocks. func (f *Feed) UpdateChannel(channelNum int, msgs []protocol.Message) { data := protocol.SerializeMessages(msgs) - blocks := protocol.SplitIntoBlocks(data) + compressed := protocol.CompressMessages(data) + blocks := protocol.SplitIntoBlocks(compressed) var lastID uint32 if len(msgs) > 0 { lastID = msgs[0].ID } + contentHash := protocol.ContentHashOf(msgs) f.mu.Lock() defer f.mu.Unlock() f.blocks[channelNum] = blocks f.lastIDs[channelNum] = lastID + f.contentHashes[channelNum] = contentHash f.updated = time.Now() f.rotateMarker() f.rebuildMetaBlocks() @@ -76,21 +87,25 @@ func (f *Feed) GetBlock(channel, block int) ([]byte, error) { } func (f *Feed) getMetadataBlock(block int) ([]byte, error) { - if len(f.metaBlocks) == 0 { + blocks := f.metaBlocks + if len(blocks) == 0 { f.rebuildMetaBlocks() + blocks = f.metaBlocks } - if block < 0 || block >= len(f.metaBlocks) { - return nil, fmt.Errorf("metadata block %d out of range (%d blocks)", block, len(f.metaBlocks)) + if block < 0 || block >= len(blocks) { + return nil, fmt.Errorf("metadata block %d out of range (%d blocks)", block, len(blocks)) } - return f.metaBlocks[block], nil + return blocks[block], nil } // rebuildMetaBlocks re-serializes the metadata and splits it into blocks. // Must be called with f.mu held. func (f *Feed) rebuildMetaBlocks() { - meta := &protocol.Metadata{ - Marker: f.marker, - Timestamp: uint32(time.Now().Unix()), + meta := protocol.Metadata{ + Marker: f.marker, + Timestamp: uint32(time.Now().Unix()), + NextFetch: f.nextFetch, + TelegramLoggedIn: f.telegramLoggedIn, } for i, name := range f.channels { @@ -101,14 +116,16 @@ func (f *Feed) rebuildMetaBlocks() { blockCount = uint16(len(blocks)) } meta.Channels = append(meta.Channels, protocol.ChannelInfo{ - Name: name, - Blocks: blockCount, - LastMsgID: f.lastIDs[chNum], + Name: name, + Blocks: blockCount, + LastMsgID: f.lastIDs[chNum], + ContentHash: f.contentHashes[chNum], + ChatType: f.chatTypes[chNum], + CanSend: f.canSend[chNum], }) } - data := protocol.SerializeMetadata(meta) - f.metaBlocks = protocol.SplitIntoBlocks(data) + f.metaBlocks = protocol.SplitIntoBlocks(protocol.SerializeMetadata(&meta)) } // ChannelNames returns the configured channel names. @@ -119,3 +136,43 @@ func (f *Feed) ChannelNames() []string { copy(result, f.channels) return result } + +// SetTelegramLoggedIn sets the flag indicating whether the server has a Telegram session. +func (f *Feed) SetTelegramLoggedIn(loggedIn bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.telegramLoggedIn = loggedIn + f.rebuildMetaBlocks() +} + +// SetNextFetch sets the unix timestamp of the next server-side fetch. +func (f *Feed) SetNextFetch(ts uint32) { + f.mu.Lock() + defer f.mu.Unlock() + f.nextFetch = ts + f.rebuildMetaBlocks() +} + +// SetChatInfo stores the chat type and send capability for a channel. +func (f *Feed) SetChatInfo(channelNum int, chatType protocol.ChatType, canSend bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.chatTypes[channelNum] = chatType + f.canSend[channelNum] = canSend + f.rebuildMetaBlocks() +} + +// IsPrivateChannel returns true if the channel has chatType == ChatTypePrivate. +func (f *Feed) IsPrivateChannel(channelNum int) bool { + f.mu.RLock() + defer f.mu.RUnlock() + return f.chatTypes[channelNum] == protocol.ChatTypePrivate +} + +// SetChannels replaces the channel list and rebuilds metadata. +func (f *Feed) SetChannels(channels []string) { + f.mu.Lock() + defer f.mu.Unlock() + f.channels = channels + f.rebuildMetaBlocks() +} diff --git a/internal/server/feed_test.go b/internal/server/feed_test.go index 09ff433..f92a272 100644 --- a/internal/server/feed_test.go +++ b/internal/server/feed_test.go @@ -20,7 +20,12 @@ func TestFeedUpdateAndGetBlock(t *testing.T) { if len(data) == 0 { t.Error("block data should not be empty") } - parsed, err := protocol.ParseMessages(data) + // Data is now compressed — decompress + parse + decompressed, err := protocol.DecompressMessages(data) + if err != nil { + t.Fatalf("DecompressMessages: %v", err) + } + parsed, err := protocol.ParseMessages(decompressed) if err != nil { t.Fatalf("ParseMessages: %v", err) } @@ -71,20 +76,20 @@ func TestFeedGetBlockUnknownChannel(t *testing.T) { func TestFeedLargeMessages(t *testing.T) { feed := NewFeed([]string{"Test"}) - // Use text large enough to span 2 blocks at DefaultBlockPayload (currently 700 bytes). - // Message serialization overhead is 10 bytes, so we need >690 bytes of text. - largeText := make([]byte, 750) + // With compression, repetitive data compresses to ~1 block. + // Use varied text so compressed size still spans multiple blocks. + largeText := make([]byte, 1500) for i := range largeText { - largeText[i] = 65 + largeText[i] = byte(i % 251) // pseudo-random pattern } msgs := []protocol.Message{{ID: 1, Timestamp: 1700000000, Text: string(largeText)}} feed.UpdateChannel(1, msgs) - _, err := feed.GetBlock(1, 0) + // Should have at least 1 block + data0, err := feed.GetBlock(1, 0) if err != nil { t.Fatalf("GetBlock(1, 0): %v", err) } - _, err = feed.GetBlock(1, 1) - if err != nil { - t.Fatalf("GetBlock(1, 1): %v", err) + if len(data0) == 0 { + t.Error("block data should not be empty") } } diff --git a/internal/server/public.go b/internal/server/public.go new file mode 100644 index 0000000..19c432e --- /dev/null +++ b/internal/server/public.go @@ -0,0 +1,372 @@ +package server + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "sync" + "time" + + "golang.org/x/net/html" + + "github.com/sartoopjj/thefeed/internal/protocol" +) + +// PublicReader fetches recent posts from public Telegram channels via the web view. +type PublicReader struct { + channels []string + feed *Feed + msgLimit int + + client *http.Client + baseURL string + + mu sync.RWMutex + cache map[string]cachedMessages + cacheTTL time.Duration +} + +// NewPublicReader creates a reader for public channels without Telegram login. +func NewPublicReader(channelUsernames []string, feed *Feed, msgLimit int) *PublicReader { + cleaned := make([]string, len(channelUsernames)) + for i, u := range channelUsernames { + cleaned[i] = strings.TrimPrefix(strings.TrimSpace(u), "@") + } + if msgLimit <= 0 { + msgLimit = 15 + } + return &PublicReader{ + channels: cleaned, + feed: feed, + msgLimit: msgLimit, + client: &http.Client{ + Timeout: 30 * time.Second, + }, + baseURL: "https://t.me/s", + cache: make(map[string]cachedMessages), + cacheTTL: 10 * time.Minute, + } +} + +// Run starts the periodic public-channel fetch loop. +func (pr *PublicReader) Run(ctx context.Context) error { + pr.feed.SetTelegramLoggedIn(false) + pr.fetchAll(ctx) + + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + pr.feed.SetNextFetch(uint32(time.Now().Add(5 * time.Minute).Unix())) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + pr.fetchAll(ctx) + pr.feed.SetNextFetch(uint32(time.Now().Add(5 * time.Minute).Unix())) + } + } +} + +func (pr *PublicReader) fetchAll(ctx context.Context) { + for i, username := range pr.channels { + chNum := i + 1 + + pr.mu.RLock() + cached, ok := pr.cache[username] + pr.mu.RUnlock() + if ok && time.Since(cached.fetched) < pr.cacheTTL { + continue + } + + msgs, err := pr.fetchChannel(ctx, username) + if err != nil { + log.Printf("[public] fetch %s: %v", username, err) + continue + } + + // Merge new messages with previously cached ones to accumulate history. + if ok && len(cached.msgs) > 0 { + msgs = mergeMessages(cached.msgs, msgs) + } + if pr.msgLimit > 0 && len(msgs) > pr.msgLimit { + msgs = msgs[:pr.msgLimit] + } + + pr.mu.Lock() + pr.cache[username] = cachedMessages{msgs: msgs, fetched: time.Now()} + pr.mu.Unlock() + + pr.feed.UpdateChannel(chNum, msgs) + pr.feed.SetChatInfo(chNum, protocol.ChatTypeChannel, false) + log.Printf("[public] updated %s: %d messages", username, len(msgs)) + } +} + +func (pr *PublicReader) fetchChannel(ctx context.Context, username string) ([]protocol.Message, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, pr.baseURL+"/"+url.PathEscape(username), nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)") + + resp, err := pr.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected HTTP status: %s", resp.Status) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return parsePublicMessages(body) +} + +type publicMessage struct { + id uint32 + timestamp uint32 + text string +} + +// mergeMessages combines old cached messages with newly fetched ones. +// New messages win on ID conflicts (edits). Result is sorted by ID descending. +func mergeMessages(old, new []protocol.Message) []protocol.Message { + byID := make(map[uint32]protocol.Message, len(old)+len(new)) + for _, m := range old { + byID[m.ID] = m + } + for _, m := range new { + byID[m.ID] = m // new overwrites old (edits) + } + merged := make([]protocol.Message, 0, len(byID)) + for _, m := range byID { + merged = append(merged, m) + } + sort.Slice(merged, func(i, j int) bool { + return merged[i].ID > merged[j].ID + }) + return merged +} + +func parsePublicMessages(body []byte) ([]protocol.Message, error) { + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return nil, fmt.Errorf("parse html: %w", err) + } + + var collected []publicMessage + visitNodes(doc, func(n *html.Node) { + post := attrValue(n, "data-post") + if post == "" { + return + } + id, err := parsePostID(post) + if err != nil { + return + } + text := strings.TrimSpace(extractMessageText(findFirstByClass(n, "tgme_widget_message_text"))) + mediaPrefix := "" + switch { + case findFirstByClass(n, "tgme_widget_message_photo_wrap") != nil: + mediaPrefix = protocol.MediaImage + case findFirstByClass(n, "tgme_widget_message_video_player") != nil || + findFirstByClass(n, "tgme_widget_message_roundvideo_player") != nil: + mediaPrefix = protocol.MediaVideo + case findFirstByClass(n, "tgme_widget_message_sticker_wrap") != nil: + mediaPrefix = protocol.MediaSticker + case findFirstByClass(n, "tgme_widget_message_voice") != nil: + mediaPrefix = protocol.MediaAudio + case findFirstByClass(n, "tgme_widget_message_poll") != nil: + mediaPrefix = protocol.MediaPoll + case findFirstByClass(n, "tgme_widget_message_location_wrap") != nil || + findFirstByClass(n, "tgme_widget_message_venue_wrap") != nil: + mediaPrefix = protocol.MediaLocation + case findFirstByClass(n, "tgme_widget_message_contact_wrap") != nil: + mediaPrefix = protocol.MediaContact + case findFirstByClass(n, "tgme_widget_message_document_wrap") != nil: + mediaPrefix = protocol.MediaFile + } + if mediaPrefix != "" { + if text != "" { + text = mediaPrefix + "\n" + text + } else { + text = mediaPrefix + } + } + if text == "" { + return + } + collected = append(collected, publicMessage{ + id: id, + timestamp: extractMessageTimestamp(n), + text: text, + }) + }) + + if len(collected) == 0 { + return nil, fmt.Errorf("no public messages found") + } + + sort.Slice(collected, func(i, j int) bool { + return collected[i].id > collected[j].id + }) + + msgs := make([]protocol.Message, 0, len(collected)) + for _, msg := range collected { + msgs = append(msgs, protocol.Message{ID: msg.id, Timestamp: msg.timestamp, Text: msg.text}) + } + return msgs, nil +} + +func visitNodes(n *html.Node, fn func(*html.Node)) { + if n == nil { + return + } + fn(n) + for child := n.FirstChild; child != nil; child = child.NextSibling { + visitNodes(child, fn) + } +} + +func findFirstByClass(n *html.Node, class string) *html.Node { + var found *html.Node + visitNodes(n, func(cur *html.Node) { + if found != nil { + return + } + if hasClass(cur, class) { + found = cur + } + }) + return found +} + +func hasClass(n *html.Node, class string) bool { + if n == nil || n.Type != html.ElementNode { + return false + } + for _, attr := range n.Attr { + if attr.Key != "class" { + continue + } + for _, token := range strings.Fields(attr.Val) { + if token == class { + return true + } + } + } + return false +} + +func attrValue(n *html.Node, key string) string { + if n == nil { + return "" + } + for _, attr := range n.Attr { + if attr.Key == key { + return attr.Val + } + } + return "" +} + +func parsePostID(post string) (uint32, error) { + idx := strings.LastIndex(post, "/") + if idx == -1 || idx+1 >= len(post) { + return 0, fmt.Errorf("invalid post id") + } + id, err := strconv.ParseUint(post[idx+1:], 10, 32) + if err != nil { + return 0, err + } + return uint32(id), nil +} + +func extractMessageTimestamp(n *html.Node) uint32 { + timeNode := findFirstByClass(n, "tgme_widget_message_date") + if timeNode == nil { + timeNode = findFirstElement(n, "time") + } + if timeNode == nil { + return uint32(time.Now().Unix()) + } + datetime := attrValue(timeNode, "datetime") + if datetime == "" { + timeChild := findFirstElement(timeNode, "time") + datetime = attrValue(timeChild, "datetime") + } + if datetime == "" { + return uint32(time.Now().Unix()) + } + ts, err := time.Parse(time.RFC3339, datetime) + if err != nil { + return uint32(time.Now().Unix()) + } + return uint32(ts.Unix()) +} + +func findFirstElement(n *html.Node, tag string) *html.Node { + var found *html.Node + visitNodes(n, func(cur *html.Node) { + if found == nil && cur.Type == html.ElementNode && cur.Data == tag { + found = cur + } + }) + return found +} + +func extractMessageText(n *html.Node) string { + if n == nil { + return "" + } + var b strings.Builder + var walk func(*html.Node) + walk = func(cur *html.Node) { + if cur == nil { + return + } + if cur.Type == html.TextNode { + text := strings.TrimSpace(cur.Data) + if text != "" { + if b.Len() > 0 { + last := b.String()[b.Len()-1] + if last != '\n' && last != ' ' { + b.WriteByte(' ') + } + } + b.WriteString(text) + } + } + if cur.Type == html.ElementNode && cur.Data == "br" { + trimTrailingSpace(&b) + if b.Len() > 0 { + b.WriteByte('\n') + } + } + for child := cur.FirstChild; child != nil; child = child.NextSibling { + walk(child) + } + } + walk(n) + return strings.TrimSpace(strings.ReplaceAll(b.String(), " \n", "\n")) +} + +func trimTrailingSpace(b *strings.Builder) { + s := b.String() + for len(s) > 0 && s[len(s)-1] == ' ' { + s = s[:len(s)-1] + } + b.Reset() + b.WriteString(s) +} diff --git a/internal/server/public_test.go b/internal/server/public_test.go new file mode 100644 index 0000000..76e3a8d --- /dev/null +++ b/internal/server/public_test.go @@ -0,0 +1,102 @@ +package server + +import ( + "testing" + + "github.com/sartoopjj/thefeed/internal/protocol" +) + +func TestParsePublicMessages(t *testing.T) { + body := []byte(` + +
+ +
hello
world
+
+
+ +
+
+
+ + +
photo caption
+
+ + `) + + msgs, err := parsePublicMessages(body) + if err != nil { + t.Fatalf("parsePublicMessages: %v", err) + } + if len(msgs) != 3 { + t.Fatalf("len(msgs) = %d, want 3", len(msgs)) + } + // Photo with caption (newest first) + if msgs[0].ID != 106 { + t.Fatalf("msgs[0].ID = %d, want 106", msgs[0].ID) + } + want := protocol.MediaImage + "\n" + "photo caption" + if msgs[0].Text != want { + t.Fatalf("msgs[0].Text = %q, want %q", msgs[0].Text, want) + } + // Photo only + if msgs[1].ID != 105 { + t.Fatalf("msgs[1].ID = %d, want 105", msgs[1].ID) + } + if msgs[1].Text != protocol.MediaImage { + t.Fatalf("msgs[1].Text = %q, want %q", msgs[1].Text, protocol.MediaImage) + } + // Text only + if msgs[2].ID != 101 { + t.Fatalf("msgs[2].ID = %d, want 101", msgs[2].ID) + } + if msgs[2].Text != "hello\nworld" { + t.Fatalf("msgs[2].Text = %q, want %q", msgs[2].Text, "hello\nworld") + } +} + +func TestParsePublicMessagesNoLimit(t *testing.T) { + body := []byte(` + +
one
+
two
+
three
+ + `) + + msgs, err := parsePublicMessages(body) + if err != nil { + t.Fatalf("parsePublicMessages: %v", err) + } + if len(msgs) != 3 { + t.Fatalf("len(msgs) = %d, want 3", len(msgs)) + } + if msgs[0].ID != 3 || msgs[1].ID != 2 || msgs[2].ID != 1 { + t.Fatalf("got ids %d,%d,%d want 3,2,1", msgs[0].ID, msgs[1].ID, msgs[2].ID) + } +} + +func TestMergeMessages(t *testing.T) { + old := []protocol.Message{ + {ID: 100, Timestamp: 1000, Text: "old100"}, + {ID: 99, Timestamp: 999, Text: "old99"}, + } + newMsgs := []protocol.Message{ + {ID: 101, Timestamp: 1001, Text: "new101"}, + {ID: 100, Timestamp: 1000, Text: "edited100"}, + } + merged := mergeMessages(old, newMsgs) + if len(merged) != 3 { + t.Fatalf("len(merged) = %d, want 3", len(merged)) + } + if merged[0].ID != 101 { + t.Fatalf("merged[0].ID = %d, want 101", merged[0].ID) + } + if merged[1].Text != "edited100" { + t.Fatalf("merged[1].Text = %q, want edited100", merged[1].Text) + } + if merged[2].ID != 99 { + t.Fatalf("merged[2].ID = %d, want 99", merged[2].ID) + } +} diff --git a/internal/server/server.go b/internal/server/server.go index dd9cdb2..46d4fc0 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -18,14 +18,17 @@ type Config struct { Passphrase string ChannelsFile string MaxPadding int - MsgLimit int // max messages per channel (0 = default 15) + MsgLimit int // max messages per channel (0 = default 15) + NoTelegram bool // if true, fetch public channels without Telegram login + AllowManage bool // if true, remote channel management and sending via DNS is allowed Telegram TelegramConfig } // Server orchestrates the DNS server and Telegram reader. type Server struct { - cfg Config - feed *Feed + cfg Config + feed *Feed + reader *TelegramReader // nil when --no-telegram } // New creates a new Server. @@ -57,24 +60,39 @@ func (s *Server) Run(ctx context.Context) error { return reader.Run(ctx) } - // Start Telegram reader in background - msgLimit := s.cfg.MsgLimit - if msgLimit <= 0 { - msgLimit = 15 - } - reader := NewTelegramReader(s.cfg.Telegram, s.feed.ChannelNames(), s.feed, msgLimit) - go func() { - if err := reader.Run(ctx); err != nil { - log.Printf("[telegram] error: %v", err) + // Start Telegram reader in background, or public web fetcher in no-login mode. + if !s.cfg.NoTelegram { + msgLimit := s.cfg.MsgLimit + if msgLimit <= 0 { + msgLimit = 15 } - }() + reader := NewTelegramReader(s.cfg.Telegram, s.feed.ChannelNames(), s.feed, msgLimit) + s.reader = reader + go func() { + if err := reader.Run(ctx); err != nil { + log.Printf("[telegram] error: %v", err) + } + }() + } else { + msgLimit := s.cfg.MsgLimit + if msgLimit <= 0 { + msgLimit = 15 + } + publicReader := NewPublicReader(s.feed.ChannelNames(), s.feed, msgLimit) + go func() { + if err := publicReader.Run(ctx); err != nil && ctx.Err() == nil { + log.Printf("[public] error: %v", err) + } + }() + log.Println("[server] running without Telegram login; fetching public channels via t.me") + } // Start DNS server (blocking, respects ctx cancellation) maxPad := s.cfg.MaxPadding if maxPad == 0 { maxPad = protocol.DefaultMaxPadding } - dnsServer := NewDNSServer(s.cfg.ListenAddr, s.cfg.Domain, s.feed, queryKey, responseKey, maxPad) + dnsServer := NewDNSServer(s.cfg.ListenAddr, s.cfg.Domain, s.feed, queryKey, responseKey, maxPad, s.reader, s.cfg.AllowManage, s.cfg.ChannelsFile) return dnsServer.ListenAndServe(ctx) } diff --git a/internal/server/telegram.go b/internal/server/telegram.go index ff7f471..ac87455 100644 --- a/internal/server/telegram.go +++ b/internal/server/telegram.go @@ -2,6 +2,7 @@ package server import ( "context" + cryptoRand "crypto/rand" "fmt" "log" "os" @@ -63,6 +64,19 @@ type TelegramReader struct { mu sync.RWMutex cache map[string]cachedMessages cacheTTL time.Duration + + // api is set once authenticated, used for sending messages. + apiMu sync.RWMutex + api *tg.Client + + refreshCh chan struct{} // signals Run() to re-fetch immediately +} + +// resolvedPeer holds the resolved Telegram peer along with its chat type. +type resolvedPeer struct { + peer tg.InputPeerClass + chatType protocol.ChatType + canSend bool } type cachedMessages struct { @@ -80,12 +94,13 @@ func NewTelegramReader(cfg TelegramConfig, channelUsernames []string, feed *Feed msgLimit = 15 } return &TelegramReader{ - cfg: cfg, - channels: cleaned, - feed: feed, - msgLimit: msgLimit, - cache: make(map[string]cachedMessages), - cacheTTL: 15 * time.Minute, + cfg: cfg, + channels: cleaned, + feed: feed, + msgLimit: msgLimit, + cache: make(map[string]cachedMessages), + cacheTTL: 10 * time.Minute, + refreshCh: make(chan struct{}, 1), } } @@ -107,6 +122,7 @@ func (tr *TelegramReader) Run(ctx context.Context) error { } log.Println("[telegram] authenticated successfully") + tr.feed.SetTelegramLoggedIn(true) // Login-only mode: just authenticate and return if tr.cfg.LoginOnly { @@ -116,24 +132,59 @@ func (tr *TelegramReader) Run(ctx context.Context) error { api := client.API() + tr.apiMu.Lock() + tr.api = api + tr.apiMu.Unlock() + // Initial fetch tr.fetchAll(ctx, api) // Periodic fetch loop - ticker := time.NewTicker(3 * time.Minute) + ticker := time.NewTicker(5 * time.Minute) defer ticker.Stop() + tr.feed.SetNextFetch(uint32(time.Now().Add(5 * time.Minute).Unix())) + for { select { case <-ctx.Done(): return ctx.Err() case <-ticker.C: tr.fetchAll(ctx, api) + tr.feed.SetNextFetch(uint32(time.Now().Add(5 * time.Minute).Unix())) + case <-tr.refreshCh: + // Invalidate cache so fetchAll re-fetches everything. + tr.mu.Lock() + tr.cache = make(map[string]cachedMessages) + tr.mu.Unlock() + tr.fetchAll(ctx, api) + ticker.Reset(5 * time.Minute) + tr.feed.SetNextFetch(uint32(time.Now().Add(5 * time.Minute).Unix())) } } }) } +// RequestRefresh signals the fetch loop to re-fetch immediately. +func (tr *TelegramReader) RequestRefresh() { + select { + case tr.refreshCh <- struct{}{}: + default: // already pending + } +} + +// UpdateChannels replaces the channel list and updates the Feed accordingly. +func (tr *TelegramReader) UpdateChannels(channels []string) { + cleaned := make([]string, len(channels)) + for i, u := range channels { + cleaned[i] = strings.TrimPrefix(strings.TrimSpace(u), "@") + } + tr.mu.Lock() + tr.channels = cleaned + tr.mu.Unlock() + tr.feed.SetChannels(cleaned) +} + func (tr *TelegramReader) authenticate(ctx context.Context, client *telegram.Client) error { status, err := client.Auth().Status(ctx) if err != nil { @@ -173,7 +224,24 @@ func (tr *TelegramReader) fetchAll(ctx context.Context, api *tg.Client) { continue } - msgs, err := tr.fetchChannel(ctx, api, username) + // Resolve peer to get chat type info + rp, err := tr.resolvePeer(ctx, api, username) + if err != nil { + log.Printf("[telegram] fetch %s: %v", username, err) + continue + } + + hist, err := api.MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{ + Peer: rp.peer, + Limit: tr.msgLimit, + }) + if err != nil { + log.Printf("[telegram] fetch %s: get history: %v", username, err) + continue + } + + userNames := buildUserMap(hist) + msgs, err := tr.extractMessages(hist, rp.chatType, userNames) if err != nil { log.Printf("[telegram] fetch %s: %v", username, err) continue @@ -184,13 +252,16 @@ func (tr *TelegramReader) fetchAll(ctx context.Context, api *tg.Client) { tr.cache[username] = cachedMessages{msgs: msgs, fetched: time.Now()} tr.mu.Unlock() - // Update feed + // Update feed with messages and chat type info tr.feed.UpdateChannel(chNum, msgs) - log.Printf("[telegram] updated %s: %d messages", username, len(msgs)) + tr.feed.SetChatInfo(chNum, rp.chatType, rp.canSend) + log.Printf("[telegram] updated %s: %d messages (type=%d, canSend=%v)", username, len(msgs), rp.chatType, rp.canSend) } } -func (tr *TelegramReader) fetchChannel(ctx context.Context, api *tg.Client, username string) ([]protocol.Message, error) { +// resolvePeer resolves a Telegram username to an InputPeer, handling channels, +// bots, and private chats. +func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, username string) (*resolvedPeer, error) { resolved, err := api.ContactsResolveUsername(ctx, &tg.ContactsResolveUsernameRequest{ Username: username, }) @@ -198,34 +269,86 @@ func (tr *TelegramReader) fetchChannel(ctx context.Context, api *tg.Client, user return nil, fmt.Errorf("resolve %s: %w", username, err) } - var channel *tg.Channel + // Check Chats first (channels, supergroups) for _, chat := range resolved.Chats { if ch, ok := chat.(*tg.Channel); ok { - channel = ch - break + canSend := !ch.Broadcast || ch.Creator || ch.AdminRights.PostMessages + return &resolvedPeer{ + peer: &tg.InputPeerChannel{ + ChannelID: ch.ID, + AccessHash: ch.AccessHash, + }, + chatType: protocol.ChatTypeChannel, + canSend: canSend, + }, nil } } - if channel == nil { - return nil, fmt.Errorf("channel %s not found in resolved chats", username) + + // Check Users (bots, private chats) + for _, u := range resolved.Users { + if user, ok := u.(*tg.User); ok { + return &resolvedPeer{ + peer: &tg.InputPeerUser{ + UserID: user.ID, + AccessHash: user.AccessHash, + }, + chatType: protocol.ChatTypePrivate, + canSend: true, + }, nil + } } - peer := &tg.InputPeerChannel{ - ChannelID: channel.ID, - AccessHash: channel.AccessHash, + return nil, fmt.Errorf("%s not found in resolved chats or users", username) +} + +func (tr *TelegramReader) fetchChannel(ctx context.Context, api *tg.Client, username string) ([]protocol.Message, error) { + rp, err := tr.resolvePeer(ctx, api, username) + if err != nil { + return nil, err } hist, err := api.MessagesGetHistory(ctx, &tg.MessagesGetHistoryRequest{ - Peer: peer, + Peer: rp.peer, Limit: tr.msgLimit, }) if err != nil { return nil, fmt.Errorf("get history %s: %w", username, err) } - return tr.extractMessages(hist) + userNames := buildUserMap(hist) + return tr.extractMessages(hist, protocol.ChatTypeChannel, userNames) } -func (tr *TelegramReader) extractMessages(hist tg.MessagesMessagesClass) ([]protocol.Message, error) { +// buildUserMap extracts a user ID → display name map from a history response. +func buildUserMap(hist tg.MessagesMessagesClass) map[int64]string { + var users []tg.UserClass + switch h := hist.(type) { + case *tg.MessagesMessages: + users = h.Users + case *tg.MessagesMessagesSlice: + users = h.Users + case *tg.MessagesChannelMessages: + users = h.Users + } + m := make(map[int64]string, len(users)) + for _, u := range users { + if user, ok := u.(*tg.User); ok { + name := user.FirstName + if user.LastName != "" { + name += " " + user.LastName + } + if name == "" { + name = user.Username + } + if name != "" { + m[user.ID] = name + } + } + } + return m +} + +func (tr *TelegramReader) extractMessages(hist tg.MessagesMessagesClass, chatType protocol.ChatType, userNames map[int64]string) ([]protocol.Message, error) { var tgMsgs []tg.MessageClass switch h := hist.(type) { @@ -251,6 +374,17 @@ func (tr *TelegramReader) extractMessages(hist tg.MessagesMessagesClass) ([]prot continue } + // For private chats, prefix with the sender's name. + if chatType == protocol.ChatTypePrivate { + if fromID, ok := msg.GetFromID(); ok { + if pu, ok := fromID.(*tg.PeerUser); ok { + if name, ok := userNames[pu.UserID]; ok { + text = name + ": " + text + } + } + } + } + msgs = append(msgs, protocol.Message{ ID: uint32(msg.ID), Timestamp: uint32(msg.Date), @@ -311,3 +445,42 @@ func (tr *TelegramReader) classifyDocument(media *tg.MessageMediaDocument) strin return protocol.MediaFile } + +// SendMessage sends a text message to the given channel/chat (1-indexed). +func (tr *TelegramReader) SendMessage(ctx context.Context, channelNum int, text string) error { + if channelNum < 1 || channelNum > len(tr.channels) { + return fmt.Errorf("invalid channel number: %d", channelNum) + } + + tr.apiMu.RLock() + api := tr.api + tr.apiMu.RUnlock() + if api == nil { + return fmt.Errorf("not authenticated") + } + + username := tr.channels[channelNum-1] + rp, err := tr.resolvePeer(ctx, api, username) + if err != nil { + return fmt.Errorf("send resolve: %w", err) + } + + var ridBuf [8]byte + if _, ridErr := cryptoRand.Read(ridBuf[:]); ridErr != nil { + return fmt.Errorf("generate random id: %w", ridErr) + } + randomID := int64(ridBuf[0]) | int64(ridBuf[1])<<8 | int64(ridBuf[2])<<16 | int64(ridBuf[3])<<24 | + int64(ridBuf[4])<<32 | int64(ridBuf[5])<<40 | int64(ridBuf[6])<<48 | int64(ridBuf[7])<<56 + + _, err = api.MessagesSendMessage(ctx, &tg.MessagesSendMessageRequest{ + Peer: rp.peer, + Message: text, + RandomID: randomID, + }) + if err != nil { + return fmt.Errorf("send to %s: %w", username, err) + } + + log.Printf("[telegram] sent message to %s (%d chars)", username, len(text)) + return nil +} diff --git a/internal/web/static/index.html b/internal/web/static/index.html index aefc00a..eeea02c 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -24,6 +24,7 @@ --text-dim: #71717a; --success: #22c55e; --error: #ef4444; + --send-color: #0ea5e9; } * { margin: 0; padding: 0; box-sizing: border-box; } @@ -47,7 +48,8 @@ } .header-left { display: flex; align-items: center; gap: 10px; } .header h1 { font-size: 16px; font-weight: 600; } - .header-actions { display: flex; gap: 8px; } + .header-actions { display: flex; gap: 8px; align-items: center; } + .next-fetch-timer { font-size: 11px; color: var(--text-dim); } .status-dot { width: 8px; height: 8px; @@ -99,6 +101,9 @@ border-left: 3px solid transparent; transition: all 0.15s; font-size: 14px; + display: flex; + justify-content: space-between; + align-items: center; } .channel-item:hover { background: var(--surface2); } .channel-item.active { @@ -106,6 +111,24 @@ border-left-color: var(--accent); color: white; } + .channel-type-badge { + font-size: 10px; + padding: 1px 5px; + border-radius: 3px; + background: var(--surface2); + color: var(--text-dim); + margin-left: 4px; + } + .channel-type-badge.private { background: #3b1f4e; color: #c084fc; } + .new-msg-badge { + background: var(--accent); + color: white; + font-size: 10px; + padding: 1px 6px; + border-radius: 10px; + min-width: 18px; + text-align: center; + } .content { flex: 1; @@ -141,6 +164,46 @@ white-space: pre-wrap; word-break: break-word; } + .media-tag { + display: inline-block; + padding: 2px 8px; + border-radius: 4px; + background: var(--surface2); + color: var(--text-dim); + font-size: 12px; + margin-bottom: 4px; + } + + .send-panel { + display: none; + padding: 8px 12px; + background: var(--surface); + border-top: 1px solid var(--border); + direction: rtl; + } + .send-panel.visible { display: flex; gap: 8px; align-items: center; } + .send-input { + flex: 1; + padding: 8px 12px; + border: 1px solid var(--border); + border-radius: 6px; + background: var(--bg); + color: var(--text); + font-family: inherit; + font-size: 14px; + direction: rtl; + } + .send-btn { + padding: 8px 16px; + border: none; + border-radius: 6px; + background: var(--send-color); + color: white; + cursor: pointer; + font-family: inherit; + font-size: 13px; + } + .send-btn:hover { background: #0284c7; } .progress-panel { height: 56px; @@ -179,6 +242,7 @@ background: var(--accent); transition: width 0.2s; } + .progress-fill.send { background: var(--send-color); } .log-panel { height: 150px; @@ -258,6 +322,30 @@ } .empty-state p { font-size: 14px; } + .footer-link { + padding: 12px 16px; + text-align: center; + font-size: 11px; + color: var(--text-dim); + border-top: 1px solid var(--border); + margin-top: auto; + } + .footer-link a { color: var(--accent); text-decoration: none; } + .footer-link a:hover { text-decoration: underline; } + .footer-link .free-iran { color: var(--success); font-weight: bold; } + + .tg-warning { + background: #3b1f1f; + border: 1px solid #7f1d1d; + border-radius: 6px; + padding: 8px 12px; + font-size: 12px; + color: #fca5a5; + margin-bottom: 16px; + display: none; + } + .tg-warning.visible { display: block; } + ::-webkit-scrollbar { width: 6px; } ::-webkit-scrollbar-track { background: transparent; } ::-webkit-scrollbar-thumb { background: var(--border); border-radius: 3px; } @@ -269,6 +357,7 @@

thefeed

+
@@ -278,10 +367,16 @@
-
Channels
+
Channels + +
No channels
+
+ GitHub + · For FREE IRAN +
@@ -291,14 +386,43 @@
+
+ + +
+ + diff --git a/internal/web/web.go b/internal/web/web.go index 735eddf..e176d45 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -2,6 +2,7 @@ package web import ( "context" + "crypto/subtle" "embed" "encoding/json" "fmt" @@ -39,15 +40,20 @@ type Config struct { // Server is the web UI server for thefeed client. type Server struct { - dataDir string - port int + dataDir string + port int + password string // admin password; empty means no auth - mu sync.RWMutex - config *Config - fetcher *client.Fetcher - cache *client.Cache - channels []protocol.ChannelInfo - messages map[int][]protocol.Message + mu sync.RWMutex + config *Config + fetcher *client.Fetcher + cache *client.Cache + channels []protocol.ChannelInfo + messages map[int][]protocol.Message + telegramLoggedIn bool + nextFetch uint32 + lastMsgIDs map[int]uint32 // last seen message IDs per channel + lastHashes map[int]uint32 // last seen content hashes per channel // fetcherCtx/fetcherCancel control the lifetime of the active fetcher's // background goroutines (rate limiter, noise, resolver checker). @@ -56,8 +62,10 @@ type Server struct { fetcherCancel context.CancelFunc // refreshMu / refreshCancel allow a new refresh to cancel an in-progress one. - refreshMu sync.Mutex - refreshCancel context.CancelFunc + // channelFetching tracks which channels are currently being fetched. + refreshMu sync.Mutex + refreshCancel context.CancelFunc + channelFetching map[int]bool // prevents duplicate fetches for same channel logMu sync.RWMutex logLines []string @@ -69,16 +77,20 @@ type Server struct { } // New creates a new web server. -func New(dataDir string, port int) (*Server, error) { +func New(dataDir string, port int, password string) (*Server, error) { if err := os.MkdirAll(dataDir, 0700); err != nil { return nil, fmt.Errorf("create data dir: %w", err) } s := &Server{ - dataDir: dataDir, - port: port, - messages: make(map[int][]protocol.Message), - clients: make(map[chan string]struct{}), + dataDir: dataDir, + port: port, + password: password, + messages: make(map[int][]protocol.Message), + clients: make(map[chan string]struct{}), + channelFetching: make(map[int]bool), + lastMsgIDs: make(map[int]uint32), + lastHashes: make(map[int]uint32), } cfg, err := s.loadConfig() @@ -104,6 +116,8 @@ func (s *Server) Run() error { mux.HandleFunc("/api/channels", s.handleChannels) mux.HandleFunc("/api/messages/", s.handleMessages) mux.HandleFunc("/api/refresh", s.handleRefresh) + mux.HandleFunc("/api/send", s.handleSend) + mux.HandleFunc("/api/admin", s.handleAdmin) mux.HandleFunc("/api/events", s.handleSSE) mux.HandleFunc("/", s.handleIndex) @@ -115,7 +129,28 @@ func (s *Server) Run() error { go s.refreshMetadataOnly() } - return http.ListenAndServe(addr, mux) + var handler http.Handler = mux + if s.password != "" { + pw := s.password + handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, pass, ok := r.BasicAuth() + if !ok || subtle.ConstantTimeCompare([]byte(pass), []byte(pw)) != 1 { + w.Header().Set("WWW-Authenticate", `Basic realm="thefeed"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + mux.ServeHTTP(w, r) + }) + } + + srv := &http.Server{ + Addr: addr, + Handler: handler, + ReadTimeout: 30 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, + } + return srv.ListenAndServe() } func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) { @@ -137,16 +172,21 @@ func (s *Server) handleStatus(w http.ResponseWriter, r *http.Request) { defer s.mu.RUnlock() status := map[string]any{ - "configured": s.config != nil, - "version": version.Version, + "configured": s.config != nil, + "version": version.Version, + "hasPassword": s.password != "", } if s.config != nil { status["domain"] = s.config.Domain status["channels"] = s.channels + status["telegramLoggedIn"] = s.telegramLoggedIn + status["nextFetch"] = s.nextFetch } writeJSON(w, status) } +// handleConfig handles GET (read) and POST (write) of client configuration. +// POST is authenticated when a global password is set (via the middleware). func (s *Server) handleConfig(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: @@ -243,6 +283,116 @@ func (s *Server) handleRefresh(w http.ResponseWriter, r *http.Request) { writeJSON(w, map[string]any{"ok": true}) } +func (s *Server) handleSend(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", 405) + return + } + + var req struct { + Channel int `json:"channel"` + Text string `json:"text"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid JSON", 400) + return + } + if req.Channel < 1 || req.Text == "" { + http.Error(w, "channel and text are required", 400) + return + } + if len(req.Text) > 4000 { + http.Error(w, "message too long (max 4000 chars)", 400) + return + } + + s.mu.RLock() + fetcher := s.fetcher + basectx := s.fetcherCtx + s.mu.RUnlock() + + if fetcher == nil || basectx == nil { + http.Error(w, "not configured", 400) + return + } + + ctx, cancel := context.WithTimeout(basectx, 5*time.Minute) + defer cancel() + + s.addLog(fmt.Sprintf("Sending message to channel %d (%d chars)...", req.Channel, len(req.Text))) + + if err := fetcher.SendMessage(ctx, req.Channel, req.Text); err != nil { + log.Printf("[web] send error ch=%d: %v", req.Channel, err) + s.addLog("Error: failed to send message") + http.Error(w, "failed to send message", 500) + return + } + + s.addLog("Message sent successfully") + writeJSON(w, map[string]any{"ok": true}) +} + +func (s *Server) handleAdmin(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", 405) + return + } + + var req struct { + Command string `json:"command"` + Arg string `json:"arg"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid JSON", 400) + return + } + if req.Command == "" { + http.Error(w, "command is required", 400) + return + } + + s.mu.RLock() + fetcher := s.fetcher + basectx := s.fetcherCtx + s.mu.RUnlock() + + if fetcher == nil || basectx == nil { + http.Error(w, "not configured", 400) + return + } + + ctx, cancel := context.WithTimeout(basectx, 5*time.Minute) + defer cancel() + + s.addLog(fmt.Sprintf("Admin command: %s %s", req.Command, req.Arg)) + + var cmd protocol.AdminCmd + switch req.Command { + case "add_channel": + cmd = protocol.AdminCmdAddChannel + case "remove_channel": + cmd = protocol.AdminCmdRemoveChannel + case "list_channels": + cmd = protocol.AdminCmdListChannels + case "refresh": + cmd = protocol.AdminCmdRefresh + default: + http.Error(w, "unknown command", 400) + return + } + + result, err := fetcher.SendAdminCommand(ctx, cmd, req.Arg) + if err != nil { + log.Printf("[web] admin error: %v", err) + s.addLog(fmt.Sprintf("Admin error: %v", err)) + http.Error(w, "admin command failed", 500) + return + } + + s.addLog(fmt.Sprintf("Admin result: %s", result)) + writeJSON(w, map[string]any{"ok": true, "result": result}) +} + func (s *Server) handleSSE(w http.ResponseWriter, r *http.Request) { flusher, ok := w.(http.Flusher) if !ok { @@ -338,8 +488,6 @@ func (s *Server) initFetcher() error { if cfg.QueryMode == "double" { fetcher.SetQueryMode(protocol.QueryMultiLabel) - } else if cfg.QueryMode == "plain" { - fetcher.SetQueryMode(protocol.QueryPlainLabel) } fetcher.SetDebug(cfg.Debug) if cfg.RateLimit > 0 { @@ -369,6 +517,7 @@ func (s *Server) initFetcher() error { checker.SetLogFunc(func(msg string) { s.addLog(msg) }) + checker.CheckNow() checker.Start(ctx) s.fetcher = fetcher @@ -412,12 +561,20 @@ func (s *Server) refreshMetadataOnly() { s.addLog("Refresh cancelled") return } - s.addLog(fmt.Sprintf("Error: %v", err)) + // Detect invalid passphrase from crypto errors + errStr := err.Error() + if strings.Contains(errStr, "integrity check failed") || strings.Contains(errStr, "message authentication failed") || strings.Contains(errStr, "cipher") { + s.addLog("Error: Invalid passphrase — check your encryption key in Settings") + } else { + s.addLog(fmt.Sprintf("Error: %v", err)) + } return } s.mu.Lock() s.channels = meta.Channels + s.telegramLoggedIn = meta.TelegramLoggedIn + s.nextFetch = meta.NextFetch s.mu.Unlock() if cache != nil { @@ -428,19 +585,26 @@ func (s *Server) refreshMetadataOnly() { } func (s *Server) refreshChannel(channelNum int) { + // Prevent duplicate fetches for the same channel s.refreshMu.Lock() + if s.channelFetching[channelNum] { + s.refreshMu.Unlock() + s.addLog(fmt.Sprintf("Channel %d is already being fetched, skipping", channelNum)) + return + } if s.refreshCancel != nil { s.refreshCancel() } + s.channelFetching[channelNum] = true s.mu.RLock() basectx := s.fetcherCtx fetcher := s.fetcher cache := s.cache - channels := s.channels s.mu.RUnlock() if fetcher == nil || basectx == nil { + delete(s.channelFetching, channelNum) s.refreshMu.Unlock() return } @@ -452,6 +616,7 @@ func (s *Server) refreshChannel(channelNum int) { cancel() s.refreshMu.Lock() s.refreshCancel = nil + delete(s.channelFetching, channelNum) s.refreshMu.Unlock() }() @@ -461,12 +626,19 @@ func (s *Server) refreshChannel(channelNum int) { s.addLog("Refresh cancelled") return } - s.addLog(fmt.Sprintf("Error: %v", err)) + errStr := err.Error() + if strings.Contains(errStr, "integrity check failed") || strings.Contains(errStr, "message authentication failed") || strings.Contains(errStr, "cipher") { + s.addLog("Error: Invalid passphrase — check your encryption key in Settings") + } else { + s.addLog(fmt.Sprintf("Error: %v", err)) + } return } s.mu.Lock() s.channels = meta.Channels + s.telegramLoggedIn = meta.TelegramLoggedIn + s.nextFetch = meta.NextFetch s.mu.Unlock() if cache != nil { @@ -474,24 +646,39 @@ func (s *Server) refreshChannel(channelNum int) { } s.broadcast("event: update\ndata: \"channels\"\n\n") - channels = meta.Channels + channels := meta.Channels if channelNum < 1 || channelNum > len(channels) { s.addLog(fmt.Sprintf("Warning: channel %d is not available", channelNum)) return } ch := channels[channelNum-1] + + // Skip refresh if the last message ID and content hash haven't changed + s.mu.RLock() + prevID := s.lastMsgIDs[channelNum] + prevHash := s.lastHashes[channelNum] + s.mu.RUnlock() + if prevID > 0 && ch.LastMsgID == prevID && ch.ContentHash == prevHash { + s.addLog(fmt.Sprintf("Channel %s: no changes (last ID: %d)", ch.Name, prevID)) + s.broadcast("event: update\ndata: \"messages\"\n\n") + return + } + blockCount := int(ch.Blocks) if blockCount <= 0 { s.mu.Lock() s.messages[channelNum] = nil + s.lastMsgIDs[channelNum] = ch.LastMsgID + s.lastHashes[channelNum] = ch.ContentHash s.mu.Unlock() s.addLog(fmt.Sprintf("Updated %s: 0 messages", ch.Name)) s.broadcast("event: update\ndata: \"messages\"\n\n") return } - msgs, err := fetcher.FetchChannel(ctx, channelNum, blockCount) + var msgs []protocol.Message + msgs, err = fetcher.FetchChannel(ctx, channelNum, blockCount) if err != nil { if ctx.Err() != nil { s.addLog("Refresh cancelled") @@ -503,6 +690,8 @@ func (s *Server) refreshChannel(channelNum int) { s.mu.Lock() s.messages[channelNum] = msgs + s.lastMsgIDs[channelNum] = ch.LastMsgID + s.lastHashes[channelNum] = ch.ContentHash s.mu.Unlock() if cache != nil { diff --git a/scripts/install.sh b/scripts/install.sh index f7e9947..5c5af03 100644 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -142,6 +142,51 @@ setup_config() { echo -e "${red}Passphrase cannot be empty${plain}" done + echo "" + echo -e "${yellow}Allow remote management (send messages, add/remove channels)?${plain}" + echo -e " If enabled, anyone with the passphrase can manage channels." + local allow_manage="" + read -rp "Enable remote management? [y/N]: " allow_manage + + local no_telegram="" + echo "" + echo -e "${green}═══════════════════════════════════════${plain}" + echo -e "${green} Telegram Mode Selection${plain}" + echo -e "${green}═══════════════════════════════════════${plain}" + echo "" + echo -e "${yellow}Option 1: Without Telegram (Recommended)${plain}" + echo -e " - Safer: no Telegram credentials stored on server" + echo -e " - Reads public channels without login" + echo -e " - Cannot read private channels or send messages" + echo "" + echo -e "${yellow}Option 2: With Telegram${plain}" + echo -e " - Required for private channels, groups, and sending messages" + echo -e " - Needs Telegram API credentials and phone number" + echo "" + read -rp "Run without Telegram login? (recommended) [Y/n]: " no_telegram + if [[ "$no_telegram" != "n" && "$no_telegram" != "N" ]]; then + local api_id="0" + local api_hash="none" + local phone="none" + local listen_addr="" + read -rp "DNS listen address [0.0.0.0:53]: " listen_addr + listen_addr="${listen_addr:-0.0.0.0:53}" + + cat > "$DATA_DIR/thefeed.env" < "$DATA_DIR/thefeed.env" < "$SERVICE_FILE" <> "$DATA_DIR/thefeed.env" + fi + fi install_service start_service fi @@ -384,6 +498,14 @@ show_help() { echo -e " ${green}--login${plain} Re-authenticate with Telegram" echo -e " ${green}--uninstall${plain} Remove thefeed" echo -e " ${green}--help${plain} Show this help" + echo "" + echo -e "No-Telegram mode (recommended for most users):" + echo -e " Reads public Telegram channels without needing Telegram credentials." + echo -e " Safer because no phone number or API keys are stored on the server." + echo "" + echo -e "Quick commands:" + echo -e " Install/Update: ${blue}sudo bash <(curl -Ls https://raw.githubusercontent.com/${GITHUB_REPO}/main/scripts/install.sh)${plain}" + echo -e " Uninstall: ${blue}sudo bash <(curl -Ls https://raw.githubusercontent.com/${GITHUB_REPO}/main/scripts/install.sh) --uninstall${plain}" } # Main diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 3aa1a4b..7263ad3 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -39,6 +39,10 @@ func findFreePort(t *testing.T, network string) int { } func startDNSServer(t *testing.T, domain, passphrase string, channels []string, messages map[int][]protocol.Message) (string, context.CancelFunc) { + return startDNSServerWithManage(t, domain, passphrase, false, channels, messages) +} + +func startDNSServerWithManage(t *testing.T, domain, passphrase string, allowManage bool, channels []string, messages map[int][]protocol.Message) (string, context.CancelFunc) { t.Helper() qk, rk, err := protocol.DeriveKeys(passphrase) @@ -54,7 +58,20 @@ func startDNSServer(t *testing.T, domain, passphrase string, channels []string, port := findFreePort(t, "udp") addr := fmt.Sprintf("127.0.0.1:%d", port) - dnsServer := server.NewDNSServer(addr, domain, feed, qk, rk, protocol.DefaultMaxPadding) + channelsFile := "" + if allowManage { + f, err := os.CreateTemp(t.TempDir(), "channels-*.txt") + if err != nil { + t.Fatalf("create temp channels file: %v", err) + } + for _, ch := range channels { + fmt.Fprintf(f, "@%s\n", ch) + } + f.Close() + channelsFile = f.Name() + } + + dnsServer := server.NewDNSServer(addr, domain, feed, qk, rk, protocol.DefaultMaxPadding, nil, allowManage, channelsFile) ctx, cancel := context.WithCancel(context.Background()) @@ -266,7 +283,7 @@ func TestE2E_LargeMessages(t *testing.T) { func TestE2E_WebAPI_ConfigAndStatus(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -331,7 +348,7 @@ func TestE2E_WebAPI_ConfigAndStatus(t *testing.T) { func TestE2E_WebAPI_InvalidConfig(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -364,7 +381,7 @@ func TestE2E_WebAPI_InvalidConfig(t *testing.T) { func TestE2E_WebAPI_Channels(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -387,7 +404,7 @@ func TestE2E_WebAPI_Channels(t *testing.T) { func TestE2E_WebAPI_Messages(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -419,7 +436,7 @@ func TestE2E_WebAPI_Messages(t *testing.T) { func TestE2E_WebAPI_IndexPage(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -445,7 +462,7 @@ func TestE2E_WebAPI_IndexPage(t *testing.T) { func TestE2E_WebAPI_NotFound(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -467,7 +484,7 @@ func TestE2E_WebAPI_NotFound(t *testing.T) { func TestE2E_WebAPI_MethodNotAllowed(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -500,7 +517,7 @@ func TestE2E_WebAPI_ConfigPersistence(t *testing.T) { dataDir := t.TempDir() port1 := findFreePort(t, "tcp") - srv1, err := web.New(dataDir, port1) + srv1, err := web.New(dataDir, port1, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -521,7 +538,7 @@ func TestE2E_WebAPI_ConfigPersistence(t *testing.T) { } port2 := findFreePort(t, "tcp") - srv2, err := web.New(dataDir, port2) + srv2, err := web.New(dataDir, port2, "") if err != nil { t.Fatalf("create second web server: %v", err) } @@ -566,7 +583,7 @@ func TestE2E_FullRoundTrip(t *testing.T) { dataDir := t.TempDir() port := findFreePort(t, "tcp") - srv, err := web.New(dataDir, port) + srv, err := web.New(dataDir, port, "") if err != nil { t.Fatalf("create web server: %v", err) } @@ -651,3 +668,126 @@ func TestE2E_FullRoundTrip(t *testing.T) { t.Errorf("msg[0].Text = %q, want %q", msgList2[0].Text, "Alert!") } } + +// --- Auth E2E Tests --- + +func TestE2E_WebAPI_GlobalAuth(t *testing.T) { + dataDir := t.TempDir() + port := findFreePort(t, "tcp") + password := "webpass123" + srv, err := web.New(dataDir, port, password) + if err != nil { + t.Fatalf("create web server: %v", err) + } + go srv.Run() + time.Sleep(200 * time.Millisecond) + + base := fmt.Sprintf("http://127.0.0.1:%d", port) + + // All endpoints should require auth when password is set. + endpoints := []struct { + method string + path string + }{ + {"GET", "/"}, + {"GET", "/api/status"}, + {"GET", "/api/config"}, + {"GET", "/api/channels"}, + {"GET", "/api/messages/1"}, + {"GET", "/api/events"}, + } + for _, ep := range endpoints { + req, _ := http.NewRequest(ep.method, base+ep.path, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", ep.method, ep.path, err) + } + resp.Body.Close() + if resp.StatusCode != 401 { + t.Errorf("%s %s without auth: expected 401, got %d", ep.method, ep.path, resp.StatusCode) + } + } + + // With correct password, should succeed. + for _, ep := range endpoints[:5] { // skip /api/events (SSE stream) + req, _ := http.NewRequest(ep.method, base+ep.path, nil) + req.SetBasicAuth("", password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", ep.method, ep.path, err) + } + resp.Body.Close() + if resp.StatusCode == 401 { + t.Errorf("%s %s with correct auth: got 401", ep.method, ep.path) + } + } + + // Wrong password should be rejected. + req, _ := http.NewRequest("GET", base+"/api/status", nil) + req.SetBasicAuth("", "wrongpass") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("GET /api/status wrong pw: %v", err) + } + resp.Body.Close() + if resp.StatusCode != 401 { + t.Errorf("wrong password: expected 401, got %d", resp.StatusCode) + } +} + +func TestE2E_AdminAllowManage(t *testing.T) { + domain := "manage.example.com" + passphrase := "manage-test" + channels := []string{"moderated"} + + msgs := map[int][]protocol.Message{ + 1: {{ID: 1, Timestamp: 1700000000, Text: "Existing"}}, + } + + resolver, cancel := startDNSServerWithManage(t, domain, passphrase, true, channels, msgs) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + + ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer ctxCancel() + + // Admin list_channels should succeed when allow-manage is enabled. + result, err := fetcher.SendAdminCommand(ctx, protocol.AdminCmdListChannels, "") + if err != nil { + t.Fatalf("expected admin command to succeed with allow-manage, got: %v", err) + } + if !strings.Contains(result, "moderated") { + t.Errorf("expected channel list to contain 'moderated', got: %q", result) + } +} + +func TestE2E_AdminNoManage(t *testing.T) { + domain := "nomanage.example.com" + passphrase := "no-manage-test" + channels := []string{"public"} + + msgs := map[int][]protocol.Message{ + 1: {{ID: 1, Timestamp: 1700000000, Text: "Public msg"}}, + } + + // Server has allow-manage disabled — admin commands should be refused. + resolver, cancel := startDNSServer(t, domain, passphrase, channels, msgs) + defer cancel() + + fetcher, err := client.NewFetcher(domain, passphrase, []string{resolver}) + if err != nil { + t.Fatalf("create fetcher: %v", err) + } + + ctx, ctxCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer ctxCancel() + + _, err = fetcher.SendAdminCommand(ctx, protocol.AdminCmdListChannels, "") + if err == nil { + t.Error("expected error when server has allow-manage disabled, got nil") + } +}