// Optional, removable backup feed. Keep this file isolated from the rest // of the web package so deleting it (plus the routes / clear-cache wiring // in web.go and the BEGIN/END telemirror block in static/index.html) is // enough to drop the feature. package web import ( "context" "encoding/json" "fmt" "io" "net/http" "net/url" "path/filepath" "strconv" "strings" "sync" "time" "github.com/sartoopjj/thefeed/internal/telemirror" ) type telemirrorHub struct { client *telemirror.Client cache *telemirror.Cache store *telemirror.Store imgs *telemirror.ImageCache // onUpdate fires after a successful refresh so the SSE channel can // nudge the frontend to re-fetch. nil-safe. onUpdate func(username string) mu sync.Mutex refreshing map[string]chan struct{} } func newTelemirrorHub(dataDir string, onUpdate func(string)) *telemirrorHub { tmDir := filepath.Join(dataDir, "telemirror") return &telemirrorHub{ client: telemirror.NewClient(), cache: telemirror.NewCache(tmDir), store: telemirror.NewStore(dataDir), imgs: telemirror.NewImageCache(filepath.Join(tmDir, "images")), onUpdate: onUpdate, refreshing: make(map[string]chan struct{}), } } func (h *telemirrorHub) handleChannels(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: list := h.store.List() type entry struct { Username string `json:"username"` Pinned bool `json:"pinned"` } out := make([]entry, 0, len(list)) for _, u := range list { out = append(out, entry{Username: u, Pinned: telemirror.IsDefault(u)}) } writeJSON(w, map[string]any{"channels": out}) case http.MethodPost: var req struct { Username string `json:"username"` Action string `json:"action"` } if err := json.NewDecoder(io.LimitReader(r.Body, 1024)).Decode(&req); err != nil { http.Error(w, "invalid JSON", 400) return } switch req.Action { case "add": if err := h.store.Add(req.Username); err != nil { http.Error(w, err.Error(), 400) return } case "remove": if err := h.store.Remove(req.Username); err != nil { http.Error(w, err.Error(), 400) return } default: http.Error(w, "unknown action", 400) return } writeJSON(w, map[string]any{"ok": true}) default: http.Error(w, "method not allowed", 405) } } // handleChannel serves /api/telemirror/channel/[?refresh=1]. // Stale-while-revalidate: serve cached content immediately, refresh in // the background when it's older than FreshTTL. func (h *telemirrorHub) handleChannel(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", 405) return } username := telemirror.SanitizeUsername(strings.TrimPrefix(r.URL.Path, "/api/telemirror/channel/")) if username == "" { http.Error(w, "missing username", 400) return } forceRefresh := r.URL.Query().Get("refresh") == "1" cached, fresh := h.cache.Get(username) if cached != nil && fresh && !forceRefresh { writeJSON(w, rewriteImageURLs(cached)) return } if cached != nil && !forceRefresh { go func() { _, _ = h.refresh(username) }() writeJSON(w, rewriteImageURLs(cached)) return } // forceRefresh + cache hit: trigger refresh in background but cap // how long we make the user wait. On flaky networks fronting can // take minutes — we'd rather return cached than time out. The // background fetch keeps running (h.refresh coalesces concurrent // callers), so the next request sees fresh data. if forceRefresh && cached != nil { ch := make(chan *telemirror.FetchResult, 1) go func() { res, err := h.refresh(username) if err != nil || res == nil { ch <- nil return } ch <- res }() select { case res := <-ch: if res != nil { writeJSON(w, rewriteImageURLs(res)) return } case <-time.After(15 * time.Second): } writeJSON(w, rewriteImageURLs(cached)) return } // No cache yet — must wait for the first fetch. res, err := h.refresh(username) if err != nil { http.Error(w, err.Error(), 502) return } writeJSON(w, rewriteImageURLs(res)) } // rewriteImageURLs points Channel.Photo at /api/telemirror/avatar/ // (stable across restarts) and Media.Thumb at the URL-hash img proxy. // Media.URL is left alone — it's a post permalink, not image bytes. func rewriteImageURLs(in *telemirror.FetchResult) *telemirror.FetchResult { if in == nil { return nil } cp := *in if u := telemirror.SanitizeUsername(cp.Channel.Username); u != "" { cp.Channel.Photo = "/api/telemirror/avatar/" + strings.ToLower(u) } else { cp.Channel.Photo = proxyImgURL(cp.Channel.Photo) } cp.Posts = make([]telemirror.Post, len(in.Posts)) for i, p := range in.Posts { p.Media = append([]telemirror.Media(nil), p.Media...) for j := range p.Media { p.Media[j].Thumb = proxyImgURL(p.Media[j].Thumb) } cp.Posts[i] = p } return &cp } // proxyImgURL routes an image URL through our /api/telemirror/img // endpoint, applying the translate.goog hostname rewrite when the // source is a Telegram CDN. Google's Translate proxy doesn't rewrite // inline background-image URLs, so the parser hands us the original // cdn4.telegram.org / cdn4.telesco.pe URLs and we have to translate // them ourselves before fetching through fronting. func proxyImgURL(raw string) string { rewritten := translateGoogify(raw) if rewritten == "" { return raw } return "/api/telemirror/img?u=" + url.QueryEscape(rewritten) } // translateGoogify maps a CDN URL to its Google-Translate-proxied form. // Returns "" if the URL is something we shouldn't proxy at all. // // Examples: // // https://cdn4.telegram.org/file/abc.jpg // → https://cdn4-telegram-org.translate.goog/file/abc.jpg // https://cdn1.telesco.pe/file/x.jpg // → https://cdn1-telesco-pe.translate.goog/file/x.jpg // https://cdn4-telegram-org.translate.goog/file/abc.jpg // → unchanged (already on translate.goog) func translateGoogify(raw string) string { if raw == "" || !strings.HasPrefix(raw, "https://") { return "" } u, err := url.Parse(raw) if err != nil || u.Host == "" { return "" } host := strings.ToLower(u.Host) switch { case strings.HasSuffix(host, ".translate.goog"): // Already on translate.goog — proxy directly. return raw case strings.HasSuffix(host, ".telegram.org"), strings.HasSuffix(host, ".telesco.pe"), host == "t.me": // Replace every "." with "-" and append ".translate.goog". // Same scheme Google's proxy uses. u.Host = strings.ReplaceAll(host, ".", "-") + ".translate.goog" return u.String() } return "" } // isProxiableHost — only the translate.goog form. Used by the /api/telemirror/img // handler to validate the `u` query parameter, after the JSON // rewrite has converted Telegram CDN URLs to their .translate.goog equivalent. func isProxiableHost(rawURL string) bool { u, err := url.Parse(rawURL) if err != nil { return false } host := strings.ToLower(u.Host) return strings.HasSuffix(host, ".translate.goog") } // handleImg proxies an image URL through fronting, persisting the // response under sha256(u) for subsequent hits. func (h *telemirrorHub) handleImg(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", 405) return } raw := r.URL.Query().Get("u") if raw == "" { http.Error(w, "missing u", 400) return } if !isProxiableHost(raw) { http.Error(w, "host not allowed", 400) return } if body, ctype, ok := h.imgs.Get(raw); ok { writeTelemirrorImg(w, body, ctype) return } ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() body, ctype, err := h.client.FetchURL(ctx, raw) if err != nil { http.Error(w, err.Error(), 502) return } // Reject HTML/error pages so the browser's onerror fires cleanly. low := strings.ToLower(ctype) if !strings.HasPrefix(low, "image/") && !strings.HasPrefix(low, "video/") && !strings.HasPrefix(low, "audio/") && low != "" { http.Error(w, "upstream not an image: "+ctype, 502) return } if ctype == "" { ctype = "application/octet-stream" } _ = h.imgs.Put(raw, ctype, body) writeTelemirrorImg(w, body, ctype) } func writeTelemirrorImg(w http.ResponseWriter, body []byte, ctype string) { if ctype == "" { ctype = "application/octet-stream" } w.Header().Set("Content-Type", ctype) w.Header().Set("Cache-Control", "private, max-age=3600") w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) _, _ = w.Write(body) } // refresh fetches and parses a channel, coalescing concurrent calls for // the same username so we don't hit the upstream more than once at a time. func (h *telemirrorHub) refresh(username string) (*telemirror.FetchResult, error) { username = strings.ToLower(telemirror.SanitizeUsername(username)) if username == "" { return nil, telemirror.ErrEmptyUsername } h.mu.Lock() if ch, ok := h.refreshing[username]; ok { h.mu.Unlock() <-ch if r, _ := h.cache.Get(username); r != nil { return r, nil } return nil, fmt.Errorf("telemirror: concurrent refresh did not produce a result") } done := make(chan struct{}) h.refreshing[username] = done h.mu.Unlock() defer func() { h.mu.Lock() delete(h.refreshing, username) h.mu.Unlock() close(done) }() ctx, cancel := context.WithTimeout(context.Background(), 3*time.Minute) defer cancel() body, err := h.client.FetchHTML(ctx, username) if err != nil { return nil, err } chInfo, posts, err := telemirror.ParseHTML(body) if err != nil { return nil, err } // Reject "successful" responses that have no widget content — usually // a captcha / rate-limit / soft-error page returned with status 200. if len(posts) == 0 && chInfo.Title == "" && chInfo.Description == "" { return nil, fmt.Errorf("telemirror: empty widget for %q", username) } if chInfo.Username == "" { chInfo.Username = username } res := &telemirror.FetchResult{Channel: *chInfo, Posts: posts} _ = h.cache.Put(username, res) if h.onUpdate != nil { h.onUpdate(username) } go h.ensureAvatarCached(chInfo.Username, chInfo.Photo) return res, nil } // ensureAvatarCached downloads and persists the avatar under username // only when no entry is on disk yet. func (h *telemirrorHub) ensureAvatarCached(username, photoURL string) { username = strings.ToLower(telemirror.SanitizeUsername(username)) if username == "" || photoURL == "" { return } if _, _, ok := h.imgs.GetByKey(username); ok { return } proxyURL := translateGoogify(photoURL) if proxyURL == "" { return } ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() body, ctype, err := h.client.FetchURL(ctx, proxyURL) if err != nil { return } low := strings.ToLower(ctype) if !strings.HasPrefix(low, "image/") && low != "" { return } if ctype == "" { ctype = "image/jpeg" } _ = h.imgs.PutByKey(username, ctype, body) } // handleAvatar serves /api/telemirror/avatar/ from disk; // on miss, recovers the URL from the cached channel JSON, fetches, // persists, and serves. func (h *telemirrorHub) handleAvatar(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", 405) return } username := strings.ToLower(telemirror.SanitizeUsername( strings.TrimPrefix(r.URL.Path, "/api/telemirror/avatar/"))) if username == "" { http.Error(w, "missing username", 400) return } if body, ctype, ok := h.imgs.GetByKey(username); ok { writeTelemirrorImg(w, body, ctype) return } cached, _ := h.cache.Get(username) if cached == nil || cached.Channel.Photo == "" { http.Error(w, "no cached avatar", 404) return } proxyURL := translateGoogify(cached.Channel.Photo) if proxyURL == "" { http.Error(w, "no cached avatar", 404) return } ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() body, ctype, err := h.client.FetchURL(ctx, proxyURL) if err != nil { http.Error(w, err.Error(), 502) return } low := strings.ToLower(ctype) if !strings.HasPrefix(low, "image/") && low != "" { http.Error(w, "upstream not an image: "+ctype, 502) return } if ctype == "" { ctype = "image/jpeg" } _ = h.imgs.PutByKey(username, ctype, body) writeTelemirrorImg(w, body, ctype) } func (h *telemirrorHub) ClearCache() { h.cache.Clear() h.imgs.Clear() } // handleOlder serves /api/telemirror/older/?before=. // Fetches a fresh widget filtered to posts older than the given id. // Not cached — every call hits upstream because pagination data // otherwise grows unbounded per channel. func (h *telemirrorHub) handleOlder(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { http.Error(w, "method not allowed", 405) return } username := strings.ToLower(telemirror.SanitizeUsername( strings.TrimPrefix(r.URL.Path, "/api/telemirror/older/"))) if username == "" { http.Error(w, "missing username", 400) return } beforeID, _ := strconv.Atoi(r.URL.Query().Get("before")) if beforeID <= 0 { http.Error(w, "missing before", 400) return } ctx, cancel := context.WithTimeout(r.Context(), 60*time.Second) defer cancel() body, err := h.client.FetchHTMLBefore(ctx, username, beforeID) if err != nil { http.Error(w, err.Error(), 502) return } chInfo, posts, err := telemirror.ParseHTML(body) if err != nil { http.Error(w, err.Error(), 502) return } if chInfo.Username == "" { chInfo.Username = username } res := &telemirror.FetchResult{Channel: *chInfo, Posts: posts} writeJSON(w, rewriteImageURLs(res)) }