From 625b00d8159db644c0e658155734bd96364343db Mon Sep 17 00:00:00 2001 From: Sarto Date: Tue, 5 May 2026 20:00:27 +0330 Subject: [PATCH] feat: downlaod profile images and fix some bugs ( #75 #76 #77 #78 #79 ) --- README-FA.md | 1 + README.md | 1 + internal/client/profile_pics.go | 161 +++++++ internal/client/profile_pics_test.go | 78 ++++ internal/protocol/dns.go | 7 + internal/protocol/profile_pics.go | 218 ++++++++++ internal/protocol/profile_pics_test.go | 168 ++++++++ internal/server/feed.go | 232 ++++++++++ internal/server/media.go | 28 +- internal/server/profile_pic_telegram.go | 91 ++++ internal/server/profile_pics_test.go | 274 ++++++++++++ internal/server/public.go | 3 + internal/server/public_profile_pics.go | 156 +++++++ internal/server/telegram.go | 16 + internal/server/xpublic.go | 4 + internal/server/xpublic_profile_pics.go | 133 ++++++ internal/telemirror/parser.go | 220 +++++++++- internal/telemirror/parser_test.go | 252 +++++++++++ internal/telemirror/types.go | 29 +- internal/web/profile_pics.go | 539 ++++++++++++++++++++++++ internal/web/relay_info.go | 76 +++- internal/web/static/index.html | 223 +++++++++- internal/web/static/telemirror.js | 96 ++++- internal/web/web.go | 77 +++- 24 files changed, 3045 insertions(+), 38 deletions(-) create mode 100644 internal/client/profile_pics.go create mode 100644 internal/client/profile_pics_test.go create mode 100644 internal/protocol/profile_pics.go create mode 100644 internal/protocol/profile_pics_test.go create mode 100644 internal/server/profile_pic_telegram.go create mode 100644 internal/server/profile_pics_test.go create mode 100644 internal/server/public_profile_pics.go create mode 100644 internal/server/xpublic_profile_pics.go create mode 100644 internal/web/profile_pics.go diff --git a/README-FA.md b/README-FA.md index 86c8272..0f6a7b3 100644 --- a/README-FA.md +++ b/README-FA.md @@ -146,6 +146,7 @@ thefeed یک سیستم تونل DNS است که به شما اجازه می‌ - کانال کانفیگ عمومی دفید: [@thefeedconfig](https://t.me/thefeedconfig) - راهنمای نصب سرور دفید: [@networkti](https://t.me/networkti/25) - راهنمای نصب سرور دفید با اسلیپ گیت: [@networkti](https://t.me/networkti/200) +- لیست تسک‌ها و رودمپ پروژه: [بورد گیتهاب](https://github.com/users/sartoopjj/projects/1/views/1) ## ⚡ نصب سریع سرور diff --git a/README.md b/README.md index 7285752..d96f0e8 100644 --- a/README.md +++ b/README.md @@ -122,6 +122,7 @@ Thank you for your support ❤️ - Public TheFeed Configs: [@thefeedconfig](https://t.me/thefeedconfig) - Setup TheFeed server guide: [@networkti](https://t.me/networkti/25) - Setup TheFeed server with SlipGate guide: [@networkti](https://t.me/networkti/200) +- Roadmap / task board: [GitHub project](https://github.com/users/sartoopjj/projects/1/views/1) ## Quick Install (Server) diff --git a/internal/client/profile_pics.go b/internal/client/profile_pics.go new file mode 100644 index 0000000..512b05f --- /dev/null +++ b/internal/client/profile_pics.go @@ -0,0 +1,161 @@ +package client + +import ( + "context" + "encoding/binary" + "fmt" + "sync" + "time" + + "github.com/sartoopjj/thefeed/internal/protocol" +) + +// ProfilePicsBundle is the client-side view of the profile-pic +// directory. The bundle (Size/CRC/Relays) describes the GitHub-served +// concatenated blob; per-entry DNSChannel/DNSBlocks describe an +// independent DNS fallback for that single avatar. +type ProfilePicsBundle struct { + BundleSize uint32 + BundleCRC uint32 + // Relays describes where the bundle is reachable, indexed by + // RelayDNS / RelayGitHub. RelayGitHub means the bundle is on + // GitHub. RelayDNS for the bundle is rarely true — the standard + // DNS path uses per-entry channels (see ProfilePicEntry). + Relays []bool + + Entries []ProfilePicEntry +} + +// HasRelay forwards to the relay availability bit at idx. +func (b ProfilePicsBundle) HasRelay(idx int) bool { + if idx < 0 || idx >= len(b.Relays) { + return false + } + return b.Relays[idx] +} + +// ProfilePicEntry points at one avatar in two ways: +// +// GitHub bundle path: bytes are bundle[Offset:Offset+Size]; CRC must +// equal CRC32-IEEE of that slice (use protocol.VerifyEntry). +// Per-entry DNS path: bytes live on DNS channel DNSChannel with +// DNSBlocks blocks. CRC and Size are checked the same way. +// +// The client picks whichever path is reachable. With the bundle path +// one HTTPS request fetches every avatar; with the DNS path each +// avatar is fetched independently so partial sets still show up. +type ProfilePicEntry struct { + Username string + Offset uint32 + Size uint32 + CRC uint32 + MIME uint8 + DNSChannel uint16 + DNSBlocks uint16 +} + +// MimeString returns "image/jpeg" / "image/png" / "image/webp" for the +// MIME tag, suitable for use as an HTTP Content-Type. +func (p ProfilePicEntry) MimeString() string { + switch p.MIME { + case protocol.ProfilePicMimePNG: + return "image/png" + case protocol.ProfilePicMimeWebP: + return "image/webp" + default: + return "image/jpeg" + } +} + +// Extension returns ".jpg" / ".png" / ".webp" for caching on disk. +func (p ProfilePicEntry) Extension() string { + switch p.MIME { + case protocol.ProfilePicMimePNG: + return ".png" + case protocol.ProfilePicMimeWebP: + return ".webp" + default: + return ".jpg" + } +} + +// FetchProfilePicDirectory pulls the bundle directory from +// ProfilePicsChannel — header (bundle metadata + relay availability) and +// per-username entries. The bundle bytes themselves are NOT fetched here; +// callers do that with FetchMedia(BundleChannel, BundleBlocks, BundleCRC) +// once and then slice locally. +// +// Returns (zero-value bundle, nil) when the server has no profile pics +// configured (or is older and doesn't know the channel). +func (f *Fetcher) FetchProfilePicDirectory(ctx context.Context) (ProfilePicsBundle, error) { + fetchCtx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + + block0, err := f.FetchBlock(fetchCtx, protocol.ProfilePicsChannel, 0) + if err != nil { + return ProfilePicsBundle{}, fmt.Errorf("fetch profile-pics: %w", err) + } + if len(block0) < 2 { + return ProfilePicsBundle{}, nil + } + totalBlocks := int(binary.BigEndian.Uint16(block0)) + payload0 := block0[2:] + + if totalBlocks <= 1 { + return decodeProfilePicsBundle(payload0) + } + + type res struct { + data []byte + err error + } + results := make([]res, totalBlocks) + results[0] = res{data: payload0} + var wg sync.WaitGroup + for blk := 1; blk < totalBlocks; blk++ { + wg.Add(1) + go func(blk int) { + defer wg.Done() + data, e := f.FetchBlock(fetchCtx, protocol.ProfilePicsChannel, uint16(blk)) + results[blk] = res{data: data, err: e} + }(blk) + } + wg.Wait() + + var all []byte + for _, r := range results { + if r.err != nil { + return ProfilePicsBundle{}, fmt.Errorf("fetch profile-pics block: %w", r.err) + } + all = append(all, r.data...) + } + return decodeProfilePicsBundle(all) +} + +func decodeProfilePicsBundle(data []byte) (ProfilePicsBundle, error) { + if len(data) == 0 { + return ProfilePicsBundle{}, nil + } + pb, err := protocol.DecodeProfilePicsBundle(data) + if err != nil { + return ProfilePicsBundle{}, fmt.Errorf("decode profile-pics: %w", err) + } + out := ProfilePicsBundle{ + BundleSize: pb.Header.BundleSize, + BundleCRC: pb.Header.BundleCRC, + Relays: append([]bool(nil), pb.Header.Relays...), + } + out.Entries = make([]ProfilePicEntry, len(pb.Entries)) + for i, e := range pb.Entries { + out.Entries[i] = ProfilePicEntry{ + Username: e.Username, + Offset: e.Offset, + Size: e.Size, + CRC: e.CRC, + MIME: e.MIME, + DNSChannel: e.DNSChannel, + DNSBlocks: e.DNSBlocks, + } + } + return out, nil +} diff --git a/internal/client/profile_pics_test.go b/internal/client/profile_pics_test.go new file mode 100644 index 0000000..fa07eab --- /dev/null +++ b/internal/client/profile_pics_test.go @@ -0,0 +1,78 @@ +package client + +import ( + "hash/crc32" + "testing" + + "github.com/sartoopjj/thefeed/internal/protocol" +) + +func TestProfilePicMimeAndExtension(t *testing.T) { + cases := []struct { + mime uint8 + wantStr string + wantExt string + }{ + {protocol.ProfilePicMimeJPEG, "image/jpeg", ".jpg"}, + {protocol.ProfilePicMimePNG, "image/png", ".png"}, + {protocol.ProfilePicMimeWebP, "image/webp", ".webp"}, + {255, "image/jpeg", ".jpg"}, // unknown → JPEG fallback + } + for _, c := range cases { + p := ProfilePicEntry{MIME: c.mime} + if got := p.MimeString(); got != c.wantStr { + t.Errorf("MimeString(%d) = %q, want %q", c.mime, got, c.wantStr) + } + if got := p.Extension(); got != c.wantExt { + t.Errorf("Extension(%d) = %q, want %q", c.mime, got, c.wantExt) + } + } +} + +func TestDecodeProfilePicsBundleRoundTrip(t *testing.T) { + // Build a real bundle so the Verify check the caller will run later + // would still succeed. + a := []byte("hello-alice") + b := []byte("hello-bob-bob-bob") + bundle := append(append([]byte{}, a...), b...) + wire := protocol.EncodeProfilePicsBundle(protocol.ProfilePicsBundle{ + Header: protocol.ProfilePicsBundleHeader{ + BundleSize: uint32(len(bundle)), + BundleCRC: crc32.ChecksumIEEE(bundle), + Relays: []bool{false, true}, + }, + Entries: []protocol.ProfilePicEntry{ + {Username: "alice", Offset: 0, Size: uint32(len(a)), CRC: crc32.ChecksumIEEE(a), MIME: protocol.ProfilePicMimeJPEG, DNSChannel: 10001, DNSBlocks: 1}, + {Username: "bob", Offset: uint32(len(a)), Size: uint32(len(b)), CRC: crc32.ChecksumIEEE(b), MIME: protocol.ProfilePicMimePNG, DNSChannel: 10002, DNSBlocks: 2}, + }, + }) + got, err := decodeProfilePicsBundle(wire) + if err != nil { + t.Fatalf("decode: %v", err) + } + if got.BundleSize != uint32(len(bundle)) { + t.Errorf("bundle metadata wrong: %+v", got) + } + if got.Entries[0].DNSChannel != 10001 || got.Entries[1].DNSChannel != 10002 { + t.Errorf("dns channels lost: %+v", got.Entries) + } + if got.HasRelay(protocol.RelayDNS) || !got.HasRelay(protocol.RelayGitHub) { + t.Errorf("relays = %v, want GitHub-only", got.Relays) + } + if len(got.Entries) != 2 { + t.Fatalf("entries = %d, want 2", len(got.Entries)) + } + if got.Entries[0].Username != "alice" || got.Entries[1].Username != "bob" { + t.Errorf("entries: %+v", got.Entries) + } + if got.Entries[1].MIME != protocol.ProfilePicMimePNG { + t.Errorf("bob MIME = %d, want PNG", got.Entries[1].MIME) + } +} + +func TestDecodeProfilePicsBundleEmpty(t *testing.T) { + got, err := decodeProfilePicsBundle(nil) + if err != nil || len(got.Entries) != 0 { + t.Errorf("decode(nil) = %v, %v", got, err) + } +} diff --git a/internal/protocol/dns.go b/internal/protocol/dns.go index 1c47f2d..813d822 100644 --- a/internal/protocol/dns.go +++ b/internal/protocol/dns.go @@ -39,6 +39,13 @@ const ( // owner/repo + domain segment). Block 0 carries it. RelayInfoChannel uint16 = 0xFFF8 + // ProfilePicsChannel serves the per-channel profile-picture index: + // for every Telegram channel that has a profile photo we emit + // (username, mediaCh, size, crc32). Bytes themselves live on the + // referenced mediaCh and are fetched via the regular media path. + // Off by default on the client. + ProfilePicsChannel uint16 = 0xFFF7 + // MaxUpstreamBlockPayload keeps uploaded query chunks comfortably below DNS // name limits across typical domains and resolver paths. MaxUpstreamBlockPayload = 8 diff --git a/internal/protocol/profile_pics.go b/internal/protocol/profile_pics.go new file mode 100644 index 0000000..00ed0e7 --- /dev/null +++ b/internal/protocol/profile_pics.go @@ -0,0 +1,218 @@ +package protocol + +import ( + "encoding/binary" + "fmt" + "hash/crc32" +) + +// Profile pictures use a hybrid layout: every avatar is concatenated +// into one bundle uploaded to the GitHub relay (one file → no +// per-file rate limit), and each avatar also gets its own DNS media +// channel so partial fetches over DNS still display. +// +// Wire layout of ProfilePicsChannel (after the block-count prefix the +// Feed layer adds): +// +// bundleSize uint32 +// bundleCRC uint32 +// relayCount uint8 — N +// relays [N]u8 — bool per relay (RelayDNS=0, RelayGitHub=1, …) +// count uint16 +// entries: +// usernameLen uint8 +// username [usernameLen]byte +// offset uint32 — within the GitHub bundle +// size uint32 +// crc uint32 — CRC32 of bundle[offset:offset+size] +// mime uint8 — 0=jpeg, 1=png, 2=webp +// dnsChannel uint16 — 0 if not on DNS +// dnsBlocks uint16 +type ProfilePicsBundleHeader struct { + BundleSize uint32 + BundleCRC uint32 + // One bool per relay constant. RelayGitHub here means the bundle + // is on GitHub; RelayDNS for the bundle is rare (per-entry DNS + // channels handle the DNS path). + Relays []bool +} + +// HasRelay reports whether the relay at idx is set. Out of range returns false. +func (h ProfilePicsBundleHeader) HasRelay(idx int) bool { + if idx < 0 || idx >= len(h.Relays) { + return false + } + return h.Relays[idx] +} + +// ProfilePicEntry points at one avatar via either the GitHub bundle +// (Offset/Size into the concatenated blob) or its own DNS channel +// (DNSChannel/DNSBlocks). Both paths verify the same Size + CRC. +type ProfilePicEntry struct { + Username string + Offset uint32 + Size uint32 + CRC uint32 + MIME uint8 + DNSChannel uint16 + DNSBlocks uint16 +} + +// MIME tag values. +const ( + ProfilePicMimeJPEG uint8 = 0 + ProfilePicMimePNG uint8 = 1 + ProfilePicMimeWebP uint8 = 2 +) + +// On-the-wire byte counts. +const ( + profilePicEntryFixed = 4 + 4 + 4 + 1 + 2 + 2 // offset+size+crc+mime+dnsCh+dnsBlk + profilePicsHeaderFixed = 4 + 4 + 1 // bundleSize+bundleCRC+relayCount +) + +// ProfilePicsBundle is the directory (header + entries). The bundle +// bytes themselves live in the referenced media channel / relay. +type ProfilePicsBundle struct { + Header ProfilePicsBundleHeader + Entries []ProfilePicEntry +} + +// EncodeProfilePicsBundle serialises the directory. +func EncodeProfilePicsBundle(b ProfilePicsBundle) []byte { + relayCount := len(b.Header.Relays) + if relayCount > 255 { + relayCount = 255 + } + size := profilePicsHeaderFixed + relayCount + 2 /*entry count*/ + for _, e := range b.Entries { + n := len(e.Username) + if n > 255 { + n = 255 + } + size += 1 + n + profilePicEntryFixed + } + buf := make([]byte, size) + off := 0 + binary.BigEndian.PutUint32(buf[off:], b.Header.BundleSize) + off += 4 + binary.BigEndian.PutUint32(buf[off:], b.Header.BundleCRC) + off += 4 + buf[off] = byte(relayCount) + off++ + for i := 0; i < relayCount; i++ { + if b.Header.Relays[i] { + buf[off] = 1 + } + off++ + } + binary.BigEndian.PutUint16(buf[off:], uint16(len(b.Entries))) + off += 2 + for _, e := range b.Entries { + nb := []byte(e.Username) + if len(nb) > 255 { + nb = nb[:255] + } + buf[off] = byte(len(nb)) + off++ + copy(buf[off:], nb) + off += len(nb) + binary.BigEndian.PutUint32(buf[off:], e.Offset) + off += 4 + binary.BigEndian.PutUint32(buf[off:], e.Size) + off += 4 + binary.BigEndian.PutUint32(buf[off:], e.CRC) + off += 4 + buf[off] = e.MIME + off++ + binary.BigEndian.PutUint16(buf[off:], e.DNSChannel) + off += 2 + binary.BigEndian.PutUint16(buf[off:], e.DNSBlocks) + off += 2 + } + return buf +} + +// DecodeProfilePicsBundle parses bytes produced by EncodeProfilePicsBundle. +func DecodeProfilePicsBundle(data []byte) (ProfilePicsBundle, error) { + var out ProfilePicsBundle + if len(data) < profilePicsHeaderFixed+2 { + return out, fmt.Errorf("profile-pics bundle too short: %d bytes", len(data)) + } + off := 0 + out.Header.BundleSize = binary.BigEndian.Uint32(data[off:]) + off += 4 + out.Header.BundleCRC = binary.BigEndian.Uint32(data[off:]) + off += 4 + relayCount := int(data[off]) + off++ + if off+relayCount+2 > len(data) { + return out, fmt.Errorf("profile-pics bundle: truncated relay list") + } + if relayCount > 0 { + out.Header.Relays = make([]bool, relayCount) + for i := 0; i < relayCount; i++ { + out.Header.Relays[i] = data[off] != 0 + off++ + } + } + count := int(binary.BigEndian.Uint16(data[off:])) + off += 2 + out.Entries = make([]ProfilePicEntry, 0, count) + for i := 0; i < count; i++ { + if off >= len(data) { + return out, fmt.Errorf("profile-pics: truncated at entry %d", i) + } + nameLen := int(data[off]) + off++ + if off+nameLen+profilePicEntryFixed > len(data) { + return out, fmt.Errorf("profile-pics: truncated entry %d body", i) + } + name := string(data[off : off+nameLen]) + off += nameLen + offset := binary.BigEndian.Uint32(data[off:]) + off += 4 + sz := binary.BigEndian.Uint32(data[off:]) + off += 4 + cr := binary.BigEndian.Uint32(data[off:]) + off += 4 + mime := data[off] + off++ + dnsCh := binary.BigEndian.Uint16(data[off:]) + off += 2 + dnsBlk := binary.BigEndian.Uint16(data[off:]) + off += 2 + out.Entries = append(out.Entries, ProfilePicEntry{ + Username: name, + Offset: offset, + Size: sz, + CRC: cr, + MIME: mime, + DNSChannel: dnsCh, + DNSBlocks: dnsBlk, + }) + } + return out, nil +} + +// VerifyEntry returns bundle[entry.Offset:entry.Offset+entry.Size] if +// the slice is in-range and its CRC32-IEEE matches entry.CRC. The +// hash check is what stops a misaligned bundle from serving the wrong +// avatar under a username. +func VerifyEntry(bundle []byte, entry ProfilePicEntry) ([]byte, error) { + end := uint64(entry.Offset) + uint64(entry.Size) + if end > uint64(len(bundle)) { + return nil, fmt.Errorf("entry %q out of range: offset=%d size=%d bundle=%d", + entry.Username, entry.Offset, entry.Size, len(bundle)) + } + slice := bundle[entry.Offset:end] + if uint32(len(slice)) != entry.Size { + return nil, fmt.Errorf("entry %q size mismatch: have %d want %d", + entry.Username, len(slice), entry.Size) + } + if got := crc32.ChecksumIEEE(slice); got != entry.CRC { + return nil, fmt.Errorf("entry %q crc mismatch: have %08x want %08x", + entry.Username, got, entry.CRC) + } + return slice, nil +} diff --git a/internal/protocol/profile_pics_test.go b/internal/protocol/profile_pics_test.go new file mode 100644 index 0000000..13a2422 --- /dev/null +++ b/internal/protocol/profile_pics_test.go @@ -0,0 +1,168 @@ +package protocol + +import ( + "hash/crc32" + "strings" + "testing" +) + +func TestEncodeDecodeProfilePicsBundleRoundTrip(t *testing.T) { + // Three avatars concatenated into a fake bundle, with offsets/CRCs + // computed for real so VerifyEntry doesn't trip on them. + a := []byte("aaaaaaaaaa") // 10 bytes + b := []byte("bbbbbbbbbbbbbb") // 14 bytes + c := []byte("ccccc") // 5 bytes + bundle := append(append(append([]byte{}, a...), b...), c...) + + in := ProfilePicsBundle{ + Header: ProfilePicsBundleHeader{ + BundleSize: uint32(len(bundle)), + BundleCRC: crc32.ChecksumIEEE(bundle), + Relays: []bool{false, true}, // bundle on GitHub only; per-entry DNS handles DNS path + }, + Entries: []ProfilePicEntry{ + {Username: "alice", Offset: 0, Size: uint32(len(a)), CRC: crc32.ChecksumIEEE(a), MIME: ProfilePicMimeJPEG, DNSChannel: 10001, DNSBlocks: 1}, + {Username: "bob", Offset: uint32(len(a)), Size: uint32(len(b)), CRC: crc32.ChecksumIEEE(b), MIME: ProfilePicMimePNG, DNSChannel: 10002, DNSBlocks: 2}, + {Username: "carol", Offset: uint32(len(a) + len(b)), Size: uint32(len(c)), CRC: crc32.ChecksumIEEE(c), MIME: ProfilePicMimeWebP, DNSChannel: 10003, DNSBlocks: 1}, + }, + } + wire := EncodeProfilePicsBundle(in) + got, err := DecodeProfilePicsBundle(wire) + if err != nil { + t.Fatalf("decode: %v", err) + } + if got.Header.BundleSize != in.Header.BundleSize || + got.Header.BundleCRC != in.Header.BundleCRC { + t.Errorf("header mismatch: got %+v want %+v", got.Header, in.Header) + } + if len(got.Header.Relays) != len(in.Header.Relays) { + t.Fatalf("relays len = %d, want %d", len(got.Header.Relays), len(in.Header.Relays)) + } + for i, r := range in.Header.Relays { + if got.Header.Relays[i] != r { + t.Errorf("relays[%d] = %v, want %v", i, got.Header.Relays[i], r) + } + } + if len(got.Entries) != len(in.Entries) { + t.Fatalf("entries len = %d, want %d", len(got.Entries), len(in.Entries)) + } + for i, want := range in.Entries { + if got.Entries[i] != want { + t.Errorf("entry %d = %+v want %+v", i, got.Entries[i], want) + } + } + + // VerifyEntry should accept the real bytes… + for _, e := range got.Entries { + if _, err := VerifyEntry(bundle, e); err != nil { + t.Errorf("VerifyEntry(%s) = %v, want ok", e.Username, err) + } + } + // …and reject a tampered bundle. + tampered := append([]byte{}, bundle...) + tampered[0] ^= 0xFF + if _, err := VerifyEntry(tampered, got.Entries[0]); err == nil { + t.Errorf("VerifyEntry should fail on tampered bundle") + } +} + +func TestVerifyEntryOutOfRange(t *testing.T) { + bundle := []byte("hello") + e := ProfilePicEntry{Username: "x", Offset: 0, Size: 100, CRC: 0} + if _, err := VerifyEntry(bundle, e); err == nil { + t.Errorf("VerifyEntry should fail when entry runs past bundle end") + } +} + +func TestVerifyEntrySizeMismatch(t *testing.T) { + bundle := []byte("0123456789") + // CRC is right for the real slice, but Size is wrong → mismatch. + right := bundle[2:6] + e := ProfilePicEntry{ + Username: "y", + Offset: 2, + Size: 5, // claim 5 but slice is 4 + CRC: crc32.ChecksumIEEE(right), + } + // In this case Size=5 + Offset=2 → end=7, in range. CRC will be checked + // over bundle[2:7] which differs from right → mismatch. + if _, err := VerifyEntry(bundle, e); err == nil { + t.Errorf("VerifyEntry should fail when claimed size doesn't match recorded crc") + } +} + +func TestProfilePicsBundleEmpty(t *testing.T) { + in := ProfilePicsBundle{ + Header: ProfilePicsBundleHeader{Relays: []bool{false, false}}, + } + wire := EncodeProfilePicsBundle(in) + got, err := DecodeProfilePicsBundle(wire) + if err != nil { + t.Fatalf("decode empty: %v", err) + } + if len(got.Entries) != 0 { + t.Errorf("entries = %d want 0", len(got.Entries)) + } +} + +func TestProfilePicsTruncatesLongUsername(t *testing.T) { + long := strings.Repeat("x", 300) + in := ProfilePicsBundle{ + Header: ProfilePicsBundleHeader{Relays: []bool{true}}, + Entries: []ProfilePicEntry{ + {Username: long, Offset: 0, Size: 100, CRC: 1, MIME: 0}, + }, + } + wire := EncodeProfilePicsBundle(in) + got, err := DecodeProfilePicsBundle(wire) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(got.Entries) != 1 || len(got.Entries[0].Username) != 255 { + t.Errorf("expected 1 entry with 255-char username, got %+v", got.Entries) + } +} + +func TestProfilePicsTruncatedDataReturnsError(t *testing.T) { + in := ProfilePicsBundle{ + Header: ProfilePicsBundleHeader{Relays: []bool{true}}, + Entries: []ProfilePicEntry{ + {Username: "a", Offset: 0, Size: 1, CRC: 1, MIME: 0}, + }, + } + wire := EncodeProfilePicsBundle(in) + for _, cut := range []int{0, 1, 2, 3, len(wire) - 1} { + _, err := DecodeProfilePicsBundle(wire[:cut]) + if err == nil { + t.Errorf("expected error on cut=%d", cut) + } + } +} + +func TestProfilePicsChannelConstant(t *testing.T) { + if ProfilePicsChannel != 0xFFF7 { + t.Errorf("ProfilePicsChannel = 0x%X, want 0xFFF7", ProfilePicsChannel) + } + others := []uint16{ + SendChannel, AdminChannel, UpstreamInitChannel, UpstreamDataChannel, + VersionChannel, TitlesChannel, RelayInfoChannel, + } + for _, o := range others { + if o == ProfilePicsChannel { + t.Fatalf("ProfilePicsChannel collides with another control channel: 0x%X", o) + } + } +} + +func TestBundleHasRelay(t *testing.T) { + h := ProfilePicsBundleHeader{Relays: []bool{false, true}} + if h.HasRelay(RelayDNS) { + t.Errorf("RelayDNS should be false") + } + if !h.HasRelay(RelayGitHub) { + t.Errorf("RelayGitHub should be true") + } + if h.HasRelay(99) || h.HasRelay(-1) { + t.Errorf("out-of-range should return false") + } +} diff --git a/internal/server/feed.go b/internal/server/feed.go index 6ff37c0..56fcb4e 100644 --- a/internal/server/feed.go +++ b/internal/server/feed.go @@ -4,7 +4,9 @@ import ( "context" "crypto/rand" "fmt" + "hash/crc32" "log" + "sort" "sync" "time" @@ -44,6 +46,16 @@ type Feed struct { // (RelayInfoChannel) — block 0 contains the GitHub "owner/repo" // string, or an empty payload if the relay is off. relayInfoBlocks [][]byte + + // ProfilePicsChannel serves the directory; the bundle bytes live + // in one media-cache entry, with each entry also reachable on its + // own DNS channel. + profilePicsBlocks [][]byte + profilePicsBundle protocol.ProfilePicsBundle + profilePicsBundleBytes []byte // last-built bundle, for MergeProfilePics + // Serialises MergeProfilePics so concurrent readers can't lose + // each other's writes through the read-modify-write sequence. + profilePicsMergeMu sync.Mutex } // NewFeed creates a new Feed with the given channel names. @@ -108,6 +120,9 @@ func (f *Feed) GetBlock(channel, block int) ([]byte, error) { if channel == int(protocol.RelayInfoChannel) { return f.getRelayInfoBlock(block) } + if channel == int(protocol.ProfilePicsChannel) { + return f.getProfilePicsBlock(block) + } // Channel sits in the binary media range — delegate to MediaCache. We // drop the read lock first because MediaCache uses its own lock and we // don't want to hold f.mu across that path. @@ -282,6 +297,223 @@ func (f *Feed) getRelayInfoBlock(block int) ([]byte, error) { return blocks[block], nil } +func (f *Feed) getProfilePicsBlock(block int) ([]byte, error) { + blocks := f.profilePicsBlocks + if len(blocks) == 0 { + // Empty payload still has to be a single non-nil block so the + // usual block-count prefix path stays consistent. + f.rebuildProfilePicsBlocksLocked() + blocks = f.profilePicsBlocks + } + if block < 0 || block >= len(blocks) { + return nil, fmt.Errorf("profile-pics block %d out of range (%d blocks)", block, len(blocks)) + } + return blocks[block], nil +} + +// rebuildProfilePicsBlocksLocked encodes the bundle and splits into +// blocks; block 0 is prefixed with the uint16 block count (same +// convention as titles). Caller holds f.mu. +func (f *Feed) rebuildProfilePicsBlocksLocked() { + payload := protocol.EncodeProfilePicsBundle(f.profilePicsBundle) + blocks := protocol.SplitIntoBlocks(payload) + if len(blocks) == 0 { + blocks = [][]byte{nil} + } + prefix := []byte{byte(len(blocks) >> 8), byte(len(blocks))} + blocks[0] = append(prefix, blocks[0]...) + f.profilePicsBlocks = blocks +} + +// SetProfilePics replaces the profile-pic bundle with the given +// username → image-bytes map. Other usernames currently in the bundle +// are dropped; use MergeProfilePics for additive behaviour. Empty +// values are skipped. Requires SetMediaCache. Returns the number of +// avatars in the resulting bundle. +func (f *Feed) SetProfilePics(pics map[string][]byte) int { + return f.replaceProfilePicsBundle(pics) +} + +// MergeProfilePics is SetProfilePics that retains the existing bundle's +// entries (re-extracted and re-verified) and overlays pics on top. +// Used by readers that only know a subset of channels (Telegram-only, +// X-only) so each one contributes without wiping the others. +// +// Serialised so two readers merging from the same prior state can't +// lose each other's writes. +func (f *Feed) MergeProfilePics(pics map[string][]byte) int { + f.profilePicsMergeMu.Lock() + defer f.profilePicsMergeMu.Unlock() + + merged := make(map[string][]byte, len(pics)) + + f.mu.RLock() + prev := f.profilePicsBundle + prevBytes := f.profilePicsBundleBytes + f.mu.RUnlock() + if len(prev.Entries) > 0 && len(prevBytes) > 0 { + for _, e := range prev.Entries { + slice, err := protocol.VerifyEntry(prevBytes, e) + if err != nil { + log.Printf("[profile-pics] merge: skipping %s (%v)", e.Username, err) + continue + } + cp := make([]byte, len(slice)) + copy(cp, slice) + merged[e.Username] = cp + } + } + for k, v := range pics { + if k == "" { + continue + } + if len(v) == 0 { + delete(merged, k) + continue + } + merged[k] = v + } + return f.replaceProfilePicsBundle(merged) +} + +// replaceProfilePicsBundle is the shared encode-and-store path. Each +// individual avatar gets its own DNS channel via SkipGitHub=true (so +// the DNS path can fetch one at a time without triggering N GitHub +// uploads), then the concatenated bundle is stored once with default +// opts to trigger the single GitHub upload that covers everything. +func (f *Feed) replaceProfilePicsBundle(pics map[string][]byte) int { + f.mu.Lock() + media := f.media + f.mu.Unlock() + if media == nil { + return 0 + } + + type kv struct { + name string + b []byte + } + ordered := make([]kv, 0, len(pics)) + for name, b := range pics { + if name == "" || len(b) == 0 { + continue + } + ordered = append(ordered, kv{name, b}) + } + sort.Slice(ordered, func(i, j int) bool { return ordered[i].name < ordered[j].name }) + + if len(ordered) == 0 { + f.mu.Lock() + f.profilePicsBundle = protocol.ProfilePicsBundle{} + f.profilePicsBundleBytes = nil + f.rebuildProfilePicsBlocksLocked() + f.mu.Unlock() + return 0 + } + + // Build the bundle bytes + per-entry directory. Each entry gets its + // own DNS channel via a SkipGitHub store so the DNS-path client can + // fetch one avatar at a time without dragging the whole bundle. + bundle := make([]byte, 0, 8192) + entries := make([]protocol.ProfilePicEntry, 0, len(ordered)) + for _, e := range ordered { + _, mimeTag := sniffProfilePicMime(e.b) + offset := uint32(len(bundle)) + bundle = append(bundle, e.b...) + + entry := protocol.ProfilePicEntry{ + Username: e.name, + Offset: offset, + Size: uint32(len(e.b)), + CRC: crc32.ChecksumIEEE(e.b), + MIME: mimeTag, + } + // Per-pic DNS channel, no GitHub upload (the bundle covers GitHub). + key := "profile-pic:" + e.name + fname := e.name + switch mimeTag { + case protocol.ProfilePicMimePNG: + fname += ".png" + case protocol.ProfilePicMimeWebP: + fname += ".webp" + default: + fname += ".jpg" + } + picMeta, err := media.StoreWithOptions(key, "[PROFILE]", e.b, mimeStringForTag(mimeTag), fname, + MediaCacheStoreOptions{SkipGitHub: true}) + if err != nil { + // No DNS channel for this entry; bundle path still works. + log.Printf("[profile-pics] store individual %s: %v", e.name, err) + } else { + entry.DNSChannel = picMeta.Channel + entry.DNSBlocks = picMeta.Blocks + } + entries = append(entries, entry) + } + + // One bundle store → one GitHub upload covering every avatar. + bundleMeta, err := media.Store("profile-pics-bundle", "[PROFILE-BUNDLE]", + bundle, "application/octet-stream", "profile-pics.bin") + if err != nil { + log.Printf("[profile-pics] store bundle: %v", err) + return 0 + } + + header := protocol.ProfilePicsBundleHeader{ + BundleSize: uint32(bundleMeta.Size), + BundleCRC: bundleMeta.CRC32, + Relays: append([]bool(nil), bundleMeta.Relays...), + } + + f.mu.Lock() + f.profilePicsBundle = protocol.ProfilePicsBundle{ + Header: header, + Entries: entries, + } + f.profilePicsBundleBytes = bundle + f.rebuildProfilePicsBlocksLocked() + f.mu.Unlock() + return len(entries) +} + +func mimeStringForTag(tag uint8) string { + switch tag { + case protocol.ProfilePicMimePNG: + return "image/png" + case protocol.ProfilePicMimeWebP: + return "image/webp" + default: + return "image/jpeg" + } +} + +// ProfilePicsBundle returns a copy of the current directory. +func (f *Feed) ProfilePicsBundle() protocol.ProfilePicsBundle { + f.mu.RLock() + defer f.mu.RUnlock() + out := protocol.ProfilePicsBundle{ + Header: f.profilePicsBundle.Header, + Entries: make([]protocol.ProfilePicEntry, len(f.profilePicsBundle.Entries)), + } + if len(f.profilePicsBundle.Header.Relays) > 0 { + out.Header.Relays = append([]bool(nil), f.profilePicsBundle.Header.Relays...) + } + copy(out.Entries, f.profilePicsBundle.Entries) + return out +} + +// sniffProfilePicMime returns (rfc-mime, ProfilePicMime tag) by looking +// at the first few bytes. Falls back to JPEG for anything unrecognised. +func sniffProfilePicMime(b []byte) (string, uint8) { + if len(b) >= 4 && b[0] == 0x89 && b[1] == 'P' && b[2] == 'N' && b[3] == 'G' { + return "image/png", protocol.ProfilePicMimePNG + } + if len(b) >= 12 && string(b[0:4]) == "RIFF" && string(b[8:12]) == "WEBP" { + return "image/webp", protocol.ProfilePicMimeWebP + } + return "image/jpeg", protocol.ProfilePicMimeJPEG +} + // rebuildTitlesBlocks re-serializes the display name map and splits it into blocks. // Block 0 is prefixed with a uint16 total-block count so the client can fetch all // remaining blocks in parallel after reading the first one. diff --git a/internal/server/media.go b/internal/server/media.go index cd8a13f..4af0d34 100644 --- a/internal/server/media.go +++ b/internal/server/media.go @@ -119,6 +119,19 @@ func NewMediaCache(cfg MediaCacheConfig) *MediaCache { // client. content is the raw file bytes; the caller may pass a slice it // continues to use after the call (Store copies into block-sized chunks). func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filename string) (protocol.MediaMeta, error) { + return c.StoreWithOptions(cacheKey, tag, content, mimeType, filename, MediaCacheStoreOptions{}) +} + +// MediaCacheStoreOptions toggles relay paths for a single Store call. +// Zero value = both DNS channel and (if a relay is configured) GitHub +// upload. SkipGitHub keeps the DNS allocation but skips the upload — +// used when many small siblings share one bundled GitHub upload. +type MediaCacheStoreOptions struct { + SkipGitHub bool +} + +// StoreWithOptions is Store with selective relay control. +func (c *MediaCache) StoreWithOptions(cacheKey, tag string, content []byte, mimeType, filename string, opts MediaCacheStoreOptions) (protocol.MediaMeta, error) { if cacheKey == "" { return protocol.MediaMeta{}, errors.New("media: empty cache key") } @@ -126,9 +139,6 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen tag = protocol.MediaFile } size := int64(len(content)) - // Reject only when no enabled relay could host this file. A file too big - // for DNS but small enough for GitHub still belongs in the cache — - // MaxAcceptableBytes() collapses both caps into a single ceiling. if max := c.MaxAcceptableBytes(); max > 0 && size > max { atomic.AddUint64(&c.storeRejected, 1) return protocol.MediaMeta{ @@ -246,9 +256,9 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen atomic.AddInt64(&c.currentBytes, size) c.logf("media: store tag=%s key=%s ch=%d size=%d blocks=%d", tag, cacheKey, channel, size, len(blocks)) - // Best-effort relay upload — copy of `content` because the caller may - // reuse the slice. Failures are logged but never block the DNS path. - if c.gh != nil { + // Best-effort relay upload. Copy because the caller may reuse the + // slice. Failures don't block the DNS path. + if c.gh != nil && !opts.SkipGitHub { gh := c.gh body := append([]byte(nil), content...) go func() { @@ -260,7 +270,11 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen }() } - return c.metaForLocked(entry), nil + meta := c.metaForLocked(entry) + if opts.SkipGitHub && len(meta.Relays) > protocol.RelayGitHub { + meta.Relays[protocol.RelayGitHub] = false + } + return meta, nil } // LookupByChannel returns the cached entry's transport metadata (mime, diff --git a/internal/server/profile_pic_telegram.go b/internal/server/profile_pic_telegram.go new file mode 100644 index 0000000..1ffd804 --- /dev/null +++ b/internal/server/profile_pic_telegram.go @@ -0,0 +1,91 @@ +package server + +import ( + "context" + "fmt" + "log" + "strings" + + "github.com/gotd/td/tg" +) + +// extractChatPhotoID returns (photoID, dcID), or (0, 0) when there is none. +func extractChatPhotoID(p tg.ChatPhotoClass) (int64, int) { + switch ph := p.(type) { + case *tg.ChatPhoto: + return ph.PhotoID, ph.DCID + } + return 0, 0 +} + +// extractUserPhotoID is the User-profile equivalent. +func extractUserPhotoID(p tg.UserProfilePhotoClass) (int64, int) { + switch ph := p.(type) { + case *tg.UserProfilePhoto: + return ph.PhotoID, ph.DCID + } + return 0, 0 +} + +// fetchProfilePhoto downloads the small (~5KB / 160px) thumb. Big:false +// is Telegram's "a" thumb size. +func (tr *TelegramReader) fetchProfilePhoto(ctx context.Context, api *tg.Client, peer tg.InputPeerClass, photoID int64) ([]byte, error) { + if photoID == 0 { + return nil, fmt.Errorf("no photo") + } + loc := &tg.InputPeerPhotoFileLocation{ + Peer: peer, + PhotoID: photoID, + } + loc.Big = false + return tr.downloadTelegramFile(ctx, api, loc, 0) +} + +// fetchAllProfilePhotos downloads each channel's avatar (skipping +// unchanged photoIDs) and merges them into the feed bundle. +// Best-effort: per-channel failures are logged and skipped. +func (tr *TelegramReader) fetchAllProfilePhotos(ctx context.Context, api *tg.Client) { + pics := make(map[string][]byte, len(tr.channels)) + for _, username := range tr.channels { + if ctx.Err() != nil { + return + } + username = strings.TrimSpace(username) + if username == "" { + continue + } + rp, err := tr.resolvePeer(ctx, api, username) + if err != nil { + log.Printf("[profile-pics] resolve %s: %v", username, err) + continue + } + if rp.photoID == 0 { + continue + } + // Skip the download if we already pushed this photoID. + tr.mu.Lock() + if tr.lastPhotoID == nil { + tr.lastPhotoID = map[string]int64{} + } + prevID, hadPrev := tr.lastPhotoID[username] + tr.mu.Unlock() + if hadPrev && prevID == rp.photoID { + continue + } + body, err := tr.fetchProfilePhoto(ctx, api, rp.peer, rp.photoID) + if err != nil { + log.Printf("[profile-pics] download %s (id=%d): %v", username, rp.photoID, err) + continue + } + pics[username] = body + tr.mu.Lock() + tr.lastPhotoID[username] = rp.photoID + tr.mu.Unlock() + } + if len(pics) == 0 { + return + } + // Merge so other readers' contributions (e.g. X "x:" entries) survive. + total := tr.feed.MergeProfilePics(pics) + log.Printf("[profile-pics] cycle done: %d new, %d total in bundle", len(pics), total) +} diff --git a/internal/server/profile_pics_test.go b/internal/server/profile_pics_test.go new file mode 100644 index 0000000..68dbcf2 --- /dev/null +++ b/internal/server/profile_pics_test.go @@ -0,0 +1,274 @@ +package server + +import ( + "crypto/rand" + "hash/crc32" + "testing" + "time" + + "github.com/sartoopjj/thefeed/internal/protocol" +) + +// fakeJPEG returns n bytes that start with the JPEG SOI marker so +// sniffProfilePicMime tags them as JPEG. +func fakeJPEG(n int) []byte { + out := make([]byte, n) + rand.Read(out) + if n >= 3 { + out[0] = 0xFF + out[1] = 0xD8 + out[2] = 0xFF + } + return out +} + +func fakePNG(n int) []byte { + out := make([]byte, n) + rand.Read(out) + if n >= 4 { + out[0] = 0x89 + out[1] = 'P' + out[2] = 'N' + out[3] = 'G' + } + return out +} + +func newFeedWithMedia(t *testing.T) *Feed { + t.Helper() + f := NewFeed([]string{"a", "b"}) + mc := NewMediaCache(MediaCacheConfig{ + MaxFileBytes: 64 * 1024, + TTL: time.Hour, + DNSRelayEnabled: true, + }) + f.SetMediaCache(mc) + return f +} + +func TestSetProfilePicsBundlesAvatarsAndExposesDirectory(t *testing.T) { + f := newFeedWithMedia(t) + alice := fakeJPEG(1024) + bob := fakePNG(2048) + + stored := f.SetProfilePics(map[string][]byte{"alice": alice, "bob": bob}) + if stored != 2 { + t.Fatalf("stored = %d, want 2", stored) + } + + got := f.ProfilePicsBundle() + if len(got.Entries) != 2 { + t.Fatalf("entries = %d, want 2", len(got.Entries)) + } + // Sorted by username. + if got.Entries[0].Username != "alice" || got.Entries[1].Username != "bob" { + t.Errorf("entries = %s,%s want alice,bob", + got.Entries[0].Username, got.Entries[1].Username) + } + // MIME survived the sniff. + if got.Entries[0].MIME != protocol.ProfilePicMimeJPEG { + t.Errorf("alice MIME = %d want JPEG", got.Entries[0].MIME) + } + if got.Entries[1].MIME != protocol.ProfilePicMimePNG { + t.Errorf("bob MIME = %d want PNG", got.Entries[1].MIME) + } + // Each per-entry CRC matches the original bytes. + if got.Entries[0].CRC != crc32.ChecksumIEEE(alice) { + t.Errorf("alice crc mismatch") + } + if got.Entries[1].CRC != crc32.ChecksumIEEE(bob) { + t.Errorf("bob crc mismatch") + } + // Offsets are tightly packed. + if got.Entries[0].Offset != 0 { + t.Errorf("alice offset = %d, want 0", got.Entries[0].Offset) + } + if got.Entries[1].Offset != got.Entries[0].Size { + t.Errorf("bob offset = %d, want %d (right after alice)", + got.Entries[1].Offset, got.Entries[0].Size) + } + + // Bundle metadata: total bytes = sum of avatars. + if got.Header.BundleSize != got.Entries[0].Size+got.Entries[1].Size { + t.Errorf("BundleSize = %d, want sum of entries (%d)", + got.Header.BundleSize, got.Entries[0].Size+got.Entries[1].Size) + } + // Each entry has its own DNS channel inside the media range. + for i, e := range got.Entries { + if e.DNSChannel < protocol.MediaChannelStart || e.DNSChannel > protocol.MediaChannelEnd { + t.Errorf("entries[%d].DNSChannel = %d, outside media range", + i, e.DNSChannel) + } + if e.DNSBlocks == 0 { + t.Errorf("entries[%d].DNSBlocks = 0", i) + } + } + // And those per-entry DNS channels are fetchable as ordinary media. + for i, e := range got.Entries { + blk0, err := f.GetBlock(int(e.DNSChannel), 0) + if err != nil || len(blk0) == 0 { + t.Errorf("entries[%d] DNS block 0: %v / %d bytes", i, err, len(blk0)) + } + } + + // And we can pull the directory back over the wire and re-derive + // each avatar's slice with VerifyEntry. + dirBlk, err := f.GetBlock(int(protocol.ProfilePicsChannel), 0) + if err != nil { + t.Fatalf("get profile-pics block 0: %v", err) + } + if len(dirBlk) < 4 { + t.Fatalf("dir block too small: %d", len(dirBlk)) + } + totalBlocks := int(dirBlk[0])<<8 | int(dirBlk[1]) + all := dirBlk[2:] + for i := 1; i < totalBlocks; i++ { + next, err := f.GetBlock(int(protocol.ProfilePicsChannel), i) + if err != nil { + t.Fatalf("get profile-pics block %d: %v", i, err) + } + all = append(all, next...) + } + decoded, err := protocol.DecodeProfilePicsBundle(all) + if err != nil { + t.Fatalf("decode dir: %v", err) + } + if len(decoded.Entries) != 2 { + t.Fatalf("decoded entries = %d, want 2", len(decoded.Entries)) + } + + // Each entry's DNS channel is independently fetchable. This is the + // "if even one DNS channel works, that one avatar still shows" path. + // Full byte-level round-trip is exercised in the client tests. + for _, e := range decoded.Entries { + blk0, err := f.GetBlock(int(e.DNSChannel), 0) + if err != nil || len(blk0) == 0 { + t.Errorf("entry %s: channel %d block 0: %v", + e.Username, e.DNSChannel, err) + } + } +} + +func TestSetProfilePicsExposesRelayBits(t *testing.T) { + f := newFeedWithMedia(t) // DNSRelayEnabled: true, no GitHub relay attached + f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(1024)}) + got := f.ProfilePicsBundle() + if !got.Header.HasRelay(protocol.RelayDNS) { + t.Errorf("RelayDNS bit should be set when DNS is enabled, got Relays=%v", got.Header.Relays) + } + if got.Header.HasRelay(protocol.RelayGitHub) { + t.Errorf("RelayGitHub bit should not be set without a GitHub relay, got Relays=%v", got.Header.Relays) + } +} + +func TestSetProfilePicsSkipsEmpty(t *testing.T) { + f := newFeedWithMedia(t) + stored := f.SetProfilePics(map[string][]byte{ + "": fakeJPEG(100), // empty username + "empty": nil, // empty bytes + "good": fakeJPEG(100), + }) + if stored != 1 { + t.Errorf("stored = %d, want 1", stored) + } +} + +func TestSetProfilePicsReplaceUpdatesBundle(t *testing.T) { + f := newFeedWithMedia(t) + first := fakeJPEG(1024) + f.SetProfilePics(map[string][]byte{"alice": first}) + b1 := f.ProfilePicsBundle() + + second := fakeJPEG(2048) + f.SetProfilePics(map[string][]byte{"alice": second}) + b2 := f.ProfilePicsBundle() + + if len(b1.Entries) != 1 || len(b2.Entries) != 1 { + t.Fatalf("expected one entry each, got %d / %d", len(b1.Entries), len(b2.Entries)) + } + if b1.Entries[0].CRC == b2.Entries[0].CRC { + t.Errorf("entry CRC didn't change after replacing bytes") + } + if b1.Header.BundleCRC == b2.Header.BundleCRC { + t.Errorf("bundle CRC didn't change after replacing bytes") + } +} + +func TestSetProfilePicsClearsOnEmpty(t *testing.T) { + f := newFeedWithMedia(t) + f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(1024)}) + if got := f.ProfilePicsBundle(); len(got.Entries) != 1 { + t.Fatalf("setup: got %d entries", len(got.Entries)) + } + stored := f.SetProfilePics(nil) + if stored != 0 { + t.Errorf("stored = %d, want 0", stored) + } + got := f.ProfilePicsBundle() + if len(got.Entries) != 0 { + t.Errorf("entries = %d, want 0 after empty refresh", len(got.Entries)) + } +} + +func TestMergeProfilePicsKeepsOtherEntries(t *testing.T) { + f := newFeedWithMedia(t) + // First reader contributes Telegram avatars. + tg := map[string][]byte{ + "alice": fakeJPEG(800), + "bob": fakeJPEG(900), + } + if got := f.SetProfilePics(tg); got != 2 { + t.Fatalf("seed Set: got %d want 2", got) + } + + // Second reader contributes X avatars under the "x:" namespace — + // the Telegram entries should survive. + xs := map[string][]byte{ + "x:elonmusk": fakePNG(700), + } + if got := f.MergeProfilePics(xs); got != 3 { + t.Fatalf("merge: got %d want 3", got) + } + + got := f.ProfilePicsBundle() + if len(got.Entries) != 3 { + t.Fatalf("entries = %d want 3", len(got.Entries)) + } + have := map[string]bool{} + for _, e := range got.Entries { + have[e.Username] = true + } + for _, want := range []string{"alice", "bob", "x:elonmusk"} { + if !have[want] { + t.Errorf("missing entry %q in merged bundle", want) + } + } +} + +func TestMergeProfilePicsDropEntry(t *testing.T) { + f := newFeedWithMedia(t) + f.SetProfilePics(map[string][]byte{ + "alice": fakeJPEG(800), + "bob": fakeJPEG(900), + }) + // nil bytes for an existing key drops it. + if got := f.MergeProfilePics(map[string][]byte{"alice": nil}); got != 1 { + t.Fatalf("merge: got %d want 1", got) + } + got := f.ProfilePicsBundle() + if len(got.Entries) != 1 || got.Entries[0].Username != "bob" { + t.Errorf("entries after drop = %+v", got.Entries) + } +} + +func TestSetProfilePicsNoMediaCache(t *testing.T) { + f := NewFeed([]string{"a"}) + stored := f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(100)}) + if stored != 0 { + t.Errorf("stored = %d, want 0 (media cache not configured)", stored) + } + if got := f.ProfilePicsBundle(); len(got.Entries) != 0 { + t.Errorf("entries = %v, want empty", got.Entries) + } +} + diff --git a/internal/server/public.go b/internal/server/public.go index 5c60e05..711e7a5 100644 --- a/internal/server/public.go +++ b/internal/server/public.go @@ -169,6 +169,9 @@ func (pr *PublicReader) fetchAll(ctx context.Context) { } log.Printf("[public] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total", time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(pr.channels)) + // Fetch avatars for the same channel set — same cadence, best-effort. + // Public mode has no MTProto session so we scrape them off t.me/s/. + pr.fetchAllPublicProfilePhotos(ctx) pr.feed.AfterFetchCycle(ctx) } diff --git a/internal/server/public_profile_pics.go b/internal/server/public_profile_pics.go new file mode 100644 index 0000000..ff69bbe --- /dev/null +++ b/internal/server/public_profile_pics.go @@ -0,0 +1,156 @@ +package server + +import ( + "context" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + "sync" + + "golang.org/x/net/html" +) + +// extractPublicAvatarURL finds the channel avatar URL on a t.me/s page. +// Returns "" when the channel has no photo or the layout is unfamiliar. +func extractPublicAvatarURL(body []byte) string { + doc, err := html.Parse(strings.NewReader(string(body))) + if err != nil { + return "" + } + if i := findFirstByClass(doc, "tgme_page_photo_image"); i != nil { + if img := firstImgChild(i); img != nil { + if src := attrValue(img, "src"); src != "" { + return src + } + } + if attrValue(i, "src") != "" { + return attrValue(i, "src") + } + } + if u := findFirstByClass(doc, "tgme_widget_message_user_photo"); u != nil { + if src := attrValue(u, "src"); src != "" { + return src + } + if img := firstImgChild(u); img != nil { + return attrValue(img, "src") + } + } + return "" +} + +func firstImgChild(n *html.Node) *html.Node { + if n == nil { + return nil + } + var found *html.Node + visitNodes(n, func(c *html.Node) { + if found != nil || c.Type != html.ElementNode { + return + } + if c.Data == "img" { + found = c + } + }) + return found +} + +// fetchPublicAvatar downloads the avatar from t.me/s/. +// Returns nil bytes when the channel has no photo. +func (pr *PublicReader) fetchPublicAvatar(ctx context.Context, username string) ([]byte, error) { + username = strings.TrimSpace(username) + if username == "" { + return nil, nil + } + pageURL := pr.baseURL + "/" + url.PathEscape(username) + body, err := httpGetBody(ctx, pr.client, pageURL) + if err != nil { + return nil, fmt.Errorf("fetch %s: %w", pageURL, err) + } + imgURL := extractPublicAvatarURL(body) + if imgURL == "" { + return nil, nil + } + const maxAvatarBytes = 512 * 1024 + imgBytes, err := httpGetWithLimit(ctx, pr.client, imgURL, maxAvatarBytes) + if err != nil { + return nil, fmt.Errorf("download avatar %s: %w", imgURL, err) + } + return imgBytes, nil +} + +// fetchAllPublicProfilePhotos scrapes each channel's avatar from +// t.me/s/ and merges into the bundle. Best-effort. +func (pr *PublicReader) fetchAllPublicProfilePhotos(ctx context.Context) { + pr.mu.RLock() + channels := append([]string(nil), pr.channels...) + pr.mu.RUnlock() + + pics := make(map[string][]byte, len(channels)) + var picsMu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, 4) // cap concurrency vs t.me + for _, u := range channels { + if ctx.Err() != nil { + return + } + u = strings.TrimSpace(u) + if u == "" { + continue + } + wg.Add(1) + go func(u string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + body, err := pr.fetchPublicAvatar(ctx, u) + if err != nil { + log.Printf("[public profile-pic] %s: %v", u, err) + return + } + if len(body) == 0 { + return + } + picsMu.Lock() + pics[u] = body + picsMu.Unlock() + }(u) + } + wg.Wait() + if len(pics) == 0 { + return + } + total := pr.feed.MergeProfilePics(pics) + log.Printf("[public profile-pic] cycle done: %d new, %d total in bundle", len(pics), total) +} + +// httpGetBody is a tiny GET helper. Cap is 8 MB. +func httpGetBody(ctx context.Context, c *http.Client, url string) ([]byte, error) { + return httpGetWithLimit(ctx, c, url, 8*1024*1024) +} + +func httpGetWithLimit(ctx context.Context, c *http.Client, u string, limit int64) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)") + resp, err := c.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("http %s: status %s", u, resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > limit { + return nil, fmt.Errorf("http %s: response exceeds %d bytes", u, limit) + } + return body, nil +} diff --git a/internal/server/telegram.go b/internal/server/telegram.go index acef006..711d3d1 100644 --- a/internal/server/telegram.go +++ b/internal/server/telegram.go @@ -68,6 +68,10 @@ type TelegramReader struct { cacheTTL time.Duration fetchInterval time.Duration + // lastPhotoID dedups profile-pic downloads across cycles. username → + // last seen Telegram photo ID; skip download when unchanged. + lastPhotoID map[string]int64 + // api is set once authenticated, used for sending messages. apiMu sync.RWMutex api *tg.Client @@ -92,6 +96,10 @@ type resolvedPeer struct { chatType protocol.ChatType canSend bool title string + // photoID is the Telegram-assigned ID of the peer's profile photo, + // 0 when the channel has none. Used to skip the download path when + // the photo hasn't changed since last fetch. + photoID int64 } type cachedMessages struct { @@ -289,6 +297,10 @@ func (tr *TelegramReader) fetchAll(ctx context.Context, api *tg.Client) { } log.Printf("[telegram] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total", time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(tr.channels)) + // Profile pics piggyback the regular cycle: best-effort, doesn't + // gate channel data. Each channel's photo is downloaded once and + // only re-fetched when Telegram reports a different photo ID. + tr.fetchAllProfilePhotos(ctx, api) tr.feed.AfterFetchCycle(ctx) } @@ -306,6 +318,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern for _, chat := range resolved.Chats { if ch, ok := chat.(*tg.Channel); ok { canSend := !ch.Broadcast || ch.Creator || ch.AdminRights.PostMessages + pid, _ := extractChatPhotoID(ch.Photo) return &resolvedPeer{ peer: &tg.InputPeerChannel{ ChannelID: ch.ID, @@ -314,6 +327,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern chatType: protocol.ChatTypeChannel, canSend: canSend, title: ch.Title, + photoID: pid, }, nil } } @@ -321,6 +335,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern // Check Users (bots, private chats) for _, u := range resolved.Users { if user, ok := u.(*tg.User); ok { + pid, _ := extractUserPhotoID(user.Photo) return &resolvedPeer{ peer: &tg.InputPeerUser{ UserID: user.ID, @@ -329,6 +344,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern chatType: protocol.ChatTypePrivate, canSend: true, title: user.FirstName, + photoID: pid, }, nil } } diff --git a/internal/server/xpublic.go b/internal/server/xpublic.go index 2a53216..e8c638f 100644 --- a/internal/server/xpublic.go +++ b/internal/server/xpublic.go @@ -226,6 +226,10 @@ func (xr *XPublicReader) fetchAll(ctx context.Context) { } log.Printf("[x] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total", time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(xr.accounts)) + // Avatars: same cycle, merged into the feed's profile-pic bundle. We + // merge rather than overwrite because TelegramReader / PublicReader + // may also be feeding into the same bundle. + xr.fetchAllXProfilePhotos(ctx) xr.feed.AfterFetchCycle(ctx) } diff --git a/internal/server/xpublic_profile_pics.go b/internal/server/xpublic_profile_pics.go new file mode 100644 index 0000000..ed0a7a4 --- /dev/null +++ b/internal/server/xpublic_profile_pics.go @@ -0,0 +1,133 @@ +package server + +import ( + "context" + "encoding/xml" + "fmt" + "io" + "log" + "net/http" + "net/url" + "strings" + "sync" +) + +// Nitter / RSS feeds put the channel image at . +type xRSSImageEnvelope struct { + XMLName xml.Name `xml:"rss"` + Channel struct { + Image struct { + URL string `xml:"url"` + } `xml:"image"` + } `xml:"channel"` +} + +func extractXAvatarURL(body []byte) string { + var env xRSSImageEnvelope + if err := xml.Unmarshal(body, &env); err != nil { + return "" + } + return strings.TrimSpace(env.Channel.Image.URL) +} + +// fetchXAvatar tries each Nitter instance and returns the first +// avatar that downloads. (nil, nil) when no instance had one. +func (xr *XPublicReader) fetchXAvatar(ctx context.Context, account string) ([]byte, error) { + const maxAvatarBytes = 512 * 1024 + var lastErr error + for _, instance := range xr.instances { + rssURL := strings.TrimSuffix(instance, "/") + "/" + url.PathEscape(account) + "/rss" + body, err := xRSSGet(ctx, xr.client, rssURL) + if err != nil { + lastErr = err + continue + } + avatarURL := extractXAvatarURL(body) + if avatarURL == "" { + continue + } + // Some Nitter builds return a relative "/pic/..." path. + if strings.HasPrefix(avatarURL, "/") { + avatarURL = strings.TrimSuffix(instance, "/") + avatarURL + } + imgBytes, err := httpGetWithLimit(ctx, xr.client, avatarURL, maxAvatarBytes) + if err != nil { + lastErr = err + continue + } + return imgBytes, nil + } + if lastErr != nil { + return nil, lastErr + } + return nil, nil +} + +// fetchAllXProfilePhotos downloads each account's avatar (via Nitter) +// and merges into the bundle under "x:" keys. +func (xr *XPublicReader) fetchAllXProfilePhotos(ctx context.Context) { + xr.mu.RLock() + accounts := append([]string(nil), xr.accounts...) + xr.mu.RUnlock() + + pics := make(map[string][]byte, len(accounts)) + var picsMu sync.Mutex + var wg sync.WaitGroup + sem := make(chan struct{}, 4) + for _, a := range accounts { + if ctx.Err() != nil { + return + } + a = strings.TrimSpace(a) + if a == "" { + continue + } + wg.Add(1) + go func(account string) { + defer wg.Done() + sem <- struct{}{} + defer func() { <-sem }() + body, err := xr.fetchXAvatar(ctx, account) + if err != nil { + log.Printf("[x profile-pic] @%s: %v", account, err) + return + } + if len(body) == 0 { + return + } + // "x:" prefix avoids colliding with same-name Telegram channels. + picsMu.Lock() + pics["x:"+account] = body + picsMu.Unlock() + }(a) + } + wg.Wait() + if len(pics) == 0 { + return + } + total := xr.feed.MergeProfilePics(pics) + log.Printf("[x profile-pic] cycle done: %d new, %d total in bundle", len(pics), total) +} + +// xRSSGet mirrors the per-instance RSS request (same UA + Accept). +func xRSSGet(ctx context.Context, c *http.Client, u string) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)") + req.Header.Set("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8") + resp, err := c.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode/100 != 2 { + return nil, fmt.Errorf("rss %s: status %s", u, resp.Status) + } + body, err := io.ReadAll(io.LimitReader(resp.Body, maxXRSSBodyBytes)) + if err != nil { + return nil, err + } + return body, nil +} diff --git a/internal/telemirror/parser.go b/internal/telemirror/parser.go index e7406c3..ae22811 100644 --- a/internal/telemirror/parser.go +++ b/internal/telemirror/parser.go @@ -1,6 +1,7 @@ package telemirror import ( + "net/url" "regexp" "strings" "time" @@ -36,7 +37,7 @@ func parseChannelInfo(doc *html.Node) *Channel { } } if descEl := findFirstByClass(doc, "tgme_channel_info_description"); descEl != nil { - ch.Description = innerHTML(descEl) + ch.Description = rewriteTranslateLinksInHTML(innerHTML(descEl)) } if header := findFirstByClass(doc, "tgme_channel_info_header"); header != nil { if img := findFirstByTag(header, "img"); img != nil { @@ -73,20 +74,28 @@ func parseSinglePost(wrap *html.Node) *Post { if owner := findFirstByClass(msg, "tgme_widget_message_owner_name"); owner != nil { p.Author = textOf(owner) } - if textEl := findFirstByClass(msg, "tgme_widget_message_text"); textEl != nil { - p.Text = innerHTML(textEl) + // The reply preview also has a `.tgme_widget_message_text` (the + // quoted snippet) which appears BEFORE the body in the DOM, so + // findFirstByClass would grab the wrong one. + if textEl := findMessageBodyText(msg); textEl != nil { + p.Text = rewriteTranslateLinksInHTML(innerHTML(textEl)) } + p.Reply = parseReply(msg) + p.Forward = parseForward(msg) + visit(msg, func(n *html.Node) bool { switch { case hasClass(n, "tgme_widget_message_photo_wrap"): + // Source-post URL → real t.me. Thumb stays on the proxy + // (that's how the bytes reach a blocked client). p.Media = append(p.Media, Media{ Type: "photo", - URL: attrOf(n, "href"), + URL: rewriteTranslateLink(attrOf(n, "href")), Thumb: extractBgImage(attrOf(n, "style")), }) case hasClass(n, "tgme_widget_message_video_player"): - m := Media{Type: "video", URL: attrOf(n, "href")} + m := Media{Type: "video", URL: rewriteTranslateLink(attrOf(n, "href"))} if t := findFirstByClass(n, "tgme_widget_message_video_thumb"); t != nil { m.Thumb = extractBgImage(attrOf(t, "style")) } @@ -138,6 +147,16 @@ func parseSinglePost(wrap *html.Node) *Post { if t := findFirstByClass(n, "tgme_widget_message_poll_type"); t != nil { m.Subtitle = textOf(t) } + visit(n, func(opt *html.Node) bool { + if hasClass(opt, "tgme_widget_message_poll_option_text") { + txt := strings.TrimSpace(textOf(opt)) + if txt != "" { + m.Options = append(m.Options, txt) + } + return false + } + return true + }) p.Media = append(p.Media, m) } return true @@ -193,6 +212,72 @@ func parseSinglePost(wrap *html.Node) *Post { return p } +// findMessageBodyText: first `.tgme_widget_message_text` not nested +// inside a `.tgme_widget_message_reply` (which would be the snippet). +func findMessageBodyText(msg *html.Node) *html.Node { + var found *html.Node + visit(msg, func(n *html.Node) bool { + if found != nil { + return false + } + if !hasClass(n, "tgme_widget_message_text") { + return true + } + for p := n.Parent; p != nil; p = p.Parent { + if hasClass(p, "tgme_widget_message_reply") { + return false + } + } + found = n + return false + }) + return found +} + +// parseReply extracts author + snippet + URL from a reply preview. +func parseReply(msg *html.Node) *Reply { + rNode := findFirstByClass(msg, "tgme_widget_message_reply") + if rNode == nil { + return nil + } + r := &Reply{ + URL: rewriteTranslateLink(attrOf(rNode, "href")), + } + if a := findFirstByClass(rNode, "tgme_widget_message_author_name"); a != nil { + r.Author = textOf(a) + } + if t := findFirstByClass(rNode, "tgme_widget_message_text"); t != nil { + r.Text = rewriteTranslateLinksInHTML(innerHTML(t)) + } + if r.Author == "" && r.Text == "" { + return nil + } + return r +} + +// parseForward extracts the "Forwarded from " header. +func parseForward(msg *html.Node) *Forward { + fNode := findFirstByClass(msg, "tgme_widget_message_forwarded_from") + if fNode == nil { + return nil + } + f := &Forward{} + if a := findFirstByClass(fNode, "tgme_widget_message_forwarded_from_name"); a != nil { + f.Author = textOf(a) + f.URL = rewriteTranslateLink(attrOf(a, "href")) + } else if a := findFirstByTag(fNode, "a"); a != nil { + // Bare without the _name class. + f.Author = textOf(a) + f.URL = rewriteTranslateLink(attrOf(a, "href")) + } else { + f.Author = textOf(fNode) + } + if f.Author == "" { + return nil + } + return f +} + // ===== DOM helpers ===== func visit(n *html.Node, fn func(*html.Node) bool) { @@ -313,3 +398,128 @@ func extractBgImage(style string) string { } return "" } + +// Google Translate proxy URLs come in two forms; we rewrite hrefs +// back to the originals so links work when Translate is blocked / the +// site is region-locked. Image src attributes are left alone since +// the proxy is how those bytes actually reach the user. + +const translateHostSuffix = ".translate.goog" + +// decodeTranslateHost: '.' ↔ '-', '-' ↔ '--'. +// +// t-me → t.me +// cdn4-telegram-org → cdn4.telegram.org +// my--domain-com → my-domain.com +func decodeTranslateHost(s string) string { + s = strings.ReplaceAll(s, "--", "\x00") + s = strings.ReplaceAll(s, "-", ".") + s = strings.ReplaceAll(s, "\x00", "-") + return s +} + +// rewriteTranslateLink handles two forms: +// 1. .translate.goog/?_x_tr_*=… — inline-proxy form; +// decode host, strip tracking params. +// 2. translate.google.com/website?u= (also /translate) — +// wrapper form; pull `u` and recurse once (Google occasionally +// double-wraps). +func rewriteTranslateLink(raw string) string { + if raw == "" { + return raw + } + u, err := url.Parse(raw) + if err != nil { + return raw + } + + // Form 2. + hostLower := strings.ToLower(u.Host) + if (hostLower == "translate.google.com" || hostLower == "www.translate.google.com") && + (u.Path == "/website" || u.Path == "/translate") { + if orig := u.Query().Get("u"); orig != "" { + return rewriteTranslateLink(orig) + } + } + + // Form 1. + if !strings.HasSuffix(hostLower, translateHostSuffix) { + return raw + } + encoded := u.Host[:len(u.Host)-len(translateHostSuffix)] + u.Host = decodeTranslateHost(encoded) + if q := u.Query(); len(q) > 0 { + for k := range q { + if strings.HasPrefix(k, "_x_tr_") { + q.Del(k) + } + } + u.RawQuery = q.Encode() + } + return u.String() +} + +// rewriteTranslateLinksInHTML rewrites in an HTML fragment. +// is intentionally left alone (we serve images via the proxy). +func rewriteTranslateLinksInHTML(htmlStr string) string { + if htmlStr == "" || !containsAnyTranslateMarker(htmlStr) { + return htmlStr + } + // Sentinel div so we can locate the fragment in the parsed tree + // (html.Parse injects …). + doc, err := html.Parse(strings.NewReader("
" + htmlStr + "
")) + if err != nil { + return htmlStr + } + rewriteHrefsInTree(doc) + root := findFirstByID(doc, "tm-rewrite-root") + if root == nil { + return htmlStr + } + var b strings.Builder + for c := root.FirstChild; c != nil; c = c.NextSibling { + if err := html.Render(&b, c); err != nil { + return htmlStr + } + } + return b.String() +} + +// containsAnyTranslateMarker is a cheap pre-check that catches both +// the inline-proxy and wrapper forms. +func containsAnyTranslateMarker(s string) bool { + return strings.Contains(s, translateHostSuffix) || + strings.Contains(s, "translate.google.com/") +} + +func rewriteHrefsInTree(n *html.Node) { + if n.Type == html.ElementNode && n.Data == "a" { + for i, a := range n.Attr { + if a.Key == "href" { + n.Attr[i].Val = rewriteTranslateLink(a.Val) + } + } + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + rewriteHrefsInTree(c) + } +} + +func findFirstByID(root *html.Node, id string) *html.Node { + var found *html.Node + visit(root, func(n *html.Node) bool { + if found != nil { + return false + } + if n.Type == html.ElementNode { + for _, a := range n.Attr { + if a.Key == "id" && a.Val == id { + found = n + return false + } + } + } + return true + }) + return found +} diff --git a/internal/telemirror/parser_test.go b/internal/telemirror/parser_test.go index cfcc8de..312652e 100644 --- a/internal/telemirror/parser_test.go +++ b/internal/telemirror/parser_test.go @@ -109,6 +109,258 @@ func TestParseHTMLPosts(t *testing.T) { } } +// Telegram embeds the reply preview INSIDE the message wrapper, BEFORE +// the actual message body. Both the snippet and the body use the +// `tgme_widget_message_text` class. The parser must pick the body, not +// the snippet, and must surface the reply metadata as Post.Reply. +const replyPostHTML = ` +
+
+ + Original Author +
quoted snippet of original
+ +
actual reply body
+
+
+` + +func TestParseHTMLReply(t *testing.T) { + _, posts, err := ParseHTML(replyPostHTML) + if err != nil { + t.Fatalf("ParseHTML: %v", err) + } + if len(posts) != 1 { + t.Fatalf("posts = %d, want 1", len(posts)) + } + p := posts[0] + if !strings.Contains(p.Text, "actual reply body") { + t.Errorf("Post.Text = %q, want main body, not snippet", p.Text) + } + if strings.Contains(p.Text, "quoted snippet") { + t.Errorf("Post.Text leaked the reply snippet: %q", p.Text) + } + if p.Reply == nil { + t.Fatalf("Post.Reply nil; want populated reply preview") + } + if p.Reply.Author != "Original Author" { + t.Errorf("Reply.Author = %q", p.Reply.Author) + } + if !strings.Contains(p.Reply.Text, "quoted snippet") { + t.Errorf("Reply.Text = %q", p.Reply.Text) + } + if p.Reply.URL != "https://t.me/sample/198" { + t.Errorf("Reply.URL = %q", p.Reply.URL) + } +} + +const forwardPostHTML = ` +
+
+
+ Forwarded from Source Channel +
+
forwarded post body
+
+
+` + +func TestParseHTMLForward(t *testing.T) { + _, posts, err := ParseHTML(forwardPostHTML) + if err != nil { + t.Fatalf("ParseHTML: %v", err) + } + if len(posts) != 1 { + t.Fatalf("posts = %d, want 1", len(posts)) + } + p := posts[0] + if p.Forward == nil { + t.Fatalf("Post.Forward nil; want populated forward header") + } + if p.Forward.Author != "Source Channel" { + t.Errorf("Forward.Author = %q", p.Forward.Author) + } + if p.Forward.URL != "https://t.me/source" { + t.Errorf("Forward.URL = %q", p.Forward.URL) + } + if !strings.Contains(p.Text, "forwarded post body") { + t.Errorf("Post.Text = %q", p.Text) + } +} + +const pollPostHTML = ` +
+
+
+
Favourite colour?
+
Public vote
+
+
Red
+
+
+
Blue
+
+
+
Green
+
+
+
+
+` + +func TestParseHTMLPollOptions(t *testing.T) { + _, posts, err := ParseHTML(pollPostHTML) + if err != nil { + t.Fatalf("ParseHTML: %v", err) + } + if len(posts) != 1 || len(posts[0].Media) != 1 { + t.Fatalf("posts/media wrong shape: posts=%d media=%d", len(posts), len(posts[0].Media)) + } + m := posts[0].Media[0] + if m.Type != "poll" { + t.Errorf("Media.Type = %q, want poll", m.Type) + } + if m.Title != "Favourite colour?" { + t.Errorf("Media.Title = %q", m.Title) + } + if m.Subtitle != "Public vote" { + t.Errorf("Media.Subtitle = %q", m.Subtitle) + } + wantOpts := []string{"Red", "Blue", "Green"} + if len(m.Options) != len(wantOpts) { + t.Fatalf("Options = %v, want %v", m.Options, wantOpts) + } + for i, want := range wantOpts { + if m.Options[i] != want { + t.Errorf("Options[%d] = %q, want %q", i, m.Options[i], want) + } + } +} + +func TestDecodeTranslateHost(t *testing.T) { + cases := []struct { + in, want string + }{ + {"t-me", "t.me"}, + {"cdn4-telegram-org", "cdn4.telegram.org"}, + {"my--domain-com", "my-domain.com"}, + {"a--b--c-d", "a-b-c.d"}, + {"plain", "plain"}, + {"", ""}, + } + for _, c := range cases { + if got := decodeTranslateHost(c.in); got != c.want { + t.Errorf("decodeTranslateHost(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestRewriteTranslateLink(t *testing.T) { + cases := []struct { + in, want string + }{ + { + "https://t-me.translate.goog/sample/123", + "https://t.me/sample/123", + }, + { + "https://example-com.translate.goog/path?_x_tr_sl=auto&_x_tr_tl=en&q=keep", + "https://example.com/path?q=keep", + }, + { + "https://my--domain-com.translate.goog/x", + "https://my-domain.com/x", + }, + // Untouched: not a translate.goog URL. + {"https://example.com/foo", "https://example.com/foo"}, + // Untouched: empty. + {"", ""}, + // Tracking-only query string collapses to no query string. + { + "https://t-me.translate.goog/sample?_x_tr_pto=wapp", + "https://t.me/sample", + }, + // Wrapper form: translate.google.com/website?u= + { + "https://translate.google.com/website?sl=auto&tl=fa&hl=en&client=webapp&u=https://seup.shop/f/rdzq", + "https://seup.shop/f/rdzq", + }, + // Wrapper form, /translate path also works. + { + "https://translate.google.com/translate?sl=en&tl=fa&u=https://example.com/page", + "https://example.com/page", + }, + // Wrapper form pointing at a goog inline-proxy URL — recurse. + { + "https://translate.google.com/website?u=https://t-me.translate.goog/networkti", + "https://t.me/networkti", + }, + } + for _, c := range cases { + if got := rewriteTranslateLink(c.in); got != c.want { + t.Errorf("rewriteTranslateLink(%q) = %q, want %q", c.in, got, c.want) + } + } +} + +func TestRewriteTranslateLinksInHTML(t *testing.T) { + in := `Check this post ` + + `and this site. ` + + ` stays.` + got := rewriteTranslateLinksInHTML(in) + if !strings.Contains(got, `href="https://t.me/sample/123"`) { + t.Errorf("translate.goog href not rewritten: %q", got) + } + if !strings.Contains(got, `href="https://example.com/"`) { + t.Errorf("example-com.translate.goog href not rewritten or query not stripped: %q", got) + } + if !strings.Contains(got, `src="https://cdn-translate.goog/img.jpg"`) { + t.Errorf("img src should NOT be rewritten (we serve images via the proxy): %q", got) + } + // Sanity check: untouched HTML round-trips without the rewriter + // damaging it. + plain := "no proxy links here, just text" + if got := rewriteTranslateLinksInHTML(plain); got != plain { + t.Errorf("plain html mutated: %q", got) + } +} + +func TestParseHTMLRewritesPostLinks(t *testing.T) { + const sample = ` +
+
+
+ Visit @networkti and + read, + and try this site. +
+
+
+` + _, posts, err := ParseHTML(sample) + if err != nil { + t.Fatalf("ParseHTML: %v", err) + } + if len(posts) != 1 { + t.Fatalf("posts = %d, want 1", len(posts)) + } + if !strings.Contains(posts[0].Text, `href="https://t.me/networkti"`) { + t.Errorf("Post.Text didn't rewrite t-me link: %q", posts[0].Text) + } + if !strings.Contains(posts[0].Text, `href="https://seup.shop/f/rdzq"`) { + t.Errorf("Post.Text didn't unwrap translate.google.com/website wrapper: %q", posts[0].Text) + } + if strings.Contains(posts[0].Text, "translate.goog") { + t.Errorf("translate.goog leaked in Post.Text: %q", posts[0].Text) + } + if strings.Contains(posts[0].Text, "translate.google.com") { + t.Errorf("translate.google.com wrapper leaked in Post.Text: %q", posts[0].Text) + } + if strings.Contains(posts[0].Text, "_x_tr_") { + t.Errorf("_x_tr_ tracking params leaked: %q", posts[0].Text) + } +} + func TestParseHTMLEmpty(t *testing.T) { ch, posts, err := ParseHTML("") if err != nil { diff --git a/internal/telemirror/types.go b/internal/telemirror/types.go index a4efec8..9660a6c 100644 --- a/internal/telemirror/types.go +++ b/internal/telemirror/types.go @@ -26,25 +26,40 @@ type Channel struct { // Media is one attachment on a post. type Media struct { - Type string `json:"type"` // "photo" | "video" | "voice" | "audio" | "document" | "sticker" | "poll" - URL string `json:"url,omitempty"` - Thumb string `json:"thumb,omitempty"` - Duration string `json:"duration,omitempty"` - Title string `json:"title,omitempty"` // file name / poll question / audio title - Subtitle string `json:"subtitle,omitempty"` + Type string `json:"type"` // photo | video | voice | audio | document | sticker | poll + URL string `json:"url,omitempty"` + Thumb string `json:"thumb,omitempty"` + Duration string `json:"duration,omitempty"` + Title string `json:"title,omitempty"` // file name / poll question / audio title + Subtitle string `json:"subtitle,omitempty"` + Options []string `json:"options,omitempty"` // poll options } -// Reaction is one emoji + count on a post. type Reaction struct { Emoji string `json:"emoji"` Count string `json:"count,omitempty"` } +// Reply is the quoted preview shown above a reply. +type Reply struct { + Author string `json:"author,omitempty"` + Text string `json:"text,omitempty"` // inner HTML of the snippet + URL string `json:"url,omitempty"` // link to the replied-to post +} + +// Forward is the "Forwarded from " header. +type Forward struct { + Author string `json:"author,omitempty"` + URL string `json:"url,omitempty"` +} + // Post is a single message from the channel feed. type Post struct { ID string `json:"id"` // "/" Author string `json:"author,omitempty"` Text string `json:"text,omitempty"` // sanitised inner HTML + Reply *Reply `json:"reply,omitempty"` + Forward *Forward `json:"forward,omitempty"` Media []Media `json:"media,omitempty"` Reactions []Reaction `json:"reactions,omitempty"` Time time.Time `json:"time,omitempty"` diff --git a/internal/web/profile_pics.go b/internal/web/profile_pics.go new file mode 100644 index 0000000..b1222b1 --- /dev/null +++ b/internal/web/profile_pics.go @@ -0,0 +1,539 @@ +package web + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "hash/crc32" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/sartoopjj/thefeed/internal/client" + "github.com/sartoopjj/thefeed/internal/protocol" +) + +// profilePicsHub caches channel avatars on disk and coordinates the +// background fetch. GitHub bundle is automatic; per-entry DNS path +// requires the ProfilePicsEnabled toggle. +type profilePicsHub struct { + dataDir string + + mu sync.Mutex + index map[string]profilePicCacheEntry // username (lower) -> entry + fetching bool + progress profilePicProgress + // CRC of the bundle that produced our current cache; if the next + // directory advertises the same value we skip the download. + bundleCRC uint32 +} + +type profilePicCacheEntry struct { + CRC uint32 `json:"crc"` + Size uint32 `json:"size"` + MIME uint8 `json:"mime"` + Extension string `json:"ext"` + StoredAt int64 `json:"storedAt"` +} + +// profilePicProgress is polled by the UI during refresh. +type profilePicProgress struct { + Active bool `json:"active"` + Total int `json:"total"` + Done int `json:"done"` + Failed int `json:"failed"` + Username string `json:"username,omitempty"` + Error string `json:"error,omitempty"` +} + +type profilePicsIndexFile struct { + BundleCRC uint32 `json:"bundleCrc"` + Users map[string]profilePicCacheEntry `json:"users"` +} + +func newProfilePicsHub(dataDir string) *profilePicsHub { + h := &profilePicsHub{ + dataDir: dataDir, + index: make(map[string]profilePicCacheEntry), + } + h.loadIndex() + return h +} + +func (h *profilePicsHub) cacheDir() string { + return filepath.Join(h.dataDir, "profile_pics") +} + +func (h *profilePicsHub) indexPath() string { + return filepath.Join(h.cacheDir(), "index.json") +} + +func (h *profilePicsHub) imagePath(username, ext string) string { + // "x:handle" → "x__handle" — Windows can't use ':' in filenames. + safe := strings.ReplaceAll(strings.ToLower(username), ":", "__") + return filepath.Join(h.cacheDir(), safe+ext) +} + +func (h *profilePicsHub) loadIndex() { + b, err := os.ReadFile(h.indexPath()) + if err != nil { + return + } + var idx profilePicsIndexFile + if err := json.Unmarshal(b, &idx); err != nil { + return + } + h.mu.Lock() + h.index = idx.Users + if h.index == nil { + h.index = make(map[string]profilePicCacheEntry) + } + h.bundleCRC = idx.BundleCRC + h.mu.Unlock() +} + +func (h *profilePicsHub) saveIndexLocked() { + if err := os.MkdirAll(h.cacheDir(), 0700); err != nil { + return + } + idx := profilePicsIndexFile{ + BundleCRC: h.bundleCRC, + Users: h.index, + } + b, err := json.MarshalIndent(idx, "", " ") + if err != nil { + return + } + _ = os.WriteFile(h.indexPath(), b, 0600) +} + +// Store writes bytes to disk and updates the index. Any previous file +// for this username (different extension) is removed. +func (h *profilePicsHub) Store(username string, content []byte, crc uint32, mime uint8) error { + if username == "" || len(content) == 0 { + return errors.New("profile-pics: empty input") + } + if err := os.MkdirAll(h.cacheDir(), 0700); err != nil { + return err + } + ext := extensionFor(mime) + tmp := h.imagePath(username, ext) + ".tmp" + if err := os.WriteFile(tmp, content, 0600); err != nil { + return err + } + final := h.imagePath(username, ext) + if err := os.Rename(tmp, final); err != nil { + _ = os.Remove(tmp) + return err + } + + h.mu.Lock() + if old, ok := h.index[strings.ToLower(username)]; ok && old.Extension != ext { + _ = os.Remove(h.imagePath(username, old.Extension)) + } + h.index[strings.ToLower(username)] = profilePicCacheEntry{ + CRC: crc, + Size: uint32(len(content)), + MIME: mime, + Extension: ext, + StoredAt: time.Now().Unix(), + } + h.saveIndexLocked() + h.mu.Unlock() + return nil +} + +// Get returns the cached bytes + content type, or os.ErrNotExist. +func (h *profilePicsHub) Get(username string) ([]byte, string, error) { + h.mu.Lock() + e, ok := h.index[strings.ToLower(username)] + h.mu.Unlock() + if !ok { + return nil, "", os.ErrNotExist + } + b, err := os.ReadFile(h.imagePath(username, e.Extension)) + if err != nil { + return nil, "", err + } + return b, contentTypeFor(e.MIME), nil +} + +// Clear wipes both the on-disk cache and the index. Hooked by /api/cache/clear. +func (h *profilePicsHub) Clear() { + h.mu.Lock() + defer h.mu.Unlock() + h.index = make(map[string]profilePicCacheEntry) + h.bundleCRC = 0 + if entries, err := os.ReadDir(h.cacheDir()); err == nil { + for _, e := range entries { + if !e.IsDir() { + _ = os.Remove(filepath.Join(h.cacheDir(), e.Name())) + } + } + } +} + +func (h *profilePicsHub) Progress() profilePicProgress { + h.mu.Lock() + defer h.mu.Unlock() + return h.progress +} + +// relayFetcher pulls the bundle bytes from the GitHub relay. nil means +// "skip the GitHub path". +type relayFetcher func(ctx context.Context, size int64, crc uint32) ([]byte, error) + +// storedCallback fires after each successful persist so the server can +// push an SSE event mid-refresh. +type storedCallback func(username string) + +// Refresh is the test entry point (DNS only, no GitHub fetcher). +func (h *profilePicsHub) Refresh(ctx context.Context, fetcher *client.Fetcher, dnsAllowed bool) error { + return h.refresh(ctx, fetcher, dnsAllowed, nil, nil) +} + +// refresh tries the GitHub bundle first, then per-entry DNS for +// anything still missing. Coalesces concurrent calls. +func (h *profilePicsHub) refresh(ctx context.Context, fetcher *client.Fetcher, dnsAllowed bool, viaGitHub relayFetcher, onStored storedCallback) error { + h.mu.Lock() + if h.fetching { + h.mu.Unlock() + return errors.New("profile-pics: refresh already in progress") + } + h.fetching = true + h.progress = profilePicProgress{Active: true} + h.mu.Unlock() + + defer func() { + h.mu.Lock() + h.fetching = false + h.progress.Active = false + h.mu.Unlock() + }() + + bundle, err := fetcher.FetchProfilePicDirectory(ctx) + if err != nil { + h.setProgressErr(fmt.Sprintf("fetch directory: %v", err)) + return err + } + if len(bundle.Entries) == 0 { + return nil + } + + h.mu.Lock() + h.progress.Total = len(bundle.Entries) + prevCRC := h.bundleCRC + h.mu.Unlock() + + // Same bundle, all entries cached → no download. + if prevCRC != 0 && prevCRC == bundle.BundleCRC && h.allEntriesCached(bundle.Entries) { + h.mu.Lock() + h.progress.Done = len(bundle.Entries) + h.mu.Unlock() + return nil + } + + // Phase 1: GitHub bundle (one fetch, all entries). + missing := bundle.Entries + hasGitHub := bundle.HasRelay(protocol.RelayGitHub) && viaGitHub != nil && bundle.BundleSize > 0 + if hasGitHub { + body, ghErr := viaGitHub(ctx, int64(bundle.BundleSize), bundle.BundleCRC) + if ghErr != nil { + h.setProgressErr(fmt.Sprintf("github bundle: %v", ghErr)) + } else if body != nil { + if uint32(len(body)) == bundle.BundleSize && crc32.ChecksumIEEE(body) == bundle.BundleCRC { + missing = h.persistFromBundle(ctx, body, bundle.Entries, onStored) + h.mu.Lock() + h.bundleCRC = bundle.BundleCRC + h.saveIndexLocked() + h.mu.Unlock() + } else { + h.setProgressErr(fmt.Sprintf("github bundle size/crc mismatch: have %d/%08x want %d/%08x", + len(body), crc32.ChecksumIEEE(body), bundle.BundleSize, bundle.BundleCRC)) + } + } + } + + // Phase 2: per-entry DNS for whatever the bundle didn't cover. + if dnsAllowed && len(missing) > 0 { + h.fetchMissingViaDNS(ctx, fetcher, missing, onStored) + } + return nil +} + +// persistFromBundle slices each entry out, verifies, writes to disk. +// Returns the entries that didn't land (so the DNS phase can retry). +func (h *profilePicsHub) persistFromBundle(ctx context.Context, body []byte, entries []client.ProfilePicEntry, onStored storedCallback) []client.ProfilePicEntry { + missing := make([]client.ProfilePicEntry, 0, len(entries)) + for _, entry := range entries { + if ctx.Err() != nil { + return missing + } + h.markProgressUsername(entry.Username) + if h.HasFresh(entry.Username, entry.CRC) { + h.bumpProgress(true) + continue + } + slice, err := protocol.VerifyEntry(body, protocol.ProfilePicEntry{ + Username: entry.Username, + Offset: entry.Offset, + Size: entry.Size, + CRC: entry.CRC, + MIME: entry.MIME, + DNSChannel: entry.DNSChannel, + DNSBlocks: entry.DNSBlocks, + }) + if err != nil { + missing = append(missing, entry) + continue + } + if err := h.Store(entry.Username, slice, entry.CRC, entry.MIME); err != nil { + missing = append(missing, entry) + continue + } + h.bumpProgress(true) + if onStored != nil { + onStored(entry.Username) + } + } + return missing +} + +// fetchMissingViaDNS fetches each entry on its own DNS channel, +// verifies, persists. Per-entry independent: one failure doesn't +// affect the others. +func (h *profilePicsHub) fetchMissingViaDNS(ctx context.Context, fetcher *client.Fetcher, entries []client.ProfilePicEntry, onStored storedCallback) { + for _, entry := range entries { + if ctx.Err() != nil { + return + } + h.markProgressUsername(entry.Username) + if entry.DNSChannel == 0 || entry.DNSBlocks == 0 { + h.bumpProgress(false) + continue + } + // HasFresh is checked again here — a previous Phase-1 success + // could already have populated this entry from another path. + if h.HasFresh(entry.Username, entry.CRC) { + h.bumpProgress(true) + continue + } + body, err := fetcher.FetchMedia(ctx, entry.DNSChannel, entry.DNSBlocks, entry.CRC, nil) + if err != nil { + h.bumpProgress(false) + continue + } + if uint32(len(body)) != entry.Size { + h.bumpProgress(false) + continue + } + if crc32.ChecksumIEEE(body) != entry.CRC { + h.bumpProgress(false) + continue + } + if err := h.Store(entry.Username, body, entry.CRC, entry.MIME); err != nil { + h.bumpProgress(false) + continue + } + h.bumpProgress(true) + if onStored != nil { + onStored(entry.Username) + } + } +} + +// HasFresh: cached pic with this CRC is already on disk. +func (h *profilePicsHub) HasFresh(username string, crc uint32) bool { + h.mu.Lock() + defer h.mu.Unlock() + e, ok := h.index[strings.ToLower(username)] + if !ok || e.CRC != crc { + return false + } + _, err := os.Stat(h.imagePath(username, e.Extension)) + return err == nil +} + +func (h *profilePicsHub) allEntriesCached(entries []client.ProfilePicEntry) bool { + for _, e := range entries { + if !h.HasFresh(e.Username, e.CRC) { + return false + } + } + return true +} + +func (h *profilePicsHub) markProgressUsername(username string) { + h.mu.Lock() + h.progress.Username = username + h.mu.Unlock() +} + +func (h *profilePicsHub) bumpProgress(ok bool) { + h.mu.Lock() + h.progress.Done++ + if !ok { + h.progress.Failed++ + } + h.mu.Unlock() +} + +func (h *profilePicsHub) setProgressErr(msg string) { + h.mu.Lock() + h.progress.Error = msg + h.mu.Unlock() +} + +// ===== HTTP handlers ===== + +// handleProfilePic serves /api/profile-pics/. Key is "" +// or "x:". 404 → front-end falls back to the letter avatar. +func (h *profilePicsHub) handleProfilePic(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", 405) + return + } + key := strings.TrimPrefix(r.URL.Path, "/api/profile-pics/") + key = strings.TrimSpace(strings.Trim(key, "/")) + if key == "" || !isValidProfilePicKey(key) { + http.Error(w, "missing username", 400) + return + } + body, ctype, err := h.Get(key) + if err != nil { + http.Error(w, "not cached", 404) + return + } + w.Header().Set("Content-Type", ctype) + w.Header().Set("Cache-Control", "private, max-age=86400") + w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body))) + _, _ = w.Write(body) +} + +// isValidProfilePicKey: "" or ":", alphanumeric + +// _-. Bars slashes/back-slashes/dots so the key can't escape the cache dir. +func isValidProfilePicKey(s string) bool { + if strings.ContainsAny(s, "/\\.") { + return false + } + parts := strings.Split(s, ":") + if len(parts) > 2 { + return false + } + for _, p := range parts { + if p == "" { + return false + } + for _, r := range p { + if !(r == '_' || r == '-' || (r >= '0' && r <= '9') || + (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')) { + return false + } + } + } + return true +} + +// handleProfilePicsRefresh kicks off a background refresh and returns +// immediately; UI polls /api/profile-pics/progress for the progress bar. +func (s *Server) handleProfilePicsRefresh(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", 405) + return + } + s.mu.RLock() + fetcher := s.fetcher + hub := s.profilePics + s.mu.RUnlock() + if fetcher == nil || hub == nil { + http.Error(w, "fetcher not ready", 503) + return + } + dnsAllowed := s.profilePicsEnabled() + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + // SSE per stored avatar so the UI updates mid-batch. + onStored := func(string) { + s.broadcast("event: update\ndata: \"profile-pics\"\n\n") + } + if err := hub.refresh(ctx, fetcher, dnsAllowed, s.fetchFromGitHubRelayBytes, onStored); err == nil { + s.broadcast("event: update\ndata: \"profile-pics\"\n\n") + } + }() + writeJSON(w, map[string]any{"ok": true}) +} + +func (s *Server) handleProfilePicsProgress(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", 405) + return + } + hub := s.profilePics + if hub == nil { + writeJSON(w, profilePicProgress{}) + return + } + writeJSON(w, hub.Progress()) +} + +// handleProfilePicsList returns which usernames have a cached avatar. +// Cache-Control: no-store so SSE-driven reloads see fresh data. +func (s *Server) handleProfilePicsList(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", 405) + return + } + w.Header().Set("Cache-Control", "no-store") + hub := s.profilePics + if hub == nil { + writeJSON(w, map[string]any{"enabled": false, "users": []string{}}) + return + } + hub.mu.Lock() + users := make([]string, 0, len(hub.index)) + for u := range hub.index { + users = append(users, u) + } + hub.mu.Unlock() + writeJSON(w, map[string]any{ + "enabled": s.profilePicsEnabled(), + "users": users, + }) +} + +func (s *Server) profilePicsEnabled() bool { + pl, err := s.loadProfiles() + if err != nil || pl == nil { + return false + } + return pl.ProfilePicsEnabled +} + +// ===== mime helpers ===== + +func extensionFor(mime uint8) string { + switch mime { + case 1: + return ".png" + case 2: + return ".webp" + default: + return ".jpg" + } +} + +func contentTypeFor(mime uint8) string { + switch mime { + case 1: + return "image/png" + case 2: + return "image/webp" + default: + return "image/jpeg" + } +} diff --git a/internal/web/relay_info.go b/internal/web/relay_info.go index 732406d..7000150 100644 --- a/internal/web/relay_info.go +++ b/internal/web/relay_info.go @@ -40,19 +40,22 @@ func (e *ghRateLimitError) Error() string { // (see .github/workflows/build.yml), so net.Lookup* goes through // bionic libc → netd → the device's actual DNS, the same path any other // Android app uses. On desktop the OS resolver is similarly fine. +// +// Tight connect / TLS / header timeouts so a blocked GitHub fails fast +// and we fall back to DNS quickly. Body read gets the full 90 s budget +// to cover multi-MB downloads on slow links. var relayHTTPClient = &http.Client{ - // Per-request budget — large enough to cover multi-MB downloads - // over a slow link without hugging short-circuit timeouts. - Timeout: 5 * time.Minute, + Timeout: 90 * time.Second, Transport: &http.Transport{ DialContext: (&net.Dialer{ - Timeout: 15 * time.Second, + Timeout: 5 * time.Second, KeepAlive: 30 * time.Second, }).DialContext, ForceAttemptHTTP2: true, MaxIdleConns: 10, IdleConnTimeout: 90 * time.Second, - TLSHandshakeTimeout: 15 * time.Second, + TLSHandshakeTimeout: 5 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, }, } @@ -116,6 +119,64 @@ func (c *relayCache) get(ctx context.Context, fetcher *client.Fetcher) (client.R return info, nil } +// fetchFromGitHubRelayBytes is the byte-returning twin of +// serveFromGitHubRelay (cache lookup + GitHub fetch + decrypt + CRC +// check). Returns (nil, nil) when the relay isn't configured. +func (s *Server) fetchFromGitHubRelayBytes(ctx context.Context, size int64, crc uint32) ([]byte, error) { + if size <= 0 || crc == 0 { + return nil, nil + } + s.mu.RLock() + fetcher := s.fetcher + rc := s.relayInfo + cache := s.mediaCache + cfg := s.config + s.mu.RUnlock() + if fetcher == nil || rc == nil || cfg == nil || cfg.Domain == "" { + return nil, nil + } + + info, err := rc.get(ctx, fetcher) + if err != nil || info.GitHubRepo == "" { + return nil, nil + } + if cfg.Key == "" { + return nil, nil + } + relayKey, err := protocol.DeriveRelayKey(cfg.Key) + if err != nil { + return nil, err + } + domainSeg := protocol.RelayDomainSegment(cfg.Domain, cfg.Key) + objectSeg := protocol.RelayObjectName(size, crc, cfg.Key) + + if cache != nil { + if body, _, ok := cache.Get(size, crc); ok { + return body, nil + } + } + + url := fmt.Sprintf("https://api.github.com/repos/%s/contents/%s/%s", + info.GitHubRepo, domainSeg, objectSeg) + const aeadOverhead = protocol.NonceSize + 16 + encBody, _, err := fetchGitHubRaw(ctx, relayHTTPClient, url, size+int64(aeadOverhead)) + if err != nil { + return nil, err + } + body, err := protocol.DecryptRelayBlob(relayKey, encBody) + if err != nil { + return nil, err + } + if int64(len(body)) != size || crc32.ChecksumIEEE(body) != crc { + return nil, errors.New("relay: hash/size mismatch") + } + if cache != nil { + mime := http.DetectContentType(body) + _ = cache.Put(size, crc, body, mime) + } + return body, nil +} + // serveFromGitHubRelay tries to stream the file from raw.githubusercontent.com // Returns true if the request was fully handled (success or terminal error // already written). Returns false to let the caller fall back to DNS. @@ -133,9 +194,8 @@ func (s *Server) serveFromGitHubRelay(w http.ResponseWriter, r *http.Request, si return false } - // Long enough to cover a multi-MB GitHub fetch over a slow link, plus - // a multi-block DNS-tunneled relay-info lookup if the cache is empty. - ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute) + // Covers a cold-cache relay-info DNS lookup + GitHub fetch. + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute) defer cancel() info, err := rc.get(ctx, fetcher) diff --git a/internal/web/static/index.html b/internal/web/static/index.html index 6714813..7484fbe 100644 --- a/internal/web/static/index.html +++ b/internal/web/static/index.html @@ -313,6 +313,7 @@ } .ch-avatar { + position: relative; width: 44px; height: 44px; border-radius: 50%; @@ -323,7 +324,17 @@ font-size: 17px; color: var(--text-dim); flex-shrink: 0; - font-weight: 600 + font-weight: 600; + overflow: hidden + } + .ch-avatar-letter { line-height: 1 } + /* Overlays the initial-letter span; img.onerror removes it and + adds .ch-avatar-noimg so the letter shows through. */ + .ch-avatar-img { + position: absolute; inset: 0; + width: 100%; height: 100%; + object-fit: cover; + display: block } .ch-item.active .ch-avatar { @@ -1311,6 +1322,53 @@ a permanent broken-image icon. */ .tm-photo-failed { display: none } + /* Reply quote — clickable, opens source in new tab. */ + .tm-post-reply { + margin: 6px 0 8px; + padding: 6px 10px; + border-inline-start: 3px solid var(--accent); + background: color-mix(in oklab, var(--accent) 6%, transparent); + border-radius: 4px; + font-size: 12.5px; + max-height: 100px; overflow: hidden + } + .tm-post-reply-author { + color: var(--accent); + font-weight: 600; + margin-bottom: 2px + } + .tm-post-reply-text { + color: var(--text-dim); + line-height: 1.4 + } + .tm-post-reply-text br { display: block; margin-bottom: 2px } + + /* "Forwarded from X" header. */ + .tm-post-forward { + font-size: 12px; + color: var(--text-dim); + margin-bottom: 4px + } + .tm-post-forward a { + color: var(--accent); + text-decoration: none + } + .tm-post-forward a:hover { text-decoration: underline } + + /* Poll options under the question. */ + .tm-poll-options { + list-style: none; padding: 0; margin: 8px 0 0; + display: flex; flex-direction: column; gap: 4px + } + .tm-poll-options li { + padding: 4px 8px; + background: var(--border); + border-radius: 4px; + font-size: 12.5px; + color: var(--text) + } + .tm-poll-options li::before { content: "○ "; color: var(--text-dim) } + /* Confirm dialog rendered inside the telemirror modal so it sits on top of the drawer / backdrop / posts at the right stacking. */ .tm-confirm-overlay { @@ -3051,6 +3109,20 @@ + +
+
+ + +
+
+ GitHub relay path is always used automatically. This option additionally pulls avatars over DNS when the relay is unavailable. +
+
+ + +
+
@@ -3510,10 +3582,18 @@ telemirror_audio: 'فایل صوتی', telemirror_file: 'فایل', telemirror_poll: 'نظرسنجی', + telemirror_forwarded_from: 'فروارد شده از', telemirror_about: 'درباره‌ی این کانال', download: 'دانلود', copy: 'کپی', // END telemirror + profile_pics_enable: 'دریافت عکس پروفایل از طریق DNS', + profile_pics_dns_note: 'عکس‌هایی که از طریق رله گیتهاب در دسترس باشند به‌صورت خودکار دریافت می‌شوند. این گزینه فقط برای دریافت عکس‌هایی است که رله نمی‌تواند سرویس بدهد و باید از طریق DNS کند گرفته شوند.', + profile_pics_refresh: 'به‌روز کردن عکس‌ها', + profile_pics_starting: 'در حال شروع...', + profile_pics_progress: 'در حال دریافت عکس‌ها: {n}/{m}', + profile_pics_done: 'انجام شد ({n} عکس)', + profile_pics_failed: 'دریافت ناموفق بود', font_size: 'اندازه قلم', debug_mode: 'حالت دیباگ', language: 'زبان', next_fetch_info: 'زمان باقی‌مانده تا دریافت بعدی محتوا توسط سرور', no_profiles: 'هنوز پروفایلی وجود ندارد', add_profile: '+ پروفایل جدید', @@ -3743,6 +3823,7 @@ telemirror_refresh: 'Refresh', telemirror_refresh_warn: 'This channel was refreshed {n} sec ago. Refreshing too often can hit a rate limit and stop working for a few minutes. Refresh anyway?', telemirror_refresh_yes: 'Refresh', + telemirror_forwarded_from: 'Forwarded from', telemirror_voice: 'Voice message', telemirror_audio: 'Audio', telemirror_file: 'File', @@ -3751,6 +3832,13 @@ download: 'Download', copy: 'Copy', // END telemirror + profile_pics_enable: 'Also fetch profile pictures over DNS', + profile_pics_dns_note: 'Avatars served by the GitHub relay are fetched automatically. Enable this to additionally fall back to DNS for avatars the relay can\'t serve (slower).', + profile_pics_refresh: 'Refresh avatars', + profile_pics_starting: 'Starting…', + profile_pics_progress: 'Refreshing avatars: {n}/{m}', + profile_pics_done: 'Done ({n} avatars)', + profile_pics_failed: 'Refresh failed', font_size: 'Font Size', debug_mode: 'Debug mode', language: 'Language', next_fetch_info: 'Time until the server next fetches fresh channel content', no_profiles: 'No profiles yet', add_profile: '+ Add Profile', @@ -4049,6 +4137,9 @@ loadBgImage(); connectSSE(); refreshResolversBadge(); + // Populate profilePicCache so the channel list renders avatars + // without a per-item probe. + loadProfilePicState().catch(function () { }); // Quietly ask GitHub for the latest published client version. Runs in // the background so a slow github.com response can't delay startup — // if there's an update, the dialog shows up a few seconds later. @@ -4312,6 +4403,74 @@ // settings is opened. var promptEl = document.getElementById('cfgShowScanPrompt'); if (promptEl) promptEl.checked = localStorage.getItem('thefeed_scan_prompt_off') !== '1'; + // Sync the profile-pics toggle from the server (it's persisted in + // profiles.json so survives app restarts). + fetch('/api/settings').then(function (r) { return r.ok ? r.json() : null; }).then(function (s) { + if (!s) return; + var pp = document.getElementById('cfgProfilePics'); + if (pp) pp.checked = !!s.profilePicsEnabled; + }).catch(function () { }); + } + + // ===== PROFILE PICTURES ===== + // Cached lowercase usernames so renderChannels can decide whether + // to overlay an over the initial-letter circle. + var profilePicCache = { enabled: false, users: {} }; + var profilePicsPollTimer = null; + // SSE throttle state — server fires one event per stored avatar. + var profilePicsReloadTimer = null; + var profilePicsLastReloadAt = 0; + + function loadProfilePicState() { + // no-store so SSE reloads see fresh data, not a cached entry. + return fetch('/api/profile-pics', { cache: 'no-store' }).then(function (r) { return r.ok ? r.json() : null; }).then(function (d) { + if (!d) return; + profilePicCache.enabled = !!d.enabled; + profilePicCache.users = {}; + (d.users || []).forEach(function (u) { profilePicCache.users[u.toLowerCase()] = true; }); + try { renderChannels(); } catch (e) { } + }).catch(function () { }); + } + + function setProfilePicsEnabled(enabled) { + // Toggle only controls the DNS path; cached GitHub avatars keep + // displaying either way. + fetch('/api/settings', { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ profilePicsEnabled: !!enabled }) + }).then(function () { + profilePicCache.enabled = !!enabled; + if (enabled) profilePicsRefresh(); + }).catch(function () { }); + } + + function profilePicsRefresh() { + var prog = document.getElementById('profilePicsProgress'); + if (prog) prog.textContent = t('profile_pics_starting') || 'Starting…'; + fetch('/api/profile-pics/refresh', { method: 'POST' }).then(function (r) { + if (!r.ok) { + if (prog) prog.textContent = t('profile_pics_failed') || 'Failed'; + return; + } + // Poll progress until inactive. + if (profilePicsPollTimer) clearInterval(profilePicsPollTimer); + profilePicsPollTimer = setInterval(function () { + fetch('/api/profile-pics/progress').then(function (r) { return r.ok ? r.json() : null; }).then(function (p) { + if (!p) return; + if (prog) { + if (p.active) { + var label = (t('profile_pics_progress') || 'Refreshing avatars: {n}/{m}').replace('{n}', p.done || 0).replace('{m}', p.total || 0); + prog.textContent = label + (p.username ? ' (@' + p.username + ')' : ''); + } else { + prog.textContent = (t('profile_pics_done') || 'Done').replace('{n}', p.done || 0); + clearInterval(profilePicsPollTimer); + profilePicsPollTimer = null; + loadProfilePicState(); + } + } + }); + }, 800); + }).catch(function () { }); } // Toggle the "show startup scan prompt" preference. Persists @@ -5055,6 +5214,25 @@ } return; } + // Throttle (not debounce): first event fires immediately, + // subsequent events at most every 600 ms during a burst. + if (data === 'profile-pics') { + var THROTTLE_MS = 600; + var now = Date.now(); + var since = now - profilePicsLastReloadAt; + if (since >= THROTTLE_MS) { + profilePicsLastReloadAt = now; + loadProfilePicState().catch(function () { }); + } else if (!profilePicsReloadTimer) { + // Trailing call for the last avatar in the burst. + profilePicsReloadTimer = setTimeout(function () { + profilePicsReloadTimer = null; + profilePicsLastReloadAt = Date.now(); + loadProfilePicState().catch(function () { }); + }, THROTTLE_MS - since); + } + return; + } // Another tab/device switched the active profile. if (data === 'profiles') { var prevActive = activeProfileId; @@ -5676,6 +5854,35 @@ var key = chNm.replace(/^@/, '').trim(); autoBtn.classList.toggle('on', autoUpdateChannels.has(key)); } + // Avatar diff against profilePicCache. Without this the + // fast path skips avatar updates and SSE-driven reloads + // don't reach the UI. + var ct2 = channels[ui].ChatType || channels[ui].chatType || 0; + var isX2 = ct2 === 2; + var bareHandle2 = chNm.replace(/^@/, '').toLowerCase(); + if (isX2 && bareHandle2.indexOf('x/') === 0) bareHandle2 = bareHandle2.substring(2); + var unameKey2 = isX2 ? ('x:' + bareHandle2) : bareHandle2; + var avatarEl = existingItems[ui].querySelector('.ch-avatar'); + if (avatarEl) { + var imgEl = avatarEl.querySelector('.ch-avatar-img'); + var shouldShow = !!profilePicCache.users[unameKey2]; + if (shouldShow && !imgEl) { + var img = document.createElement('img'); + img.className = 'ch-avatar-img'; + img.src = '/api/profile-pics/' + encodeURIComponent(unameKey2); + img.loading = 'lazy'; + img.alt = ''; + img.onerror = function () { + this.parentNode.classList.add('ch-avatar-noimg'); + this.remove(); + }; + avatarEl.classList.remove('ch-avatar-noimg'); + avatarEl.insertBefore(img, avatarEl.firstChild); + } else if (!shouldShow && imgEl) { + imgEl.remove(); + avatarEl.classList.add('ch-avatar-noimg'); + } + } } _updateRefreshBadge(); return; } @@ -5706,7 +5913,19 @@ var chNm2 = e.ch.Name || e.ch.name || ''; var badge = (channelHasNew(e.ch) && num2 !== selectedChannel) ? 'NEW' : ''; h += '
'; - h += '
' + esc(avatarText) + '
'; + // X channels: e.ch.Name is "x/" (category prefix); + // strip before re-attaching the bundle's "x:" prefix. + var avatarImg = ''; + var bareHandle = (chNm2 || avatarName || '').replace(/^@/, '').toLowerCase(); + if (isX && bareHandle.indexOf('x/') === 0) bareHandle = bareHandle.substring(2); + var unameKey = isX ? ('x:' + bareHandle) : bareHandle; + if (profilePicCache.users[unameKey]) { + avatarImg = ''; + } + h += '
' + avatarImg + '' + esc(avatarText) + '
'; var chSubText = !isX ? (handle.charAt(0) === '@' ? handle : '@' + handle) : ''; h += '
' + formatIranTitleHtml(label) + (isPriv ? '' + t('private') + '' : (isX ? '' + t('x_label') + '' : '')) + '
'; if (chSubText) h += '
' + esc(chSubText) + '
'; diff --git a/internal/web/static/telemirror.js b/internal/web/static/telemirror.js index e9d34d7..68f1db7 100644 --- a/internal/web/static/telemirror.js +++ b/internal/web/static/telemirror.js @@ -360,6 +360,28 @@ } html += '
'; + if (p.forward && p.forward.author) { + var fwdLabel = tmI18n('telemirror_forwarded_from', 'Forwarded from'); + var fwdName = tmEsc(p.forward.author); + if (p.forward.url) { + fwdName = '' + + fwdName + ''; + } + html += '
↪ ' + tmEsc(fwdLabel) + ' ' + fwdName + '
'; + } + + if (p.reply) { + var rAuth = p.reply.author ? tmEsc(p.reply.author) : ''; + var rText = p.reply.text || ''; + html += '
'; + if (rAuth) html += '
' + rAuth + '
'; + if (rText) html += '
' + rText + '
'; + html += '
'; + } + if (p.text) html += '
' + p.text + '
'; if (p.media && p.media.length) { @@ -450,8 +472,9 @@ + ' onerror="this.parentNode.classList.add(\'tm-photo-failed\')">' + '⬇' + + ' onclick="return tmDownloadPhoto(this, event)">⬇' + '
'; } if (m.type === 'video') { @@ -486,14 +509,81 @@ + '
'; } if (m.type === 'poll') { - return '
📊' + var optionsHtml = ''; + if (m.options && m.options.length) { + optionsHtml = '
    '; + for (var k = 0; k < m.options.length; k++) { + optionsHtml += '
  • ' + tmEsc(m.options[k]) + '
    • '; + } + optionsHtml += '
'; + } + return '
📊' + '
' + tmEsc(m.title || tmI18n('telemirror_poll', 'Poll')) - + '
' + tmEsc(m.subtitle || '') + '
'; + + '
' + tmEsc(m.subtitle || '') + '
' + + optionsHtml + + '
'; } return ''; } + // tmDownloadPhoto fetches the bytes and either hands them to the + // Android bridge (saveMedia) or builds a blob-URL on + // desktop. alone doesn't work on Android WebView for + // cross-origin URLs. + window.tmDownloadPhoto = function (anchor, ev) { + if (ev) ev.stopPropagation(); + var url = anchor.getAttribute('href'); + var fname = anchor.getAttribute('data-fname') || 'photo.jpg'; + var bridge = (typeof window !== 'undefined' && window.Android) ? window.Android : null; + + var doFetch = function () { + return fetch(url, { referrerPolicy: 'no-referrer' }).then(function (r) { + if (!r.ok) throw new Error('http ' + r.status); + return r.blob(); + }); + }; + + if (bridge && typeof bridge.saveMedia === 'function') { + doFetch().then(function (blob) { + return tmBlobToBase64(blob).then(function (b64) { + try { bridge.saveMedia(b64, blob.type || 'image/jpeg', fname); } + catch (e) { tmFallbackOpen(url); } + }); + }).catch(function () { tmFallbackOpen(url); }); + return false; + } + + doFetch().then(function (blob) { + var objectUrl = URL.createObjectURL(blob); + var a = document.createElement('a'); + a.href = objectUrl; + a.download = fname; + a.style.display = 'none'; + document.body.appendChild(a); + a.click(); + setTimeout(function () { URL.revokeObjectURL(objectUrl); a.remove(); }, 100); + }).catch(function () { window.location.href = url; }); + return false; + }; + + function tmBlobToBase64(blob) { + return new Promise(function (resolve, reject) { + var fr = new FileReader(); + fr.onload = function () { + var s = fr.result || ''; + var i = s.indexOf(','); + resolve(i >= 0 ? s.substring(i + 1) : s); + }; + fr.onerror = function () { reject(fr.error); }; + fr.readAsDataURL(blob); + }); + } + + function tmFallbackOpen(url) { + try { window.open(url, '_blank'); } catch (e) { window.location.href = url; } + } + window.tmCopyPost = function (btn) { var post = btn.closest ? btn.closest('.tm-post') : null; var pid = post ? post.getAttribute('data-pid') : ''; diff --git a/internal/web/web.go b/internal/web/web.go index 197e3df..93b40e4 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -117,6 +117,10 @@ type ProfileList struct { // changes (each launch picks a fresh port → different localStorage // origin → flag was lost on every restart). ScanPromptOff bool `json:"scanPromptOff,omitempty"` + + // ProfilePicsEnabled enables fetching avatars over DNS when the + // GitHub relay can't serve them. Off by default. + ProfilePicsEnabled bool `json:"profilePicsEnabled,omitempty"` } // lastScanData is the on-disk structure for last_scan.json. @@ -216,6 +220,9 @@ type Server struct { // Optional, removable backup feed (Telegram-via-Translate proxy). telemirror *telemirrorHub + + // Optional per-channel profile pictures cache. + profilePics *profilePicsHub } // New creates a new web server. @@ -253,6 +260,7 @@ func New(dataDir string, port int, host string, password string) (*Server, error dlProgress: make(map[string]*mediaDLProgress), relayInfo: newRelayCache(), telemirror: newTelemirrorHub(dataDir), + profilePics: newProfilePicsHub(dataDir), } if mediaCache != nil { @@ -345,6 +353,11 @@ func (s *Server) Run() error { mux.HandleFunc("/api/telemirror/channels", s.telemirror.handleChannels) mux.HandleFunc("/api/telemirror/channel/", s.telemirror.handleChannel) mux.HandleFunc("/api/telemirror/img", s.telemirror.handleImg) + // Profile-pics cache + control endpoints. + mux.HandleFunc("/api/profile-pics/", s.profilePics.handleProfilePic) + mux.HandleFunc("/api/profile-pics", s.handleProfilePicsList) + mux.HandleFunc("/api/profile-pics/refresh", s.handleProfilePicsRefresh) + mux.HandleFunc("/api/profile-pics/progress", s.handleProfilePicsProgress) mux.HandleFunc("/", s.handleIndex) // Listen on the specified host (default 127.0.0.1) @@ -1319,6 +1332,42 @@ func (s *Server) refreshMetadataOnly() { if needsFetch { go s.ensureTitlesFetched(basectx) } + + go s.maybeRefreshProfilePics(basectx) +} + +// maybeRefreshProfilePics fires a refresh when GitHub relay is up or +// the user has opted into the DNS path. No-op otherwise; hub coalesces. +func (s *Server) maybeRefreshProfilePics(parentCtx context.Context) { + s.mu.RLock() + hub := s.profilePics + fetcher := s.fetcher + rc := s.relayInfo + s.mu.RUnlock() + if hub == nil || fetcher == nil { + return + } + dnsAllowed := s.profilePicsEnabled() + githubLikelyUp := false + if rc != nil { + ctx, cancel := context.WithTimeout(parentCtx, 5*time.Second) + info, err := rc.get(ctx, fetcher) + cancel() + if err == nil && info.GitHubRepo != "" { + githubLikelyUp = true + } + } + if !dnsAllowed && !githubLikelyUp { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + onStored := func(string) { + s.broadcast("event: update\ndata: \"profile-pics\"\n\n") + } + if err := hub.refresh(ctx, fetcher, dnsAllowed, s.fetchFromGitHubRelayBytes, onStored); err == nil { + s.broadcast("event: update\ndata: \"profile-pics\"\n\n") + } } // ensureTitlesFetched fetches channel display names from TitlesChannel in the background. @@ -2758,16 +2807,26 @@ func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) { if pl == nil { pl = &ProfileList{} } - writeJSON(w, map[string]any{"fontSize": pl.FontSize, "debug": pl.Debug, "theme": pl.Theme, "lang": pl.Lang, "scanPromptOff": pl.ScanPromptOff, "version": version.Version, "commit": version.Commit}) + writeJSON(w, map[string]any{ + "fontSize": pl.FontSize, + "debug": pl.Debug, + "theme": pl.Theme, + "lang": pl.Lang, + "scanPromptOff": pl.ScanPromptOff, + "profilePicsEnabled": pl.ProfilePicsEnabled, + "version": version.Version, + "commit": version.Commit, + }) case http.MethodPost: // Optional pointers so partial requests don't reset other fields. var req struct { - FontSize *int `json:"fontSize"` - Debug *bool `json:"debug"` - Theme *string `json:"theme"` - Lang *string `json:"lang"` - ScanPromptOff *bool `json:"scanPromptOff"` + FontSize *int `json:"fontSize"` + Debug *bool `json:"debug"` + Theme *string `json:"theme"` + Lang *string `json:"lang"` + ScanPromptOff *bool `json:"scanPromptOff"` + ProfilePicsEnabled *bool `json:"profilePicsEnabled"` } if err := json.NewDecoder(r.Body).Decode(&req); err != nil { http.Error(w, "invalid JSON", 400) @@ -2805,6 +2864,9 @@ func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) { if req.ScanPromptOff != nil { pl.ScanPromptOff = *req.ScanPromptOff } + if req.ProfilePicsEnabled != nil { + pl.ProfilePicsEnabled = *req.ProfilePicsEnabled + } if err := s.saveProfiles(pl); err != nil { http.Error(w, fmt.Sprintf("save: %v", err), 500) return @@ -3585,6 +3647,9 @@ func (s *Server) handleClearCache(w http.ResponseWriter, r *http.Request) { if s.telemirror != nil { s.telemirror.ClearCache() } + if s.profilePics != nil { + s.profilePics.Clear() + } mediaDeleted := 0 if s.mediaCache != nil { mediaDeleted = s.mediaCache.Clear()