feat: downlaod profile images and fix some bugs ( #75 #76 #77 #78 #79 )

This commit is contained in:
Sarto
2026-05-05 20:00:27 +03:30
parent 1e953489e7
commit 625b00d815
24 changed files with 3045 additions and 38 deletions
+1
View File
@@ -146,6 +146,7 @@ thefeed یک سیستم تونل DNS است که به شما اجازه می‌
- کانال کانفیگ عمومی دفید: [@thefeedconfig](https://t.me/thefeedconfig)
- راهنمای نصب سرور دفید: [@networkti](https://t.me/networkti/25)
- راهنمای نصب سرور دفید با اسلیپ گیت: [@networkti](https://t.me/networkti/200)
- لیست تسک‌ها و رودمپ پروژه: [بورد گیتهاب](https://github.com/users/sartoopjj/projects/1/views/1)
## ⚡ نصب سریع سرور
+1
View File
@@ -122,6 +122,7 @@ Thank you for your support ❤️
- Public TheFeed Configs: [@thefeedconfig](https://t.me/thefeedconfig)
- Setup TheFeed server guide: [@networkti](https://t.me/networkti/25)
- Setup TheFeed server with SlipGate guide: [@networkti](https://t.me/networkti/200)
- Roadmap / task board: [GitHub project](https://github.com/users/sartoopjj/projects/1/views/1)
## Quick Install (Server)
+161
View File
@@ -0,0 +1,161 @@
package client
import (
"context"
"encoding/binary"
"fmt"
"sync"
"time"
"github.com/sartoopjj/thefeed/internal/protocol"
)
// ProfilePicsBundle is the client-side view of the profile-pic
// directory. The bundle (Size/CRC/Relays) describes the GitHub-served
// concatenated blob; per-entry DNSChannel/DNSBlocks describe an
// independent DNS fallback for that single avatar.
type ProfilePicsBundle struct {
BundleSize uint32
BundleCRC uint32
// Relays describes where the bundle is reachable, indexed by
// RelayDNS / RelayGitHub. RelayGitHub means the bundle is on
// GitHub. RelayDNS for the bundle is rarely true — the standard
// DNS path uses per-entry channels (see ProfilePicEntry).
Relays []bool
Entries []ProfilePicEntry
}
// HasRelay forwards to the relay availability bit at idx.
func (b ProfilePicsBundle) HasRelay(idx int) bool {
if idx < 0 || idx >= len(b.Relays) {
return false
}
return b.Relays[idx]
}
// ProfilePicEntry points at one avatar in two ways:
//
// GitHub bundle path: bytes are bundle[Offset:Offset+Size]; CRC must
// equal CRC32-IEEE of that slice (use protocol.VerifyEntry).
// Per-entry DNS path: bytes live on DNS channel DNSChannel with
// DNSBlocks blocks. CRC and Size are checked the same way.
//
// The client picks whichever path is reachable. With the bundle path
// one HTTPS request fetches every avatar; with the DNS path each
// avatar is fetched independently so partial sets still show up.
type ProfilePicEntry struct {
Username string
Offset uint32
Size uint32
CRC uint32
MIME uint8
DNSChannel uint16
DNSBlocks uint16
}
// MimeString returns "image/jpeg" / "image/png" / "image/webp" for the
// MIME tag, suitable for use as an HTTP Content-Type.
func (p ProfilePicEntry) MimeString() string {
switch p.MIME {
case protocol.ProfilePicMimePNG:
return "image/png"
case protocol.ProfilePicMimeWebP:
return "image/webp"
default:
return "image/jpeg"
}
}
// Extension returns ".jpg" / ".png" / ".webp" for caching on disk.
func (p ProfilePicEntry) Extension() string {
switch p.MIME {
case protocol.ProfilePicMimePNG:
return ".png"
case protocol.ProfilePicMimeWebP:
return ".webp"
default:
return ".jpg"
}
}
// FetchProfilePicDirectory pulls the bundle directory from
// ProfilePicsChannel — header (bundle metadata + relay availability) and
// per-username entries. The bundle bytes themselves are NOT fetched here;
// callers do that with FetchMedia(BundleChannel, BundleBlocks, BundleCRC)
// once and then slice locally.
//
// Returns (zero-value bundle, nil) when the server has no profile pics
// configured (or is older and doesn't know the channel).
func (f *Fetcher) FetchProfilePicDirectory(ctx context.Context) (ProfilePicsBundle, error) {
fetchCtx, cancel := context.WithTimeout(ctx, time.Minute)
defer cancel()
block0, err := f.FetchBlock(fetchCtx, protocol.ProfilePicsChannel, 0)
if err != nil {
return ProfilePicsBundle{}, fmt.Errorf("fetch profile-pics: %w", err)
}
if len(block0) < 2 {
return ProfilePicsBundle{}, nil
}
totalBlocks := int(binary.BigEndian.Uint16(block0))
payload0 := block0[2:]
if totalBlocks <= 1 {
return decodeProfilePicsBundle(payload0)
}
type res struct {
data []byte
err error
}
results := make([]res, totalBlocks)
results[0] = res{data: payload0}
var wg sync.WaitGroup
for blk := 1; blk < totalBlocks; blk++ {
wg.Add(1)
go func(blk int) {
defer wg.Done()
data, e := f.FetchBlock(fetchCtx, protocol.ProfilePicsChannel, uint16(blk))
results[blk] = res{data: data, err: e}
}(blk)
}
wg.Wait()
var all []byte
for _, r := range results {
if r.err != nil {
return ProfilePicsBundle{}, fmt.Errorf("fetch profile-pics block: %w", r.err)
}
all = append(all, r.data...)
}
return decodeProfilePicsBundle(all)
}
func decodeProfilePicsBundle(data []byte) (ProfilePicsBundle, error) {
if len(data) == 0 {
return ProfilePicsBundle{}, nil
}
pb, err := protocol.DecodeProfilePicsBundle(data)
if err != nil {
return ProfilePicsBundle{}, fmt.Errorf("decode profile-pics: %w", err)
}
out := ProfilePicsBundle{
BundleSize: pb.Header.BundleSize,
BundleCRC: pb.Header.BundleCRC,
Relays: append([]bool(nil), pb.Header.Relays...),
}
out.Entries = make([]ProfilePicEntry, len(pb.Entries))
for i, e := range pb.Entries {
out.Entries[i] = ProfilePicEntry{
Username: e.Username,
Offset: e.Offset,
Size: e.Size,
CRC: e.CRC,
MIME: e.MIME,
DNSChannel: e.DNSChannel,
DNSBlocks: e.DNSBlocks,
}
}
return out, nil
}
+78
View File
@@ -0,0 +1,78 @@
package client
import (
"hash/crc32"
"testing"
"github.com/sartoopjj/thefeed/internal/protocol"
)
func TestProfilePicMimeAndExtension(t *testing.T) {
cases := []struct {
mime uint8
wantStr string
wantExt string
}{
{protocol.ProfilePicMimeJPEG, "image/jpeg", ".jpg"},
{protocol.ProfilePicMimePNG, "image/png", ".png"},
{protocol.ProfilePicMimeWebP, "image/webp", ".webp"},
{255, "image/jpeg", ".jpg"}, // unknown → JPEG fallback
}
for _, c := range cases {
p := ProfilePicEntry{MIME: c.mime}
if got := p.MimeString(); got != c.wantStr {
t.Errorf("MimeString(%d) = %q, want %q", c.mime, got, c.wantStr)
}
if got := p.Extension(); got != c.wantExt {
t.Errorf("Extension(%d) = %q, want %q", c.mime, got, c.wantExt)
}
}
}
func TestDecodeProfilePicsBundleRoundTrip(t *testing.T) {
// Build a real bundle so the Verify check the caller will run later
// would still succeed.
a := []byte("hello-alice")
b := []byte("hello-bob-bob-bob")
bundle := append(append([]byte{}, a...), b...)
wire := protocol.EncodeProfilePicsBundle(protocol.ProfilePicsBundle{
Header: protocol.ProfilePicsBundleHeader{
BundleSize: uint32(len(bundle)),
BundleCRC: crc32.ChecksumIEEE(bundle),
Relays: []bool{false, true},
},
Entries: []protocol.ProfilePicEntry{
{Username: "alice", Offset: 0, Size: uint32(len(a)), CRC: crc32.ChecksumIEEE(a), MIME: protocol.ProfilePicMimeJPEG, DNSChannel: 10001, DNSBlocks: 1},
{Username: "bob", Offset: uint32(len(a)), Size: uint32(len(b)), CRC: crc32.ChecksumIEEE(b), MIME: protocol.ProfilePicMimePNG, DNSChannel: 10002, DNSBlocks: 2},
},
})
got, err := decodeProfilePicsBundle(wire)
if err != nil {
t.Fatalf("decode: %v", err)
}
if got.BundleSize != uint32(len(bundle)) {
t.Errorf("bundle metadata wrong: %+v", got)
}
if got.Entries[0].DNSChannel != 10001 || got.Entries[1].DNSChannel != 10002 {
t.Errorf("dns channels lost: %+v", got.Entries)
}
if got.HasRelay(protocol.RelayDNS) || !got.HasRelay(protocol.RelayGitHub) {
t.Errorf("relays = %v, want GitHub-only", got.Relays)
}
if len(got.Entries) != 2 {
t.Fatalf("entries = %d, want 2", len(got.Entries))
}
if got.Entries[0].Username != "alice" || got.Entries[1].Username != "bob" {
t.Errorf("entries: %+v", got.Entries)
}
if got.Entries[1].MIME != protocol.ProfilePicMimePNG {
t.Errorf("bob MIME = %d, want PNG", got.Entries[1].MIME)
}
}
func TestDecodeProfilePicsBundleEmpty(t *testing.T) {
got, err := decodeProfilePicsBundle(nil)
if err != nil || len(got.Entries) != 0 {
t.Errorf("decode(nil) = %v, %v", got, err)
}
}
+7
View File
@@ -39,6 +39,13 @@ const (
// owner/repo + domain segment). Block 0 carries it.
RelayInfoChannel uint16 = 0xFFF8
// ProfilePicsChannel serves the per-channel profile-picture index:
// for every Telegram channel that has a profile photo we emit
// (username, mediaCh, size, crc32). Bytes themselves live on the
// referenced mediaCh and are fetched via the regular media path.
// Off by default on the client.
ProfilePicsChannel uint16 = 0xFFF7
// MaxUpstreamBlockPayload keeps uploaded query chunks comfortably below DNS
// name limits across typical domains and resolver paths.
MaxUpstreamBlockPayload = 8
+218
View File
@@ -0,0 +1,218 @@
package protocol
import (
"encoding/binary"
"fmt"
"hash/crc32"
)
// Profile pictures use a hybrid layout: every avatar is concatenated
// into one bundle uploaded to the GitHub relay (one file → no
// per-file rate limit), and each avatar also gets its own DNS media
// channel so partial fetches over DNS still display.
//
// Wire layout of ProfilePicsChannel (after the block-count prefix the
// Feed layer adds):
//
// bundleSize uint32
// bundleCRC uint32
// relayCount uint8 — N
// relays [N]u8 — bool per relay (RelayDNS=0, RelayGitHub=1, …)
// count uint16
// entries:
// usernameLen uint8
// username [usernameLen]byte
// offset uint32 — within the GitHub bundle
// size uint32
// crc uint32 — CRC32 of bundle[offset:offset+size]
// mime uint8 — 0=jpeg, 1=png, 2=webp
// dnsChannel uint16 — 0 if not on DNS
// dnsBlocks uint16
type ProfilePicsBundleHeader struct {
BundleSize uint32
BundleCRC uint32
// One bool per relay constant. RelayGitHub here means the bundle
// is on GitHub; RelayDNS for the bundle is rare (per-entry DNS
// channels handle the DNS path).
Relays []bool
}
// HasRelay reports whether the relay at idx is set. Out of range returns false.
func (h ProfilePicsBundleHeader) HasRelay(idx int) bool {
if idx < 0 || idx >= len(h.Relays) {
return false
}
return h.Relays[idx]
}
// ProfilePicEntry points at one avatar via either the GitHub bundle
// (Offset/Size into the concatenated blob) or its own DNS channel
// (DNSChannel/DNSBlocks). Both paths verify the same Size + CRC.
type ProfilePicEntry struct {
Username string
Offset uint32
Size uint32
CRC uint32
MIME uint8
DNSChannel uint16
DNSBlocks uint16
}
// MIME tag values.
const (
ProfilePicMimeJPEG uint8 = 0
ProfilePicMimePNG uint8 = 1
ProfilePicMimeWebP uint8 = 2
)
// On-the-wire byte counts.
const (
profilePicEntryFixed = 4 + 4 + 4 + 1 + 2 + 2 // offset+size+crc+mime+dnsCh+dnsBlk
profilePicsHeaderFixed = 4 + 4 + 1 // bundleSize+bundleCRC+relayCount
)
// ProfilePicsBundle is the directory (header + entries). The bundle
// bytes themselves live in the referenced media channel / relay.
type ProfilePicsBundle struct {
Header ProfilePicsBundleHeader
Entries []ProfilePicEntry
}
// EncodeProfilePicsBundle serialises the directory.
func EncodeProfilePicsBundle(b ProfilePicsBundle) []byte {
relayCount := len(b.Header.Relays)
if relayCount > 255 {
relayCount = 255
}
size := profilePicsHeaderFixed + relayCount + 2 /*entry count*/
for _, e := range b.Entries {
n := len(e.Username)
if n > 255 {
n = 255
}
size += 1 + n + profilePicEntryFixed
}
buf := make([]byte, size)
off := 0
binary.BigEndian.PutUint32(buf[off:], b.Header.BundleSize)
off += 4
binary.BigEndian.PutUint32(buf[off:], b.Header.BundleCRC)
off += 4
buf[off] = byte(relayCount)
off++
for i := 0; i < relayCount; i++ {
if b.Header.Relays[i] {
buf[off] = 1
}
off++
}
binary.BigEndian.PutUint16(buf[off:], uint16(len(b.Entries)))
off += 2
for _, e := range b.Entries {
nb := []byte(e.Username)
if len(nb) > 255 {
nb = nb[:255]
}
buf[off] = byte(len(nb))
off++
copy(buf[off:], nb)
off += len(nb)
binary.BigEndian.PutUint32(buf[off:], e.Offset)
off += 4
binary.BigEndian.PutUint32(buf[off:], e.Size)
off += 4
binary.BigEndian.PutUint32(buf[off:], e.CRC)
off += 4
buf[off] = e.MIME
off++
binary.BigEndian.PutUint16(buf[off:], e.DNSChannel)
off += 2
binary.BigEndian.PutUint16(buf[off:], e.DNSBlocks)
off += 2
}
return buf
}
// DecodeProfilePicsBundle parses bytes produced by EncodeProfilePicsBundle.
func DecodeProfilePicsBundle(data []byte) (ProfilePicsBundle, error) {
var out ProfilePicsBundle
if len(data) < profilePicsHeaderFixed+2 {
return out, fmt.Errorf("profile-pics bundle too short: %d bytes", len(data))
}
off := 0
out.Header.BundleSize = binary.BigEndian.Uint32(data[off:])
off += 4
out.Header.BundleCRC = binary.BigEndian.Uint32(data[off:])
off += 4
relayCount := int(data[off])
off++
if off+relayCount+2 > len(data) {
return out, fmt.Errorf("profile-pics bundle: truncated relay list")
}
if relayCount > 0 {
out.Header.Relays = make([]bool, relayCount)
for i := 0; i < relayCount; i++ {
out.Header.Relays[i] = data[off] != 0
off++
}
}
count := int(binary.BigEndian.Uint16(data[off:]))
off += 2
out.Entries = make([]ProfilePicEntry, 0, count)
for i := 0; i < count; i++ {
if off >= len(data) {
return out, fmt.Errorf("profile-pics: truncated at entry %d", i)
}
nameLen := int(data[off])
off++
if off+nameLen+profilePicEntryFixed > len(data) {
return out, fmt.Errorf("profile-pics: truncated entry %d body", i)
}
name := string(data[off : off+nameLen])
off += nameLen
offset := binary.BigEndian.Uint32(data[off:])
off += 4
sz := binary.BigEndian.Uint32(data[off:])
off += 4
cr := binary.BigEndian.Uint32(data[off:])
off += 4
mime := data[off]
off++
dnsCh := binary.BigEndian.Uint16(data[off:])
off += 2
dnsBlk := binary.BigEndian.Uint16(data[off:])
off += 2
out.Entries = append(out.Entries, ProfilePicEntry{
Username: name,
Offset: offset,
Size: sz,
CRC: cr,
MIME: mime,
DNSChannel: dnsCh,
DNSBlocks: dnsBlk,
})
}
return out, nil
}
// VerifyEntry returns bundle[entry.Offset:entry.Offset+entry.Size] if
// the slice is in-range and its CRC32-IEEE matches entry.CRC. The
// hash check is what stops a misaligned bundle from serving the wrong
// avatar under a username.
func VerifyEntry(bundle []byte, entry ProfilePicEntry) ([]byte, error) {
end := uint64(entry.Offset) + uint64(entry.Size)
if end > uint64(len(bundle)) {
return nil, fmt.Errorf("entry %q out of range: offset=%d size=%d bundle=%d",
entry.Username, entry.Offset, entry.Size, len(bundle))
}
slice := bundle[entry.Offset:end]
if uint32(len(slice)) != entry.Size {
return nil, fmt.Errorf("entry %q size mismatch: have %d want %d",
entry.Username, len(slice), entry.Size)
}
if got := crc32.ChecksumIEEE(slice); got != entry.CRC {
return nil, fmt.Errorf("entry %q crc mismatch: have %08x want %08x",
entry.Username, got, entry.CRC)
}
return slice, nil
}
+168
View File
@@ -0,0 +1,168 @@
package protocol
import (
"hash/crc32"
"strings"
"testing"
)
func TestEncodeDecodeProfilePicsBundleRoundTrip(t *testing.T) {
// Three avatars concatenated into a fake bundle, with offsets/CRCs
// computed for real so VerifyEntry doesn't trip on them.
a := []byte("aaaaaaaaaa") // 10 bytes
b := []byte("bbbbbbbbbbbbbb") // 14 bytes
c := []byte("ccccc") // 5 bytes
bundle := append(append(append([]byte{}, a...), b...), c...)
in := ProfilePicsBundle{
Header: ProfilePicsBundleHeader{
BundleSize: uint32(len(bundle)),
BundleCRC: crc32.ChecksumIEEE(bundle),
Relays: []bool{false, true}, // bundle on GitHub only; per-entry DNS handles DNS path
},
Entries: []ProfilePicEntry{
{Username: "alice", Offset: 0, Size: uint32(len(a)), CRC: crc32.ChecksumIEEE(a), MIME: ProfilePicMimeJPEG, DNSChannel: 10001, DNSBlocks: 1},
{Username: "bob", Offset: uint32(len(a)), Size: uint32(len(b)), CRC: crc32.ChecksumIEEE(b), MIME: ProfilePicMimePNG, DNSChannel: 10002, DNSBlocks: 2},
{Username: "carol", Offset: uint32(len(a) + len(b)), Size: uint32(len(c)), CRC: crc32.ChecksumIEEE(c), MIME: ProfilePicMimeWebP, DNSChannel: 10003, DNSBlocks: 1},
},
}
wire := EncodeProfilePicsBundle(in)
got, err := DecodeProfilePicsBundle(wire)
if err != nil {
t.Fatalf("decode: %v", err)
}
if got.Header.BundleSize != in.Header.BundleSize ||
got.Header.BundleCRC != in.Header.BundleCRC {
t.Errorf("header mismatch: got %+v want %+v", got.Header, in.Header)
}
if len(got.Header.Relays) != len(in.Header.Relays) {
t.Fatalf("relays len = %d, want %d", len(got.Header.Relays), len(in.Header.Relays))
}
for i, r := range in.Header.Relays {
if got.Header.Relays[i] != r {
t.Errorf("relays[%d] = %v, want %v", i, got.Header.Relays[i], r)
}
}
if len(got.Entries) != len(in.Entries) {
t.Fatalf("entries len = %d, want %d", len(got.Entries), len(in.Entries))
}
for i, want := range in.Entries {
if got.Entries[i] != want {
t.Errorf("entry %d = %+v want %+v", i, got.Entries[i], want)
}
}
// VerifyEntry should accept the real bytes…
for _, e := range got.Entries {
if _, err := VerifyEntry(bundle, e); err != nil {
t.Errorf("VerifyEntry(%s) = %v, want ok", e.Username, err)
}
}
// …and reject a tampered bundle.
tampered := append([]byte{}, bundle...)
tampered[0] ^= 0xFF
if _, err := VerifyEntry(tampered, got.Entries[0]); err == nil {
t.Errorf("VerifyEntry should fail on tampered bundle")
}
}
func TestVerifyEntryOutOfRange(t *testing.T) {
bundle := []byte("hello")
e := ProfilePicEntry{Username: "x", Offset: 0, Size: 100, CRC: 0}
if _, err := VerifyEntry(bundle, e); err == nil {
t.Errorf("VerifyEntry should fail when entry runs past bundle end")
}
}
func TestVerifyEntrySizeMismatch(t *testing.T) {
bundle := []byte("0123456789")
// CRC is right for the real slice, but Size is wrong → mismatch.
right := bundle[2:6]
e := ProfilePicEntry{
Username: "y",
Offset: 2,
Size: 5, // claim 5 but slice is 4
CRC: crc32.ChecksumIEEE(right),
}
// In this case Size=5 + Offset=2 → end=7, in range. CRC will be checked
// over bundle[2:7] which differs from right → mismatch.
if _, err := VerifyEntry(bundle, e); err == nil {
t.Errorf("VerifyEntry should fail when claimed size doesn't match recorded crc")
}
}
func TestProfilePicsBundleEmpty(t *testing.T) {
in := ProfilePicsBundle{
Header: ProfilePicsBundleHeader{Relays: []bool{false, false}},
}
wire := EncodeProfilePicsBundle(in)
got, err := DecodeProfilePicsBundle(wire)
if err != nil {
t.Fatalf("decode empty: %v", err)
}
if len(got.Entries) != 0 {
t.Errorf("entries = %d want 0", len(got.Entries))
}
}
func TestProfilePicsTruncatesLongUsername(t *testing.T) {
long := strings.Repeat("x", 300)
in := ProfilePicsBundle{
Header: ProfilePicsBundleHeader{Relays: []bool{true}},
Entries: []ProfilePicEntry{
{Username: long, Offset: 0, Size: 100, CRC: 1, MIME: 0},
},
}
wire := EncodeProfilePicsBundle(in)
got, err := DecodeProfilePicsBundle(wire)
if err != nil {
t.Fatalf("decode: %v", err)
}
if len(got.Entries) != 1 || len(got.Entries[0].Username) != 255 {
t.Errorf("expected 1 entry with 255-char username, got %+v", got.Entries)
}
}
func TestProfilePicsTruncatedDataReturnsError(t *testing.T) {
in := ProfilePicsBundle{
Header: ProfilePicsBundleHeader{Relays: []bool{true}},
Entries: []ProfilePicEntry{
{Username: "a", Offset: 0, Size: 1, CRC: 1, MIME: 0},
},
}
wire := EncodeProfilePicsBundle(in)
for _, cut := range []int{0, 1, 2, 3, len(wire) - 1} {
_, err := DecodeProfilePicsBundle(wire[:cut])
if err == nil {
t.Errorf("expected error on cut=%d", cut)
}
}
}
func TestProfilePicsChannelConstant(t *testing.T) {
if ProfilePicsChannel != 0xFFF7 {
t.Errorf("ProfilePicsChannel = 0x%X, want 0xFFF7", ProfilePicsChannel)
}
others := []uint16{
SendChannel, AdminChannel, UpstreamInitChannel, UpstreamDataChannel,
VersionChannel, TitlesChannel, RelayInfoChannel,
}
for _, o := range others {
if o == ProfilePicsChannel {
t.Fatalf("ProfilePicsChannel collides with another control channel: 0x%X", o)
}
}
}
func TestBundleHasRelay(t *testing.T) {
h := ProfilePicsBundleHeader{Relays: []bool{false, true}}
if h.HasRelay(RelayDNS) {
t.Errorf("RelayDNS should be false")
}
if !h.HasRelay(RelayGitHub) {
t.Errorf("RelayGitHub should be true")
}
if h.HasRelay(99) || h.HasRelay(-1) {
t.Errorf("out-of-range should return false")
}
}
+232
View File
@@ -4,7 +4,9 @@ import (
"context"
"crypto/rand"
"fmt"
"hash/crc32"
"log"
"sort"
"sync"
"time"
@@ -44,6 +46,16 @@ type Feed struct {
// (RelayInfoChannel) — block 0 contains the GitHub "owner/repo"
// string, or an empty payload if the relay is off.
relayInfoBlocks [][]byte
// ProfilePicsChannel serves the directory; the bundle bytes live
// in one media-cache entry, with each entry also reachable on its
// own DNS channel.
profilePicsBlocks [][]byte
profilePicsBundle protocol.ProfilePicsBundle
profilePicsBundleBytes []byte // last-built bundle, for MergeProfilePics
// Serialises MergeProfilePics so concurrent readers can't lose
// each other's writes through the read-modify-write sequence.
profilePicsMergeMu sync.Mutex
}
// NewFeed creates a new Feed with the given channel names.
@@ -108,6 +120,9 @@ func (f *Feed) GetBlock(channel, block int) ([]byte, error) {
if channel == int(protocol.RelayInfoChannel) {
return f.getRelayInfoBlock(block)
}
if channel == int(protocol.ProfilePicsChannel) {
return f.getProfilePicsBlock(block)
}
// Channel sits in the binary media range — delegate to MediaCache. We
// drop the read lock first because MediaCache uses its own lock and we
// don't want to hold f.mu across that path.
@@ -282,6 +297,223 @@ func (f *Feed) getRelayInfoBlock(block int) ([]byte, error) {
return blocks[block], nil
}
func (f *Feed) getProfilePicsBlock(block int) ([]byte, error) {
blocks := f.profilePicsBlocks
if len(blocks) == 0 {
// Empty payload still has to be a single non-nil block so the
// usual block-count prefix path stays consistent.
f.rebuildProfilePicsBlocksLocked()
blocks = f.profilePicsBlocks
}
if block < 0 || block >= len(blocks) {
return nil, fmt.Errorf("profile-pics block %d out of range (%d blocks)", block, len(blocks))
}
return blocks[block], nil
}
// rebuildProfilePicsBlocksLocked encodes the bundle and splits into
// blocks; block 0 is prefixed with the uint16 block count (same
// convention as titles). Caller holds f.mu.
func (f *Feed) rebuildProfilePicsBlocksLocked() {
payload := protocol.EncodeProfilePicsBundle(f.profilePicsBundle)
blocks := protocol.SplitIntoBlocks(payload)
if len(blocks) == 0 {
blocks = [][]byte{nil}
}
prefix := []byte{byte(len(blocks) >> 8), byte(len(blocks))}
blocks[0] = append(prefix, blocks[0]...)
f.profilePicsBlocks = blocks
}
// SetProfilePics replaces the profile-pic bundle with the given
// username → image-bytes map. Other usernames currently in the bundle
// are dropped; use MergeProfilePics for additive behaviour. Empty
// values are skipped. Requires SetMediaCache. Returns the number of
// avatars in the resulting bundle.
func (f *Feed) SetProfilePics(pics map[string][]byte) int {
return f.replaceProfilePicsBundle(pics)
}
// MergeProfilePics is SetProfilePics that retains the existing bundle's
// entries (re-extracted and re-verified) and overlays pics on top.
// Used by readers that only know a subset of channels (Telegram-only,
// X-only) so each one contributes without wiping the others.
//
// Serialised so two readers merging from the same prior state can't
// lose each other's writes.
func (f *Feed) MergeProfilePics(pics map[string][]byte) int {
f.profilePicsMergeMu.Lock()
defer f.profilePicsMergeMu.Unlock()
merged := make(map[string][]byte, len(pics))
f.mu.RLock()
prev := f.profilePicsBundle
prevBytes := f.profilePicsBundleBytes
f.mu.RUnlock()
if len(prev.Entries) > 0 && len(prevBytes) > 0 {
for _, e := range prev.Entries {
slice, err := protocol.VerifyEntry(prevBytes, e)
if err != nil {
log.Printf("[profile-pics] merge: skipping %s (%v)", e.Username, err)
continue
}
cp := make([]byte, len(slice))
copy(cp, slice)
merged[e.Username] = cp
}
}
for k, v := range pics {
if k == "" {
continue
}
if len(v) == 0 {
delete(merged, k)
continue
}
merged[k] = v
}
return f.replaceProfilePicsBundle(merged)
}
// replaceProfilePicsBundle is the shared encode-and-store path. Each
// individual avatar gets its own DNS channel via SkipGitHub=true (so
// the DNS path can fetch one at a time without triggering N GitHub
// uploads), then the concatenated bundle is stored once with default
// opts to trigger the single GitHub upload that covers everything.
func (f *Feed) replaceProfilePicsBundle(pics map[string][]byte) int {
f.mu.Lock()
media := f.media
f.mu.Unlock()
if media == nil {
return 0
}
type kv struct {
name string
b []byte
}
ordered := make([]kv, 0, len(pics))
for name, b := range pics {
if name == "" || len(b) == 0 {
continue
}
ordered = append(ordered, kv{name, b})
}
sort.Slice(ordered, func(i, j int) bool { return ordered[i].name < ordered[j].name })
if len(ordered) == 0 {
f.mu.Lock()
f.profilePicsBundle = protocol.ProfilePicsBundle{}
f.profilePicsBundleBytes = nil
f.rebuildProfilePicsBlocksLocked()
f.mu.Unlock()
return 0
}
// Build the bundle bytes + per-entry directory. Each entry gets its
// own DNS channel via a SkipGitHub store so the DNS-path client can
// fetch one avatar at a time without dragging the whole bundle.
bundle := make([]byte, 0, 8192)
entries := make([]protocol.ProfilePicEntry, 0, len(ordered))
for _, e := range ordered {
_, mimeTag := sniffProfilePicMime(e.b)
offset := uint32(len(bundle))
bundle = append(bundle, e.b...)
entry := protocol.ProfilePicEntry{
Username: e.name,
Offset: offset,
Size: uint32(len(e.b)),
CRC: crc32.ChecksumIEEE(e.b),
MIME: mimeTag,
}
// Per-pic DNS channel, no GitHub upload (the bundle covers GitHub).
key := "profile-pic:" + e.name
fname := e.name
switch mimeTag {
case protocol.ProfilePicMimePNG:
fname += ".png"
case protocol.ProfilePicMimeWebP:
fname += ".webp"
default:
fname += ".jpg"
}
picMeta, err := media.StoreWithOptions(key, "[PROFILE]", e.b, mimeStringForTag(mimeTag), fname,
MediaCacheStoreOptions{SkipGitHub: true})
if err != nil {
// No DNS channel for this entry; bundle path still works.
log.Printf("[profile-pics] store individual %s: %v", e.name, err)
} else {
entry.DNSChannel = picMeta.Channel
entry.DNSBlocks = picMeta.Blocks
}
entries = append(entries, entry)
}
// One bundle store → one GitHub upload covering every avatar.
bundleMeta, err := media.Store("profile-pics-bundle", "[PROFILE-BUNDLE]",
bundle, "application/octet-stream", "profile-pics.bin")
if err != nil {
log.Printf("[profile-pics] store bundle: %v", err)
return 0
}
header := protocol.ProfilePicsBundleHeader{
BundleSize: uint32(bundleMeta.Size),
BundleCRC: bundleMeta.CRC32,
Relays: append([]bool(nil), bundleMeta.Relays...),
}
f.mu.Lock()
f.profilePicsBundle = protocol.ProfilePicsBundle{
Header: header,
Entries: entries,
}
f.profilePicsBundleBytes = bundle
f.rebuildProfilePicsBlocksLocked()
f.mu.Unlock()
return len(entries)
}
func mimeStringForTag(tag uint8) string {
switch tag {
case protocol.ProfilePicMimePNG:
return "image/png"
case protocol.ProfilePicMimeWebP:
return "image/webp"
default:
return "image/jpeg"
}
}
// ProfilePicsBundle returns a copy of the current directory.
func (f *Feed) ProfilePicsBundle() protocol.ProfilePicsBundle {
f.mu.RLock()
defer f.mu.RUnlock()
out := protocol.ProfilePicsBundle{
Header: f.profilePicsBundle.Header,
Entries: make([]protocol.ProfilePicEntry, len(f.profilePicsBundle.Entries)),
}
if len(f.profilePicsBundle.Header.Relays) > 0 {
out.Header.Relays = append([]bool(nil), f.profilePicsBundle.Header.Relays...)
}
copy(out.Entries, f.profilePicsBundle.Entries)
return out
}
// sniffProfilePicMime returns (rfc-mime, ProfilePicMime tag) by looking
// at the first few bytes. Falls back to JPEG for anything unrecognised.
func sniffProfilePicMime(b []byte) (string, uint8) {
if len(b) >= 4 && b[0] == 0x89 && b[1] == 'P' && b[2] == 'N' && b[3] == 'G' {
return "image/png", protocol.ProfilePicMimePNG
}
if len(b) >= 12 && string(b[0:4]) == "RIFF" && string(b[8:12]) == "WEBP" {
return "image/webp", protocol.ProfilePicMimeWebP
}
return "image/jpeg", protocol.ProfilePicMimeJPEG
}
// rebuildTitlesBlocks re-serializes the display name map and splits it into blocks.
// Block 0 is prefixed with a uint16 total-block count so the client can fetch all
// remaining blocks in parallel after reading the first one.
+21 -7
View File
@@ -119,6 +119,19 @@ func NewMediaCache(cfg MediaCacheConfig) *MediaCache {
// client. content is the raw file bytes; the caller may pass a slice it
// continues to use after the call (Store copies into block-sized chunks).
func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filename string) (protocol.MediaMeta, error) {
return c.StoreWithOptions(cacheKey, tag, content, mimeType, filename, MediaCacheStoreOptions{})
}
// MediaCacheStoreOptions toggles relay paths for a single Store call.
// Zero value = both DNS channel and (if a relay is configured) GitHub
// upload. SkipGitHub keeps the DNS allocation but skips the upload —
// used when many small siblings share one bundled GitHub upload.
type MediaCacheStoreOptions struct {
SkipGitHub bool
}
// StoreWithOptions is Store with selective relay control.
func (c *MediaCache) StoreWithOptions(cacheKey, tag string, content []byte, mimeType, filename string, opts MediaCacheStoreOptions) (protocol.MediaMeta, error) {
if cacheKey == "" {
return protocol.MediaMeta{}, errors.New("media: empty cache key")
}
@@ -126,9 +139,6 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen
tag = protocol.MediaFile
}
size := int64(len(content))
// Reject only when no enabled relay could host this file. A file too big
// for DNS but small enough for GitHub still belongs in the cache —
// MaxAcceptableBytes() collapses both caps into a single ceiling.
if max := c.MaxAcceptableBytes(); max > 0 && size > max {
atomic.AddUint64(&c.storeRejected, 1)
return protocol.MediaMeta{
@@ -246,9 +256,9 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen
atomic.AddInt64(&c.currentBytes, size)
c.logf("media: store tag=%s key=%s ch=%d size=%d blocks=%d", tag, cacheKey, channel, size, len(blocks))
// Best-effort relay upload — copy of `content` because the caller may
// reuse the slice. Failures are logged but never block the DNS path.
if c.gh != nil {
// Best-effort relay upload. Copy because the caller may reuse the
// slice. Failures don't block the DNS path.
if c.gh != nil && !opts.SkipGitHub {
gh := c.gh
body := append([]byte(nil), content...)
go func() {
@@ -260,7 +270,11 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen
}()
}
return c.metaForLocked(entry), nil
meta := c.metaForLocked(entry)
if opts.SkipGitHub && len(meta.Relays) > protocol.RelayGitHub {
meta.Relays[protocol.RelayGitHub] = false
}
return meta, nil
}
// LookupByChannel returns the cached entry's transport metadata (mime,
+91
View File
@@ -0,0 +1,91 @@
package server
import (
"context"
"fmt"
"log"
"strings"
"github.com/gotd/td/tg"
)
// extractChatPhotoID returns (photoID, dcID), or (0, 0) when there is none.
func extractChatPhotoID(p tg.ChatPhotoClass) (int64, int) {
switch ph := p.(type) {
case *tg.ChatPhoto:
return ph.PhotoID, ph.DCID
}
return 0, 0
}
// extractUserPhotoID is the User-profile equivalent.
func extractUserPhotoID(p tg.UserProfilePhotoClass) (int64, int) {
switch ph := p.(type) {
case *tg.UserProfilePhoto:
return ph.PhotoID, ph.DCID
}
return 0, 0
}
// fetchProfilePhoto downloads the small (~5KB / 160px) thumb. Big:false
// is Telegram's "a" thumb size.
func (tr *TelegramReader) fetchProfilePhoto(ctx context.Context, api *tg.Client, peer tg.InputPeerClass, photoID int64) ([]byte, error) {
if photoID == 0 {
return nil, fmt.Errorf("no photo")
}
loc := &tg.InputPeerPhotoFileLocation{
Peer: peer,
PhotoID: photoID,
}
loc.Big = false
return tr.downloadTelegramFile(ctx, api, loc, 0)
}
// fetchAllProfilePhotos downloads each channel's avatar (skipping
// unchanged photoIDs) and merges them into the feed bundle.
// Best-effort: per-channel failures are logged and skipped.
func (tr *TelegramReader) fetchAllProfilePhotos(ctx context.Context, api *tg.Client) {
pics := make(map[string][]byte, len(tr.channels))
for _, username := range tr.channels {
if ctx.Err() != nil {
return
}
username = strings.TrimSpace(username)
if username == "" {
continue
}
rp, err := tr.resolvePeer(ctx, api, username)
if err != nil {
log.Printf("[profile-pics] resolve %s: %v", username, err)
continue
}
if rp.photoID == 0 {
continue
}
// Skip the download if we already pushed this photoID.
tr.mu.Lock()
if tr.lastPhotoID == nil {
tr.lastPhotoID = map[string]int64{}
}
prevID, hadPrev := tr.lastPhotoID[username]
tr.mu.Unlock()
if hadPrev && prevID == rp.photoID {
continue
}
body, err := tr.fetchProfilePhoto(ctx, api, rp.peer, rp.photoID)
if err != nil {
log.Printf("[profile-pics] download %s (id=%d): %v", username, rp.photoID, err)
continue
}
pics[username] = body
tr.mu.Lock()
tr.lastPhotoID[username] = rp.photoID
tr.mu.Unlock()
}
if len(pics) == 0 {
return
}
// Merge so other readers' contributions (e.g. X "x:" entries) survive.
total := tr.feed.MergeProfilePics(pics)
log.Printf("[profile-pics] cycle done: %d new, %d total in bundle", len(pics), total)
}
+274
View File
@@ -0,0 +1,274 @@
package server
import (
"crypto/rand"
"hash/crc32"
"testing"
"time"
"github.com/sartoopjj/thefeed/internal/protocol"
)
// fakeJPEG returns n bytes that start with the JPEG SOI marker so
// sniffProfilePicMime tags them as JPEG.
func fakeJPEG(n int) []byte {
out := make([]byte, n)
rand.Read(out)
if n >= 3 {
out[0] = 0xFF
out[1] = 0xD8
out[2] = 0xFF
}
return out
}
func fakePNG(n int) []byte {
out := make([]byte, n)
rand.Read(out)
if n >= 4 {
out[0] = 0x89
out[1] = 'P'
out[2] = 'N'
out[3] = 'G'
}
return out
}
func newFeedWithMedia(t *testing.T) *Feed {
t.Helper()
f := NewFeed([]string{"a", "b"})
mc := NewMediaCache(MediaCacheConfig{
MaxFileBytes: 64 * 1024,
TTL: time.Hour,
DNSRelayEnabled: true,
})
f.SetMediaCache(mc)
return f
}
func TestSetProfilePicsBundlesAvatarsAndExposesDirectory(t *testing.T) {
f := newFeedWithMedia(t)
alice := fakeJPEG(1024)
bob := fakePNG(2048)
stored := f.SetProfilePics(map[string][]byte{"alice": alice, "bob": bob})
if stored != 2 {
t.Fatalf("stored = %d, want 2", stored)
}
got := f.ProfilePicsBundle()
if len(got.Entries) != 2 {
t.Fatalf("entries = %d, want 2", len(got.Entries))
}
// Sorted by username.
if got.Entries[0].Username != "alice" || got.Entries[1].Username != "bob" {
t.Errorf("entries = %s,%s want alice,bob",
got.Entries[0].Username, got.Entries[1].Username)
}
// MIME survived the sniff.
if got.Entries[0].MIME != protocol.ProfilePicMimeJPEG {
t.Errorf("alice MIME = %d want JPEG", got.Entries[0].MIME)
}
if got.Entries[1].MIME != protocol.ProfilePicMimePNG {
t.Errorf("bob MIME = %d want PNG", got.Entries[1].MIME)
}
// Each per-entry CRC matches the original bytes.
if got.Entries[0].CRC != crc32.ChecksumIEEE(alice) {
t.Errorf("alice crc mismatch")
}
if got.Entries[1].CRC != crc32.ChecksumIEEE(bob) {
t.Errorf("bob crc mismatch")
}
// Offsets are tightly packed.
if got.Entries[0].Offset != 0 {
t.Errorf("alice offset = %d, want 0", got.Entries[0].Offset)
}
if got.Entries[1].Offset != got.Entries[0].Size {
t.Errorf("bob offset = %d, want %d (right after alice)",
got.Entries[1].Offset, got.Entries[0].Size)
}
// Bundle metadata: total bytes = sum of avatars.
if got.Header.BundleSize != got.Entries[0].Size+got.Entries[1].Size {
t.Errorf("BundleSize = %d, want sum of entries (%d)",
got.Header.BundleSize, got.Entries[0].Size+got.Entries[1].Size)
}
// Each entry has its own DNS channel inside the media range.
for i, e := range got.Entries {
if e.DNSChannel < protocol.MediaChannelStart || e.DNSChannel > protocol.MediaChannelEnd {
t.Errorf("entries[%d].DNSChannel = %d, outside media range",
i, e.DNSChannel)
}
if e.DNSBlocks == 0 {
t.Errorf("entries[%d].DNSBlocks = 0", i)
}
}
// And those per-entry DNS channels are fetchable as ordinary media.
for i, e := range got.Entries {
blk0, err := f.GetBlock(int(e.DNSChannel), 0)
if err != nil || len(blk0) == 0 {
t.Errorf("entries[%d] DNS block 0: %v / %d bytes", i, err, len(blk0))
}
}
// And we can pull the directory back over the wire and re-derive
// each avatar's slice with VerifyEntry.
dirBlk, err := f.GetBlock(int(protocol.ProfilePicsChannel), 0)
if err != nil {
t.Fatalf("get profile-pics block 0: %v", err)
}
if len(dirBlk) < 4 {
t.Fatalf("dir block too small: %d", len(dirBlk))
}
totalBlocks := int(dirBlk[0])<<8 | int(dirBlk[1])
all := dirBlk[2:]
for i := 1; i < totalBlocks; i++ {
next, err := f.GetBlock(int(protocol.ProfilePicsChannel), i)
if err != nil {
t.Fatalf("get profile-pics block %d: %v", i, err)
}
all = append(all, next...)
}
decoded, err := protocol.DecodeProfilePicsBundle(all)
if err != nil {
t.Fatalf("decode dir: %v", err)
}
if len(decoded.Entries) != 2 {
t.Fatalf("decoded entries = %d, want 2", len(decoded.Entries))
}
// Each entry's DNS channel is independently fetchable. This is the
// "if even one DNS channel works, that one avatar still shows" path.
// Full byte-level round-trip is exercised in the client tests.
for _, e := range decoded.Entries {
blk0, err := f.GetBlock(int(e.DNSChannel), 0)
if err != nil || len(blk0) == 0 {
t.Errorf("entry %s: channel %d block 0: %v",
e.Username, e.DNSChannel, err)
}
}
}
func TestSetProfilePicsExposesRelayBits(t *testing.T) {
f := newFeedWithMedia(t) // DNSRelayEnabled: true, no GitHub relay attached
f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(1024)})
got := f.ProfilePicsBundle()
if !got.Header.HasRelay(protocol.RelayDNS) {
t.Errorf("RelayDNS bit should be set when DNS is enabled, got Relays=%v", got.Header.Relays)
}
if got.Header.HasRelay(protocol.RelayGitHub) {
t.Errorf("RelayGitHub bit should not be set without a GitHub relay, got Relays=%v", got.Header.Relays)
}
}
func TestSetProfilePicsSkipsEmpty(t *testing.T) {
f := newFeedWithMedia(t)
stored := f.SetProfilePics(map[string][]byte{
"": fakeJPEG(100), // empty username
"empty": nil, // empty bytes
"good": fakeJPEG(100),
})
if stored != 1 {
t.Errorf("stored = %d, want 1", stored)
}
}
func TestSetProfilePicsReplaceUpdatesBundle(t *testing.T) {
f := newFeedWithMedia(t)
first := fakeJPEG(1024)
f.SetProfilePics(map[string][]byte{"alice": first})
b1 := f.ProfilePicsBundle()
second := fakeJPEG(2048)
f.SetProfilePics(map[string][]byte{"alice": second})
b2 := f.ProfilePicsBundle()
if len(b1.Entries) != 1 || len(b2.Entries) != 1 {
t.Fatalf("expected one entry each, got %d / %d", len(b1.Entries), len(b2.Entries))
}
if b1.Entries[0].CRC == b2.Entries[0].CRC {
t.Errorf("entry CRC didn't change after replacing bytes")
}
if b1.Header.BundleCRC == b2.Header.BundleCRC {
t.Errorf("bundle CRC didn't change after replacing bytes")
}
}
func TestSetProfilePicsClearsOnEmpty(t *testing.T) {
f := newFeedWithMedia(t)
f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(1024)})
if got := f.ProfilePicsBundle(); len(got.Entries) != 1 {
t.Fatalf("setup: got %d entries", len(got.Entries))
}
stored := f.SetProfilePics(nil)
if stored != 0 {
t.Errorf("stored = %d, want 0", stored)
}
got := f.ProfilePicsBundle()
if len(got.Entries) != 0 {
t.Errorf("entries = %d, want 0 after empty refresh", len(got.Entries))
}
}
func TestMergeProfilePicsKeepsOtherEntries(t *testing.T) {
f := newFeedWithMedia(t)
// First reader contributes Telegram avatars.
tg := map[string][]byte{
"alice": fakeJPEG(800),
"bob": fakeJPEG(900),
}
if got := f.SetProfilePics(tg); got != 2 {
t.Fatalf("seed Set: got %d want 2", got)
}
// Second reader contributes X avatars under the "x:" namespace —
// the Telegram entries should survive.
xs := map[string][]byte{
"x:elonmusk": fakePNG(700),
}
if got := f.MergeProfilePics(xs); got != 3 {
t.Fatalf("merge: got %d want 3", got)
}
got := f.ProfilePicsBundle()
if len(got.Entries) != 3 {
t.Fatalf("entries = %d want 3", len(got.Entries))
}
have := map[string]bool{}
for _, e := range got.Entries {
have[e.Username] = true
}
for _, want := range []string{"alice", "bob", "x:elonmusk"} {
if !have[want] {
t.Errorf("missing entry %q in merged bundle", want)
}
}
}
func TestMergeProfilePicsDropEntry(t *testing.T) {
f := newFeedWithMedia(t)
f.SetProfilePics(map[string][]byte{
"alice": fakeJPEG(800),
"bob": fakeJPEG(900),
})
// nil bytes for an existing key drops it.
if got := f.MergeProfilePics(map[string][]byte{"alice": nil}); got != 1 {
t.Fatalf("merge: got %d want 1", got)
}
got := f.ProfilePicsBundle()
if len(got.Entries) != 1 || got.Entries[0].Username != "bob" {
t.Errorf("entries after drop = %+v", got.Entries)
}
}
func TestSetProfilePicsNoMediaCache(t *testing.T) {
f := NewFeed([]string{"a"})
stored := f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(100)})
if stored != 0 {
t.Errorf("stored = %d, want 0 (media cache not configured)", stored)
}
if got := f.ProfilePicsBundle(); len(got.Entries) != 0 {
t.Errorf("entries = %v, want empty", got.Entries)
}
}
+3
View File
@@ -169,6 +169,9 @@ func (pr *PublicReader) fetchAll(ctx context.Context) {
}
log.Printf("[public] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total",
time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(pr.channels))
// Fetch avatars for the same channel set — same cadence, best-effort.
// Public mode has no MTProto session so we scrape them off t.me/s/<u>.
pr.fetchAllPublicProfilePhotos(ctx)
pr.feed.AfterFetchCycle(ctx)
}
+156
View File
@@ -0,0 +1,156 @@
package server
import (
"context"
"fmt"
"io"
"log"
"net/http"
"net/url"
"strings"
"sync"
"golang.org/x/net/html"
)
// extractPublicAvatarURL finds the channel avatar URL on a t.me/s page.
// Returns "" when the channel has no photo or the layout is unfamiliar.
func extractPublicAvatarURL(body []byte) string {
doc, err := html.Parse(strings.NewReader(string(body)))
if err != nil {
return ""
}
if i := findFirstByClass(doc, "tgme_page_photo_image"); i != nil {
if img := firstImgChild(i); img != nil {
if src := attrValue(img, "src"); src != "" {
return src
}
}
if attrValue(i, "src") != "" {
return attrValue(i, "src")
}
}
if u := findFirstByClass(doc, "tgme_widget_message_user_photo"); u != nil {
if src := attrValue(u, "src"); src != "" {
return src
}
if img := firstImgChild(u); img != nil {
return attrValue(img, "src")
}
}
return ""
}
func firstImgChild(n *html.Node) *html.Node {
if n == nil {
return nil
}
var found *html.Node
visitNodes(n, func(c *html.Node) {
if found != nil || c.Type != html.ElementNode {
return
}
if c.Data == "img" {
found = c
}
})
return found
}
// fetchPublicAvatar downloads the avatar from t.me/s/<username>.
// Returns nil bytes when the channel has no photo.
func (pr *PublicReader) fetchPublicAvatar(ctx context.Context, username string) ([]byte, error) {
username = strings.TrimSpace(username)
if username == "" {
return nil, nil
}
pageURL := pr.baseURL + "/" + url.PathEscape(username)
body, err := httpGetBody(ctx, pr.client, pageURL)
if err != nil {
return nil, fmt.Errorf("fetch %s: %w", pageURL, err)
}
imgURL := extractPublicAvatarURL(body)
if imgURL == "" {
return nil, nil
}
const maxAvatarBytes = 512 * 1024
imgBytes, err := httpGetWithLimit(ctx, pr.client, imgURL, maxAvatarBytes)
if err != nil {
return nil, fmt.Errorf("download avatar %s: %w", imgURL, err)
}
return imgBytes, nil
}
// fetchAllPublicProfilePhotos scrapes each channel's avatar from
// t.me/s/<u> and merges into the bundle. Best-effort.
func (pr *PublicReader) fetchAllPublicProfilePhotos(ctx context.Context) {
pr.mu.RLock()
channels := append([]string(nil), pr.channels...)
pr.mu.RUnlock()
pics := make(map[string][]byte, len(channels))
var picsMu sync.Mutex
var wg sync.WaitGroup
sem := make(chan struct{}, 4) // cap concurrency vs t.me
for _, u := range channels {
if ctx.Err() != nil {
return
}
u = strings.TrimSpace(u)
if u == "" {
continue
}
wg.Add(1)
go func(u string) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
body, err := pr.fetchPublicAvatar(ctx, u)
if err != nil {
log.Printf("[public profile-pic] %s: %v", u, err)
return
}
if len(body) == 0 {
return
}
picsMu.Lock()
pics[u] = body
picsMu.Unlock()
}(u)
}
wg.Wait()
if len(pics) == 0 {
return
}
total := pr.feed.MergeProfilePics(pics)
log.Printf("[public profile-pic] cycle done: %d new, %d total in bundle", len(pics), total)
}
// httpGetBody is a tiny GET helper. Cap is 8 MB.
func httpGetBody(ctx context.Context, c *http.Client, url string) ([]byte, error) {
return httpGetWithLimit(ctx, c, url, 8*1024*1024)
}
func httpGetWithLimit(ctx context.Context, c *http.Client, u string, limit int64) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)")
resp, err := c.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("http %s: status %s", u, resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
if err != nil {
return nil, err
}
if int64(len(body)) > limit {
return nil, fmt.Errorf("http %s: response exceeds %d bytes", u, limit)
}
return body, nil
}
+16
View File
@@ -68,6 +68,10 @@ type TelegramReader struct {
cacheTTL time.Duration
fetchInterval time.Duration
// lastPhotoID dedups profile-pic downloads across cycles. username →
// last seen Telegram photo ID; skip download when unchanged.
lastPhotoID map[string]int64
// api is set once authenticated, used for sending messages.
apiMu sync.RWMutex
api *tg.Client
@@ -92,6 +96,10 @@ type resolvedPeer struct {
chatType protocol.ChatType
canSend bool
title string
// photoID is the Telegram-assigned ID of the peer's profile photo,
// 0 when the channel has none. Used to skip the download path when
// the photo hasn't changed since last fetch.
photoID int64
}
type cachedMessages struct {
@@ -289,6 +297,10 @@ func (tr *TelegramReader) fetchAll(ctx context.Context, api *tg.Client) {
}
log.Printf("[telegram] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total",
time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(tr.channels))
// Profile pics piggyback the regular cycle: best-effort, doesn't
// gate channel data. Each channel's photo is downloaded once and
// only re-fetched when Telegram reports a different photo ID.
tr.fetchAllProfilePhotos(ctx, api)
tr.feed.AfterFetchCycle(ctx)
}
@@ -306,6 +318,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
for _, chat := range resolved.Chats {
if ch, ok := chat.(*tg.Channel); ok {
canSend := !ch.Broadcast || ch.Creator || ch.AdminRights.PostMessages
pid, _ := extractChatPhotoID(ch.Photo)
return &resolvedPeer{
peer: &tg.InputPeerChannel{
ChannelID: ch.ID,
@@ -314,6 +327,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
chatType: protocol.ChatTypeChannel,
canSend: canSend,
title: ch.Title,
photoID: pid,
}, nil
}
}
@@ -321,6 +335,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
// Check Users (bots, private chats)
for _, u := range resolved.Users {
if user, ok := u.(*tg.User); ok {
pid, _ := extractUserPhotoID(user.Photo)
return &resolvedPeer{
peer: &tg.InputPeerUser{
UserID: user.ID,
@@ -329,6 +344,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
chatType: protocol.ChatTypePrivate,
canSend: true,
title: user.FirstName,
photoID: pid,
}, nil
}
}
+4
View File
@@ -226,6 +226,10 @@ func (xr *XPublicReader) fetchAll(ctx context.Context) {
}
log.Printf("[x] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total",
time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(xr.accounts))
// Avatars: same cycle, merged into the feed's profile-pic bundle. We
// merge rather than overwrite because TelegramReader / PublicReader
// may also be feeding into the same bundle.
xr.fetchAllXProfilePhotos(ctx)
xr.feed.AfterFetchCycle(ctx)
}
+133
View File
@@ -0,0 +1,133 @@
package server
import (
"context"
"encoding/xml"
"fmt"
"io"
"log"
"net/http"
"net/url"
"strings"
"sync"
)
// Nitter / RSS feeds put the channel image at <rss><channel><image><url>.
type xRSSImageEnvelope struct {
XMLName xml.Name `xml:"rss"`
Channel struct {
Image struct {
URL string `xml:"url"`
} `xml:"image"`
} `xml:"channel"`
}
func extractXAvatarURL(body []byte) string {
var env xRSSImageEnvelope
if err := xml.Unmarshal(body, &env); err != nil {
return ""
}
return strings.TrimSpace(env.Channel.Image.URL)
}
// fetchXAvatar tries each Nitter instance and returns the first
// avatar that downloads. (nil, nil) when no instance had one.
func (xr *XPublicReader) fetchXAvatar(ctx context.Context, account string) ([]byte, error) {
const maxAvatarBytes = 512 * 1024
var lastErr error
for _, instance := range xr.instances {
rssURL := strings.TrimSuffix(instance, "/") + "/" + url.PathEscape(account) + "/rss"
body, err := xRSSGet(ctx, xr.client, rssURL)
if err != nil {
lastErr = err
continue
}
avatarURL := extractXAvatarURL(body)
if avatarURL == "" {
continue
}
// Some Nitter builds return a relative "/pic/..." path.
if strings.HasPrefix(avatarURL, "/") {
avatarURL = strings.TrimSuffix(instance, "/") + avatarURL
}
imgBytes, err := httpGetWithLimit(ctx, xr.client, avatarURL, maxAvatarBytes)
if err != nil {
lastErr = err
continue
}
return imgBytes, nil
}
if lastErr != nil {
return nil, lastErr
}
return nil, nil
}
// fetchAllXProfilePhotos downloads each account's avatar (via Nitter)
// and merges into the bundle under "x:<handle>" keys.
func (xr *XPublicReader) fetchAllXProfilePhotos(ctx context.Context) {
xr.mu.RLock()
accounts := append([]string(nil), xr.accounts...)
xr.mu.RUnlock()
pics := make(map[string][]byte, len(accounts))
var picsMu sync.Mutex
var wg sync.WaitGroup
sem := make(chan struct{}, 4)
for _, a := range accounts {
if ctx.Err() != nil {
return
}
a = strings.TrimSpace(a)
if a == "" {
continue
}
wg.Add(1)
go func(account string) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
body, err := xr.fetchXAvatar(ctx, account)
if err != nil {
log.Printf("[x profile-pic] @%s: %v", account, err)
return
}
if len(body) == 0 {
return
}
// "x:" prefix avoids colliding with same-name Telegram channels.
picsMu.Lock()
pics["x:"+account] = body
picsMu.Unlock()
}(a)
}
wg.Wait()
if len(pics) == 0 {
return
}
total := xr.feed.MergeProfilePics(pics)
log.Printf("[x profile-pic] cycle done: %d new, %d total in bundle", len(pics), total)
}
// xRSSGet mirrors the per-instance RSS request (same UA + Accept).
func xRSSGet(ctx context.Context, c *http.Client, u string) ([]byte, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)")
req.Header.Set("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8")
resp, err := c.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode/100 != 2 {
return nil, fmt.Errorf("rss %s: status %s", u, resp.Status)
}
body, err := io.ReadAll(io.LimitReader(resp.Body, maxXRSSBodyBytes))
if err != nil {
return nil, err
}
return body, nil
}
+215 -5
View File
@@ -1,6 +1,7 @@
package telemirror
import (
"net/url"
"regexp"
"strings"
"time"
@@ -36,7 +37,7 @@ func parseChannelInfo(doc *html.Node) *Channel {
}
}
if descEl := findFirstByClass(doc, "tgme_channel_info_description"); descEl != nil {
ch.Description = innerHTML(descEl)
ch.Description = rewriteTranslateLinksInHTML(innerHTML(descEl))
}
if header := findFirstByClass(doc, "tgme_channel_info_header"); header != nil {
if img := findFirstByTag(header, "img"); img != nil {
@@ -73,20 +74,28 @@ func parseSinglePost(wrap *html.Node) *Post {
if owner := findFirstByClass(msg, "tgme_widget_message_owner_name"); owner != nil {
p.Author = textOf(owner)
}
if textEl := findFirstByClass(msg, "tgme_widget_message_text"); textEl != nil {
p.Text = innerHTML(textEl)
// The reply preview also has a `.tgme_widget_message_text` (the
// quoted snippet) which appears BEFORE the body in the DOM, so
// findFirstByClass would grab the wrong one.
if textEl := findMessageBodyText(msg); textEl != nil {
p.Text = rewriteTranslateLinksInHTML(innerHTML(textEl))
}
p.Reply = parseReply(msg)
p.Forward = parseForward(msg)
visit(msg, func(n *html.Node) bool {
switch {
case hasClass(n, "tgme_widget_message_photo_wrap"):
// Source-post URL → real t.me. Thumb stays on the proxy
// (that's how the bytes reach a blocked client).
p.Media = append(p.Media, Media{
Type: "photo",
URL: attrOf(n, "href"),
URL: rewriteTranslateLink(attrOf(n, "href")),
Thumb: extractBgImage(attrOf(n, "style")),
})
case hasClass(n, "tgme_widget_message_video_player"):
m := Media{Type: "video", URL: attrOf(n, "href")}
m := Media{Type: "video", URL: rewriteTranslateLink(attrOf(n, "href"))}
if t := findFirstByClass(n, "tgme_widget_message_video_thumb"); t != nil {
m.Thumb = extractBgImage(attrOf(t, "style"))
}
@@ -138,6 +147,16 @@ func parseSinglePost(wrap *html.Node) *Post {
if t := findFirstByClass(n, "tgme_widget_message_poll_type"); t != nil {
m.Subtitle = textOf(t)
}
visit(n, func(opt *html.Node) bool {
if hasClass(opt, "tgme_widget_message_poll_option_text") {
txt := strings.TrimSpace(textOf(opt))
if txt != "" {
m.Options = append(m.Options, txt)
}
return false
}
return true
})
p.Media = append(p.Media, m)
}
return true
@@ -193,6 +212,72 @@ func parseSinglePost(wrap *html.Node) *Post {
return p
}
// findMessageBodyText: first `.tgme_widget_message_text` not nested
// inside a `.tgme_widget_message_reply` (which would be the snippet).
func findMessageBodyText(msg *html.Node) *html.Node {
var found *html.Node
visit(msg, func(n *html.Node) bool {
if found != nil {
return false
}
if !hasClass(n, "tgme_widget_message_text") {
return true
}
for p := n.Parent; p != nil; p = p.Parent {
if hasClass(p, "tgme_widget_message_reply") {
return false
}
}
found = n
return false
})
return found
}
// parseReply extracts author + snippet + URL from a reply preview.
func parseReply(msg *html.Node) *Reply {
rNode := findFirstByClass(msg, "tgme_widget_message_reply")
if rNode == nil {
return nil
}
r := &Reply{
URL: rewriteTranslateLink(attrOf(rNode, "href")),
}
if a := findFirstByClass(rNode, "tgme_widget_message_author_name"); a != nil {
r.Author = textOf(a)
}
if t := findFirstByClass(rNode, "tgme_widget_message_text"); t != nil {
r.Text = rewriteTranslateLinksInHTML(innerHTML(t))
}
if r.Author == "" && r.Text == "" {
return nil
}
return r
}
// parseForward extracts the "Forwarded from <name>" header.
func parseForward(msg *html.Node) *Forward {
fNode := findFirstByClass(msg, "tgme_widget_message_forwarded_from")
if fNode == nil {
return nil
}
f := &Forward{}
if a := findFirstByClass(fNode, "tgme_widget_message_forwarded_from_name"); a != nil {
f.Author = textOf(a)
f.URL = rewriteTranslateLink(attrOf(a, "href"))
} else if a := findFirstByTag(fNode, "a"); a != nil {
// Bare <a> without the _name class.
f.Author = textOf(a)
f.URL = rewriteTranslateLink(attrOf(a, "href"))
} else {
f.Author = textOf(fNode)
}
if f.Author == "" {
return nil
}
return f
}
// ===== DOM helpers =====
func visit(n *html.Node, fn func(*html.Node) bool) {
@@ -313,3 +398,128 @@ func extractBgImage(style string) string {
}
return ""
}
// Google Translate proxy URLs come in two forms; we rewrite hrefs
// back to the originals so links work when Translate is blocked / the
// site is region-locked. Image src attributes are left alone since
// the proxy is how those bytes actually reach the user.
const translateHostSuffix = ".translate.goog"
// decodeTranslateHost: '.' ↔ '-', '-' ↔ '--'.
//
// t-me → t.me
// cdn4-telegram-org → cdn4.telegram.org
// my--domain-com → my-domain.com
func decodeTranslateHost(s string) string {
s = strings.ReplaceAll(s, "--", "\x00")
s = strings.ReplaceAll(s, "-", ".")
s = strings.ReplaceAll(s, "\x00", "-")
return s
}
// rewriteTranslateLink handles two forms:
// 1. <encoded>.translate.goog/<path>?_x_tr_*=… — inline-proxy form;
// decode host, strip tracking params.
// 2. translate.google.com/website?u=<original> (also /translate) —
// wrapper form; pull `u` and recurse once (Google occasionally
// double-wraps).
func rewriteTranslateLink(raw string) string {
if raw == "" {
return raw
}
u, err := url.Parse(raw)
if err != nil {
return raw
}
// Form 2.
hostLower := strings.ToLower(u.Host)
if (hostLower == "translate.google.com" || hostLower == "www.translate.google.com") &&
(u.Path == "/website" || u.Path == "/translate") {
if orig := u.Query().Get("u"); orig != "" {
return rewriteTranslateLink(orig)
}
}
// Form 1.
if !strings.HasSuffix(hostLower, translateHostSuffix) {
return raw
}
encoded := u.Host[:len(u.Host)-len(translateHostSuffix)]
u.Host = decodeTranslateHost(encoded)
if q := u.Query(); len(q) > 0 {
for k := range q {
if strings.HasPrefix(k, "_x_tr_") {
q.Del(k)
}
}
u.RawQuery = q.Encode()
}
return u.String()
}
// rewriteTranslateLinksInHTML rewrites <a href> in an HTML fragment.
// <img src> is intentionally left alone (we serve images via the proxy).
func rewriteTranslateLinksInHTML(htmlStr string) string {
if htmlStr == "" || !containsAnyTranslateMarker(htmlStr) {
return htmlStr
}
// Sentinel div so we can locate the fragment in the parsed tree
// (html.Parse injects <html><head><body>…).
doc, err := html.Parse(strings.NewReader("<div id=\"tm-rewrite-root\">" + htmlStr + "</div>"))
if err != nil {
return htmlStr
}
rewriteHrefsInTree(doc)
root := findFirstByID(doc, "tm-rewrite-root")
if root == nil {
return htmlStr
}
var b strings.Builder
for c := root.FirstChild; c != nil; c = c.NextSibling {
if err := html.Render(&b, c); err != nil {
return htmlStr
}
}
return b.String()
}
// containsAnyTranslateMarker is a cheap pre-check that catches both
// the inline-proxy and wrapper forms.
func containsAnyTranslateMarker(s string) bool {
return strings.Contains(s, translateHostSuffix) ||
strings.Contains(s, "translate.google.com/")
}
func rewriteHrefsInTree(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "a" {
for i, a := range n.Attr {
if a.Key == "href" {
n.Attr[i].Val = rewriteTranslateLink(a.Val)
}
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
rewriteHrefsInTree(c)
}
}
func findFirstByID(root *html.Node, id string) *html.Node {
var found *html.Node
visit(root, func(n *html.Node) bool {
if found != nil {
return false
}
if n.Type == html.ElementNode {
for _, a := range n.Attr {
if a.Key == "id" && a.Val == id {
found = n
return false
}
}
}
return true
})
return found
}
+252
View File
@@ -109,6 +109,258 @@ func TestParseHTMLPosts(t *testing.T) {
}
}
// Telegram embeds the reply preview INSIDE the message wrapper, BEFORE
// the actual message body. Both the snippet and the body use the
// `tgme_widget_message_text` class. The parser must pick the body, not
// the snippet, and must surface the reply metadata as Post.Reply.
const replyPostHTML = `<!DOCTYPE html><html><body>
<div class="tgme_widget_message_wrap">
<div class="tgme_widget_message" data-post="sample/200">
<a class="tgme_widget_message_reply" href="https://t.me/sample/198">
<span class="tgme_widget_message_author_name">Original Author</span>
<div class="tgme_widget_message_text">quoted snippet of original</div>
</a>
<div class="tgme_widget_message_text">actual reply body</div>
</div>
</div>
</body></html>`
func TestParseHTMLReply(t *testing.T) {
_, posts, err := ParseHTML(replyPostHTML)
if err != nil {
t.Fatalf("ParseHTML: %v", err)
}
if len(posts) != 1 {
t.Fatalf("posts = %d, want 1", len(posts))
}
p := posts[0]
if !strings.Contains(p.Text, "actual reply body") {
t.Errorf("Post.Text = %q, want main body, not snippet", p.Text)
}
if strings.Contains(p.Text, "quoted snippet") {
t.Errorf("Post.Text leaked the reply snippet: %q", p.Text)
}
if p.Reply == nil {
t.Fatalf("Post.Reply nil; want populated reply preview")
}
if p.Reply.Author != "Original Author" {
t.Errorf("Reply.Author = %q", p.Reply.Author)
}
if !strings.Contains(p.Reply.Text, "quoted snippet") {
t.Errorf("Reply.Text = %q", p.Reply.Text)
}
if p.Reply.URL != "https://t.me/sample/198" {
t.Errorf("Reply.URL = %q", p.Reply.URL)
}
}
const forwardPostHTML = `<!DOCTYPE html><html><body>
<div class="tgme_widget_message_wrap">
<div class="tgme_widget_message" data-post="sample/201">
<div class="tgme_widget_message_forwarded_from">
Forwarded from <a class="tgme_widget_message_forwarded_from_name" href="https://t.me/source">Source Channel</a>
</div>
<div class="tgme_widget_message_text">forwarded post body</div>
</div>
</div>
</body></html>`
func TestParseHTMLForward(t *testing.T) {
_, posts, err := ParseHTML(forwardPostHTML)
if err != nil {
t.Fatalf("ParseHTML: %v", err)
}
if len(posts) != 1 {
t.Fatalf("posts = %d, want 1", len(posts))
}
p := posts[0]
if p.Forward == nil {
t.Fatalf("Post.Forward nil; want populated forward header")
}
if p.Forward.Author != "Source Channel" {
t.Errorf("Forward.Author = %q", p.Forward.Author)
}
if p.Forward.URL != "https://t.me/source" {
t.Errorf("Forward.URL = %q", p.Forward.URL)
}
if !strings.Contains(p.Text, "forwarded post body") {
t.Errorf("Post.Text = %q", p.Text)
}
}
const pollPostHTML = `<!DOCTYPE html><html><body>
<div class="tgme_widget_message_wrap">
<div class="tgme_widget_message" data-post="sample/202">
<div class="tgme_widget_message_poll">
<div class="tgme_widget_message_poll_question">Favourite colour?</div>
<div class="tgme_widget_message_poll_type">Public vote</div>
<div class="tgme_widget_message_poll_option">
<div class="tgme_widget_message_poll_option_text">Red</div>
</div>
<div class="tgme_widget_message_poll_option">
<div class="tgme_widget_message_poll_option_text">Blue</div>
</div>
<div class="tgme_widget_message_poll_option">
<div class="tgme_widget_message_poll_option_text">Green</div>
</div>
</div>
</div>
</div>
</body></html>`
func TestParseHTMLPollOptions(t *testing.T) {
_, posts, err := ParseHTML(pollPostHTML)
if err != nil {
t.Fatalf("ParseHTML: %v", err)
}
if len(posts) != 1 || len(posts[0].Media) != 1 {
t.Fatalf("posts/media wrong shape: posts=%d media=%d", len(posts), len(posts[0].Media))
}
m := posts[0].Media[0]
if m.Type != "poll" {
t.Errorf("Media.Type = %q, want poll", m.Type)
}
if m.Title != "Favourite colour?" {
t.Errorf("Media.Title = %q", m.Title)
}
if m.Subtitle != "Public vote" {
t.Errorf("Media.Subtitle = %q", m.Subtitle)
}
wantOpts := []string{"Red", "Blue", "Green"}
if len(m.Options) != len(wantOpts) {
t.Fatalf("Options = %v, want %v", m.Options, wantOpts)
}
for i, want := range wantOpts {
if m.Options[i] != want {
t.Errorf("Options[%d] = %q, want %q", i, m.Options[i], want)
}
}
}
func TestDecodeTranslateHost(t *testing.T) {
cases := []struct {
in, want string
}{
{"t-me", "t.me"},
{"cdn4-telegram-org", "cdn4.telegram.org"},
{"my--domain-com", "my-domain.com"},
{"a--b--c-d", "a-b-c.d"},
{"plain", "plain"},
{"", ""},
}
for _, c := range cases {
if got := decodeTranslateHost(c.in); got != c.want {
t.Errorf("decodeTranslateHost(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestRewriteTranslateLink(t *testing.T) {
cases := []struct {
in, want string
}{
{
"https://t-me.translate.goog/sample/123",
"https://t.me/sample/123",
},
{
"https://example-com.translate.goog/path?_x_tr_sl=auto&_x_tr_tl=en&q=keep",
"https://example.com/path?q=keep",
},
{
"https://my--domain-com.translate.goog/x",
"https://my-domain.com/x",
},
// Untouched: not a translate.goog URL.
{"https://example.com/foo", "https://example.com/foo"},
// Untouched: empty.
{"", ""},
// Tracking-only query string collapses to no query string.
{
"https://t-me.translate.goog/sample?_x_tr_pto=wapp",
"https://t.me/sample",
},
// Wrapper form: translate.google.com/website?u=<original>
{
"https://translate.google.com/website?sl=auto&tl=fa&hl=en&client=webapp&u=https://seup.shop/f/rdzq",
"https://seup.shop/f/rdzq",
},
// Wrapper form, /translate path also works.
{
"https://translate.google.com/translate?sl=en&tl=fa&u=https://example.com/page",
"https://example.com/page",
},
// Wrapper form pointing at a goog inline-proxy URL — recurse.
{
"https://translate.google.com/website?u=https://t-me.translate.goog/networkti",
"https://t.me/networkti",
},
}
for _, c := range cases {
if got := rewriteTranslateLink(c.in); got != c.want {
t.Errorf("rewriteTranslateLink(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestRewriteTranslateLinksInHTML(t *testing.T) {
in := `Check <a href="https://t-me.translate.goog/sample/123">this post</a> ` +
`and <a href="https://example-com.translate.goog/?_x_tr_sl=auto">this site</a>. ` +
`<img src="https://cdn-translate.goog/img.jpg"> stays.`
got := rewriteTranslateLinksInHTML(in)
if !strings.Contains(got, `href="https://t.me/sample/123"`) {
t.Errorf("translate.goog href not rewritten: %q", got)
}
if !strings.Contains(got, `href="https://example.com/"`) {
t.Errorf("example-com.translate.goog href not rewritten or query not stripped: %q", got)
}
if !strings.Contains(got, `src="https://cdn-translate.goog/img.jpg"`) {
t.Errorf("img src should NOT be rewritten (we serve images via the proxy): %q", got)
}
// Sanity check: untouched HTML round-trips without the rewriter
// damaging it.
plain := "no proxy links here, just <b>text</b>"
if got := rewriteTranslateLinksInHTML(plain); got != plain {
t.Errorf("plain html mutated: %q", got)
}
}
func TestParseHTMLRewritesPostLinks(t *testing.T) {
const sample = `<!DOCTYPE html><html><body>
<div class="tgme_widget_message_wrap">
<div class="tgme_widget_message" data-post="sample/300">
<div class="tgme_widget_message_text">
Visit <a href="https://t-me.translate.goog/networkti">@networkti</a> and
<a href="https://example-com.translate.goog/article?_x_tr_sl=auto&_x_tr_tl=en">read</a>,
and try <a href="https://translate.google.com/website?sl=auto&tl=fa&u=https://seup.shop/f/rdzq">this site</a>.
</div>
</div>
</div>
</body></html>`
_, posts, err := ParseHTML(sample)
if err != nil {
t.Fatalf("ParseHTML: %v", err)
}
if len(posts) != 1 {
t.Fatalf("posts = %d, want 1", len(posts))
}
if !strings.Contains(posts[0].Text, `href="https://t.me/networkti"`) {
t.Errorf("Post.Text didn't rewrite t-me link: %q", posts[0].Text)
}
if !strings.Contains(posts[0].Text, `href="https://seup.shop/f/rdzq"`) {
t.Errorf("Post.Text didn't unwrap translate.google.com/website wrapper: %q", posts[0].Text)
}
if strings.Contains(posts[0].Text, "translate.goog") {
t.Errorf("translate.goog leaked in Post.Text: %q", posts[0].Text)
}
if strings.Contains(posts[0].Text, "translate.google.com") {
t.Errorf("translate.google.com wrapper leaked in Post.Text: %q", posts[0].Text)
}
if strings.Contains(posts[0].Text, "_x_tr_") {
t.Errorf("_x_tr_ tracking params leaked: %q", posts[0].Text)
}
}
func TestParseHTMLEmpty(t *testing.T) {
ch, posts, err := ParseHTML("<html></html>")
if err != nil {
+22 -7
View File
@@ -26,25 +26,40 @@ type Channel struct {
// Media is one attachment on a post.
type Media struct {
Type string `json:"type"` // "photo" | "video" | "voice" | "audio" | "document" | "sticker" | "poll"
URL string `json:"url,omitempty"`
Thumb string `json:"thumb,omitempty"`
Duration string `json:"duration,omitempty"`
Title string `json:"title,omitempty"` // file name / poll question / audio title
Subtitle string `json:"subtitle,omitempty"`
Type string `json:"type"` // photo | video | voice | audio | document | sticker | poll
URL string `json:"url,omitempty"`
Thumb string `json:"thumb,omitempty"`
Duration string `json:"duration,omitempty"`
Title string `json:"title,omitempty"` // file name / poll question / audio title
Subtitle string `json:"subtitle,omitempty"`
Options []string `json:"options,omitempty"` // poll options
}
// Reaction is one emoji + count on a post.
type Reaction struct {
Emoji string `json:"emoji"`
Count string `json:"count,omitempty"`
}
// Reply is the quoted preview shown above a reply.
type Reply struct {
Author string `json:"author,omitempty"`
Text string `json:"text,omitempty"` // inner HTML of the snippet
URL string `json:"url,omitempty"` // link to the replied-to post
}
// Forward is the "Forwarded from <name>" header.
type Forward struct {
Author string `json:"author,omitempty"`
URL string `json:"url,omitempty"`
}
// Post is a single message from the channel feed.
type Post struct {
ID string `json:"id"` // "<channel>/<msgid>"
Author string `json:"author,omitempty"`
Text string `json:"text,omitempty"` // sanitised inner HTML
Reply *Reply `json:"reply,omitempty"`
Forward *Forward `json:"forward,omitempty"`
Media []Media `json:"media,omitempty"`
Reactions []Reaction `json:"reactions,omitempty"`
Time time.Time `json:"time,omitempty"`
+539
View File
@@ -0,0 +1,539 @@
package web
import (
"context"
"encoding/json"
"errors"
"fmt"
"hash/crc32"
"net/http"
"os"
"path/filepath"
"strings"
"sync"
"time"
"github.com/sartoopjj/thefeed/internal/client"
"github.com/sartoopjj/thefeed/internal/protocol"
)
// profilePicsHub caches channel avatars on disk and coordinates the
// background fetch. GitHub bundle is automatic; per-entry DNS path
// requires the ProfilePicsEnabled toggle.
type profilePicsHub struct {
dataDir string
mu sync.Mutex
index map[string]profilePicCacheEntry // username (lower) -> entry
fetching bool
progress profilePicProgress
// CRC of the bundle that produced our current cache; if the next
// directory advertises the same value we skip the download.
bundleCRC uint32
}
type profilePicCacheEntry struct {
CRC uint32 `json:"crc"`
Size uint32 `json:"size"`
MIME uint8 `json:"mime"`
Extension string `json:"ext"`
StoredAt int64 `json:"storedAt"`
}
// profilePicProgress is polled by the UI during refresh.
type profilePicProgress struct {
Active bool `json:"active"`
Total int `json:"total"`
Done int `json:"done"`
Failed int `json:"failed"`
Username string `json:"username,omitempty"`
Error string `json:"error,omitempty"`
}
type profilePicsIndexFile struct {
BundleCRC uint32 `json:"bundleCrc"`
Users map[string]profilePicCacheEntry `json:"users"`
}
func newProfilePicsHub(dataDir string) *profilePicsHub {
h := &profilePicsHub{
dataDir: dataDir,
index: make(map[string]profilePicCacheEntry),
}
h.loadIndex()
return h
}
func (h *profilePicsHub) cacheDir() string {
return filepath.Join(h.dataDir, "profile_pics")
}
func (h *profilePicsHub) indexPath() string {
return filepath.Join(h.cacheDir(), "index.json")
}
func (h *profilePicsHub) imagePath(username, ext string) string {
// "x:handle" → "x__handle" — Windows can't use ':' in filenames.
safe := strings.ReplaceAll(strings.ToLower(username), ":", "__")
return filepath.Join(h.cacheDir(), safe+ext)
}
func (h *profilePicsHub) loadIndex() {
b, err := os.ReadFile(h.indexPath())
if err != nil {
return
}
var idx profilePicsIndexFile
if err := json.Unmarshal(b, &idx); err != nil {
return
}
h.mu.Lock()
h.index = idx.Users
if h.index == nil {
h.index = make(map[string]profilePicCacheEntry)
}
h.bundleCRC = idx.BundleCRC
h.mu.Unlock()
}
func (h *profilePicsHub) saveIndexLocked() {
if err := os.MkdirAll(h.cacheDir(), 0700); err != nil {
return
}
idx := profilePicsIndexFile{
BundleCRC: h.bundleCRC,
Users: h.index,
}
b, err := json.MarshalIndent(idx, "", " ")
if err != nil {
return
}
_ = os.WriteFile(h.indexPath(), b, 0600)
}
// Store writes bytes to disk and updates the index. Any previous file
// for this username (different extension) is removed.
func (h *profilePicsHub) Store(username string, content []byte, crc uint32, mime uint8) error {
if username == "" || len(content) == 0 {
return errors.New("profile-pics: empty input")
}
if err := os.MkdirAll(h.cacheDir(), 0700); err != nil {
return err
}
ext := extensionFor(mime)
tmp := h.imagePath(username, ext) + ".tmp"
if err := os.WriteFile(tmp, content, 0600); err != nil {
return err
}
final := h.imagePath(username, ext)
if err := os.Rename(tmp, final); err != nil {
_ = os.Remove(tmp)
return err
}
h.mu.Lock()
if old, ok := h.index[strings.ToLower(username)]; ok && old.Extension != ext {
_ = os.Remove(h.imagePath(username, old.Extension))
}
h.index[strings.ToLower(username)] = profilePicCacheEntry{
CRC: crc,
Size: uint32(len(content)),
MIME: mime,
Extension: ext,
StoredAt: time.Now().Unix(),
}
h.saveIndexLocked()
h.mu.Unlock()
return nil
}
// Get returns the cached bytes + content type, or os.ErrNotExist.
func (h *profilePicsHub) Get(username string) ([]byte, string, error) {
h.mu.Lock()
e, ok := h.index[strings.ToLower(username)]
h.mu.Unlock()
if !ok {
return nil, "", os.ErrNotExist
}
b, err := os.ReadFile(h.imagePath(username, e.Extension))
if err != nil {
return nil, "", err
}
return b, contentTypeFor(e.MIME), nil
}
// Clear wipes both the on-disk cache and the index. Hooked by /api/cache/clear.
func (h *profilePicsHub) Clear() {
h.mu.Lock()
defer h.mu.Unlock()
h.index = make(map[string]profilePicCacheEntry)
h.bundleCRC = 0
if entries, err := os.ReadDir(h.cacheDir()); err == nil {
for _, e := range entries {
if !e.IsDir() {
_ = os.Remove(filepath.Join(h.cacheDir(), e.Name()))
}
}
}
}
func (h *profilePicsHub) Progress() profilePicProgress {
h.mu.Lock()
defer h.mu.Unlock()
return h.progress
}
// relayFetcher pulls the bundle bytes from the GitHub relay. nil means
// "skip the GitHub path".
type relayFetcher func(ctx context.Context, size int64, crc uint32) ([]byte, error)
// storedCallback fires after each successful persist so the server can
// push an SSE event mid-refresh.
type storedCallback func(username string)
// Refresh is the test entry point (DNS only, no GitHub fetcher).
func (h *profilePicsHub) Refresh(ctx context.Context, fetcher *client.Fetcher, dnsAllowed bool) error {
return h.refresh(ctx, fetcher, dnsAllowed, nil, nil)
}
// refresh tries the GitHub bundle first, then per-entry DNS for
// anything still missing. Coalesces concurrent calls.
func (h *profilePicsHub) refresh(ctx context.Context, fetcher *client.Fetcher, dnsAllowed bool, viaGitHub relayFetcher, onStored storedCallback) error {
h.mu.Lock()
if h.fetching {
h.mu.Unlock()
return errors.New("profile-pics: refresh already in progress")
}
h.fetching = true
h.progress = profilePicProgress{Active: true}
h.mu.Unlock()
defer func() {
h.mu.Lock()
h.fetching = false
h.progress.Active = false
h.mu.Unlock()
}()
bundle, err := fetcher.FetchProfilePicDirectory(ctx)
if err != nil {
h.setProgressErr(fmt.Sprintf("fetch directory: %v", err))
return err
}
if len(bundle.Entries) == 0 {
return nil
}
h.mu.Lock()
h.progress.Total = len(bundle.Entries)
prevCRC := h.bundleCRC
h.mu.Unlock()
// Same bundle, all entries cached → no download.
if prevCRC != 0 && prevCRC == bundle.BundleCRC && h.allEntriesCached(bundle.Entries) {
h.mu.Lock()
h.progress.Done = len(bundle.Entries)
h.mu.Unlock()
return nil
}
// Phase 1: GitHub bundle (one fetch, all entries).
missing := bundle.Entries
hasGitHub := bundle.HasRelay(protocol.RelayGitHub) && viaGitHub != nil && bundle.BundleSize > 0
if hasGitHub {
body, ghErr := viaGitHub(ctx, int64(bundle.BundleSize), bundle.BundleCRC)
if ghErr != nil {
h.setProgressErr(fmt.Sprintf("github bundle: %v", ghErr))
} else if body != nil {
if uint32(len(body)) == bundle.BundleSize && crc32.ChecksumIEEE(body) == bundle.BundleCRC {
missing = h.persistFromBundle(ctx, body, bundle.Entries, onStored)
h.mu.Lock()
h.bundleCRC = bundle.BundleCRC
h.saveIndexLocked()
h.mu.Unlock()
} else {
h.setProgressErr(fmt.Sprintf("github bundle size/crc mismatch: have %d/%08x want %d/%08x",
len(body), crc32.ChecksumIEEE(body), bundle.BundleSize, bundle.BundleCRC))
}
}
}
// Phase 2: per-entry DNS for whatever the bundle didn't cover.
if dnsAllowed && len(missing) > 0 {
h.fetchMissingViaDNS(ctx, fetcher, missing, onStored)
}
return nil
}
// persistFromBundle slices each entry out, verifies, writes to disk.
// Returns the entries that didn't land (so the DNS phase can retry).
func (h *profilePicsHub) persistFromBundle(ctx context.Context, body []byte, entries []client.ProfilePicEntry, onStored storedCallback) []client.ProfilePicEntry {
missing := make([]client.ProfilePicEntry, 0, len(entries))
for _, entry := range entries {
if ctx.Err() != nil {
return missing
}
h.markProgressUsername(entry.Username)
if h.HasFresh(entry.Username, entry.CRC) {
h.bumpProgress(true)
continue
}
slice, err := protocol.VerifyEntry(body, protocol.ProfilePicEntry{
Username: entry.Username,
Offset: entry.Offset,
Size: entry.Size,
CRC: entry.CRC,
MIME: entry.MIME,
DNSChannel: entry.DNSChannel,
DNSBlocks: entry.DNSBlocks,
})
if err != nil {
missing = append(missing, entry)
continue
}
if err := h.Store(entry.Username, slice, entry.CRC, entry.MIME); err != nil {
missing = append(missing, entry)
continue
}
h.bumpProgress(true)
if onStored != nil {
onStored(entry.Username)
}
}
return missing
}
// fetchMissingViaDNS fetches each entry on its own DNS channel,
// verifies, persists. Per-entry independent: one failure doesn't
// affect the others.
func (h *profilePicsHub) fetchMissingViaDNS(ctx context.Context, fetcher *client.Fetcher, entries []client.ProfilePicEntry, onStored storedCallback) {
for _, entry := range entries {
if ctx.Err() != nil {
return
}
h.markProgressUsername(entry.Username)
if entry.DNSChannel == 0 || entry.DNSBlocks == 0 {
h.bumpProgress(false)
continue
}
// HasFresh is checked again here — a previous Phase-1 success
// could already have populated this entry from another path.
if h.HasFresh(entry.Username, entry.CRC) {
h.bumpProgress(true)
continue
}
body, err := fetcher.FetchMedia(ctx, entry.DNSChannel, entry.DNSBlocks, entry.CRC, nil)
if err != nil {
h.bumpProgress(false)
continue
}
if uint32(len(body)) != entry.Size {
h.bumpProgress(false)
continue
}
if crc32.ChecksumIEEE(body) != entry.CRC {
h.bumpProgress(false)
continue
}
if err := h.Store(entry.Username, body, entry.CRC, entry.MIME); err != nil {
h.bumpProgress(false)
continue
}
h.bumpProgress(true)
if onStored != nil {
onStored(entry.Username)
}
}
}
// HasFresh: cached pic with this CRC is already on disk.
func (h *profilePicsHub) HasFresh(username string, crc uint32) bool {
h.mu.Lock()
defer h.mu.Unlock()
e, ok := h.index[strings.ToLower(username)]
if !ok || e.CRC != crc {
return false
}
_, err := os.Stat(h.imagePath(username, e.Extension))
return err == nil
}
func (h *profilePicsHub) allEntriesCached(entries []client.ProfilePicEntry) bool {
for _, e := range entries {
if !h.HasFresh(e.Username, e.CRC) {
return false
}
}
return true
}
func (h *profilePicsHub) markProgressUsername(username string) {
h.mu.Lock()
h.progress.Username = username
h.mu.Unlock()
}
func (h *profilePicsHub) bumpProgress(ok bool) {
h.mu.Lock()
h.progress.Done++
if !ok {
h.progress.Failed++
}
h.mu.Unlock()
}
func (h *profilePicsHub) setProgressErr(msg string) {
h.mu.Lock()
h.progress.Error = msg
h.mu.Unlock()
}
// ===== HTTP handlers =====
// handleProfilePic serves /api/profile-pics/<key>. Key is "<handle>"
// or "x:<handle>". 404 → front-end falls back to the letter avatar.
func (h *profilePicsHub) handleProfilePic(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
key := strings.TrimPrefix(r.URL.Path, "/api/profile-pics/")
key = strings.TrimSpace(strings.Trim(key, "/"))
if key == "" || !isValidProfilePicKey(key) {
http.Error(w, "missing username", 400)
return
}
body, ctype, err := h.Get(key)
if err != nil {
http.Error(w, "not cached", 404)
return
}
w.Header().Set("Content-Type", ctype)
w.Header().Set("Cache-Control", "private, max-age=86400")
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body)))
_, _ = w.Write(body)
}
// isValidProfilePicKey: "<handle>" or "<type>:<handle>", alphanumeric +
// _-. Bars slashes/back-slashes/dots so the key can't escape the cache dir.
func isValidProfilePicKey(s string) bool {
if strings.ContainsAny(s, "/\\.") {
return false
}
parts := strings.Split(s, ":")
if len(parts) > 2 {
return false
}
for _, p := range parts {
if p == "" {
return false
}
for _, r := range p {
if !(r == '_' || r == '-' || (r >= '0' && r <= '9') ||
(r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')) {
return false
}
}
}
return true
}
// handleProfilePicsRefresh kicks off a background refresh and returns
// immediately; UI polls /api/profile-pics/progress for the progress bar.
func (s *Server) handleProfilePicsRefresh(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
http.Error(w, "method not allowed", 405)
return
}
s.mu.RLock()
fetcher := s.fetcher
hub := s.profilePics
s.mu.RUnlock()
if fetcher == nil || hub == nil {
http.Error(w, "fetcher not ready", 503)
return
}
dnsAllowed := s.profilePicsEnabled()
go func() {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
// SSE per stored avatar so the UI updates mid-batch.
onStored := func(string) {
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
}
if err := hub.refresh(ctx, fetcher, dnsAllowed, s.fetchFromGitHubRelayBytes, onStored); err == nil {
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
}
}()
writeJSON(w, map[string]any{"ok": true})
}
func (s *Server) handleProfilePicsProgress(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
hub := s.profilePics
if hub == nil {
writeJSON(w, profilePicProgress{})
return
}
writeJSON(w, hub.Progress())
}
// handleProfilePicsList returns which usernames have a cached avatar.
// Cache-Control: no-store so SSE-driven reloads see fresh data.
func (s *Server) handleProfilePicsList(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "method not allowed", 405)
return
}
w.Header().Set("Cache-Control", "no-store")
hub := s.profilePics
if hub == nil {
writeJSON(w, map[string]any{"enabled": false, "users": []string{}})
return
}
hub.mu.Lock()
users := make([]string, 0, len(hub.index))
for u := range hub.index {
users = append(users, u)
}
hub.mu.Unlock()
writeJSON(w, map[string]any{
"enabled": s.profilePicsEnabled(),
"users": users,
})
}
func (s *Server) profilePicsEnabled() bool {
pl, err := s.loadProfiles()
if err != nil || pl == nil {
return false
}
return pl.ProfilePicsEnabled
}
// ===== mime helpers =====
func extensionFor(mime uint8) string {
switch mime {
case 1:
return ".png"
case 2:
return ".webp"
default:
return ".jpg"
}
}
func contentTypeFor(mime uint8) string {
switch mime {
case 1:
return "image/png"
case 2:
return "image/webp"
default:
return "image/jpeg"
}
}
+68 -8
View File
@@ -40,19 +40,22 @@ func (e *ghRateLimitError) Error() string {
// (see .github/workflows/build.yml), so net.Lookup* goes through
// bionic libc → netd → the device's actual DNS, the same path any other
// Android app uses. On desktop the OS resolver is similarly fine.
//
// Tight connect / TLS / header timeouts so a blocked GitHub fails fast
// and we fall back to DNS quickly. Body read gets the full 90 s budget
// to cover multi-MB downloads on slow links.
var relayHTTPClient = &http.Client{
// Per-request budget — large enough to cover multi-MB downloads
// over a slow link without hugging short-circuit timeouts.
Timeout: 5 * time.Minute,
Timeout: 90 * time.Second,
Transport: &http.Transport{
DialContext: (&net.Dialer{
Timeout: 15 * time.Second,
Timeout: 5 * time.Second,
KeepAlive: 30 * time.Second,
}).DialContext,
ForceAttemptHTTP2: true,
MaxIdleConns: 10,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 15 * time.Second,
TLSHandshakeTimeout: 5 * time.Second,
ResponseHeaderTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
},
}
@@ -116,6 +119,64 @@ func (c *relayCache) get(ctx context.Context, fetcher *client.Fetcher) (client.R
return info, nil
}
// fetchFromGitHubRelayBytes is the byte-returning twin of
// serveFromGitHubRelay (cache lookup + GitHub fetch + decrypt + CRC
// check). Returns (nil, nil) when the relay isn't configured.
func (s *Server) fetchFromGitHubRelayBytes(ctx context.Context, size int64, crc uint32) ([]byte, error) {
if size <= 0 || crc == 0 {
return nil, nil
}
s.mu.RLock()
fetcher := s.fetcher
rc := s.relayInfo
cache := s.mediaCache
cfg := s.config
s.mu.RUnlock()
if fetcher == nil || rc == nil || cfg == nil || cfg.Domain == "" {
return nil, nil
}
info, err := rc.get(ctx, fetcher)
if err != nil || info.GitHubRepo == "" {
return nil, nil
}
if cfg.Key == "" {
return nil, nil
}
relayKey, err := protocol.DeriveRelayKey(cfg.Key)
if err != nil {
return nil, err
}
domainSeg := protocol.RelayDomainSegment(cfg.Domain, cfg.Key)
objectSeg := protocol.RelayObjectName(size, crc, cfg.Key)
if cache != nil {
if body, _, ok := cache.Get(size, crc); ok {
return body, nil
}
}
url := fmt.Sprintf("https://api.github.com/repos/%s/contents/%s/%s",
info.GitHubRepo, domainSeg, objectSeg)
const aeadOverhead = protocol.NonceSize + 16
encBody, _, err := fetchGitHubRaw(ctx, relayHTTPClient, url, size+int64(aeadOverhead))
if err != nil {
return nil, err
}
body, err := protocol.DecryptRelayBlob(relayKey, encBody)
if err != nil {
return nil, err
}
if int64(len(body)) != size || crc32.ChecksumIEEE(body) != crc {
return nil, errors.New("relay: hash/size mismatch")
}
if cache != nil {
mime := http.DetectContentType(body)
_ = cache.Put(size, crc, body, mime)
}
return body, nil
}
// serveFromGitHubRelay tries to stream the file from raw.githubusercontent.com
// Returns true if the request was fully handled (success or terminal error
// already written). Returns false to let the caller fall back to DNS.
@@ -133,9 +194,8 @@ func (s *Server) serveFromGitHubRelay(w http.ResponseWriter, r *http.Request, si
return false
}
// Long enough to cover a multi-MB GitHub fetch over a slow link, plus
// a multi-block DNS-tunneled relay-info lookup if the cache is empty.
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
// Covers a cold-cache relay-info DNS lookup + GitHub fetch.
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
defer cancel()
info, err := rc.get(ctx, fetcher)
+221 -2
View File
@@ -313,6 +313,7 @@
}
.ch-avatar {
position: relative;
width: 44px;
height: 44px;
border-radius: 50%;
@@ -323,7 +324,17 @@
font-size: 17px;
color: var(--text-dim);
flex-shrink: 0;
font-weight: 600
font-weight: 600;
overflow: hidden
}
.ch-avatar-letter { line-height: 1 }
/* Overlays the initial-letter span; img.onerror removes it and
adds .ch-avatar-noimg so the letter shows through. */
.ch-avatar-img {
position: absolute; inset: 0;
width: 100%; height: 100%;
object-fit: cover;
display: block
}
.ch-item.active .ch-avatar {
@@ -1311,6 +1322,53 @@
a permanent broken-image icon. */
.tm-photo-failed { display: none }
/* Reply quote — clickable, opens source in new tab. */
.tm-post-reply {
margin: 6px 0 8px;
padding: 6px 10px;
border-inline-start: 3px solid var(--accent);
background: color-mix(in oklab, var(--accent) 6%, transparent);
border-radius: 4px;
font-size: 12.5px;
max-height: 100px; overflow: hidden
}
.tm-post-reply-author {
color: var(--accent);
font-weight: 600;
margin-bottom: 2px
}
.tm-post-reply-text {
color: var(--text-dim);
line-height: 1.4
}
.tm-post-reply-text br { display: block; margin-bottom: 2px }
/* "Forwarded from X" header. */
.tm-post-forward {
font-size: 12px;
color: var(--text-dim);
margin-bottom: 4px
}
.tm-post-forward a {
color: var(--accent);
text-decoration: none
}
.tm-post-forward a:hover { text-decoration: underline }
/* Poll options under the question. */
.tm-poll-options {
list-style: none; padding: 0; margin: 8px 0 0;
display: flex; flex-direction: column; gap: 4px
}
.tm-poll-options li {
padding: 4px 8px;
background: var(--border);
border-radius: 4px;
font-size: 12.5px;
color: var(--text)
}
.tm-poll-options li::before { content: "○ "; color: var(--text-dim) }
/* Confirm dialog rendered inside the telemirror modal so it sits
on top of the drawer / backdrop / posts at the right stacking. */
.tm-confirm-overlay {
@@ -3051,6 +3109,20 @@
<label for="cfgShowScanPrompt" data-i18n="show_scan_prompt">Show scan prompt on startup</label>
</div>
</div>
<!-- Profile pictures. GitHub path is automatic; toggle opts into DNS. -->
<div class="form-group">
<div class="row">
<input type="checkbox" id="cfgProfilePics" onchange="setProfilePicsEnabled(this.checked)">
<label for="cfgProfilePics" data-i18n="profile_pics_enable">Also fetch profile pictures over DNS</label>
</div>
<div class="row" style="margin-top:4px;font-size:11px;color:var(--text-dim)">
<span data-i18n="profile_pics_dns_note">GitHub relay path is always used automatically. This option additionally pulls avatars over DNS when the relay is unavailable.</span>
</div>
<div class="row" style="margin-top:6px;gap:8px;align-items:center">
<button type="button" class="btn btn-flat" id="profilePicsRefreshBtn" onclick="profilePicsRefresh()" data-i18n="profile_pics_refresh">Refresh avatars</button>
<span id="profilePicsProgress" style="font-size:11px;color:var(--text-dim)"></span>
</div>
</div>
<div class="form-group">
<label data-i18n="language">Language</label>
<div class="lang-row">
@@ -3510,10 +3582,18 @@
telemirror_audio: 'فایل صوتی',
telemirror_file: 'فایل',
telemirror_poll: 'نظرسنجی',
telemirror_forwarded_from: 'فروارد شده از',
telemirror_about: 'درباره‌ی این کانال',
download: 'دانلود',
copy: 'کپی',
// END telemirror
profile_pics_enable: 'دریافت عکس پروفایل از طریق DNS',
profile_pics_dns_note: 'عکس‌هایی که از طریق رله گیتهاب در دسترس باشند به‌صورت خودکار دریافت می‌شوند. این گزینه فقط برای دریافت عکس‌هایی است که رله نمی‌تواند سرویس بدهد و باید از طریق DNS کند گرفته شوند.',
profile_pics_refresh: 'به‌روز کردن عکس‌ها',
profile_pics_starting: 'در حال شروع...',
profile_pics_progress: 'در حال دریافت عکس‌ها: {n}/{m}',
profile_pics_done: 'انجام شد ({n} عکس)',
profile_pics_failed: 'دریافت ناموفق بود',
font_size: 'اندازه قلم', debug_mode: 'حالت دیباگ', language: 'زبان',
next_fetch_info: 'زمان باقی‌مانده تا دریافت بعدی محتوا توسط سرور',
no_profiles: 'هنوز پروفایلی وجود ندارد', add_profile: '+ پروفایل جدید',
@@ -3743,6 +3823,7 @@
telemirror_refresh: 'Refresh',
telemirror_refresh_warn: 'This channel was refreshed {n} sec ago. Refreshing too often can hit a rate limit and stop working for a few minutes. Refresh anyway?',
telemirror_refresh_yes: 'Refresh',
telemirror_forwarded_from: 'Forwarded from',
telemirror_voice: 'Voice message',
telemirror_audio: 'Audio',
telemirror_file: 'File',
@@ -3751,6 +3832,13 @@
download: 'Download',
copy: 'Copy',
// END telemirror
profile_pics_enable: 'Also fetch profile pictures over DNS',
profile_pics_dns_note: 'Avatars served by the GitHub relay are fetched automatically. Enable this to additionally fall back to DNS for avatars the relay can\'t serve (slower).',
profile_pics_refresh: 'Refresh avatars',
profile_pics_starting: 'Starting…',
profile_pics_progress: 'Refreshing avatars: {n}/{m}',
profile_pics_done: 'Done ({n} avatars)',
profile_pics_failed: 'Refresh failed',
font_size: 'Font Size', debug_mode: 'Debug mode', language: 'Language',
next_fetch_info: 'Time until the server next fetches fresh channel content',
no_profiles: 'No profiles yet', add_profile: '+ Add Profile',
@@ -4049,6 +4137,9 @@
loadBgImage();
connectSSE();
refreshResolversBadge();
// Populate profilePicCache so the channel list renders avatars
// without a per-item probe.
loadProfilePicState().catch(function () { });
// Quietly ask GitHub for the latest published client version. Runs in
// the background so a slow github.com response can't delay startup —
// if there's an update, the dialog shows up a few seconds later.
@@ -4312,6 +4403,74 @@
// settings is opened.
var promptEl = document.getElementById('cfgShowScanPrompt');
if (promptEl) promptEl.checked = localStorage.getItem('thefeed_scan_prompt_off') !== '1';
// Sync the profile-pics toggle from the server (it's persisted in
// profiles.json so survives app restarts).
fetch('/api/settings').then(function (r) { return r.ok ? r.json() : null; }).then(function (s) {
if (!s) return;
var pp = document.getElementById('cfgProfilePics');
if (pp) pp.checked = !!s.profilePicsEnabled;
}).catch(function () { });
}
// ===== PROFILE PICTURES =====
// Cached lowercase usernames so renderChannels can decide whether
// to overlay an <img> over the initial-letter circle.
var profilePicCache = { enabled: false, users: {} };
var profilePicsPollTimer = null;
// SSE throttle state — server fires one event per stored avatar.
var profilePicsReloadTimer = null;
var profilePicsLastReloadAt = 0;
function loadProfilePicState() {
// no-store so SSE reloads see fresh data, not a cached entry.
return fetch('/api/profile-pics', { cache: 'no-store' }).then(function (r) { return r.ok ? r.json() : null; }).then(function (d) {
if (!d) return;
profilePicCache.enabled = !!d.enabled;
profilePicCache.users = {};
(d.users || []).forEach(function (u) { profilePicCache.users[u.toLowerCase()] = true; });
try { renderChannels(); } catch (e) { }
}).catch(function () { });
}
function setProfilePicsEnabled(enabled) {
// Toggle only controls the DNS path; cached GitHub avatars keep
// displaying either way.
fetch('/api/settings', {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ profilePicsEnabled: !!enabled })
}).then(function () {
profilePicCache.enabled = !!enabled;
if (enabled) profilePicsRefresh();
}).catch(function () { });
}
function profilePicsRefresh() {
var prog = document.getElementById('profilePicsProgress');
if (prog) prog.textContent = t('profile_pics_starting') || 'Starting…';
fetch('/api/profile-pics/refresh', { method: 'POST' }).then(function (r) {
if (!r.ok) {
if (prog) prog.textContent = t('profile_pics_failed') || 'Failed';
return;
}
// Poll progress until inactive.
if (profilePicsPollTimer) clearInterval(profilePicsPollTimer);
profilePicsPollTimer = setInterval(function () {
fetch('/api/profile-pics/progress').then(function (r) { return r.ok ? r.json() : null; }).then(function (p) {
if (!p) return;
if (prog) {
if (p.active) {
var label = (t('profile_pics_progress') || 'Refreshing avatars: {n}/{m}').replace('{n}', p.done || 0).replace('{m}', p.total || 0);
prog.textContent = label + (p.username ? ' (@' + p.username + ')' : '');
} else {
prog.textContent = (t('profile_pics_done') || 'Done').replace('{n}', p.done || 0);
clearInterval(profilePicsPollTimer);
profilePicsPollTimer = null;
loadProfilePicState();
}
}
});
}, 800);
}).catch(function () { });
}
// Toggle the "show startup scan prompt" preference. Persists
@@ -5055,6 +5214,25 @@
}
return;
}
// Throttle (not debounce): first event fires immediately,
// subsequent events at most every 600 ms during a burst.
if (data === 'profile-pics') {
var THROTTLE_MS = 600;
var now = Date.now();
var since = now - profilePicsLastReloadAt;
if (since >= THROTTLE_MS) {
profilePicsLastReloadAt = now;
loadProfilePicState().catch(function () { });
} else if (!profilePicsReloadTimer) {
// Trailing call for the last avatar in the burst.
profilePicsReloadTimer = setTimeout(function () {
profilePicsReloadTimer = null;
profilePicsLastReloadAt = Date.now();
loadProfilePicState().catch(function () { });
}, THROTTLE_MS - since);
}
return;
}
// Another tab/device switched the active profile.
if (data === 'profiles') {
var prevActive = activeProfileId;
@@ -5676,6 +5854,35 @@
var key = chNm.replace(/^@/, '').trim();
autoBtn.classList.toggle('on', autoUpdateChannels.has(key));
}
// Avatar diff against profilePicCache. Without this the
// fast path skips avatar updates and SSE-driven reloads
// don't reach the UI.
var ct2 = channels[ui].ChatType || channels[ui].chatType || 0;
var isX2 = ct2 === 2;
var bareHandle2 = chNm.replace(/^@/, '').toLowerCase();
if (isX2 && bareHandle2.indexOf('x/') === 0) bareHandle2 = bareHandle2.substring(2);
var unameKey2 = isX2 ? ('x:' + bareHandle2) : bareHandle2;
var avatarEl = existingItems[ui].querySelector('.ch-avatar');
if (avatarEl) {
var imgEl = avatarEl.querySelector('.ch-avatar-img');
var shouldShow = !!profilePicCache.users[unameKey2];
if (shouldShow && !imgEl) {
var img = document.createElement('img');
img.className = 'ch-avatar-img';
img.src = '/api/profile-pics/' + encodeURIComponent(unameKey2);
img.loading = 'lazy';
img.alt = '';
img.onerror = function () {
this.parentNode.classList.add('ch-avatar-noimg');
this.remove();
};
avatarEl.classList.remove('ch-avatar-noimg');
avatarEl.insertBefore(img, avatarEl.firstChild);
} else if (!shouldShow && imgEl) {
imgEl.remove();
avatarEl.classList.add('ch-avatar-noimg');
}
}
}
_updateRefreshBadge(); return;
}
@@ -5706,7 +5913,19 @@
var chNm2 = e.ch.Name || e.ch.name || '';
var badge = (channelHasNew(e.ch) && num2 !== selectedChannel) ? '<span class="ch-badge">NEW</span>' : '';
h += '<div class="ch-item' + active + '" data-name="' + escAttr(handle) + '" data-label="' + escAttr(label) + '" onclick="selectChannel(' + num2 + ')">';
h += '<div class="ch-avatar">' + esc(avatarText) + '</div>';
// X channels: e.ch.Name is "x/<handle>" (category prefix);
// strip before re-attaching the bundle's "x:" prefix.
var avatarImg = '';
var bareHandle = (chNm2 || avatarName || '').replace(/^@/, '').toLowerCase();
if (isX && bareHandle.indexOf('x/') === 0) bareHandle = bareHandle.substring(2);
var unameKey = isX ? ('x:' + bareHandle) : bareHandle;
if (profilePicCache.users[unameKey]) {
avatarImg = '<img class="ch-avatar-img" src="/api/profile-pics/'
+ encodeURIComponent(unameKey)
+ '" loading="lazy" alt=""'
+ ' onerror="this.parentNode.classList.add(\'ch-avatar-noimg\');this.remove()">';
}
h += '<div class="ch-avatar">' + avatarImg + '<span class="ch-avatar-letter">' + esc(avatarText) + '</span></div>';
var chSubText = !isX ? (handle.charAt(0) === '@' ? handle : '@' + handle) : '';
h += '<div class="ch-info"><div class="ch-name">' + formatIranTitleHtml(label) + (isPriv ? '<span class="ch-type-tag">' + t('private') + '</span>' : (isX ? '<span class="ch-type-tag x-tag">' + t('x_label') + '</span>' : '')) + '</div>';
if (chSubText) h += '<div class="ch-sub">' + esc(chSubText) + '</div>';
+93 -3
View File
@@ -360,6 +360,28 @@
}
html += '</div>';
if (p.forward && p.forward.author) {
var fwdLabel = tmI18n('telemirror_forwarded_from', 'Forwarded from');
var fwdName = tmEsc(p.forward.author);
if (p.forward.url) {
fwdName = '<a href="' + tmEscAttr(p.forward.url) + '" target="_blank" rel="noopener noreferrer">'
+ fwdName + '</a>';
}
html += '<div class="tm-post-forward">↪ ' + tmEsc(fwdLabel) + ' ' + fwdName + '</div>';
}
if (p.reply) {
var rAuth = p.reply.author ? tmEsc(p.reply.author) : '';
var rText = p.reply.text || '';
html += '<div class="tm-post-reply"'
+ (p.reply.url ? ' onclick="window.open(\'' + tmEscAttr(p.reply.url) + '\', \'_blank\')"' : '')
+ (p.reply.url ? ' style="cursor:pointer"' : '')
+ '>';
if (rAuth) html += '<div class="tm-post-reply-author">' + rAuth + '</div>';
if (rText) html += '<div class="tm-post-reply-text">' + rText + '</div>';
html += '</div>';
}
if (p.text) html += '<div class="tm-post-text">' + p.text + '</div>';
if (p.media && p.media.length) {
@@ -450,8 +472,9 @@
+ ' onerror="this.parentNode.classList.add(\'tm-photo-failed\')">'
+ '<a class="tm-photo-dl" href="' + tmEscAttr(m.thumb) + '"'
+ ' download="' + tmEscAttr(fname) + '"'
+ ' data-fname="' + tmEscAttr(fname) + '"'
+ ' title="' + tmEscAttr(tmI18n('download', 'Download')) + '"'
+ ' onclick="event.stopPropagation()">⬇</a>'
+ ' onclick="return tmDownloadPhoto(this, event)">⬇</a>'
+ '</div>';
}
if (m.type === 'video') {
@@ -486,14 +509,81 @@
+ '</div>';
}
if (m.type === 'poll') {
return '<div class="tm-media-tile"><span class="tm-media-icon">📊</span>'
var optionsHtml = '';
if (m.options && m.options.length) {
optionsHtml = '<ul class="tm-poll-options">';
for (var k = 0; k < m.options.length; k++) {
optionsHtml += '<li>' + tmEsc(m.options[k]) + '</li>';
}
optionsHtml += '</ul>';
}
return '<div class="tm-media-tile tm-media-poll"><span class="tm-media-icon">📊</span>'
+ '<div class="tm-media-meta"><div class="tm-media-title">'
+ tmEsc(m.title || tmI18n('telemirror_poll', 'Poll'))
+ '</div><div class="tm-media-sub">' + tmEsc(m.subtitle || '') + '</div></div></div>';
+ '</div><div class="tm-media-sub">' + tmEsc(m.subtitle || '') + '</div>'
+ optionsHtml
+ '</div></div>';
}
return '';
}
// tmDownloadPhoto fetches the bytes and either hands them to the
// Android bridge (saveMedia) or builds a blob-URL <a download> on
// desktop. <a download> alone doesn't work on Android WebView for
// cross-origin URLs.
window.tmDownloadPhoto = function (anchor, ev) {
if (ev) ev.stopPropagation();
var url = anchor.getAttribute('href');
var fname = anchor.getAttribute('data-fname') || 'photo.jpg';
var bridge = (typeof window !== 'undefined' && window.Android) ? window.Android : null;
var doFetch = function () {
return fetch(url, { referrerPolicy: 'no-referrer' }).then(function (r) {
if (!r.ok) throw new Error('http ' + r.status);
return r.blob();
});
};
if (bridge && typeof bridge.saveMedia === 'function') {
doFetch().then(function (blob) {
return tmBlobToBase64(blob).then(function (b64) {
try { bridge.saveMedia(b64, blob.type || 'image/jpeg', fname); }
catch (e) { tmFallbackOpen(url); }
});
}).catch(function () { tmFallbackOpen(url); });
return false;
}
doFetch().then(function (blob) {
var objectUrl = URL.createObjectURL(blob);
var a = document.createElement('a');
a.href = objectUrl;
a.download = fname;
a.style.display = 'none';
document.body.appendChild(a);
a.click();
setTimeout(function () { URL.revokeObjectURL(objectUrl); a.remove(); }, 100);
}).catch(function () { window.location.href = url; });
return false;
};
function tmBlobToBase64(blob) {
return new Promise(function (resolve, reject) {
var fr = new FileReader();
fr.onload = function () {
var s = fr.result || '';
var i = s.indexOf(',');
resolve(i >= 0 ? s.substring(i + 1) : s);
};
fr.onerror = function () { reject(fr.error); };
fr.readAsDataURL(blob);
});
}
function tmFallbackOpen(url) {
try { window.open(url, '_blank'); } catch (e) { window.location.href = url; }
}
window.tmCopyPost = function (btn) {
var post = btn.closest ? btn.closest('.tm-post') : null;
var pid = post ? post.getAttribute('data-pid') : '';
+71 -6
View File
@@ -117,6 +117,10 @@ type ProfileList struct {
// changes (each launch picks a fresh port → different localStorage
// origin → flag was lost on every restart).
ScanPromptOff bool `json:"scanPromptOff,omitempty"`
// ProfilePicsEnabled enables fetching avatars over DNS when the
// GitHub relay can't serve them. Off by default.
ProfilePicsEnabled bool `json:"profilePicsEnabled,omitempty"`
}
// lastScanData is the on-disk structure for last_scan.json.
@@ -216,6 +220,9 @@ type Server struct {
// Optional, removable backup feed (Telegram-via-Translate proxy).
telemirror *telemirrorHub
// Optional per-channel profile pictures cache.
profilePics *profilePicsHub
}
// New creates a new web server.
@@ -253,6 +260,7 @@ func New(dataDir string, port int, host string, password string) (*Server, error
dlProgress: make(map[string]*mediaDLProgress),
relayInfo: newRelayCache(),
telemirror: newTelemirrorHub(dataDir),
profilePics: newProfilePicsHub(dataDir),
}
if mediaCache != nil {
@@ -345,6 +353,11 @@ func (s *Server) Run() error {
mux.HandleFunc("/api/telemirror/channels", s.telemirror.handleChannels)
mux.HandleFunc("/api/telemirror/channel/", s.telemirror.handleChannel)
mux.HandleFunc("/api/telemirror/img", s.telemirror.handleImg)
// Profile-pics cache + control endpoints.
mux.HandleFunc("/api/profile-pics/", s.profilePics.handleProfilePic)
mux.HandleFunc("/api/profile-pics", s.handleProfilePicsList)
mux.HandleFunc("/api/profile-pics/refresh", s.handleProfilePicsRefresh)
mux.HandleFunc("/api/profile-pics/progress", s.handleProfilePicsProgress)
mux.HandleFunc("/", s.handleIndex)
// Listen on the specified host (default 127.0.0.1)
@@ -1319,6 +1332,42 @@ func (s *Server) refreshMetadataOnly() {
if needsFetch {
go s.ensureTitlesFetched(basectx)
}
go s.maybeRefreshProfilePics(basectx)
}
// maybeRefreshProfilePics fires a refresh when GitHub relay is up or
// the user has opted into the DNS path. No-op otherwise; hub coalesces.
func (s *Server) maybeRefreshProfilePics(parentCtx context.Context) {
s.mu.RLock()
hub := s.profilePics
fetcher := s.fetcher
rc := s.relayInfo
s.mu.RUnlock()
if hub == nil || fetcher == nil {
return
}
dnsAllowed := s.profilePicsEnabled()
githubLikelyUp := false
if rc != nil {
ctx, cancel := context.WithTimeout(parentCtx, 5*time.Second)
info, err := rc.get(ctx, fetcher)
cancel()
if err == nil && info.GitHubRepo != "" {
githubLikelyUp = true
}
}
if !dnsAllowed && !githubLikelyUp {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
defer cancel()
onStored := func(string) {
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
}
if err := hub.refresh(ctx, fetcher, dnsAllowed, s.fetchFromGitHubRelayBytes, onStored); err == nil {
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
}
}
// ensureTitlesFetched fetches channel display names from TitlesChannel in the background.
@@ -2758,16 +2807,26 @@ func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) {
if pl == nil {
pl = &ProfileList{}
}
writeJSON(w, map[string]any{"fontSize": pl.FontSize, "debug": pl.Debug, "theme": pl.Theme, "lang": pl.Lang, "scanPromptOff": pl.ScanPromptOff, "version": version.Version, "commit": version.Commit})
writeJSON(w, map[string]any{
"fontSize": pl.FontSize,
"debug": pl.Debug,
"theme": pl.Theme,
"lang": pl.Lang,
"scanPromptOff": pl.ScanPromptOff,
"profilePicsEnabled": pl.ProfilePicsEnabled,
"version": version.Version,
"commit": version.Commit,
})
case http.MethodPost:
// Optional pointers so partial requests don't reset other fields.
var req struct {
FontSize *int `json:"fontSize"`
Debug *bool `json:"debug"`
Theme *string `json:"theme"`
Lang *string `json:"lang"`
ScanPromptOff *bool `json:"scanPromptOff"`
FontSize *int `json:"fontSize"`
Debug *bool `json:"debug"`
Theme *string `json:"theme"`
Lang *string `json:"lang"`
ScanPromptOff *bool `json:"scanPromptOff"`
ProfilePicsEnabled *bool `json:"profilePicsEnabled"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, "invalid JSON", 400)
@@ -2805,6 +2864,9 @@ func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) {
if req.ScanPromptOff != nil {
pl.ScanPromptOff = *req.ScanPromptOff
}
if req.ProfilePicsEnabled != nil {
pl.ProfilePicsEnabled = *req.ProfilePicsEnabled
}
if err := s.saveProfiles(pl); err != nil {
http.Error(w, fmt.Sprintf("save: %v", err), 500)
return
@@ -3585,6 +3647,9 @@ func (s *Server) handleClearCache(w http.ResponseWriter, r *http.Request) {
if s.telemirror != nil {
s.telemirror.ClearCache()
}
if s.profilePics != nil {
s.profilePics.Clear()
}
mediaDeleted := 0
if s.mediaCache != nil {
mediaDeleted = s.mediaCache.Clear()