mirror of
https://github.com/sartoopjj/thefeed.git
synced 2026-05-19 08:44:34 +03:00
This commit is contained in:
@@ -146,6 +146,7 @@ thefeed یک سیستم تونل DNS است که به شما اجازه می
|
||||
- کانال کانفیگ عمومی دفید: [@thefeedconfig](https://t.me/thefeedconfig)
|
||||
- راهنمای نصب سرور دفید: [@networkti](https://t.me/networkti/25)
|
||||
- راهنمای نصب سرور دفید با اسلیپ گیت: [@networkti](https://t.me/networkti/200)
|
||||
- لیست تسکها و رودمپ پروژه: [بورد گیتهاب](https://github.com/users/sartoopjj/projects/1/views/1)
|
||||
|
||||
## ⚡ نصب سریع سرور
|
||||
|
||||
|
||||
@@ -122,6 +122,7 @@ Thank you for your support ❤️
|
||||
- Public TheFeed Configs: [@thefeedconfig](https://t.me/thefeedconfig)
|
||||
- Setup TheFeed server guide: [@networkti](https://t.me/networkti/25)
|
||||
- Setup TheFeed server with SlipGate guide: [@networkti](https://t.me/networkti/200)
|
||||
- Roadmap / task board: [GitHub project](https://github.com/users/sartoopjj/projects/1/views/1)
|
||||
|
||||
## Quick Install (Server)
|
||||
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/sartoopjj/thefeed/internal/protocol"
|
||||
)
|
||||
|
||||
// ProfilePicsBundle is the client-side view of the profile-pic
|
||||
// directory. The bundle (Size/CRC/Relays) describes the GitHub-served
|
||||
// concatenated blob; per-entry DNSChannel/DNSBlocks describe an
|
||||
// independent DNS fallback for that single avatar.
|
||||
type ProfilePicsBundle struct {
|
||||
BundleSize uint32
|
||||
BundleCRC uint32
|
||||
// Relays describes where the bundle is reachable, indexed by
|
||||
// RelayDNS / RelayGitHub. RelayGitHub means the bundle is on
|
||||
// GitHub. RelayDNS for the bundle is rarely true — the standard
|
||||
// DNS path uses per-entry channels (see ProfilePicEntry).
|
||||
Relays []bool
|
||||
|
||||
Entries []ProfilePicEntry
|
||||
}
|
||||
|
||||
// HasRelay forwards to the relay availability bit at idx.
|
||||
func (b ProfilePicsBundle) HasRelay(idx int) bool {
|
||||
if idx < 0 || idx >= len(b.Relays) {
|
||||
return false
|
||||
}
|
||||
return b.Relays[idx]
|
||||
}
|
||||
|
||||
// ProfilePicEntry points at one avatar in two ways:
|
||||
//
|
||||
// GitHub bundle path: bytes are bundle[Offset:Offset+Size]; CRC must
|
||||
// equal CRC32-IEEE of that slice (use protocol.VerifyEntry).
|
||||
// Per-entry DNS path: bytes live on DNS channel DNSChannel with
|
||||
// DNSBlocks blocks. CRC and Size are checked the same way.
|
||||
//
|
||||
// The client picks whichever path is reachable. With the bundle path
|
||||
// one HTTPS request fetches every avatar; with the DNS path each
|
||||
// avatar is fetched independently so partial sets still show up.
|
||||
type ProfilePicEntry struct {
|
||||
Username string
|
||||
Offset uint32
|
||||
Size uint32
|
||||
CRC uint32
|
||||
MIME uint8
|
||||
DNSChannel uint16
|
||||
DNSBlocks uint16
|
||||
}
|
||||
|
||||
// MimeString returns "image/jpeg" / "image/png" / "image/webp" for the
|
||||
// MIME tag, suitable for use as an HTTP Content-Type.
|
||||
func (p ProfilePicEntry) MimeString() string {
|
||||
switch p.MIME {
|
||||
case protocol.ProfilePicMimePNG:
|
||||
return "image/png"
|
||||
case protocol.ProfilePicMimeWebP:
|
||||
return "image/webp"
|
||||
default:
|
||||
return "image/jpeg"
|
||||
}
|
||||
}
|
||||
|
||||
// Extension returns ".jpg" / ".png" / ".webp" for caching on disk.
|
||||
func (p ProfilePicEntry) Extension() string {
|
||||
switch p.MIME {
|
||||
case protocol.ProfilePicMimePNG:
|
||||
return ".png"
|
||||
case protocol.ProfilePicMimeWebP:
|
||||
return ".webp"
|
||||
default:
|
||||
return ".jpg"
|
||||
}
|
||||
}
|
||||
|
||||
// FetchProfilePicDirectory pulls the bundle directory from
|
||||
// ProfilePicsChannel — header (bundle metadata + relay availability) and
|
||||
// per-username entries. The bundle bytes themselves are NOT fetched here;
|
||||
// callers do that with FetchMedia(BundleChannel, BundleBlocks, BundleCRC)
|
||||
// once and then slice locally.
|
||||
//
|
||||
// Returns (zero-value bundle, nil) when the server has no profile pics
|
||||
// configured (or is older and doesn't know the channel).
|
||||
func (f *Fetcher) FetchProfilePicDirectory(ctx context.Context) (ProfilePicsBundle, error) {
|
||||
fetchCtx, cancel := context.WithTimeout(ctx, time.Minute)
|
||||
defer cancel()
|
||||
|
||||
block0, err := f.FetchBlock(fetchCtx, protocol.ProfilePicsChannel, 0)
|
||||
if err != nil {
|
||||
return ProfilePicsBundle{}, fmt.Errorf("fetch profile-pics: %w", err)
|
||||
}
|
||||
if len(block0) < 2 {
|
||||
return ProfilePicsBundle{}, nil
|
||||
}
|
||||
totalBlocks := int(binary.BigEndian.Uint16(block0))
|
||||
payload0 := block0[2:]
|
||||
|
||||
if totalBlocks <= 1 {
|
||||
return decodeProfilePicsBundle(payload0)
|
||||
}
|
||||
|
||||
type res struct {
|
||||
data []byte
|
||||
err error
|
||||
}
|
||||
results := make([]res, totalBlocks)
|
||||
results[0] = res{data: payload0}
|
||||
var wg sync.WaitGroup
|
||||
for blk := 1; blk < totalBlocks; blk++ {
|
||||
wg.Add(1)
|
||||
go func(blk int) {
|
||||
defer wg.Done()
|
||||
data, e := f.FetchBlock(fetchCtx, protocol.ProfilePicsChannel, uint16(blk))
|
||||
results[blk] = res{data: data, err: e}
|
||||
}(blk)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var all []byte
|
||||
for _, r := range results {
|
||||
if r.err != nil {
|
||||
return ProfilePicsBundle{}, fmt.Errorf("fetch profile-pics block: %w", r.err)
|
||||
}
|
||||
all = append(all, r.data...)
|
||||
}
|
||||
return decodeProfilePicsBundle(all)
|
||||
}
|
||||
|
||||
func decodeProfilePicsBundle(data []byte) (ProfilePicsBundle, error) {
|
||||
if len(data) == 0 {
|
||||
return ProfilePicsBundle{}, nil
|
||||
}
|
||||
pb, err := protocol.DecodeProfilePicsBundle(data)
|
||||
if err != nil {
|
||||
return ProfilePicsBundle{}, fmt.Errorf("decode profile-pics: %w", err)
|
||||
}
|
||||
out := ProfilePicsBundle{
|
||||
BundleSize: pb.Header.BundleSize,
|
||||
BundleCRC: pb.Header.BundleCRC,
|
||||
Relays: append([]bool(nil), pb.Header.Relays...),
|
||||
}
|
||||
out.Entries = make([]ProfilePicEntry, len(pb.Entries))
|
||||
for i, e := range pb.Entries {
|
||||
out.Entries[i] = ProfilePicEntry{
|
||||
Username: e.Username,
|
||||
Offset: e.Offset,
|
||||
Size: e.Size,
|
||||
CRC: e.CRC,
|
||||
MIME: e.MIME,
|
||||
DNSChannel: e.DNSChannel,
|
||||
DNSBlocks: e.DNSBlocks,
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"hash/crc32"
|
||||
"testing"
|
||||
|
||||
"github.com/sartoopjj/thefeed/internal/protocol"
|
||||
)
|
||||
|
||||
func TestProfilePicMimeAndExtension(t *testing.T) {
|
||||
cases := []struct {
|
||||
mime uint8
|
||||
wantStr string
|
||||
wantExt string
|
||||
}{
|
||||
{protocol.ProfilePicMimeJPEG, "image/jpeg", ".jpg"},
|
||||
{protocol.ProfilePicMimePNG, "image/png", ".png"},
|
||||
{protocol.ProfilePicMimeWebP, "image/webp", ".webp"},
|
||||
{255, "image/jpeg", ".jpg"}, // unknown → JPEG fallback
|
||||
}
|
||||
for _, c := range cases {
|
||||
p := ProfilePicEntry{MIME: c.mime}
|
||||
if got := p.MimeString(); got != c.wantStr {
|
||||
t.Errorf("MimeString(%d) = %q, want %q", c.mime, got, c.wantStr)
|
||||
}
|
||||
if got := p.Extension(); got != c.wantExt {
|
||||
t.Errorf("Extension(%d) = %q, want %q", c.mime, got, c.wantExt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeProfilePicsBundleRoundTrip(t *testing.T) {
|
||||
// Build a real bundle so the Verify check the caller will run later
|
||||
// would still succeed.
|
||||
a := []byte("hello-alice")
|
||||
b := []byte("hello-bob-bob-bob")
|
||||
bundle := append(append([]byte{}, a...), b...)
|
||||
wire := protocol.EncodeProfilePicsBundle(protocol.ProfilePicsBundle{
|
||||
Header: protocol.ProfilePicsBundleHeader{
|
||||
BundleSize: uint32(len(bundle)),
|
||||
BundleCRC: crc32.ChecksumIEEE(bundle),
|
||||
Relays: []bool{false, true},
|
||||
},
|
||||
Entries: []protocol.ProfilePicEntry{
|
||||
{Username: "alice", Offset: 0, Size: uint32(len(a)), CRC: crc32.ChecksumIEEE(a), MIME: protocol.ProfilePicMimeJPEG, DNSChannel: 10001, DNSBlocks: 1},
|
||||
{Username: "bob", Offset: uint32(len(a)), Size: uint32(len(b)), CRC: crc32.ChecksumIEEE(b), MIME: protocol.ProfilePicMimePNG, DNSChannel: 10002, DNSBlocks: 2},
|
||||
},
|
||||
})
|
||||
got, err := decodeProfilePicsBundle(wire)
|
||||
if err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.BundleSize != uint32(len(bundle)) {
|
||||
t.Errorf("bundle metadata wrong: %+v", got)
|
||||
}
|
||||
if got.Entries[0].DNSChannel != 10001 || got.Entries[1].DNSChannel != 10002 {
|
||||
t.Errorf("dns channels lost: %+v", got.Entries)
|
||||
}
|
||||
if got.HasRelay(protocol.RelayDNS) || !got.HasRelay(protocol.RelayGitHub) {
|
||||
t.Errorf("relays = %v, want GitHub-only", got.Relays)
|
||||
}
|
||||
if len(got.Entries) != 2 {
|
||||
t.Fatalf("entries = %d, want 2", len(got.Entries))
|
||||
}
|
||||
if got.Entries[0].Username != "alice" || got.Entries[1].Username != "bob" {
|
||||
t.Errorf("entries: %+v", got.Entries)
|
||||
}
|
||||
if got.Entries[1].MIME != protocol.ProfilePicMimePNG {
|
||||
t.Errorf("bob MIME = %d, want PNG", got.Entries[1].MIME)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeProfilePicsBundleEmpty(t *testing.T) {
|
||||
got, err := decodeProfilePicsBundle(nil)
|
||||
if err != nil || len(got.Entries) != 0 {
|
||||
t.Errorf("decode(nil) = %v, %v", got, err)
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,13 @@ const (
|
||||
// owner/repo + domain segment). Block 0 carries it.
|
||||
RelayInfoChannel uint16 = 0xFFF8
|
||||
|
||||
// ProfilePicsChannel serves the per-channel profile-picture index:
|
||||
// for every Telegram channel that has a profile photo we emit
|
||||
// (username, mediaCh, size, crc32). Bytes themselves live on the
|
||||
// referenced mediaCh and are fetched via the regular media path.
|
||||
// Off by default on the client.
|
||||
ProfilePicsChannel uint16 = 0xFFF7
|
||||
|
||||
// MaxUpstreamBlockPayload keeps uploaded query chunks comfortably below DNS
|
||||
// name limits across typical domains and resolver paths.
|
||||
MaxUpstreamBlockPayload = 8
|
||||
|
||||
@@ -0,0 +1,218 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
)
|
||||
|
||||
// Profile pictures use a hybrid layout: every avatar is concatenated
|
||||
// into one bundle uploaded to the GitHub relay (one file → no
|
||||
// per-file rate limit), and each avatar also gets its own DNS media
|
||||
// channel so partial fetches over DNS still display.
|
||||
//
|
||||
// Wire layout of ProfilePicsChannel (after the block-count prefix the
|
||||
// Feed layer adds):
|
||||
//
|
||||
// bundleSize uint32
|
||||
// bundleCRC uint32
|
||||
// relayCount uint8 — N
|
||||
// relays [N]u8 — bool per relay (RelayDNS=0, RelayGitHub=1, …)
|
||||
// count uint16
|
||||
// entries:
|
||||
// usernameLen uint8
|
||||
// username [usernameLen]byte
|
||||
// offset uint32 — within the GitHub bundle
|
||||
// size uint32
|
||||
// crc uint32 — CRC32 of bundle[offset:offset+size]
|
||||
// mime uint8 — 0=jpeg, 1=png, 2=webp
|
||||
// dnsChannel uint16 — 0 if not on DNS
|
||||
// dnsBlocks uint16
|
||||
type ProfilePicsBundleHeader struct {
|
||||
BundleSize uint32
|
||||
BundleCRC uint32
|
||||
// One bool per relay constant. RelayGitHub here means the bundle
|
||||
// is on GitHub; RelayDNS for the bundle is rare (per-entry DNS
|
||||
// channels handle the DNS path).
|
||||
Relays []bool
|
||||
}
|
||||
|
||||
// HasRelay reports whether the relay at idx is set. Out of range returns false.
|
||||
func (h ProfilePicsBundleHeader) HasRelay(idx int) bool {
|
||||
if idx < 0 || idx >= len(h.Relays) {
|
||||
return false
|
||||
}
|
||||
return h.Relays[idx]
|
||||
}
|
||||
|
||||
// ProfilePicEntry points at one avatar via either the GitHub bundle
|
||||
// (Offset/Size into the concatenated blob) or its own DNS channel
|
||||
// (DNSChannel/DNSBlocks). Both paths verify the same Size + CRC.
|
||||
type ProfilePicEntry struct {
|
||||
Username string
|
||||
Offset uint32
|
||||
Size uint32
|
||||
CRC uint32
|
||||
MIME uint8
|
||||
DNSChannel uint16
|
||||
DNSBlocks uint16
|
||||
}
|
||||
|
||||
// MIME tag values.
|
||||
const (
|
||||
ProfilePicMimeJPEG uint8 = 0
|
||||
ProfilePicMimePNG uint8 = 1
|
||||
ProfilePicMimeWebP uint8 = 2
|
||||
)
|
||||
|
||||
// On-the-wire byte counts.
|
||||
const (
|
||||
profilePicEntryFixed = 4 + 4 + 4 + 1 + 2 + 2 // offset+size+crc+mime+dnsCh+dnsBlk
|
||||
profilePicsHeaderFixed = 4 + 4 + 1 // bundleSize+bundleCRC+relayCount
|
||||
)
|
||||
|
||||
// ProfilePicsBundle is the directory (header + entries). The bundle
|
||||
// bytes themselves live in the referenced media channel / relay.
|
||||
type ProfilePicsBundle struct {
|
||||
Header ProfilePicsBundleHeader
|
||||
Entries []ProfilePicEntry
|
||||
}
|
||||
|
||||
// EncodeProfilePicsBundle serialises the directory.
|
||||
func EncodeProfilePicsBundle(b ProfilePicsBundle) []byte {
|
||||
relayCount := len(b.Header.Relays)
|
||||
if relayCount > 255 {
|
||||
relayCount = 255
|
||||
}
|
||||
size := profilePicsHeaderFixed + relayCount + 2 /*entry count*/
|
||||
for _, e := range b.Entries {
|
||||
n := len(e.Username)
|
||||
if n > 255 {
|
||||
n = 255
|
||||
}
|
||||
size += 1 + n + profilePicEntryFixed
|
||||
}
|
||||
buf := make([]byte, size)
|
||||
off := 0
|
||||
binary.BigEndian.PutUint32(buf[off:], b.Header.BundleSize)
|
||||
off += 4
|
||||
binary.BigEndian.PutUint32(buf[off:], b.Header.BundleCRC)
|
||||
off += 4
|
||||
buf[off] = byte(relayCount)
|
||||
off++
|
||||
for i := 0; i < relayCount; i++ {
|
||||
if b.Header.Relays[i] {
|
||||
buf[off] = 1
|
||||
}
|
||||
off++
|
||||
}
|
||||
binary.BigEndian.PutUint16(buf[off:], uint16(len(b.Entries)))
|
||||
off += 2
|
||||
for _, e := range b.Entries {
|
||||
nb := []byte(e.Username)
|
||||
if len(nb) > 255 {
|
||||
nb = nb[:255]
|
||||
}
|
||||
buf[off] = byte(len(nb))
|
||||
off++
|
||||
copy(buf[off:], nb)
|
||||
off += len(nb)
|
||||
binary.BigEndian.PutUint32(buf[off:], e.Offset)
|
||||
off += 4
|
||||
binary.BigEndian.PutUint32(buf[off:], e.Size)
|
||||
off += 4
|
||||
binary.BigEndian.PutUint32(buf[off:], e.CRC)
|
||||
off += 4
|
||||
buf[off] = e.MIME
|
||||
off++
|
||||
binary.BigEndian.PutUint16(buf[off:], e.DNSChannel)
|
||||
off += 2
|
||||
binary.BigEndian.PutUint16(buf[off:], e.DNSBlocks)
|
||||
off += 2
|
||||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
// DecodeProfilePicsBundle parses bytes produced by EncodeProfilePicsBundle.
|
||||
func DecodeProfilePicsBundle(data []byte) (ProfilePicsBundle, error) {
|
||||
var out ProfilePicsBundle
|
||||
if len(data) < profilePicsHeaderFixed+2 {
|
||||
return out, fmt.Errorf("profile-pics bundle too short: %d bytes", len(data))
|
||||
}
|
||||
off := 0
|
||||
out.Header.BundleSize = binary.BigEndian.Uint32(data[off:])
|
||||
off += 4
|
||||
out.Header.BundleCRC = binary.BigEndian.Uint32(data[off:])
|
||||
off += 4
|
||||
relayCount := int(data[off])
|
||||
off++
|
||||
if off+relayCount+2 > len(data) {
|
||||
return out, fmt.Errorf("profile-pics bundle: truncated relay list")
|
||||
}
|
||||
if relayCount > 0 {
|
||||
out.Header.Relays = make([]bool, relayCount)
|
||||
for i := 0; i < relayCount; i++ {
|
||||
out.Header.Relays[i] = data[off] != 0
|
||||
off++
|
||||
}
|
||||
}
|
||||
count := int(binary.BigEndian.Uint16(data[off:]))
|
||||
off += 2
|
||||
out.Entries = make([]ProfilePicEntry, 0, count)
|
||||
for i := 0; i < count; i++ {
|
||||
if off >= len(data) {
|
||||
return out, fmt.Errorf("profile-pics: truncated at entry %d", i)
|
||||
}
|
||||
nameLen := int(data[off])
|
||||
off++
|
||||
if off+nameLen+profilePicEntryFixed > len(data) {
|
||||
return out, fmt.Errorf("profile-pics: truncated entry %d body", i)
|
||||
}
|
||||
name := string(data[off : off+nameLen])
|
||||
off += nameLen
|
||||
offset := binary.BigEndian.Uint32(data[off:])
|
||||
off += 4
|
||||
sz := binary.BigEndian.Uint32(data[off:])
|
||||
off += 4
|
||||
cr := binary.BigEndian.Uint32(data[off:])
|
||||
off += 4
|
||||
mime := data[off]
|
||||
off++
|
||||
dnsCh := binary.BigEndian.Uint16(data[off:])
|
||||
off += 2
|
||||
dnsBlk := binary.BigEndian.Uint16(data[off:])
|
||||
off += 2
|
||||
out.Entries = append(out.Entries, ProfilePicEntry{
|
||||
Username: name,
|
||||
Offset: offset,
|
||||
Size: sz,
|
||||
CRC: cr,
|
||||
MIME: mime,
|
||||
DNSChannel: dnsCh,
|
||||
DNSBlocks: dnsBlk,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// VerifyEntry returns bundle[entry.Offset:entry.Offset+entry.Size] if
|
||||
// the slice is in-range and its CRC32-IEEE matches entry.CRC. The
|
||||
// hash check is what stops a misaligned bundle from serving the wrong
|
||||
// avatar under a username.
|
||||
func VerifyEntry(bundle []byte, entry ProfilePicEntry) ([]byte, error) {
|
||||
end := uint64(entry.Offset) + uint64(entry.Size)
|
||||
if end > uint64(len(bundle)) {
|
||||
return nil, fmt.Errorf("entry %q out of range: offset=%d size=%d bundle=%d",
|
||||
entry.Username, entry.Offset, entry.Size, len(bundle))
|
||||
}
|
||||
slice := bundle[entry.Offset:end]
|
||||
if uint32(len(slice)) != entry.Size {
|
||||
return nil, fmt.Errorf("entry %q size mismatch: have %d want %d",
|
||||
entry.Username, len(slice), entry.Size)
|
||||
}
|
||||
if got := crc32.ChecksumIEEE(slice); got != entry.CRC {
|
||||
return nil, fmt.Errorf("entry %q crc mismatch: have %08x want %08x",
|
||||
entry.Username, got, entry.CRC)
|
||||
}
|
||||
return slice, nil
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
package protocol
|
||||
|
||||
import (
|
||||
"hash/crc32"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestEncodeDecodeProfilePicsBundleRoundTrip(t *testing.T) {
|
||||
// Three avatars concatenated into a fake bundle, with offsets/CRCs
|
||||
// computed for real so VerifyEntry doesn't trip on them.
|
||||
a := []byte("aaaaaaaaaa") // 10 bytes
|
||||
b := []byte("bbbbbbbbbbbbbb") // 14 bytes
|
||||
c := []byte("ccccc") // 5 bytes
|
||||
bundle := append(append(append([]byte{}, a...), b...), c...)
|
||||
|
||||
in := ProfilePicsBundle{
|
||||
Header: ProfilePicsBundleHeader{
|
||||
BundleSize: uint32(len(bundle)),
|
||||
BundleCRC: crc32.ChecksumIEEE(bundle),
|
||||
Relays: []bool{false, true}, // bundle on GitHub only; per-entry DNS handles DNS path
|
||||
},
|
||||
Entries: []ProfilePicEntry{
|
||||
{Username: "alice", Offset: 0, Size: uint32(len(a)), CRC: crc32.ChecksumIEEE(a), MIME: ProfilePicMimeJPEG, DNSChannel: 10001, DNSBlocks: 1},
|
||||
{Username: "bob", Offset: uint32(len(a)), Size: uint32(len(b)), CRC: crc32.ChecksumIEEE(b), MIME: ProfilePicMimePNG, DNSChannel: 10002, DNSBlocks: 2},
|
||||
{Username: "carol", Offset: uint32(len(a) + len(b)), Size: uint32(len(c)), CRC: crc32.ChecksumIEEE(c), MIME: ProfilePicMimeWebP, DNSChannel: 10003, DNSBlocks: 1},
|
||||
},
|
||||
}
|
||||
wire := EncodeProfilePicsBundle(in)
|
||||
got, err := DecodeProfilePicsBundle(wire)
|
||||
if err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if got.Header.BundleSize != in.Header.BundleSize ||
|
||||
got.Header.BundleCRC != in.Header.BundleCRC {
|
||||
t.Errorf("header mismatch: got %+v want %+v", got.Header, in.Header)
|
||||
}
|
||||
if len(got.Header.Relays) != len(in.Header.Relays) {
|
||||
t.Fatalf("relays len = %d, want %d", len(got.Header.Relays), len(in.Header.Relays))
|
||||
}
|
||||
for i, r := range in.Header.Relays {
|
||||
if got.Header.Relays[i] != r {
|
||||
t.Errorf("relays[%d] = %v, want %v", i, got.Header.Relays[i], r)
|
||||
}
|
||||
}
|
||||
if len(got.Entries) != len(in.Entries) {
|
||||
t.Fatalf("entries len = %d, want %d", len(got.Entries), len(in.Entries))
|
||||
}
|
||||
for i, want := range in.Entries {
|
||||
if got.Entries[i] != want {
|
||||
t.Errorf("entry %d = %+v want %+v", i, got.Entries[i], want)
|
||||
}
|
||||
}
|
||||
|
||||
// VerifyEntry should accept the real bytes…
|
||||
for _, e := range got.Entries {
|
||||
if _, err := VerifyEntry(bundle, e); err != nil {
|
||||
t.Errorf("VerifyEntry(%s) = %v, want ok", e.Username, err)
|
||||
}
|
||||
}
|
||||
// …and reject a tampered bundle.
|
||||
tampered := append([]byte{}, bundle...)
|
||||
tampered[0] ^= 0xFF
|
||||
if _, err := VerifyEntry(tampered, got.Entries[0]); err == nil {
|
||||
t.Errorf("VerifyEntry should fail on tampered bundle")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyEntryOutOfRange(t *testing.T) {
|
||||
bundle := []byte("hello")
|
||||
e := ProfilePicEntry{Username: "x", Offset: 0, Size: 100, CRC: 0}
|
||||
if _, err := VerifyEntry(bundle, e); err == nil {
|
||||
t.Errorf("VerifyEntry should fail when entry runs past bundle end")
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyEntrySizeMismatch(t *testing.T) {
|
||||
bundle := []byte("0123456789")
|
||||
// CRC is right for the real slice, but Size is wrong → mismatch.
|
||||
right := bundle[2:6]
|
||||
e := ProfilePicEntry{
|
||||
Username: "y",
|
||||
Offset: 2,
|
||||
Size: 5, // claim 5 but slice is 4
|
||||
CRC: crc32.ChecksumIEEE(right),
|
||||
}
|
||||
// In this case Size=5 + Offset=2 → end=7, in range. CRC will be checked
|
||||
// over bundle[2:7] which differs from right → mismatch.
|
||||
if _, err := VerifyEntry(bundle, e); err == nil {
|
||||
t.Errorf("VerifyEntry should fail when claimed size doesn't match recorded crc")
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfilePicsBundleEmpty(t *testing.T) {
|
||||
in := ProfilePicsBundle{
|
||||
Header: ProfilePicsBundleHeader{Relays: []bool{false, false}},
|
||||
}
|
||||
wire := EncodeProfilePicsBundle(in)
|
||||
got, err := DecodeProfilePicsBundle(wire)
|
||||
if err != nil {
|
||||
t.Fatalf("decode empty: %v", err)
|
||||
}
|
||||
if len(got.Entries) != 0 {
|
||||
t.Errorf("entries = %d want 0", len(got.Entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfilePicsTruncatesLongUsername(t *testing.T) {
|
||||
long := strings.Repeat("x", 300)
|
||||
in := ProfilePicsBundle{
|
||||
Header: ProfilePicsBundleHeader{Relays: []bool{true}},
|
||||
Entries: []ProfilePicEntry{
|
||||
{Username: long, Offset: 0, Size: 100, CRC: 1, MIME: 0},
|
||||
},
|
||||
}
|
||||
wire := EncodeProfilePicsBundle(in)
|
||||
got, err := DecodeProfilePicsBundle(wire)
|
||||
if err != nil {
|
||||
t.Fatalf("decode: %v", err)
|
||||
}
|
||||
if len(got.Entries) != 1 || len(got.Entries[0].Username) != 255 {
|
||||
t.Errorf("expected 1 entry with 255-char username, got %+v", got.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfilePicsTruncatedDataReturnsError(t *testing.T) {
|
||||
in := ProfilePicsBundle{
|
||||
Header: ProfilePicsBundleHeader{Relays: []bool{true}},
|
||||
Entries: []ProfilePicEntry{
|
||||
{Username: "a", Offset: 0, Size: 1, CRC: 1, MIME: 0},
|
||||
},
|
||||
}
|
||||
wire := EncodeProfilePicsBundle(in)
|
||||
for _, cut := range []int{0, 1, 2, 3, len(wire) - 1} {
|
||||
_, err := DecodeProfilePicsBundle(wire[:cut])
|
||||
if err == nil {
|
||||
t.Errorf("expected error on cut=%d", cut)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestProfilePicsChannelConstant(t *testing.T) {
|
||||
if ProfilePicsChannel != 0xFFF7 {
|
||||
t.Errorf("ProfilePicsChannel = 0x%X, want 0xFFF7", ProfilePicsChannel)
|
||||
}
|
||||
others := []uint16{
|
||||
SendChannel, AdminChannel, UpstreamInitChannel, UpstreamDataChannel,
|
||||
VersionChannel, TitlesChannel, RelayInfoChannel,
|
||||
}
|
||||
for _, o := range others {
|
||||
if o == ProfilePicsChannel {
|
||||
t.Fatalf("ProfilePicsChannel collides with another control channel: 0x%X", o)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBundleHasRelay(t *testing.T) {
|
||||
h := ProfilePicsBundleHeader{Relays: []bool{false, true}}
|
||||
if h.HasRelay(RelayDNS) {
|
||||
t.Errorf("RelayDNS should be false")
|
||||
}
|
||||
if !h.HasRelay(RelayGitHub) {
|
||||
t.Errorf("RelayGitHub should be true")
|
||||
}
|
||||
if h.HasRelay(99) || h.HasRelay(-1) {
|
||||
t.Errorf("out-of-range should return false")
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,9 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"log"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -44,6 +46,16 @@ type Feed struct {
|
||||
// (RelayInfoChannel) — block 0 contains the GitHub "owner/repo"
|
||||
// string, or an empty payload if the relay is off.
|
||||
relayInfoBlocks [][]byte
|
||||
|
||||
// ProfilePicsChannel serves the directory; the bundle bytes live
|
||||
// in one media-cache entry, with each entry also reachable on its
|
||||
// own DNS channel.
|
||||
profilePicsBlocks [][]byte
|
||||
profilePicsBundle protocol.ProfilePicsBundle
|
||||
profilePicsBundleBytes []byte // last-built bundle, for MergeProfilePics
|
||||
// Serialises MergeProfilePics so concurrent readers can't lose
|
||||
// each other's writes through the read-modify-write sequence.
|
||||
profilePicsMergeMu sync.Mutex
|
||||
}
|
||||
|
||||
// NewFeed creates a new Feed with the given channel names.
|
||||
@@ -108,6 +120,9 @@ func (f *Feed) GetBlock(channel, block int) ([]byte, error) {
|
||||
if channel == int(protocol.RelayInfoChannel) {
|
||||
return f.getRelayInfoBlock(block)
|
||||
}
|
||||
if channel == int(protocol.ProfilePicsChannel) {
|
||||
return f.getProfilePicsBlock(block)
|
||||
}
|
||||
// Channel sits in the binary media range — delegate to MediaCache. We
|
||||
// drop the read lock first because MediaCache uses its own lock and we
|
||||
// don't want to hold f.mu across that path.
|
||||
@@ -282,6 +297,223 @@ func (f *Feed) getRelayInfoBlock(block int) ([]byte, error) {
|
||||
return blocks[block], nil
|
||||
}
|
||||
|
||||
func (f *Feed) getProfilePicsBlock(block int) ([]byte, error) {
|
||||
blocks := f.profilePicsBlocks
|
||||
if len(blocks) == 0 {
|
||||
// Empty payload still has to be a single non-nil block so the
|
||||
// usual block-count prefix path stays consistent.
|
||||
f.rebuildProfilePicsBlocksLocked()
|
||||
blocks = f.profilePicsBlocks
|
||||
}
|
||||
if block < 0 || block >= len(blocks) {
|
||||
return nil, fmt.Errorf("profile-pics block %d out of range (%d blocks)", block, len(blocks))
|
||||
}
|
||||
return blocks[block], nil
|
||||
}
|
||||
|
||||
// rebuildProfilePicsBlocksLocked encodes the bundle and splits into
|
||||
// blocks; block 0 is prefixed with the uint16 block count (same
|
||||
// convention as titles). Caller holds f.mu.
|
||||
func (f *Feed) rebuildProfilePicsBlocksLocked() {
|
||||
payload := protocol.EncodeProfilePicsBundle(f.profilePicsBundle)
|
||||
blocks := protocol.SplitIntoBlocks(payload)
|
||||
if len(blocks) == 0 {
|
||||
blocks = [][]byte{nil}
|
||||
}
|
||||
prefix := []byte{byte(len(blocks) >> 8), byte(len(blocks))}
|
||||
blocks[0] = append(prefix, blocks[0]...)
|
||||
f.profilePicsBlocks = blocks
|
||||
}
|
||||
|
||||
// SetProfilePics replaces the profile-pic bundle with the given
|
||||
// username → image-bytes map. Other usernames currently in the bundle
|
||||
// are dropped; use MergeProfilePics for additive behaviour. Empty
|
||||
// values are skipped. Requires SetMediaCache. Returns the number of
|
||||
// avatars in the resulting bundle.
|
||||
func (f *Feed) SetProfilePics(pics map[string][]byte) int {
|
||||
return f.replaceProfilePicsBundle(pics)
|
||||
}
|
||||
|
||||
// MergeProfilePics is SetProfilePics that retains the existing bundle's
|
||||
// entries (re-extracted and re-verified) and overlays pics on top.
|
||||
// Used by readers that only know a subset of channels (Telegram-only,
|
||||
// X-only) so each one contributes without wiping the others.
|
||||
//
|
||||
// Serialised so two readers merging from the same prior state can't
|
||||
// lose each other's writes.
|
||||
func (f *Feed) MergeProfilePics(pics map[string][]byte) int {
|
||||
f.profilePicsMergeMu.Lock()
|
||||
defer f.profilePicsMergeMu.Unlock()
|
||||
|
||||
merged := make(map[string][]byte, len(pics))
|
||||
|
||||
f.mu.RLock()
|
||||
prev := f.profilePicsBundle
|
||||
prevBytes := f.profilePicsBundleBytes
|
||||
f.mu.RUnlock()
|
||||
if len(prev.Entries) > 0 && len(prevBytes) > 0 {
|
||||
for _, e := range prev.Entries {
|
||||
slice, err := protocol.VerifyEntry(prevBytes, e)
|
||||
if err != nil {
|
||||
log.Printf("[profile-pics] merge: skipping %s (%v)", e.Username, err)
|
||||
continue
|
||||
}
|
||||
cp := make([]byte, len(slice))
|
||||
copy(cp, slice)
|
||||
merged[e.Username] = cp
|
||||
}
|
||||
}
|
||||
for k, v := range pics {
|
||||
if k == "" {
|
||||
continue
|
||||
}
|
||||
if len(v) == 0 {
|
||||
delete(merged, k)
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return f.replaceProfilePicsBundle(merged)
|
||||
}
|
||||
|
||||
// replaceProfilePicsBundle is the shared encode-and-store path. Each
|
||||
// individual avatar gets its own DNS channel via SkipGitHub=true (so
|
||||
// the DNS path can fetch one at a time without triggering N GitHub
|
||||
// uploads), then the concatenated bundle is stored once with default
|
||||
// opts to trigger the single GitHub upload that covers everything.
|
||||
func (f *Feed) replaceProfilePicsBundle(pics map[string][]byte) int {
|
||||
f.mu.Lock()
|
||||
media := f.media
|
||||
f.mu.Unlock()
|
||||
if media == nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
type kv struct {
|
||||
name string
|
||||
b []byte
|
||||
}
|
||||
ordered := make([]kv, 0, len(pics))
|
||||
for name, b := range pics {
|
||||
if name == "" || len(b) == 0 {
|
||||
continue
|
||||
}
|
||||
ordered = append(ordered, kv{name, b})
|
||||
}
|
||||
sort.Slice(ordered, func(i, j int) bool { return ordered[i].name < ordered[j].name })
|
||||
|
||||
if len(ordered) == 0 {
|
||||
f.mu.Lock()
|
||||
f.profilePicsBundle = protocol.ProfilePicsBundle{}
|
||||
f.profilePicsBundleBytes = nil
|
||||
f.rebuildProfilePicsBlocksLocked()
|
||||
f.mu.Unlock()
|
||||
return 0
|
||||
}
|
||||
|
||||
// Build the bundle bytes + per-entry directory. Each entry gets its
|
||||
// own DNS channel via a SkipGitHub store so the DNS-path client can
|
||||
// fetch one avatar at a time without dragging the whole bundle.
|
||||
bundle := make([]byte, 0, 8192)
|
||||
entries := make([]protocol.ProfilePicEntry, 0, len(ordered))
|
||||
for _, e := range ordered {
|
||||
_, mimeTag := sniffProfilePicMime(e.b)
|
||||
offset := uint32(len(bundle))
|
||||
bundle = append(bundle, e.b...)
|
||||
|
||||
entry := protocol.ProfilePicEntry{
|
||||
Username: e.name,
|
||||
Offset: offset,
|
||||
Size: uint32(len(e.b)),
|
||||
CRC: crc32.ChecksumIEEE(e.b),
|
||||
MIME: mimeTag,
|
||||
}
|
||||
// Per-pic DNS channel, no GitHub upload (the bundle covers GitHub).
|
||||
key := "profile-pic:" + e.name
|
||||
fname := e.name
|
||||
switch mimeTag {
|
||||
case protocol.ProfilePicMimePNG:
|
||||
fname += ".png"
|
||||
case protocol.ProfilePicMimeWebP:
|
||||
fname += ".webp"
|
||||
default:
|
||||
fname += ".jpg"
|
||||
}
|
||||
picMeta, err := media.StoreWithOptions(key, "[PROFILE]", e.b, mimeStringForTag(mimeTag), fname,
|
||||
MediaCacheStoreOptions{SkipGitHub: true})
|
||||
if err != nil {
|
||||
// No DNS channel for this entry; bundle path still works.
|
||||
log.Printf("[profile-pics] store individual %s: %v", e.name, err)
|
||||
} else {
|
||||
entry.DNSChannel = picMeta.Channel
|
||||
entry.DNSBlocks = picMeta.Blocks
|
||||
}
|
||||
entries = append(entries, entry)
|
||||
}
|
||||
|
||||
// One bundle store → one GitHub upload covering every avatar.
|
||||
bundleMeta, err := media.Store("profile-pics-bundle", "[PROFILE-BUNDLE]",
|
||||
bundle, "application/octet-stream", "profile-pics.bin")
|
||||
if err != nil {
|
||||
log.Printf("[profile-pics] store bundle: %v", err)
|
||||
return 0
|
||||
}
|
||||
|
||||
header := protocol.ProfilePicsBundleHeader{
|
||||
BundleSize: uint32(bundleMeta.Size),
|
||||
BundleCRC: bundleMeta.CRC32,
|
||||
Relays: append([]bool(nil), bundleMeta.Relays...),
|
||||
}
|
||||
|
||||
f.mu.Lock()
|
||||
f.profilePicsBundle = protocol.ProfilePicsBundle{
|
||||
Header: header,
|
||||
Entries: entries,
|
||||
}
|
||||
f.profilePicsBundleBytes = bundle
|
||||
f.rebuildProfilePicsBlocksLocked()
|
||||
f.mu.Unlock()
|
||||
return len(entries)
|
||||
}
|
||||
|
||||
func mimeStringForTag(tag uint8) string {
|
||||
switch tag {
|
||||
case protocol.ProfilePicMimePNG:
|
||||
return "image/png"
|
||||
case protocol.ProfilePicMimeWebP:
|
||||
return "image/webp"
|
||||
default:
|
||||
return "image/jpeg"
|
||||
}
|
||||
}
|
||||
|
||||
// ProfilePicsBundle returns a copy of the current directory.
|
||||
func (f *Feed) ProfilePicsBundle() protocol.ProfilePicsBundle {
|
||||
f.mu.RLock()
|
||||
defer f.mu.RUnlock()
|
||||
out := protocol.ProfilePicsBundle{
|
||||
Header: f.profilePicsBundle.Header,
|
||||
Entries: make([]protocol.ProfilePicEntry, len(f.profilePicsBundle.Entries)),
|
||||
}
|
||||
if len(f.profilePicsBundle.Header.Relays) > 0 {
|
||||
out.Header.Relays = append([]bool(nil), f.profilePicsBundle.Header.Relays...)
|
||||
}
|
||||
copy(out.Entries, f.profilePicsBundle.Entries)
|
||||
return out
|
||||
}
|
||||
|
||||
// sniffProfilePicMime returns (rfc-mime, ProfilePicMime tag) by looking
|
||||
// at the first few bytes. Falls back to JPEG for anything unrecognised.
|
||||
func sniffProfilePicMime(b []byte) (string, uint8) {
|
||||
if len(b) >= 4 && b[0] == 0x89 && b[1] == 'P' && b[2] == 'N' && b[3] == 'G' {
|
||||
return "image/png", protocol.ProfilePicMimePNG
|
||||
}
|
||||
if len(b) >= 12 && string(b[0:4]) == "RIFF" && string(b[8:12]) == "WEBP" {
|
||||
return "image/webp", protocol.ProfilePicMimeWebP
|
||||
}
|
||||
return "image/jpeg", protocol.ProfilePicMimeJPEG
|
||||
}
|
||||
|
||||
// rebuildTitlesBlocks re-serializes the display name map and splits it into blocks.
|
||||
// Block 0 is prefixed with a uint16 total-block count so the client can fetch all
|
||||
// remaining blocks in parallel after reading the first one.
|
||||
|
||||
@@ -119,6 +119,19 @@ func NewMediaCache(cfg MediaCacheConfig) *MediaCache {
|
||||
// client. content is the raw file bytes; the caller may pass a slice it
|
||||
// continues to use after the call (Store copies into block-sized chunks).
|
||||
func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filename string) (protocol.MediaMeta, error) {
|
||||
return c.StoreWithOptions(cacheKey, tag, content, mimeType, filename, MediaCacheStoreOptions{})
|
||||
}
|
||||
|
||||
// MediaCacheStoreOptions toggles relay paths for a single Store call.
|
||||
// Zero value = both DNS channel and (if a relay is configured) GitHub
|
||||
// upload. SkipGitHub keeps the DNS allocation but skips the upload —
|
||||
// used when many small siblings share one bundled GitHub upload.
|
||||
type MediaCacheStoreOptions struct {
|
||||
SkipGitHub bool
|
||||
}
|
||||
|
||||
// StoreWithOptions is Store with selective relay control.
|
||||
func (c *MediaCache) StoreWithOptions(cacheKey, tag string, content []byte, mimeType, filename string, opts MediaCacheStoreOptions) (protocol.MediaMeta, error) {
|
||||
if cacheKey == "" {
|
||||
return protocol.MediaMeta{}, errors.New("media: empty cache key")
|
||||
}
|
||||
@@ -126,9 +139,6 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen
|
||||
tag = protocol.MediaFile
|
||||
}
|
||||
size := int64(len(content))
|
||||
// Reject only when no enabled relay could host this file. A file too big
|
||||
// for DNS but small enough for GitHub still belongs in the cache —
|
||||
// MaxAcceptableBytes() collapses both caps into a single ceiling.
|
||||
if max := c.MaxAcceptableBytes(); max > 0 && size > max {
|
||||
atomic.AddUint64(&c.storeRejected, 1)
|
||||
return protocol.MediaMeta{
|
||||
@@ -246,9 +256,9 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen
|
||||
atomic.AddInt64(&c.currentBytes, size)
|
||||
c.logf("media: store tag=%s key=%s ch=%d size=%d blocks=%d", tag, cacheKey, channel, size, len(blocks))
|
||||
|
||||
// Best-effort relay upload — copy of `content` because the caller may
|
||||
// reuse the slice. Failures are logged but never block the DNS path.
|
||||
if c.gh != nil {
|
||||
// Best-effort relay upload. Copy because the caller may reuse the
|
||||
// slice. Failures don't block the DNS path.
|
||||
if c.gh != nil && !opts.SkipGitHub {
|
||||
gh := c.gh
|
||||
body := append([]byte(nil), content...)
|
||||
go func() {
|
||||
@@ -260,7 +270,11 @@ func (c *MediaCache) Store(cacheKey, tag string, content []byte, mimeType, filen
|
||||
}()
|
||||
}
|
||||
|
||||
return c.metaForLocked(entry), nil
|
||||
meta := c.metaForLocked(entry)
|
||||
if opts.SkipGitHub && len(meta.Relays) > protocol.RelayGitHub {
|
||||
meta.Relays[protocol.RelayGitHub] = false
|
||||
}
|
||||
return meta, nil
|
||||
}
|
||||
|
||||
// LookupByChannel returns the cached entry's transport metadata (mime,
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/gotd/td/tg"
|
||||
)
|
||||
|
||||
// extractChatPhotoID returns (photoID, dcID), or (0, 0) when there is none.
|
||||
func extractChatPhotoID(p tg.ChatPhotoClass) (int64, int) {
|
||||
switch ph := p.(type) {
|
||||
case *tg.ChatPhoto:
|
||||
return ph.PhotoID, ph.DCID
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
// extractUserPhotoID is the User-profile equivalent.
|
||||
func extractUserPhotoID(p tg.UserProfilePhotoClass) (int64, int) {
|
||||
switch ph := p.(type) {
|
||||
case *tg.UserProfilePhoto:
|
||||
return ph.PhotoID, ph.DCID
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
// fetchProfilePhoto downloads the small (~5KB / 160px) thumb. Big:false
|
||||
// is Telegram's "a" thumb size.
|
||||
func (tr *TelegramReader) fetchProfilePhoto(ctx context.Context, api *tg.Client, peer tg.InputPeerClass, photoID int64) ([]byte, error) {
|
||||
if photoID == 0 {
|
||||
return nil, fmt.Errorf("no photo")
|
||||
}
|
||||
loc := &tg.InputPeerPhotoFileLocation{
|
||||
Peer: peer,
|
||||
PhotoID: photoID,
|
||||
}
|
||||
loc.Big = false
|
||||
return tr.downloadTelegramFile(ctx, api, loc, 0)
|
||||
}
|
||||
|
||||
// fetchAllProfilePhotos downloads each channel's avatar (skipping
|
||||
// unchanged photoIDs) and merges them into the feed bundle.
|
||||
// Best-effort: per-channel failures are logged and skipped.
|
||||
func (tr *TelegramReader) fetchAllProfilePhotos(ctx context.Context, api *tg.Client) {
|
||||
pics := make(map[string][]byte, len(tr.channels))
|
||||
for _, username := range tr.channels {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
username = strings.TrimSpace(username)
|
||||
if username == "" {
|
||||
continue
|
||||
}
|
||||
rp, err := tr.resolvePeer(ctx, api, username)
|
||||
if err != nil {
|
||||
log.Printf("[profile-pics] resolve %s: %v", username, err)
|
||||
continue
|
||||
}
|
||||
if rp.photoID == 0 {
|
||||
continue
|
||||
}
|
||||
// Skip the download if we already pushed this photoID.
|
||||
tr.mu.Lock()
|
||||
if tr.lastPhotoID == nil {
|
||||
tr.lastPhotoID = map[string]int64{}
|
||||
}
|
||||
prevID, hadPrev := tr.lastPhotoID[username]
|
||||
tr.mu.Unlock()
|
||||
if hadPrev && prevID == rp.photoID {
|
||||
continue
|
||||
}
|
||||
body, err := tr.fetchProfilePhoto(ctx, api, rp.peer, rp.photoID)
|
||||
if err != nil {
|
||||
log.Printf("[profile-pics] download %s (id=%d): %v", username, rp.photoID, err)
|
||||
continue
|
||||
}
|
||||
pics[username] = body
|
||||
tr.mu.Lock()
|
||||
tr.lastPhotoID[username] = rp.photoID
|
||||
tr.mu.Unlock()
|
||||
}
|
||||
if len(pics) == 0 {
|
||||
return
|
||||
}
|
||||
// Merge so other readers' contributions (e.g. X "x:" entries) survive.
|
||||
total := tr.feed.MergeProfilePics(pics)
|
||||
log.Printf("[profile-pics] cycle done: %d new, %d total in bundle", len(pics), total)
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"hash/crc32"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/sartoopjj/thefeed/internal/protocol"
|
||||
)
|
||||
|
||||
// fakeJPEG returns n bytes that start with the JPEG SOI marker so
|
||||
// sniffProfilePicMime tags them as JPEG.
|
||||
func fakeJPEG(n int) []byte {
|
||||
out := make([]byte, n)
|
||||
rand.Read(out)
|
||||
if n >= 3 {
|
||||
out[0] = 0xFF
|
||||
out[1] = 0xD8
|
||||
out[2] = 0xFF
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func fakePNG(n int) []byte {
|
||||
out := make([]byte, n)
|
||||
rand.Read(out)
|
||||
if n >= 4 {
|
||||
out[0] = 0x89
|
||||
out[1] = 'P'
|
||||
out[2] = 'N'
|
||||
out[3] = 'G'
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func newFeedWithMedia(t *testing.T) *Feed {
|
||||
t.Helper()
|
||||
f := NewFeed([]string{"a", "b"})
|
||||
mc := NewMediaCache(MediaCacheConfig{
|
||||
MaxFileBytes: 64 * 1024,
|
||||
TTL: time.Hour,
|
||||
DNSRelayEnabled: true,
|
||||
})
|
||||
f.SetMediaCache(mc)
|
||||
return f
|
||||
}
|
||||
|
||||
func TestSetProfilePicsBundlesAvatarsAndExposesDirectory(t *testing.T) {
|
||||
f := newFeedWithMedia(t)
|
||||
alice := fakeJPEG(1024)
|
||||
bob := fakePNG(2048)
|
||||
|
||||
stored := f.SetProfilePics(map[string][]byte{"alice": alice, "bob": bob})
|
||||
if stored != 2 {
|
||||
t.Fatalf("stored = %d, want 2", stored)
|
||||
}
|
||||
|
||||
got := f.ProfilePicsBundle()
|
||||
if len(got.Entries) != 2 {
|
||||
t.Fatalf("entries = %d, want 2", len(got.Entries))
|
||||
}
|
||||
// Sorted by username.
|
||||
if got.Entries[0].Username != "alice" || got.Entries[1].Username != "bob" {
|
||||
t.Errorf("entries = %s,%s want alice,bob",
|
||||
got.Entries[0].Username, got.Entries[1].Username)
|
||||
}
|
||||
// MIME survived the sniff.
|
||||
if got.Entries[0].MIME != protocol.ProfilePicMimeJPEG {
|
||||
t.Errorf("alice MIME = %d want JPEG", got.Entries[0].MIME)
|
||||
}
|
||||
if got.Entries[1].MIME != protocol.ProfilePicMimePNG {
|
||||
t.Errorf("bob MIME = %d want PNG", got.Entries[1].MIME)
|
||||
}
|
||||
// Each per-entry CRC matches the original bytes.
|
||||
if got.Entries[0].CRC != crc32.ChecksumIEEE(alice) {
|
||||
t.Errorf("alice crc mismatch")
|
||||
}
|
||||
if got.Entries[1].CRC != crc32.ChecksumIEEE(bob) {
|
||||
t.Errorf("bob crc mismatch")
|
||||
}
|
||||
// Offsets are tightly packed.
|
||||
if got.Entries[0].Offset != 0 {
|
||||
t.Errorf("alice offset = %d, want 0", got.Entries[0].Offset)
|
||||
}
|
||||
if got.Entries[1].Offset != got.Entries[0].Size {
|
||||
t.Errorf("bob offset = %d, want %d (right after alice)",
|
||||
got.Entries[1].Offset, got.Entries[0].Size)
|
||||
}
|
||||
|
||||
// Bundle metadata: total bytes = sum of avatars.
|
||||
if got.Header.BundleSize != got.Entries[0].Size+got.Entries[1].Size {
|
||||
t.Errorf("BundleSize = %d, want sum of entries (%d)",
|
||||
got.Header.BundleSize, got.Entries[0].Size+got.Entries[1].Size)
|
||||
}
|
||||
// Each entry has its own DNS channel inside the media range.
|
||||
for i, e := range got.Entries {
|
||||
if e.DNSChannel < protocol.MediaChannelStart || e.DNSChannel > protocol.MediaChannelEnd {
|
||||
t.Errorf("entries[%d].DNSChannel = %d, outside media range",
|
||||
i, e.DNSChannel)
|
||||
}
|
||||
if e.DNSBlocks == 0 {
|
||||
t.Errorf("entries[%d].DNSBlocks = 0", i)
|
||||
}
|
||||
}
|
||||
// And those per-entry DNS channels are fetchable as ordinary media.
|
||||
for i, e := range got.Entries {
|
||||
blk0, err := f.GetBlock(int(e.DNSChannel), 0)
|
||||
if err != nil || len(blk0) == 0 {
|
||||
t.Errorf("entries[%d] DNS block 0: %v / %d bytes", i, err, len(blk0))
|
||||
}
|
||||
}
|
||||
|
||||
// And we can pull the directory back over the wire and re-derive
|
||||
// each avatar's slice with VerifyEntry.
|
||||
dirBlk, err := f.GetBlock(int(protocol.ProfilePicsChannel), 0)
|
||||
if err != nil {
|
||||
t.Fatalf("get profile-pics block 0: %v", err)
|
||||
}
|
||||
if len(dirBlk) < 4 {
|
||||
t.Fatalf("dir block too small: %d", len(dirBlk))
|
||||
}
|
||||
totalBlocks := int(dirBlk[0])<<8 | int(dirBlk[1])
|
||||
all := dirBlk[2:]
|
||||
for i := 1; i < totalBlocks; i++ {
|
||||
next, err := f.GetBlock(int(protocol.ProfilePicsChannel), i)
|
||||
if err != nil {
|
||||
t.Fatalf("get profile-pics block %d: %v", i, err)
|
||||
}
|
||||
all = append(all, next...)
|
||||
}
|
||||
decoded, err := protocol.DecodeProfilePicsBundle(all)
|
||||
if err != nil {
|
||||
t.Fatalf("decode dir: %v", err)
|
||||
}
|
||||
if len(decoded.Entries) != 2 {
|
||||
t.Fatalf("decoded entries = %d, want 2", len(decoded.Entries))
|
||||
}
|
||||
|
||||
// Each entry's DNS channel is independently fetchable. This is the
|
||||
// "if even one DNS channel works, that one avatar still shows" path.
|
||||
// Full byte-level round-trip is exercised in the client tests.
|
||||
for _, e := range decoded.Entries {
|
||||
blk0, err := f.GetBlock(int(e.DNSChannel), 0)
|
||||
if err != nil || len(blk0) == 0 {
|
||||
t.Errorf("entry %s: channel %d block 0: %v",
|
||||
e.Username, e.DNSChannel, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProfilePicsExposesRelayBits(t *testing.T) {
|
||||
f := newFeedWithMedia(t) // DNSRelayEnabled: true, no GitHub relay attached
|
||||
f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(1024)})
|
||||
got := f.ProfilePicsBundle()
|
||||
if !got.Header.HasRelay(protocol.RelayDNS) {
|
||||
t.Errorf("RelayDNS bit should be set when DNS is enabled, got Relays=%v", got.Header.Relays)
|
||||
}
|
||||
if got.Header.HasRelay(protocol.RelayGitHub) {
|
||||
t.Errorf("RelayGitHub bit should not be set without a GitHub relay, got Relays=%v", got.Header.Relays)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProfilePicsSkipsEmpty(t *testing.T) {
|
||||
f := newFeedWithMedia(t)
|
||||
stored := f.SetProfilePics(map[string][]byte{
|
||||
"": fakeJPEG(100), // empty username
|
||||
"empty": nil, // empty bytes
|
||||
"good": fakeJPEG(100),
|
||||
})
|
||||
if stored != 1 {
|
||||
t.Errorf("stored = %d, want 1", stored)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProfilePicsReplaceUpdatesBundle(t *testing.T) {
|
||||
f := newFeedWithMedia(t)
|
||||
first := fakeJPEG(1024)
|
||||
f.SetProfilePics(map[string][]byte{"alice": first})
|
||||
b1 := f.ProfilePicsBundle()
|
||||
|
||||
second := fakeJPEG(2048)
|
||||
f.SetProfilePics(map[string][]byte{"alice": second})
|
||||
b2 := f.ProfilePicsBundle()
|
||||
|
||||
if len(b1.Entries) != 1 || len(b2.Entries) != 1 {
|
||||
t.Fatalf("expected one entry each, got %d / %d", len(b1.Entries), len(b2.Entries))
|
||||
}
|
||||
if b1.Entries[0].CRC == b2.Entries[0].CRC {
|
||||
t.Errorf("entry CRC didn't change after replacing bytes")
|
||||
}
|
||||
if b1.Header.BundleCRC == b2.Header.BundleCRC {
|
||||
t.Errorf("bundle CRC didn't change after replacing bytes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProfilePicsClearsOnEmpty(t *testing.T) {
|
||||
f := newFeedWithMedia(t)
|
||||
f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(1024)})
|
||||
if got := f.ProfilePicsBundle(); len(got.Entries) != 1 {
|
||||
t.Fatalf("setup: got %d entries", len(got.Entries))
|
||||
}
|
||||
stored := f.SetProfilePics(nil)
|
||||
if stored != 0 {
|
||||
t.Errorf("stored = %d, want 0", stored)
|
||||
}
|
||||
got := f.ProfilePicsBundle()
|
||||
if len(got.Entries) != 0 {
|
||||
t.Errorf("entries = %d, want 0 after empty refresh", len(got.Entries))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeProfilePicsKeepsOtherEntries(t *testing.T) {
|
||||
f := newFeedWithMedia(t)
|
||||
// First reader contributes Telegram avatars.
|
||||
tg := map[string][]byte{
|
||||
"alice": fakeJPEG(800),
|
||||
"bob": fakeJPEG(900),
|
||||
}
|
||||
if got := f.SetProfilePics(tg); got != 2 {
|
||||
t.Fatalf("seed Set: got %d want 2", got)
|
||||
}
|
||||
|
||||
// Second reader contributes X avatars under the "x:" namespace —
|
||||
// the Telegram entries should survive.
|
||||
xs := map[string][]byte{
|
||||
"x:elonmusk": fakePNG(700),
|
||||
}
|
||||
if got := f.MergeProfilePics(xs); got != 3 {
|
||||
t.Fatalf("merge: got %d want 3", got)
|
||||
}
|
||||
|
||||
got := f.ProfilePicsBundle()
|
||||
if len(got.Entries) != 3 {
|
||||
t.Fatalf("entries = %d want 3", len(got.Entries))
|
||||
}
|
||||
have := map[string]bool{}
|
||||
for _, e := range got.Entries {
|
||||
have[e.Username] = true
|
||||
}
|
||||
for _, want := range []string{"alice", "bob", "x:elonmusk"} {
|
||||
if !have[want] {
|
||||
t.Errorf("missing entry %q in merged bundle", want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeProfilePicsDropEntry(t *testing.T) {
|
||||
f := newFeedWithMedia(t)
|
||||
f.SetProfilePics(map[string][]byte{
|
||||
"alice": fakeJPEG(800),
|
||||
"bob": fakeJPEG(900),
|
||||
})
|
||||
// nil bytes for an existing key drops it.
|
||||
if got := f.MergeProfilePics(map[string][]byte{"alice": nil}); got != 1 {
|
||||
t.Fatalf("merge: got %d want 1", got)
|
||||
}
|
||||
got := f.ProfilePicsBundle()
|
||||
if len(got.Entries) != 1 || got.Entries[0].Username != "bob" {
|
||||
t.Errorf("entries after drop = %+v", got.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetProfilePicsNoMediaCache(t *testing.T) {
|
||||
f := NewFeed([]string{"a"})
|
||||
stored := f.SetProfilePics(map[string][]byte{"alice": fakeJPEG(100)})
|
||||
if stored != 0 {
|
||||
t.Errorf("stored = %d, want 0 (media cache not configured)", stored)
|
||||
}
|
||||
if got := f.ProfilePicsBundle(); len(got.Entries) != 0 {
|
||||
t.Errorf("entries = %v, want empty", got.Entries)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +169,9 @@ func (pr *PublicReader) fetchAll(ctx context.Context) {
|
||||
}
|
||||
log.Printf("[public] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total",
|
||||
time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(pr.channels))
|
||||
// Fetch avatars for the same channel set — same cadence, best-effort.
|
||||
// Public mode has no MTProto session so we scrape them off t.me/s/<u>.
|
||||
pr.fetchAllPublicProfilePhotos(ctx)
|
||||
pr.feed.AfterFetchCycle(ctx)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"golang.org/x/net/html"
|
||||
)
|
||||
|
||||
// extractPublicAvatarURL finds the channel avatar URL on a t.me/s page.
|
||||
// Returns "" when the channel has no photo or the layout is unfamiliar.
|
||||
func extractPublicAvatarURL(body []byte) string {
|
||||
doc, err := html.Parse(strings.NewReader(string(body)))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
if i := findFirstByClass(doc, "tgme_page_photo_image"); i != nil {
|
||||
if img := firstImgChild(i); img != nil {
|
||||
if src := attrValue(img, "src"); src != "" {
|
||||
return src
|
||||
}
|
||||
}
|
||||
if attrValue(i, "src") != "" {
|
||||
return attrValue(i, "src")
|
||||
}
|
||||
}
|
||||
if u := findFirstByClass(doc, "tgme_widget_message_user_photo"); u != nil {
|
||||
if src := attrValue(u, "src"); src != "" {
|
||||
return src
|
||||
}
|
||||
if img := firstImgChild(u); img != nil {
|
||||
return attrValue(img, "src")
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func firstImgChild(n *html.Node) *html.Node {
|
||||
if n == nil {
|
||||
return nil
|
||||
}
|
||||
var found *html.Node
|
||||
visitNodes(n, func(c *html.Node) {
|
||||
if found != nil || c.Type != html.ElementNode {
|
||||
return
|
||||
}
|
||||
if c.Data == "img" {
|
||||
found = c
|
||||
}
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
// fetchPublicAvatar downloads the avatar from t.me/s/<username>.
|
||||
// Returns nil bytes when the channel has no photo.
|
||||
func (pr *PublicReader) fetchPublicAvatar(ctx context.Context, username string) ([]byte, error) {
|
||||
username = strings.TrimSpace(username)
|
||||
if username == "" {
|
||||
return nil, nil
|
||||
}
|
||||
pageURL := pr.baseURL + "/" + url.PathEscape(username)
|
||||
body, err := httpGetBody(ctx, pr.client, pageURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("fetch %s: %w", pageURL, err)
|
||||
}
|
||||
imgURL := extractPublicAvatarURL(body)
|
||||
if imgURL == "" {
|
||||
return nil, nil
|
||||
}
|
||||
const maxAvatarBytes = 512 * 1024
|
||||
imgBytes, err := httpGetWithLimit(ctx, pr.client, imgURL, maxAvatarBytes)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("download avatar %s: %w", imgURL, err)
|
||||
}
|
||||
return imgBytes, nil
|
||||
}
|
||||
|
||||
// fetchAllPublicProfilePhotos scrapes each channel's avatar from
|
||||
// t.me/s/<u> and merges into the bundle. Best-effort.
|
||||
func (pr *PublicReader) fetchAllPublicProfilePhotos(ctx context.Context) {
|
||||
pr.mu.RLock()
|
||||
channels := append([]string(nil), pr.channels...)
|
||||
pr.mu.RUnlock()
|
||||
|
||||
pics := make(map[string][]byte, len(channels))
|
||||
var picsMu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 4) // cap concurrency vs t.me
|
||||
for _, u := range channels {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
u = strings.TrimSpace(u)
|
||||
if u == "" {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(u string) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
body, err := pr.fetchPublicAvatar(ctx, u)
|
||||
if err != nil {
|
||||
log.Printf("[public profile-pic] %s: %v", u, err)
|
||||
return
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return
|
||||
}
|
||||
picsMu.Lock()
|
||||
pics[u] = body
|
||||
picsMu.Unlock()
|
||||
}(u)
|
||||
}
|
||||
wg.Wait()
|
||||
if len(pics) == 0 {
|
||||
return
|
||||
}
|
||||
total := pr.feed.MergeProfilePics(pics)
|
||||
log.Printf("[public profile-pic] cycle done: %d new, %d total in bundle", len(pics), total)
|
||||
}
|
||||
|
||||
// httpGetBody is a tiny GET helper. Cap is 8 MB.
|
||||
func httpGetBody(ctx context.Context, c *http.Client, url string) ([]byte, error) {
|
||||
return httpGetWithLimit(ctx, c, url, 8*1024*1024)
|
||||
}
|
||||
|
||||
func httpGetWithLimit(ctx context.Context, c *http.Client, u string, limit int64) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)")
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, fmt.Errorf("http %s: status %s", u, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, limit+1))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) > limit {
|
||||
return nil, fmt.Errorf("http %s: response exceeds %d bytes", u, limit)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -68,6 +68,10 @@ type TelegramReader struct {
|
||||
cacheTTL time.Duration
|
||||
fetchInterval time.Duration
|
||||
|
||||
// lastPhotoID dedups profile-pic downloads across cycles. username →
|
||||
// last seen Telegram photo ID; skip download when unchanged.
|
||||
lastPhotoID map[string]int64
|
||||
|
||||
// api is set once authenticated, used for sending messages.
|
||||
apiMu sync.RWMutex
|
||||
api *tg.Client
|
||||
@@ -92,6 +96,10 @@ type resolvedPeer struct {
|
||||
chatType protocol.ChatType
|
||||
canSend bool
|
||||
title string
|
||||
// photoID is the Telegram-assigned ID of the peer's profile photo,
|
||||
// 0 when the channel has none. Used to skip the download path when
|
||||
// the photo hasn't changed since last fetch.
|
||||
photoID int64
|
||||
}
|
||||
|
||||
type cachedMessages struct {
|
||||
@@ -289,6 +297,10 @@ func (tr *TelegramReader) fetchAll(ctx context.Context, api *tg.Client) {
|
||||
}
|
||||
log.Printf("[telegram] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total",
|
||||
time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(tr.channels))
|
||||
// Profile pics piggyback the regular cycle: best-effort, doesn't
|
||||
// gate channel data. Each channel's photo is downloaded once and
|
||||
// only re-fetched when Telegram reports a different photo ID.
|
||||
tr.fetchAllProfilePhotos(ctx, api)
|
||||
tr.feed.AfterFetchCycle(ctx)
|
||||
}
|
||||
|
||||
@@ -306,6 +318,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
|
||||
for _, chat := range resolved.Chats {
|
||||
if ch, ok := chat.(*tg.Channel); ok {
|
||||
canSend := !ch.Broadcast || ch.Creator || ch.AdminRights.PostMessages
|
||||
pid, _ := extractChatPhotoID(ch.Photo)
|
||||
return &resolvedPeer{
|
||||
peer: &tg.InputPeerChannel{
|
||||
ChannelID: ch.ID,
|
||||
@@ -314,6 +327,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
|
||||
chatType: protocol.ChatTypeChannel,
|
||||
canSend: canSend,
|
||||
title: ch.Title,
|
||||
photoID: pid,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
@@ -321,6 +335,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
|
||||
// Check Users (bots, private chats)
|
||||
for _, u := range resolved.Users {
|
||||
if user, ok := u.(*tg.User); ok {
|
||||
pid, _ := extractUserPhotoID(user.Photo)
|
||||
return &resolvedPeer{
|
||||
peer: &tg.InputPeerUser{
|
||||
UserID: user.ID,
|
||||
@@ -329,6 +344,7 @@ func (tr *TelegramReader) resolvePeer(ctx context.Context, api *tg.Client, usern
|
||||
chatType: protocol.ChatTypePrivate,
|
||||
canSend: true,
|
||||
title: user.FirstName,
|
||||
photoID: pid,
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -226,6 +226,10 @@ func (xr *XPublicReader) fetchAll(ctx context.Context) {
|
||||
}
|
||||
log.Printf("[x] fetch cycle done in %s: %d fetched, %d failed, %d skipped, %d total",
|
||||
time.Since(start).Round(time.Millisecond), fetched, failed, skipped, len(xr.accounts))
|
||||
// Avatars: same cycle, merged into the feed's profile-pic bundle. We
|
||||
// merge rather than overwrite because TelegramReader / PublicReader
|
||||
// may also be feeding into the same bundle.
|
||||
xr.fetchAllXProfilePhotos(ctx)
|
||||
xr.feed.AfterFetchCycle(ctx)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Nitter / RSS feeds put the channel image at <rss><channel><image><url>.
|
||||
type xRSSImageEnvelope struct {
|
||||
XMLName xml.Name `xml:"rss"`
|
||||
Channel struct {
|
||||
Image struct {
|
||||
URL string `xml:"url"`
|
||||
} `xml:"image"`
|
||||
} `xml:"channel"`
|
||||
}
|
||||
|
||||
func extractXAvatarURL(body []byte) string {
|
||||
var env xRSSImageEnvelope
|
||||
if err := xml.Unmarshal(body, &env); err != nil {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(env.Channel.Image.URL)
|
||||
}
|
||||
|
||||
// fetchXAvatar tries each Nitter instance and returns the first
|
||||
// avatar that downloads. (nil, nil) when no instance had one.
|
||||
func (xr *XPublicReader) fetchXAvatar(ctx context.Context, account string) ([]byte, error) {
|
||||
const maxAvatarBytes = 512 * 1024
|
||||
var lastErr error
|
||||
for _, instance := range xr.instances {
|
||||
rssURL := strings.TrimSuffix(instance, "/") + "/" + url.PathEscape(account) + "/rss"
|
||||
body, err := xRSSGet(ctx, xr.client, rssURL)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
avatarURL := extractXAvatarURL(body)
|
||||
if avatarURL == "" {
|
||||
continue
|
||||
}
|
||||
// Some Nitter builds return a relative "/pic/..." path.
|
||||
if strings.HasPrefix(avatarURL, "/") {
|
||||
avatarURL = strings.TrimSuffix(instance, "/") + avatarURL
|
||||
}
|
||||
imgBytes, err := httpGetWithLimit(ctx, xr.client, avatarURL, maxAvatarBytes)
|
||||
if err != nil {
|
||||
lastErr = err
|
||||
continue
|
||||
}
|
||||
return imgBytes, nil
|
||||
}
|
||||
if lastErr != nil {
|
||||
return nil, lastErr
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// fetchAllXProfilePhotos downloads each account's avatar (via Nitter)
|
||||
// and merges into the bundle under "x:<handle>" keys.
|
||||
func (xr *XPublicReader) fetchAllXProfilePhotos(ctx context.Context) {
|
||||
xr.mu.RLock()
|
||||
accounts := append([]string(nil), xr.accounts...)
|
||||
xr.mu.RUnlock()
|
||||
|
||||
pics := make(map[string][]byte, len(accounts))
|
||||
var picsMu sync.Mutex
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, 4)
|
||||
for _, a := range accounts {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
a = strings.TrimSpace(a)
|
||||
if a == "" {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(account string) {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
body, err := xr.fetchXAvatar(ctx, account)
|
||||
if err != nil {
|
||||
log.Printf("[x profile-pic] @%s: %v", account, err)
|
||||
return
|
||||
}
|
||||
if len(body) == 0 {
|
||||
return
|
||||
}
|
||||
// "x:" prefix avoids colliding with same-name Telegram channels.
|
||||
picsMu.Lock()
|
||||
pics["x:"+account] = body
|
||||
picsMu.Unlock()
|
||||
}(a)
|
||||
}
|
||||
wg.Wait()
|
||||
if len(pics) == 0 {
|
||||
return
|
||||
}
|
||||
total := xr.feed.MergeProfilePics(pics)
|
||||
log.Printf("[x profile-pic] cycle done: %d new, %d total in bundle", len(pics), total)
|
||||
}
|
||||
|
||||
// xRSSGet mirrors the per-instance RSS request (same UA + Accept).
|
||||
func xRSSGet(ctx context.Context, c *http.Client, u string) ([]byte, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; thefeed/1.0; +https://github.com/sartoopjj/thefeed)")
|
||||
req.Header.Set("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8")
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode/100 != 2 {
|
||||
return nil, fmt.Errorf("rss %s: status %s", u, resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxXRSSBodyBytes))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
package telemirror
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -36,7 +37,7 @@ func parseChannelInfo(doc *html.Node) *Channel {
|
||||
}
|
||||
}
|
||||
if descEl := findFirstByClass(doc, "tgme_channel_info_description"); descEl != nil {
|
||||
ch.Description = innerHTML(descEl)
|
||||
ch.Description = rewriteTranslateLinksInHTML(innerHTML(descEl))
|
||||
}
|
||||
if header := findFirstByClass(doc, "tgme_channel_info_header"); header != nil {
|
||||
if img := findFirstByTag(header, "img"); img != nil {
|
||||
@@ -73,20 +74,28 @@ func parseSinglePost(wrap *html.Node) *Post {
|
||||
if owner := findFirstByClass(msg, "tgme_widget_message_owner_name"); owner != nil {
|
||||
p.Author = textOf(owner)
|
||||
}
|
||||
if textEl := findFirstByClass(msg, "tgme_widget_message_text"); textEl != nil {
|
||||
p.Text = innerHTML(textEl)
|
||||
// The reply preview also has a `.tgme_widget_message_text` (the
|
||||
// quoted snippet) which appears BEFORE the body in the DOM, so
|
||||
// findFirstByClass would grab the wrong one.
|
||||
if textEl := findMessageBodyText(msg); textEl != nil {
|
||||
p.Text = rewriteTranslateLinksInHTML(innerHTML(textEl))
|
||||
}
|
||||
|
||||
p.Reply = parseReply(msg)
|
||||
p.Forward = parseForward(msg)
|
||||
|
||||
visit(msg, func(n *html.Node) bool {
|
||||
switch {
|
||||
case hasClass(n, "tgme_widget_message_photo_wrap"):
|
||||
// Source-post URL → real t.me. Thumb stays on the proxy
|
||||
// (that's how the bytes reach a blocked client).
|
||||
p.Media = append(p.Media, Media{
|
||||
Type: "photo",
|
||||
URL: attrOf(n, "href"),
|
||||
URL: rewriteTranslateLink(attrOf(n, "href")),
|
||||
Thumb: extractBgImage(attrOf(n, "style")),
|
||||
})
|
||||
case hasClass(n, "tgme_widget_message_video_player"):
|
||||
m := Media{Type: "video", URL: attrOf(n, "href")}
|
||||
m := Media{Type: "video", URL: rewriteTranslateLink(attrOf(n, "href"))}
|
||||
if t := findFirstByClass(n, "tgme_widget_message_video_thumb"); t != nil {
|
||||
m.Thumb = extractBgImage(attrOf(t, "style"))
|
||||
}
|
||||
@@ -138,6 +147,16 @@ func parseSinglePost(wrap *html.Node) *Post {
|
||||
if t := findFirstByClass(n, "tgme_widget_message_poll_type"); t != nil {
|
||||
m.Subtitle = textOf(t)
|
||||
}
|
||||
visit(n, func(opt *html.Node) bool {
|
||||
if hasClass(opt, "tgme_widget_message_poll_option_text") {
|
||||
txt := strings.TrimSpace(textOf(opt))
|
||||
if txt != "" {
|
||||
m.Options = append(m.Options, txt)
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
})
|
||||
p.Media = append(p.Media, m)
|
||||
}
|
||||
return true
|
||||
@@ -193,6 +212,72 @@ func parseSinglePost(wrap *html.Node) *Post {
|
||||
return p
|
||||
}
|
||||
|
||||
// findMessageBodyText: first `.tgme_widget_message_text` not nested
|
||||
// inside a `.tgme_widget_message_reply` (which would be the snippet).
|
||||
func findMessageBodyText(msg *html.Node) *html.Node {
|
||||
var found *html.Node
|
||||
visit(msg, func(n *html.Node) bool {
|
||||
if found != nil {
|
||||
return false
|
||||
}
|
||||
if !hasClass(n, "tgme_widget_message_text") {
|
||||
return true
|
||||
}
|
||||
for p := n.Parent; p != nil; p = p.Parent {
|
||||
if hasClass(p, "tgme_widget_message_reply") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
found = n
|
||||
return false
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
// parseReply extracts author + snippet + URL from a reply preview.
|
||||
func parseReply(msg *html.Node) *Reply {
|
||||
rNode := findFirstByClass(msg, "tgme_widget_message_reply")
|
||||
if rNode == nil {
|
||||
return nil
|
||||
}
|
||||
r := &Reply{
|
||||
URL: rewriteTranslateLink(attrOf(rNode, "href")),
|
||||
}
|
||||
if a := findFirstByClass(rNode, "tgme_widget_message_author_name"); a != nil {
|
||||
r.Author = textOf(a)
|
||||
}
|
||||
if t := findFirstByClass(rNode, "tgme_widget_message_text"); t != nil {
|
||||
r.Text = rewriteTranslateLinksInHTML(innerHTML(t))
|
||||
}
|
||||
if r.Author == "" && r.Text == "" {
|
||||
return nil
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// parseForward extracts the "Forwarded from <name>" header.
|
||||
func parseForward(msg *html.Node) *Forward {
|
||||
fNode := findFirstByClass(msg, "tgme_widget_message_forwarded_from")
|
||||
if fNode == nil {
|
||||
return nil
|
||||
}
|
||||
f := &Forward{}
|
||||
if a := findFirstByClass(fNode, "tgme_widget_message_forwarded_from_name"); a != nil {
|
||||
f.Author = textOf(a)
|
||||
f.URL = rewriteTranslateLink(attrOf(a, "href"))
|
||||
} else if a := findFirstByTag(fNode, "a"); a != nil {
|
||||
// Bare <a> without the _name class.
|
||||
f.Author = textOf(a)
|
||||
f.URL = rewriteTranslateLink(attrOf(a, "href"))
|
||||
} else {
|
||||
f.Author = textOf(fNode)
|
||||
}
|
||||
if f.Author == "" {
|
||||
return nil
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// ===== DOM helpers =====
|
||||
|
||||
func visit(n *html.Node, fn func(*html.Node) bool) {
|
||||
@@ -313,3 +398,128 @@ func extractBgImage(style string) string {
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Google Translate proxy URLs come in two forms; we rewrite hrefs
|
||||
// back to the originals so links work when Translate is blocked / the
|
||||
// site is region-locked. Image src attributes are left alone since
|
||||
// the proxy is how those bytes actually reach the user.
|
||||
|
||||
const translateHostSuffix = ".translate.goog"
|
||||
|
||||
// decodeTranslateHost: '.' ↔ '-', '-' ↔ '--'.
|
||||
//
|
||||
// t-me → t.me
|
||||
// cdn4-telegram-org → cdn4.telegram.org
|
||||
// my--domain-com → my-domain.com
|
||||
func decodeTranslateHost(s string) string {
|
||||
s = strings.ReplaceAll(s, "--", "\x00")
|
||||
s = strings.ReplaceAll(s, "-", ".")
|
||||
s = strings.ReplaceAll(s, "\x00", "-")
|
||||
return s
|
||||
}
|
||||
|
||||
// rewriteTranslateLink handles two forms:
|
||||
// 1. <encoded>.translate.goog/<path>?_x_tr_*=… — inline-proxy form;
|
||||
// decode host, strip tracking params.
|
||||
// 2. translate.google.com/website?u=<original> (also /translate) —
|
||||
// wrapper form; pull `u` and recurse once (Google occasionally
|
||||
// double-wraps).
|
||||
func rewriteTranslateLink(raw string) string {
|
||||
if raw == "" {
|
||||
return raw
|
||||
}
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return raw
|
||||
}
|
||||
|
||||
// Form 2.
|
||||
hostLower := strings.ToLower(u.Host)
|
||||
if (hostLower == "translate.google.com" || hostLower == "www.translate.google.com") &&
|
||||
(u.Path == "/website" || u.Path == "/translate") {
|
||||
if orig := u.Query().Get("u"); orig != "" {
|
||||
return rewriteTranslateLink(orig)
|
||||
}
|
||||
}
|
||||
|
||||
// Form 1.
|
||||
if !strings.HasSuffix(hostLower, translateHostSuffix) {
|
||||
return raw
|
||||
}
|
||||
encoded := u.Host[:len(u.Host)-len(translateHostSuffix)]
|
||||
u.Host = decodeTranslateHost(encoded)
|
||||
if q := u.Query(); len(q) > 0 {
|
||||
for k := range q {
|
||||
if strings.HasPrefix(k, "_x_tr_") {
|
||||
q.Del(k)
|
||||
}
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
}
|
||||
return u.String()
|
||||
}
|
||||
|
||||
// rewriteTranslateLinksInHTML rewrites <a href> in an HTML fragment.
|
||||
// <img src> is intentionally left alone (we serve images via the proxy).
|
||||
func rewriteTranslateLinksInHTML(htmlStr string) string {
|
||||
if htmlStr == "" || !containsAnyTranslateMarker(htmlStr) {
|
||||
return htmlStr
|
||||
}
|
||||
// Sentinel div so we can locate the fragment in the parsed tree
|
||||
// (html.Parse injects <html><head><body>…).
|
||||
doc, err := html.Parse(strings.NewReader("<div id=\"tm-rewrite-root\">" + htmlStr + "</div>"))
|
||||
if err != nil {
|
||||
return htmlStr
|
||||
}
|
||||
rewriteHrefsInTree(doc)
|
||||
root := findFirstByID(doc, "tm-rewrite-root")
|
||||
if root == nil {
|
||||
return htmlStr
|
||||
}
|
||||
var b strings.Builder
|
||||
for c := root.FirstChild; c != nil; c = c.NextSibling {
|
||||
if err := html.Render(&b, c); err != nil {
|
||||
return htmlStr
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// containsAnyTranslateMarker is a cheap pre-check that catches both
|
||||
// the inline-proxy and wrapper forms.
|
||||
func containsAnyTranslateMarker(s string) bool {
|
||||
return strings.Contains(s, translateHostSuffix) ||
|
||||
strings.Contains(s, "translate.google.com/")
|
||||
}
|
||||
|
||||
func rewriteHrefsInTree(n *html.Node) {
|
||||
if n.Type == html.ElementNode && n.Data == "a" {
|
||||
for i, a := range n.Attr {
|
||||
if a.Key == "href" {
|
||||
n.Attr[i].Val = rewriteTranslateLink(a.Val)
|
||||
}
|
||||
}
|
||||
}
|
||||
for c := n.FirstChild; c != nil; c = c.NextSibling {
|
||||
rewriteHrefsInTree(c)
|
||||
}
|
||||
}
|
||||
|
||||
func findFirstByID(root *html.Node, id string) *html.Node {
|
||||
var found *html.Node
|
||||
visit(root, func(n *html.Node) bool {
|
||||
if found != nil {
|
||||
return false
|
||||
}
|
||||
if n.Type == html.ElementNode {
|
||||
for _, a := range n.Attr {
|
||||
if a.Key == "id" && a.Val == id {
|
||||
found = n
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
return found
|
||||
}
|
||||
|
||||
@@ -109,6 +109,258 @@ func TestParseHTMLPosts(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Telegram embeds the reply preview INSIDE the message wrapper, BEFORE
|
||||
// the actual message body. Both the snippet and the body use the
|
||||
// `tgme_widget_message_text` class. The parser must pick the body, not
|
||||
// the snippet, and must surface the reply metadata as Post.Reply.
|
||||
const replyPostHTML = `<!DOCTYPE html><html><body>
|
||||
<div class="tgme_widget_message_wrap">
|
||||
<div class="tgme_widget_message" data-post="sample/200">
|
||||
<a class="tgme_widget_message_reply" href="https://t.me/sample/198">
|
||||
<span class="tgme_widget_message_author_name">Original Author</span>
|
||||
<div class="tgme_widget_message_text">quoted snippet of original</div>
|
||||
</a>
|
||||
<div class="tgme_widget_message_text">actual reply body</div>
|
||||
</div>
|
||||
</div>
|
||||
</body></html>`
|
||||
|
||||
func TestParseHTMLReply(t *testing.T) {
|
||||
_, posts, err := ParseHTML(replyPostHTML)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML: %v", err)
|
||||
}
|
||||
if len(posts) != 1 {
|
||||
t.Fatalf("posts = %d, want 1", len(posts))
|
||||
}
|
||||
p := posts[0]
|
||||
if !strings.Contains(p.Text, "actual reply body") {
|
||||
t.Errorf("Post.Text = %q, want main body, not snippet", p.Text)
|
||||
}
|
||||
if strings.Contains(p.Text, "quoted snippet") {
|
||||
t.Errorf("Post.Text leaked the reply snippet: %q", p.Text)
|
||||
}
|
||||
if p.Reply == nil {
|
||||
t.Fatalf("Post.Reply nil; want populated reply preview")
|
||||
}
|
||||
if p.Reply.Author != "Original Author" {
|
||||
t.Errorf("Reply.Author = %q", p.Reply.Author)
|
||||
}
|
||||
if !strings.Contains(p.Reply.Text, "quoted snippet") {
|
||||
t.Errorf("Reply.Text = %q", p.Reply.Text)
|
||||
}
|
||||
if p.Reply.URL != "https://t.me/sample/198" {
|
||||
t.Errorf("Reply.URL = %q", p.Reply.URL)
|
||||
}
|
||||
}
|
||||
|
||||
const forwardPostHTML = `<!DOCTYPE html><html><body>
|
||||
<div class="tgme_widget_message_wrap">
|
||||
<div class="tgme_widget_message" data-post="sample/201">
|
||||
<div class="tgme_widget_message_forwarded_from">
|
||||
Forwarded from <a class="tgme_widget_message_forwarded_from_name" href="https://t.me/source">Source Channel</a>
|
||||
</div>
|
||||
<div class="tgme_widget_message_text">forwarded post body</div>
|
||||
</div>
|
||||
</div>
|
||||
</body></html>`
|
||||
|
||||
func TestParseHTMLForward(t *testing.T) {
|
||||
_, posts, err := ParseHTML(forwardPostHTML)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML: %v", err)
|
||||
}
|
||||
if len(posts) != 1 {
|
||||
t.Fatalf("posts = %d, want 1", len(posts))
|
||||
}
|
||||
p := posts[0]
|
||||
if p.Forward == nil {
|
||||
t.Fatalf("Post.Forward nil; want populated forward header")
|
||||
}
|
||||
if p.Forward.Author != "Source Channel" {
|
||||
t.Errorf("Forward.Author = %q", p.Forward.Author)
|
||||
}
|
||||
if p.Forward.URL != "https://t.me/source" {
|
||||
t.Errorf("Forward.URL = %q", p.Forward.URL)
|
||||
}
|
||||
if !strings.Contains(p.Text, "forwarded post body") {
|
||||
t.Errorf("Post.Text = %q", p.Text)
|
||||
}
|
||||
}
|
||||
|
||||
const pollPostHTML = `<!DOCTYPE html><html><body>
|
||||
<div class="tgme_widget_message_wrap">
|
||||
<div class="tgme_widget_message" data-post="sample/202">
|
||||
<div class="tgme_widget_message_poll">
|
||||
<div class="tgme_widget_message_poll_question">Favourite colour?</div>
|
||||
<div class="tgme_widget_message_poll_type">Public vote</div>
|
||||
<div class="tgme_widget_message_poll_option">
|
||||
<div class="tgme_widget_message_poll_option_text">Red</div>
|
||||
</div>
|
||||
<div class="tgme_widget_message_poll_option">
|
||||
<div class="tgme_widget_message_poll_option_text">Blue</div>
|
||||
</div>
|
||||
<div class="tgme_widget_message_poll_option">
|
||||
<div class="tgme_widget_message_poll_option_text">Green</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body></html>`
|
||||
|
||||
func TestParseHTMLPollOptions(t *testing.T) {
|
||||
_, posts, err := ParseHTML(pollPostHTML)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML: %v", err)
|
||||
}
|
||||
if len(posts) != 1 || len(posts[0].Media) != 1 {
|
||||
t.Fatalf("posts/media wrong shape: posts=%d media=%d", len(posts), len(posts[0].Media))
|
||||
}
|
||||
m := posts[0].Media[0]
|
||||
if m.Type != "poll" {
|
||||
t.Errorf("Media.Type = %q, want poll", m.Type)
|
||||
}
|
||||
if m.Title != "Favourite colour?" {
|
||||
t.Errorf("Media.Title = %q", m.Title)
|
||||
}
|
||||
if m.Subtitle != "Public vote" {
|
||||
t.Errorf("Media.Subtitle = %q", m.Subtitle)
|
||||
}
|
||||
wantOpts := []string{"Red", "Blue", "Green"}
|
||||
if len(m.Options) != len(wantOpts) {
|
||||
t.Fatalf("Options = %v, want %v", m.Options, wantOpts)
|
||||
}
|
||||
for i, want := range wantOpts {
|
||||
if m.Options[i] != want {
|
||||
t.Errorf("Options[%d] = %q, want %q", i, m.Options[i], want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeTranslateHost(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{"t-me", "t.me"},
|
||||
{"cdn4-telegram-org", "cdn4.telegram.org"},
|
||||
{"my--domain-com", "my-domain.com"},
|
||||
{"a--b--c-d", "a-b-c.d"},
|
||||
{"plain", "plain"},
|
||||
{"", ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := decodeTranslateHost(c.in); got != c.want {
|
||||
t.Errorf("decodeTranslateHost(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteTranslateLink(t *testing.T) {
|
||||
cases := []struct {
|
||||
in, want string
|
||||
}{
|
||||
{
|
||||
"https://t-me.translate.goog/sample/123",
|
||||
"https://t.me/sample/123",
|
||||
},
|
||||
{
|
||||
"https://example-com.translate.goog/path?_x_tr_sl=auto&_x_tr_tl=en&q=keep",
|
||||
"https://example.com/path?q=keep",
|
||||
},
|
||||
{
|
||||
"https://my--domain-com.translate.goog/x",
|
||||
"https://my-domain.com/x",
|
||||
},
|
||||
// Untouched: not a translate.goog URL.
|
||||
{"https://example.com/foo", "https://example.com/foo"},
|
||||
// Untouched: empty.
|
||||
{"", ""},
|
||||
// Tracking-only query string collapses to no query string.
|
||||
{
|
||||
"https://t-me.translate.goog/sample?_x_tr_pto=wapp",
|
||||
"https://t.me/sample",
|
||||
},
|
||||
// Wrapper form: translate.google.com/website?u=<original>
|
||||
{
|
||||
"https://translate.google.com/website?sl=auto&tl=fa&hl=en&client=webapp&u=https://seup.shop/f/rdzq",
|
||||
"https://seup.shop/f/rdzq",
|
||||
},
|
||||
// Wrapper form, /translate path also works.
|
||||
{
|
||||
"https://translate.google.com/translate?sl=en&tl=fa&u=https://example.com/page",
|
||||
"https://example.com/page",
|
||||
},
|
||||
// Wrapper form pointing at a goog inline-proxy URL — recurse.
|
||||
{
|
||||
"https://translate.google.com/website?u=https://t-me.translate.goog/networkti",
|
||||
"https://t.me/networkti",
|
||||
},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := rewriteTranslateLink(c.in); got != c.want {
|
||||
t.Errorf("rewriteTranslateLink(%q) = %q, want %q", c.in, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestRewriteTranslateLinksInHTML(t *testing.T) {
|
||||
in := `Check <a href="https://t-me.translate.goog/sample/123">this post</a> ` +
|
||||
`and <a href="https://example-com.translate.goog/?_x_tr_sl=auto">this site</a>. ` +
|
||||
`<img src="https://cdn-translate.goog/img.jpg"> stays.`
|
||||
got := rewriteTranslateLinksInHTML(in)
|
||||
if !strings.Contains(got, `href="https://t.me/sample/123"`) {
|
||||
t.Errorf("translate.goog href not rewritten: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, `href="https://example.com/"`) {
|
||||
t.Errorf("example-com.translate.goog href not rewritten or query not stripped: %q", got)
|
||||
}
|
||||
if !strings.Contains(got, `src="https://cdn-translate.goog/img.jpg"`) {
|
||||
t.Errorf("img src should NOT be rewritten (we serve images via the proxy): %q", got)
|
||||
}
|
||||
// Sanity check: untouched HTML round-trips without the rewriter
|
||||
// damaging it.
|
||||
plain := "no proxy links here, just <b>text</b>"
|
||||
if got := rewriteTranslateLinksInHTML(plain); got != plain {
|
||||
t.Errorf("plain html mutated: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTMLRewritesPostLinks(t *testing.T) {
|
||||
const sample = `<!DOCTYPE html><html><body>
|
||||
<div class="tgme_widget_message_wrap">
|
||||
<div class="tgme_widget_message" data-post="sample/300">
|
||||
<div class="tgme_widget_message_text">
|
||||
Visit <a href="https://t-me.translate.goog/networkti">@networkti</a> and
|
||||
<a href="https://example-com.translate.goog/article?_x_tr_sl=auto&_x_tr_tl=en">read</a>,
|
||||
and try <a href="https://translate.google.com/website?sl=auto&tl=fa&u=https://seup.shop/f/rdzq">this site</a>.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body></html>`
|
||||
_, posts, err := ParseHTML(sample)
|
||||
if err != nil {
|
||||
t.Fatalf("ParseHTML: %v", err)
|
||||
}
|
||||
if len(posts) != 1 {
|
||||
t.Fatalf("posts = %d, want 1", len(posts))
|
||||
}
|
||||
if !strings.Contains(posts[0].Text, `href="https://t.me/networkti"`) {
|
||||
t.Errorf("Post.Text didn't rewrite t-me link: %q", posts[0].Text)
|
||||
}
|
||||
if !strings.Contains(posts[0].Text, `href="https://seup.shop/f/rdzq"`) {
|
||||
t.Errorf("Post.Text didn't unwrap translate.google.com/website wrapper: %q", posts[0].Text)
|
||||
}
|
||||
if strings.Contains(posts[0].Text, "translate.goog") {
|
||||
t.Errorf("translate.goog leaked in Post.Text: %q", posts[0].Text)
|
||||
}
|
||||
if strings.Contains(posts[0].Text, "translate.google.com") {
|
||||
t.Errorf("translate.google.com wrapper leaked in Post.Text: %q", posts[0].Text)
|
||||
}
|
||||
if strings.Contains(posts[0].Text, "_x_tr_") {
|
||||
t.Errorf("_x_tr_ tracking params leaked: %q", posts[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseHTMLEmpty(t *testing.T) {
|
||||
ch, posts, err := ParseHTML("<html></html>")
|
||||
if err != nil {
|
||||
|
||||
@@ -26,25 +26,40 @@ type Channel struct {
|
||||
|
||||
// Media is one attachment on a post.
|
||||
type Media struct {
|
||||
Type string `json:"type"` // "photo" | "video" | "voice" | "audio" | "document" | "sticker" | "poll"
|
||||
URL string `json:"url,omitempty"`
|
||||
Thumb string `json:"thumb,omitempty"`
|
||||
Duration string `json:"duration,omitempty"`
|
||||
Title string `json:"title,omitempty"` // file name / poll question / audio title
|
||||
Subtitle string `json:"subtitle,omitempty"`
|
||||
Type string `json:"type"` // photo | video | voice | audio | document | sticker | poll
|
||||
URL string `json:"url,omitempty"`
|
||||
Thumb string `json:"thumb,omitempty"`
|
||||
Duration string `json:"duration,omitempty"`
|
||||
Title string `json:"title,omitempty"` // file name / poll question / audio title
|
||||
Subtitle string `json:"subtitle,omitempty"`
|
||||
Options []string `json:"options,omitempty"` // poll options
|
||||
}
|
||||
|
||||
// Reaction is one emoji + count on a post.
|
||||
type Reaction struct {
|
||||
Emoji string `json:"emoji"`
|
||||
Count string `json:"count,omitempty"`
|
||||
}
|
||||
|
||||
// Reply is the quoted preview shown above a reply.
|
||||
type Reply struct {
|
||||
Author string `json:"author,omitempty"`
|
||||
Text string `json:"text,omitempty"` // inner HTML of the snippet
|
||||
URL string `json:"url,omitempty"` // link to the replied-to post
|
||||
}
|
||||
|
||||
// Forward is the "Forwarded from <name>" header.
|
||||
type Forward struct {
|
||||
Author string `json:"author,omitempty"`
|
||||
URL string `json:"url,omitempty"`
|
||||
}
|
||||
|
||||
// Post is a single message from the channel feed.
|
||||
type Post struct {
|
||||
ID string `json:"id"` // "<channel>/<msgid>"
|
||||
Author string `json:"author,omitempty"`
|
||||
Text string `json:"text,omitempty"` // sanitised inner HTML
|
||||
Reply *Reply `json:"reply,omitempty"`
|
||||
Forward *Forward `json:"forward,omitempty"`
|
||||
Media []Media `json:"media,omitempty"`
|
||||
Reactions []Reaction `json:"reactions,omitempty"`
|
||||
Time time.Time `json:"time,omitempty"`
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
package web
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/crc32"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/sartoopjj/thefeed/internal/client"
|
||||
"github.com/sartoopjj/thefeed/internal/protocol"
|
||||
)
|
||||
|
||||
// profilePicsHub caches channel avatars on disk and coordinates the
|
||||
// background fetch. GitHub bundle is automatic; per-entry DNS path
|
||||
// requires the ProfilePicsEnabled toggle.
|
||||
type profilePicsHub struct {
|
||||
dataDir string
|
||||
|
||||
mu sync.Mutex
|
||||
index map[string]profilePicCacheEntry // username (lower) -> entry
|
||||
fetching bool
|
||||
progress profilePicProgress
|
||||
// CRC of the bundle that produced our current cache; if the next
|
||||
// directory advertises the same value we skip the download.
|
||||
bundleCRC uint32
|
||||
}
|
||||
|
||||
type profilePicCacheEntry struct {
|
||||
CRC uint32 `json:"crc"`
|
||||
Size uint32 `json:"size"`
|
||||
MIME uint8 `json:"mime"`
|
||||
Extension string `json:"ext"`
|
||||
StoredAt int64 `json:"storedAt"`
|
||||
}
|
||||
|
||||
// profilePicProgress is polled by the UI during refresh.
|
||||
type profilePicProgress struct {
|
||||
Active bool `json:"active"`
|
||||
Total int `json:"total"`
|
||||
Done int `json:"done"`
|
||||
Failed int `json:"failed"`
|
||||
Username string `json:"username,omitempty"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type profilePicsIndexFile struct {
|
||||
BundleCRC uint32 `json:"bundleCrc"`
|
||||
Users map[string]profilePicCacheEntry `json:"users"`
|
||||
}
|
||||
|
||||
func newProfilePicsHub(dataDir string) *profilePicsHub {
|
||||
h := &profilePicsHub{
|
||||
dataDir: dataDir,
|
||||
index: make(map[string]profilePicCacheEntry),
|
||||
}
|
||||
h.loadIndex()
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) cacheDir() string {
|
||||
return filepath.Join(h.dataDir, "profile_pics")
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) indexPath() string {
|
||||
return filepath.Join(h.cacheDir(), "index.json")
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) imagePath(username, ext string) string {
|
||||
// "x:handle" → "x__handle" — Windows can't use ':' in filenames.
|
||||
safe := strings.ReplaceAll(strings.ToLower(username), ":", "__")
|
||||
return filepath.Join(h.cacheDir(), safe+ext)
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) loadIndex() {
|
||||
b, err := os.ReadFile(h.indexPath())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var idx profilePicsIndexFile
|
||||
if err := json.Unmarshal(b, &idx); err != nil {
|
||||
return
|
||||
}
|
||||
h.mu.Lock()
|
||||
h.index = idx.Users
|
||||
if h.index == nil {
|
||||
h.index = make(map[string]profilePicCacheEntry)
|
||||
}
|
||||
h.bundleCRC = idx.BundleCRC
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) saveIndexLocked() {
|
||||
if err := os.MkdirAll(h.cacheDir(), 0700); err != nil {
|
||||
return
|
||||
}
|
||||
idx := profilePicsIndexFile{
|
||||
BundleCRC: h.bundleCRC,
|
||||
Users: h.index,
|
||||
}
|
||||
b, err := json.MarshalIndent(idx, "", " ")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
_ = os.WriteFile(h.indexPath(), b, 0600)
|
||||
}
|
||||
|
||||
// Store writes bytes to disk and updates the index. Any previous file
|
||||
// for this username (different extension) is removed.
|
||||
func (h *profilePicsHub) Store(username string, content []byte, crc uint32, mime uint8) error {
|
||||
if username == "" || len(content) == 0 {
|
||||
return errors.New("profile-pics: empty input")
|
||||
}
|
||||
if err := os.MkdirAll(h.cacheDir(), 0700); err != nil {
|
||||
return err
|
||||
}
|
||||
ext := extensionFor(mime)
|
||||
tmp := h.imagePath(username, ext) + ".tmp"
|
||||
if err := os.WriteFile(tmp, content, 0600); err != nil {
|
||||
return err
|
||||
}
|
||||
final := h.imagePath(username, ext)
|
||||
if err := os.Rename(tmp, final); err != nil {
|
||||
_ = os.Remove(tmp)
|
||||
return err
|
||||
}
|
||||
|
||||
h.mu.Lock()
|
||||
if old, ok := h.index[strings.ToLower(username)]; ok && old.Extension != ext {
|
||||
_ = os.Remove(h.imagePath(username, old.Extension))
|
||||
}
|
||||
h.index[strings.ToLower(username)] = profilePicCacheEntry{
|
||||
CRC: crc,
|
||||
Size: uint32(len(content)),
|
||||
MIME: mime,
|
||||
Extension: ext,
|
||||
StoredAt: time.Now().Unix(),
|
||||
}
|
||||
h.saveIndexLocked()
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Get returns the cached bytes + content type, or os.ErrNotExist.
|
||||
func (h *profilePicsHub) Get(username string) ([]byte, string, error) {
|
||||
h.mu.Lock()
|
||||
e, ok := h.index[strings.ToLower(username)]
|
||||
h.mu.Unlock()
|
||||
if !ok {
|
||||
return nil, "", os.ErrNotExist
|
||||
}
|
||||
b, err := os.ReadFile(h.imagePath(username, e.Extension))
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
return b, contentTypeFor(e.MIME), nil
|
||||
}
|
||||
|
||||
// Clear wipes both the on-disk cache and the index. Hooked by /api/cache/clear.
|
||||
func (h *profilePicsHub) Clear() {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
h.index = make(map[string]profilePicCacheEntry)
|
||||
h.bundleCRC = 0
|
||||
if entries, err := os.ReadDir(h.cacheDir()); err == nil {
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() {
|
||||
_ = os.Remove(filepath.Join(h.cacheDir(), e.Name()))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) Progress() profilePicProgress {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.progress
|
||||
}
|
||||
|
||||
// relayFetcher pulls the bundle bytes from the GitHub relay. nil means
|
||||
// "skip the GitHub path".
|
||||
type relayFetcher func(ctx context.Context, size int64, crc uint32) ([]byte, error)
|
||||
|
||||
// storedCallback fires after each successful persist so the server can
|
||||
// push an SSE event mid-refresh.
|
||||
type storedCallback func(username string)
|
||||
|
||||
// Refresh is the test entry point (DNS only, no GitHub fetcher).
|
||||
func (h *profilePicsHub) Refresh(ctx context.Context, fetcher *client.Fetcher, dnsAllowed bool) error {
|
||||
return h.refresh(ctx, fetcher, dnsAllowed, nil, nil)
|
||||
}
|
||||
|
||||
// refresh tries the GitHub bundle first, then per-entry DNS for
|
||||
// anything still missing. Coalesces concurrent calls.
|
||||
func (h *profilePicsHub) refresh(ctx context.Context, fetcher *client.Fetcher, dnsAllowed bool, viaGitHub relayFetcher, onStored storedCallback) error {
|
||||
h.mu.Lock()
|
||||
if h.fetching {
|
||||
h.mu.Unlock()
|
||||
return errors.New("profile-pics: refresh already in progress")
|
||||
}
|
||||
h.fetching = true
|
||||
h.progress = profilePicProgress{Active: true}
|
||||
h.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
h.mu.Lock()
|
||||
h.fetching = false
|
||||
h.progress.Active = false
|
||||
h.mu.Unlock()
|
||||
}()
|
||||
|
||||
bundle, err := fetcher.FetchProfilePicDirectory(ctx)
|
||||
if err != nil {
|
||||
h.setProgressErr(fmt.Sprintf("fetch directory: %v", err))
|
||||
return err
|
||||
}
|
||||
if len(bundle.Entries) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
h.mu.Lock()
|
||||
h.progress.Total = len(bundle.Entries)
|
||||
prevCRC := h.bundleCRC
|
||||
h.mu.Unlock()
|
||||
|
||||
// Same bundle, all entries cached → no download.
|
||||
if prevCRC != 0 && prevCRC == bundle.BundleCRC && h.allEntriesCached(bundle.Entries) {
|
||||
h.mu.Lock()
|
||||
h.progress.Done = len(bundle.Entries)
|
||||
h.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Phase 1: GitHub bundle (one fetch, all entries).
|
||||
missing := bundle.Entries
|
||||
hasGitHub := bundle.HasRelay(protocol.RelayGitHub) && viaGitHub != nil && bundle.BundleSize > 0
|
||||
if hasGitHub {
|
||||
body, ghErr := viaGitHub(ctx, int64(bundle.BundleSize), bundle.BundleCRC)
|
||||
if ghErr != nil {
|
||||
h.setProgressErr(fmt.Sprintf("github bundle: %v", ghErr))
|
||||
} else if body != nil {
|
||||
if uint32(len(body)) == bundle.BundleSize && crc32.ChecksumIEEE(body) == bundle.BundleCRC {
|
||||
missing = h.persistFromBundle(ctx, body, bundle.Entries, onStored)
|
||||
h.mu.Lock()
|
||||
h.bundleCRC = bundle.BundleCRC
|
||||
h.saveIndexLocked()
|
||||
h.mu.Unlock()
|
||||
} else {
|
||||
h.setProgressErr(fmt.Sprintf("github bundle size/crc mismatch: have %d/%08x want %d/%08x",
|
||||
len(body), crc32.ChecksumIEEE(body), bundle.BundleSize, bundle.BundleCRC))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Phase 2: per-entry DNS for whatever the bundle didn't cover.
|
||||
if dnsAllowed && len(missing) > 0 {
|
||||
h.fetchMissingViaDNS(ctx, fetcher, missing, onStored)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// persistFromBundle slices each entry out, verifies, writes to disk.
|
||||
// Returns the entries that didn't land (so the DNS phase can retry).
|
||||
func (h *profilePicsHub) persistFromBundle(ctx context.Context, body []byte, entries []client.ProfilePicEntry, onStored storedCallback) []client.ProfilePicEntry {
|
||||
missing := make([]client.ProfilePicEntry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if ctx.Err() != nil {
|
||||
return missing
|
||||
}
|
||||
h.markProgressUsername(entry.Username)
|
||||
if h.HasFresh(entry.Username, entry.CRC) {
|
||||
h.bumpProgress(true)
|
||||
continue
|
||||
}
|
||||
slice, err := protocol.VerifyEntry(body, protocol.ProfilePicEntry{
|
||||
Username: entry.Username,
|
||||
Offset: entry.Offset,
|
||||
Size: entry.Size,
|
||||
CRC: entry.CRC,
|
||||
MIME: entry.MIME,
|
||||
DNSChannel: entry.DNSChannel,
|
||||
DNSBlocks: entry.DNSBlocks,
|
||||
})
|
||||
if err != nil {
|
||||
missing = append(missing, entry)
|
||||
continue
|
||||
}
|
||||
if err := h.Store(entry.Username, slice, entry.CRC, entry.MIME); err != nil {
|
||||
missing = append(missing, entry)
|
||||
continue
|
||||
}
|
||||
h.bumpProgress(true)
|
||||
if onStored != nil {
|
||||
onStored(entry.Username)
|
||||
}
|
||||
}
|
||||
return missing
|
||||
}
|
||||
|
||||
// fetchMissingViaDNS fetches each entry on its own DNS channel,
|
||||
// verifies, persists. Per-entry independent: one failure doesn't
|
||||
// affect the others.
|
||||
func (h *profilePicsHub) fetchMissingViaDNS(ctx context.Context, fetcher *client.Fetcher, entries []client.ProfilePicEntry, onStored storedCallback) {
|
||||
for _, entry := range entries {
|
||||
if ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
h.markProgressUsername(entry.Username)
|
||||
if entry.DNSChannel == 0 || entry.DNSBlocks == 0 {
|
||||
h.bumpProgress(false)
|
||||
continue
|
||||
}
|
||||
// HasFresh is checked again here — a previous Phase-1 success
|
||||
// could already have populated this entry from another path.
|
||||
if h.HasFresh(entry.Username, entry.CRC) {
|
||||
h.bumpProgress(true)
|
||||
continue
|
||||
}
|
||||
body, err := fetcher.FetchMedia(ctx, entry.DNSChannel, entry.DNSBlocks, entry.CRC, nil)
|
||||
if err != nil {
|
||||
h.bumpProgress(false)
|
||||
continue
|
||||
}
|
||||
if uint32(len(body)) != entry.Size {
|
||||
h.bumpProgress(false)
|
||||
continue
|
||||
}
|
||||
if crc32.ChecksumIEEE(body) != entry.CRC {
|
||||
h.bumpProgress(false)
|
||||
continue
|
||||
}
|
||||
if err := h.Store(entry.Username, body, entry.CRC, entry.MIME); err != nil {
|
||||
h.bumpProgress(false)
|
||||
continue
|
||||
}
|
||||
h.bumpProgress(true)
|
||||
if onStored != nil {
|
||||
onStored(entry.Username)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// HasFresh: cached pic with this CRC is already on disk.
|
||||
func (h *profilePicsHub) HasFresh(username string, crc uint32) bool {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
e, ok := h.index[strings.ToLower(username)]
|
||||
if !ok || e.CRC != crc {
|
||||
return false
|
||||
}
|
||||
_, err := os.Stat(h.imagePath(username, e.Extension))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) allEntriesCached(entries []client.ProfilePicEntry) bool {
|
||||
for _, e := range entries {
|
||||
if !h.HasFresh(e.Username, e.CRC) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) markProgressUsername(username string) {
|
||||
h.mu.Lock()
|
||||
h.progress.Username = username
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) bumpProgress(ok bool) {
|
||||
h.mu.Lock()
|
||||
h.progress.Done++
|
||||
if !ok {
|
||||
h.progress.Failed++
|
||||
}
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
func (h *profilePicsHub) setProgressErr(msg string) {
|
||||
h.mu.Lock()
|
||||
h.progress.Error = msg
|
||||
h.mu.Unlock()
|
||||
}
|
||||
|
||||
// ===== HTTP handlers =====
|
||||
|
||||
// handleProfilePic serves /api/profile-pics/<key>. Key is "<handle>"
|
||||
// or "x:<handle>". 404 → front-end falls back to the letter avatar.
|
||||
func (h *profilePicsHub) handleProfilePic(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
key := strings.TrimPrefix(r.URL.Path, "/api/profile-pics/")
|
||||
key = strings.TrimSpace(strings.Trim(key, "/"))
|
||||
if key == "" || !isValidProfilePicKey(key) {
|
||||
http.Error(w, "missing username", 400)
|
||||
return
|
||||
}
|
||||
body, ctype, err := h.Get(key)
|
||||
if err != nil {
|
||||
http.Error(w, "not cached", 404)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", ctype)
|
||||
w.Header().Set("Cache-Control", "private, max-age=86400")
|
||||
w.Header().Set("Content-Length", fmt.Sprintf("%d", len(body)))
|
||||
_, _ = w.Write(body)
|
||||
}
|
||||
|
||||
// isValidProfilePicKey: "<handle>" or "<type>:<handle>", alphanumeric +
|
||||
// _-. Bars slashes/back-slashes/dots so the key can't escape the cache dir.
|
||||
func isValidProfilePicKey(s string) bool {
|
||||
if strings.ContainsAny(s, "/\\.") {
|
||||
return false
|
||||
}
|
||||
parts := strings.Split(s, ":")
|
||||
if len(parts) > 2 {
|
||||
return false
|
||||
}
|
||||
for _, p := range parts {
|
||||
if p == "" {
|
||||
return false
|
||||
}
|
||||
for _, r := range p {
|
||||
if !(r == '_' || r == '-' || (r >= '0' && r <= '9') ||
|
||||
(r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z')) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// handleProfilePicsRefresh kicks off a background refresh and returns
|
||||
// immediately; UI polls /api/profile-pics/progress for the progress bar.
|
||||
func (s *Server) handleProfilePicsRefresh(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
s.mu.RLock()
|
||||
fetcher := s.fetcher
|
||||
hub := s.profilePics
|
||||
s.mu.RUnlock()
|
||||
if fetcher == nil || hub == nil {
|
||||
http.Error(w, "fetcher not ready", 503)
|
||||
return
|
||||
}
|
||||
dnsAllowed := s.profilePicsEnabled()
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
// SSE per stored avatar so the UI updates mid-batch.
|
||||
onStored := func(string) {
|
||||
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
|
||||
}
|
||||
if err := hub.refresh(ctx, fetcher, dnsAllowed, s.fetchFromGitHubRelayBytes, onStored); err == nil {
|
||||
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
|
||||
}
|
||||
}()
|
||||
writeJSON(w, map[string]any{"ok": true})
|
||||
}
|
||||
|
||||
func (s *Server) handleProfilePicsProgress(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
hub := s.profilePics
|
||||
if hub == nil {
|
||||
writeJSON(w, profilePicProgress{})
|
||||
return
|
||||
}
|
||||
writeJSON(w, hub.Progress())
|
||||
}
|
||||
|
||||
// handleProfilePicsList returns which usernames have a cached avatar.
|
||||
// Cache-Control: no-store so SSE-driven reloads see fresh data.
|
||||
func (s *Server) handleProfilePicsList(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "method not allowed", 405)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Cache-Control", "no-store")
|
||||
hub := s.profilePics
|
||||
if hub == nil {
|
||||
writeJSON(w, map[string]any{"enabled": false, "users": []string{}})
|
||||
return
|
||||
}
|
||||
hub.mu.Lock()
|
||||
users := make([]string, 0, len(hub.index))
|
||||
for u := range hub.index {
|
||||
users = append(users, u)
|
||||
}
|
||||
hub.mu.Unlock()
|
||||
writeJSON(w, map[string]any{
|
||||
"enabled": s.profilePicsEnabled(),
|
||||
"users": users,
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Server) profilePicsEnabled() bool {
|
||||
pl, err := s.loadProfiles()
|
||||
if err != nil || pl == nil {
|
||||
return false
|
||||
}
|
||||
return pl.ProfilePicsEnabled
|
||||
}
|
||||
|
||||
// ===== mime helpers =====
|
||||
|
||||
func extensionFor(mime uint8) string {
|
||||
switch mime {
|
||||
case 1:
|
||||
return ".png"
|
||||
case 2:
|
||||
return ".webp"
|
||||
default:
|
||||
return ".jpg"
|
||||
}
|
||||
}
|
||||
|
||||
func contentTypeFor(mime uint8) string {
|
||||
switch mime {
|
||||
case 1:
|
||||
return "image/png"
|
||||
case 2:
|
||||
return "image/webp"
|
||||
default:
|
||||
return "image/jpeg"
|
||||
}
|
||||
}
|
||||
@@ -40,19 +40,22 @@ func (e *ghRateLimitError) Error() string {
|
||||
// (see .github/workflows/build.yml), so net.Lookup* goes through
|
||||
// bionic libc → netd → the device's actual DNS, the same path any other
|
||||
// Android app uses. On desktop the OS resolver is similarly fine.
|
||||
//
|
||||
// Tight connect / TLS / header timeouts so a blocked GitHub fails fast
|
||||
// and we fall back to DNS quickly. Body read gets the full 90 s budget
|
||||
// to cover multi-MB downloads on slow links.
|
||||
var relayHTTPClient = &http.Client{
|
||||
// Per-request budget — large enough to cover multi-MB downloads
|
||||
// over a slow link without hugging short-circuit timeouts.
|
||||
Timeout: 5 * time.Minute,
|
||||
Timeout: 90 * time.Second,
|
||||
Transport: &http.Transport{
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 15 * time.Second,
|
||||
Timeout: 5 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
MaxIdleConns: 10,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 15 * time.Second,
|
||||
TLSHandshakeTimeout: 5 * time.Second,
|
||||
ResponseHeaderTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
},
|
||||
}
|
||||
@@ -116,6 +119,64 @@ func (c *relayCache) get(ctx context.Context, fetcher *client.Fetcher) (client.R
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// fetchFromGitHubRelayBytes is the byte-returning twin of
|
||||
// serveFromGitHubRelay (cache lookup + GitHub fetch + decrypt + CRC
|
||||
// check). Returns (nil, nil) when the relay isn't configured.
|
||||
func (s *Server) fetchFromGitHubRelayBytes(ctx context.Context, size int64, crc uint32) ([]byte, error) {
|
||||
if size <= 0 || crc == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
s.mu.RLock()
|
||||
fetcher := s.fetcher
|
||||
rc := s.relayInfo
|
||||
cache := s.mediaCache
|
||||
cfg := s.config
|
||||
s.mu.RUnlock()
|
||||
if fetcher == nil || rc == nil || cfg == nil || cfg.Domain == "" {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
info, err := rc.get(ctx, fetcher)
|
||||
if err != nil || info.GitHubRepo == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if cfg.Key == "" {
|
||||
return nil, nil
|
||||
}
|
||||
relayKey, err := protocol.DeriveRelayKey(cfg.Key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
domainSeg := protocol.RelayDomainSegment(cfg.Domain, cfg.Key)
|
||||
objectSeg := protocol.RelayObjectName(size, crc, cfg.Key)
|
||||
|
||||
if cache != nil {
|
||||
if body, _, ok := cache.Get(size, crc); ok {
|
||||
return body, nil
|
||||
}
|
||||
}
|
||||
|
||||
url := fmt.Sprintf("https://api.github.com/repos/%s/contents/%s/%s",
|
||||
info.GitHubRepo, domainSeg, objectSeg)
|
||||
const aeadOverhead = protocol.NonceSize + 16
|
||||
encBody, _, err := fetchGitHubRaw(ctx, relayHTTPClient, url, size+int64(aeadOverhead))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
body, err := protocol.DecryptRelayBlob(relayKey, encBody)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if int64(len(body)) != size || crc32.ChecksumIEEE(body) != crc {
|
||||
return nil, errors.New("relay: hash/size mismatch")
|
||||
}
|
||||
if cache != nil {
|
||||
mime := http.DetectContentType(body)
|
||||
_ = cache.Put(size, crc, body, mime)
|
||||
}
|
||||
return body, nil
|
||||
}
|
||||
|
||||
// serveFromGitHubRelay tries to stream the file from raw.githubusercontent.com
|
||||
// Returns true if the request was fully handled (success or terminal error
|
||||
// already written). Returns false to let the caller fall back to DNS.
|
||||
@@ -133,9 +194,8 @@ func (s *Server) serveFromGitHubRelay(w http.ResponseWriter, r *http.Request, si
|
||||
return false
|
||||
}
|
||||
|
||||
// Long enough to cover a multi-MB GitHub fetch over a slow link, plus
|
||||
// a multi-block DNS-tunneled relay-info lookup if the cache is empty.
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 10*time.Minute)
|
||||
// Covers a cold-cache relay-info DNS lookup + GitHub fetch.
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
info, err := rc.get(ctx, fetcher)
|
||||
|
||||
@@ -313,6 +313,7 @@
|
||||
}
|
||||
|
||||
.ch-avatar {
|
||||
position: relative;
|
||||
width: 44px;
|
||||
height: 44px;
|
||||
border-radius: 50%;
|
||||
@@ -323,7 +324,17 @@
|
||||
font-size: 17px;
|
||||
color: var(--text-dim);
|
||||
flex-shrink: 0;
|
||||
font-weight: 600
|
||||
font-weight: 600;
|
||||
overflow: hidden
|
||||
}
|
||||
.ch-avatar-letter { line-height: 1 }
|
||||
/* Overlays the initial-letter span; img.onerror removes it and
|
||||
adds .ch-avatar-noimg so the letter shows through. */
|
||||
.ch-avatar-img {
|
||||
position: absolute; inset: 0;
|
||||
width: 100%; height: 100%;
|
||||
object-fit: cover;
|
||||
display: block
|
||||
}
|
||||
|
||||
.ch-item.active .ch-avatar {
|
||||
@@ -1311,6 +1322,53 @@
|
||||
a permanent broken-image icon. */
|
||||
.tm-photo-failed { display: none }
|
||||
|
||||
/* Reply quote — clickable, opens source in new tab. */
|
||||
.tm-post-reply {
|
||||
margin: 6px 0 8px;
|
||||
padding: 6px 10px;
|
||||
border-inline-start: 3px solid var(--accent);
|
||||
background: color-mix(in oklab, var(--accent) 6%, transparent);
|
||||
border-radius: 4px;
|
||||
font-size: 12.5px;
|
||||
max-height: 100px; overflow: hidden
|
||||
}
|
||||
.tm-post-reply-author {
|
||||
color: var(--accent);
|
||||
font-weight: 600;
|
||||
margin-bottom: 2px
|
||||
}
|
||||
.tm-post-reply-text {
|
||||
color: var(--text-dim);
|
||||
line-height: 1.4
|
||||
}
|
||||
.tm-post-reply-text br { display: block; margin-bottom: 2px }
|
||||
|
||||
/* "Forwarded from X" header. */
|
||||
.tm-post-forward {
|
||||
font-size: 12px;
|
||||
color: var(--text-dim);
|
||||
margin-bottom: 4px
|
||||
}
|
||||
.tm-post-forward a {
|
||||
color: var(--accent);
|
||||
text-decoration: none
|
||||
}
|
||||
.tm-post-forward a:hover { text-decoration: underline }
|
||||
|
||||
/* Poll options under the question. */
|
||||
.tm-poll-options {
|
||||
list-style: none; padding: 0; margin: 8px 0 0;
|
||||
display: flex; flex-direction: column; gap: 4px
|
||||
}
|
||||
.tm-poll-options li {
|
||||
padding: 4px 8px;
|
||||
background: var(--border);
|
||||
border-radius: 4px;
|
||||
font-size: 12.5px;
|
||||
color: var(--text)
|
||||
}
|
||||
.tm-poll-options li::before { content: "○ "; color: var(--text-dim) }
|
||||
|
||||
/* Confirm dialog rendered inside the telemirror modal so it sits
|
||||
on top of the drawer / backdrop / posts at the right stacking. */
|
||||
.tm-confirm-overlay {
|
||||
@@ -3051,6 +3109,20 @@
|
||||
<label for="cfgShowScanPrompt" data-i18n="show_scan_prompt">Show scan prompt on startup</label>
|
||||
</div>
|
||||
</div>
|
||||
<!-- Profile pictures. GitHub path is automatic; toggle opts into DNS. -->
|
||||
<div class="form-group">
|
||||
<div class="row">
|
||||
<input type="checkbox" id="cfgProfilePics" onchange="setProfilePicsEnabled(this.checked)">
|
||||
<label for="cfgProfilePics" data-i18n="profile_pics_enable">Also fetch profile pictures over DNS</label>
|
||||
</div>
|
||||
<div class="row" style="margin-top:4px;font-size:11px;color:var(--text-dim)">
|
||||
<span data-i18n="profile_pics_dns_note">GitHub relay path is always used automatically. This option additionally pulls avatars over DNS when the relay is unavailable.</span>
|
||||
</div>
|
||||
<div class="row" style="margin-top:6px;gap:8px;align-items:center">
|
||||
<button type="button" class="btn btn-flat" id="profilePicsRefreshBtn" onclick="profilePicsRefresh()" data-i18n="profile_pics_refresh">Refresh avatars</button>
|
||||
<span id="profilePicsProgress" style="font-size:11px;color:var(--text-dim)"></span>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label data-i18n="language">Language</label>
|
||||
<div class="lang-row">
|
||||
@@ -3510,10 +3582,18 @@
|
||||
telemirror_audio: 'فایل صوتی',
|
||||
telemirror_file: 'فایل',
|
||||
telemirror_poll: 'نظرسنجی',
|
||||
telemirror_forwarded_from: 'فروارد شده از',
|
||||
telemirror_about: 'دربارهی این کانال',
|
||||
download: 'دانلود',
|
||||
copy: 'کپی',
|
||||
// END telemirror
|
||||
profile_pics_enable: 'دریافت عکس پروفایل از طریق DNS',
|
||||
profile_pics_dns_note: 'عکسهایی که از طریق رله گیتهاب در دسترس باشند بهصورت خودکار دریافت میشوند. این گزینه فقط برای دریافت عکسهایی است که رله نمیتواند سرویس بدهد و باید از طریق DNS کند گرفته شوند.',
|
||||
profile_pics_refresh: 'بهروز کردن عکسها',
|
||||
profile_pics_starting: 'در حال شروع...',
|
||||
profile_pics_progress: 'در حال دریافت عکسها: {n}/{m}',
|
||||
profile_pics_done: 'انجام شد ({n} عکس)',
|
||||
profile_pics_failed: 'دریافت ناموفق بود',
|
||||
font_size: 'اندازه قلم', debug_mode: 'حالت دیباگ', language: 'زبان',
|
||||
next_fetch_info: 'زمان باقیمانده تا دریافت بعدی محتوا توسط سرور',
|
||||
no_profiles: 'هنوز پروفایلی وجود ندارد', add_profile: '+ پروفایل جدید',
|
||||
@@ -3743,6 +3823,7 @@
|
||||
telemirror_refresh: 'Refresh',
|
||||
telemirror_refresh_warn: 'This channel was refreshed {n} sec ago. Refreshing too often can hit a rate limit and stop working for a few minutes. Refresh anyway?',
|
||||
telemirror_refresh_yes: 'Refresh',
|
||||
telemirror_forwarded_from: 'Forwarded from',
|
||||
telemirror_voice: 'Voice message',
|
||||
telemirror_audio: 'Audio',
|
||||
telemirror_file: 'File',
|
||||
@@ -3751,6 +3832,13 @@
|
||||
download: 'Download',
|
||||
copy: 'Copy',
|
||||
// END telemirror
|
||||
profile_pics_enable: 'Also fetch profile pictures over DNS',
|
||||
profile_pics_dns_note: 'Avatars served by the GitHub relay are fetched automatically. Enable this to additionally fall back to DNS for avatars the relay can\'t serve (slower).',
|
||||
profile_pics_refresh: 'Refresh avatars',
|
||||
profile_pics_starting: 'Starting…',
|
||||
profile_pics_progress: 'Refreshing avatars: {n}/{m}',
|
||||
profile_pics_done: 'Done ({n} avatars)',
|
||||
profile_pics_failed: 'Refresh failed',
|
||||
font_size: 'Font Size', debug_mode: 'Debug mode', language: 'Language',
|
||||
next_fetch_info: 'Time until the server next fetches fresh channel content',
|
||||
no_profiles: 'No profiles yet', add_profile: '+ Add Profile',
|
||||
@@ -4049,6 +4137,9 @@
|
||||
loadBgImage();
|
||||
connectSSE();
|
||||
refreshResolversBadge();
|
||||
// Populate profilePicCache so the channel list renders avatars
|
||||
// without a per-item probe.
|
||||
loadProfilePicState().catch(function () { });
|
||||
// Quietly ask GitHub for the latest published client version. Runs in
|
||||
// the background so a slow github.com response can't delay startup —
|
||||
// if there's an update, the dialog shows up a few seconds later.
|
||||
@@ -4312,6 +4403,74 @@
|
||||
// settings is opened.
|
||||
var promptEl = document.getElementById('cfgShowScanPrompt');
|
||||
if (promptEl) promptEl.checked = localStorage.getItem('thefeed_scan_prompt_off') !== '1';
|
||||
// Sync the profile-pics toggle from the server (it's persisted in
|
||||
// profiles.json so survives app restarts).
|
||||
fetch('/api/settings').then(function (r) { return r.ok ? r.json() : null; }).then(function (s) {
|
||||
if (!s) return;
|
||||
var pp = document.getElementById('cfgProfilePics');
|
||||
if (pp) pp.checked = !!s.profilePicsEnabled;
|
||||
}).catch(function () { });
|
||||
}
|
||||
|
||||
// ===== PROFILE PICTURES =====
|
||||
// Cached lowercase usernames so renderChannels can decide whether
|
||||
// to overlay an <img> over the initial-letter circle.
|
||||
var profilePicCache = { enabled: false, users: {} };
|
||||
var profilePicsPollTimer = null;
|
||||
// SSE throttle state — server fires one event per stored avatar.
|
||||
var profilePicsReloadTimer = null;
|
||||
var profilePicsLastReloadAt = 0;
|
||||
|
||||
function loadProfilePicState() {
|
||||
// no-store so SSE reloads see fresh data, not a cached entry.
|
||||
return fetch('/api/profile-pics', { cache: 'no-store' }).then(function (r) { return r.ok ? r.json() : null; }).then(function (d) {
|
||||
if (!d) return;
|
||||
profilePicCache.enabled = !!d.enabled;
|
||||
profilePicCache.users = {};
|
||||
(d.users || []).forEach(function (u) { profilePicCache.users[u.toLowerCase()] = true; });
|
||||
try { renderChannels(); } catch (e) { }
|
||||
}).catch(function () { });
|
||||
}
|
||||
|
||||
function setProfilePicsEnabled(enabled) {
|
||||
// Toggle only controls the DNS path; cached GitHub avatars keep
|
||||
// displaying either way.
|
||||
fetch('/api/settings', {
|
||||
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ profilePicsEnabled: !!enabled })
|
||||
}).then(function () {
|
||||
profilePicCache.enabled = !!enabled;
|
||||
if (enabled) profilePicsRefresh();
|
||||
}).catch(function () { });
|
||||
}
|
||||
|
||||
function profilePicsRefresh() {
|
||||
var prog = document.getElementById('profilePicsProgress');
|
||||
if (prog) prog.textContent = t('profile_pics_starting') || 'Starting…';
|
||||
fetch('/api/profile-pics/refresh', { method: 'POST' }).then(function (r) {
|
||||
if (!r.ok) {
|
||||
if (prog) prog.textContent = t('profile_pics_failed') || 'Failed';
|
||||
return;
|
||||
}
|
||||
// Poll progress until inactive.
|
||||
if (profilePicsPollTimer) clearInterval(profilePicsPollTimer);
|
||||
profilePicsPollTimer = setInterval(function () {
|
||||
fetch('/api/profile-pics/progress').then(function (r) { return r.ok ? r.json() : null; }).then(function (p) {
|
||||
if (!p) return;
|
||||
if (prog) {
|
||||
if (p.active) {
|
||||
var label = (t('profile_pics_progress') || 'Refreshing avatars: {n}/{m}').replace('{n}', p.done || 0).replace('{m}', p.total || 0);
|
||||
prog.textContent = label + (p.username ? ' (@' + p.username + ')' : '');
|
||||
} else {
|
||||
prog.textContent = (t('profile_pics_done') || 'Done').replace('{n}', p.done || 0);
|
||||
clearInterval(profilePicsPollTimer);
|
||||
profilePicsPollTimer = null;
|
||||
loadProfilePicState();
|
||||
}
|
||||
}
|
||||
});
|
||||
}, 800);
|
||||
}).catch(function () { });
|
||||
}
|
||||
|
||||
// Toggle the "show startup scan prompt" preference. Persists
|
||||
@@ -5055,6 +5214,25 @@
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Throttle (not debounce): first event fires immediately,
|
||||
// subsequent events at most every 600 ms during a burst.
|
||||
if (data === 'profile-pics') {
|
||||
var THROTTLE_MS = 600;
|
||||
var now = Date.now();
|
||||
var since = now - profilePicsLastReloadAt;
|
||||
if (since >= THROTTLE_MS) {
|
||||
profilePicsLastReloadAt = now;
|
||||
loadProfilePicState().catch(function () { });
|
||||
} else if (!profilePicsReloadTimer) {
|
||||
// Trailing call for the last avatar in the burst.
|
||||
profilePicsReloadTimer = setTimeout(function () {
|
||||
profilePicsReloadTimer = null;
|
||||
profilePicsLastReloadAt = Date.now();
|
||||
loadProfilePicState().catch(function () { });
|
||||
}, THROTTLE_MS - since);
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Another tab/device switched the active profile.
|
||||
if (data === 'profiles') {
|
||||
var prevActive = activeProfileId;
|
||||
@@ -5676,6 +5854,35 @@
|
||||
var key = chNm.replace(/^@/, '').trim();
|
||||
autoBtn.classList.toggle('on', autoUpdateChannels.has(key));
|
||||
}
|
||||
// Avatar diff against profilePicCache. Without this the
|
||||
// fast path skips avatar updates and SSE-driven reloads
|
||||
// don't reach the UI.
|
||||
var ct2 = channels[ui].ChatType || channels[ui].chatType || 0;
|
||||
var isX2 = ct2 === 2;
|
||||
var bareHandle2 = chNm.replace(/^@/, '').toLowerCase();
|
||||
if (isX2 && bareHandle2.indexOf('x/') === 0) bareHandle2 = bareHandle2.substring(2);
|
||||
var unameKey2 = isX2 ? ('x:' + bareHandle2) : bareHandle2;
|
||||
var avatarEl = existingItems[ui].querySelector('.ch-avatar');
|
||||
if (avatarEl) {
|
||||
var imgEl = avatarEl.querySelector('.ch-avatar-img');
|
||||
var shouldShow = !!profilePicCache.users[unameKey2];
|
||||
if (shouldShow && !imgEl) {
|
||||
var img = document.createElement('img');
|
||||
img.className = 'ch-avatar-img';
|
||||
img.src = '/api/profile-pics/' + encodeURIComponent(unameKey2);
|
||||
img.loading = 'lazy';
|
||||
img.alt = '';
|
||||
img.onerror = function () {
|
||||
this.parentNode.classList.add('ch-avatar-noimg');
|
||||
this.remove();
|
||||
};
|
||||
avatarEl.classList.remove('ch-avatar-noimg');
|
||||
avatarEl.insertBefore(img, avatarEl.firstChild);
|
||||
} else if (!shouldShow && imgEl) {
|
||||
imgEl.remove();
|
||||
avatarEl.classList.add('ch-avatar-noimg');
|
||||
}
|
||||
}
|
||||
}
|
||||
_updateRefreshBadge(); return;
|
||||
}
|
||||
@@ -5706,7 +5913,19 @@
|
||||
var chNm2 = e.ch.Name || e.ch.name || '';
|
||||
var badge = (channelHasNew(e.ch) && num2 !== selectedChannel) ? '<span class="ch-badge">NEW</span>' : '';
|
||||
h += '<div class="ch-item' + active + '" data-name="' + escAttr(handle) + '" data-label="' + escAttr(label) + '" onclick="selectChannel(' + num2 + ')">';
|
||||
h += '<div class="ch-avatar">' + esc(avatarText) + '</div>';
|
||||
// X channels: e.ch.Name is "x/<handle>" (category prefix);
|
||||
// strip before re-attaching the bundle's "x:" prefix.
|
||||
var avatarImg = '';
|
||||
var bareHandle = (chNm2 || avatarName || '').replace(/^@/, '').toLowerCase();
|
||||
if (isX && bareHandle.indexOf('x/') === 0) bareHandle = bareHandle.substring(2);
|
||||
var unameKey = isX ? ('x:' + bareHandle) : bareHandle;
|
||||
if (profilePicCache.users[unameKey]) {
|
||||
avatarImg = '<img class="ch-avatar-img" src="/api/profile-pics/'
|
||||
+ encodeURIComponent(unameKey)
|
||||
+ '" loading="lazy" alt=""'
|
||||
+ ' onerror="this.parentNode.classList.add(\'ch-avatar-noimg\');this.remove()">';
|
||||
}
|
||||
h += '<div class="ch-avatar">' + avatarImg + '<span class="ch-avatar-letter">' + esc(avatarText) + '</span></div>';
|
||||
var chSubText = !isX ? (handle.charAt(0) === '@' ? handle : '@' + handle) : '';
|
||||
h += '<div class="ch-info"><div class="ch-name">' + formatIranTitleHtml(label) + (isPriv ? '<span class="ch-type-tag">' + t('private') + '</span>' : (isX ? '<span class="ch-type-tag x-tag">' + t('x_label') + '</span>' : '')) + '</div>';
|
||||
if (chSubText) h += '<div class="ch-sub">' + esc(chSubText) + '</div>';
|
||||
|
||||
@@ -360,6 +360,28 @@
|
||||
}
|
||||
html += '</div>';
|
||||
|
||||
if (p.forward && p.forward.author) {
|
||||
var fwdLabel = tmI18n('telemirror_forwarded_from', 'Forwarded from');
|
||||
var fwdName = tmEsc(p.forward.author);
|
||||
if (p.forward.url) {
|
||||
fwdName = '<a href="' + tmEscAttr(p.forward.url) + '" target="_blank" rel="noopener noreferrer">'
|
||||
+ fwdName + '</a>';
|
||||
}
|
||||
html += '<div class="tm-post-forward">↪ ' + tmEsc(fwdLabel) + ' ' + fwdName + '</div>';
|
||||
}
|
||||
|
||||
if (p.reply) {
|
||||
var rAuth = p.reply.author ? tmEsc(p.reply.author) : '';
|
||||
var rText = p.reply.text || '';
|
||||
html += '<div class="tm-post-reply"'
|
||||
+ (p.reply.url ? ' onclick="window.open(\'' + tmEscAttr(p.reply.url) + '\', \'_blank\')"' : '')
|
||||
+ (p.reply.url ? ' style="cursor:pointer"' : '')
|
||||
+ '>';
|
||||
if (rAuth) html += '<div class="tm-post-reply-author">' + rAuth + '</div>';
|
||||
if (rText) html += '<div class="tm-post-reply-text">' + rText + '</div>';
|
||||
html += '</div>';
|
||||
}
|
||||
|
||||
if (p.text) html += '<div class="tm-post-text">' + p.text + '</div>';
|
||||
|
||||
if (p.media && p.media.length) {
|
||||
@@ -450,8 +472,9 @@
|
||||
+ ' onerror="this.parentNode.classList.add(\'tm-photo-failed\')">'
|
||||
+ '<a class="tm-photo-dl" href="' + tmEscAttr(m.thumb) + '"'
|
||||
+ ' download="' + tmEscAttr(fname) + '"'
|
||||
+ ' data-fname="' + tmEscAttr(fname) + '"'
|
||||
+ ' title="' + tmEscAttr(tmI18n('download', 'Download')) + '"'
|
||||
+ ' onclick="event.stopPropagation()">⬇</a>'
|
||||
+ ' onclick="return tmDownloadPhoto(this, event)">⬇</a>'
|
||||
+ '</div>';
|
||||
}
|
||||
if (m.type === 'video') {
|
||||
@@ -486,14 +509,81 @@
|
||||
+ '</div>';
|
||||
}
|
||||
if (m.type === 'poll') {
|
||||
return '<div class="tm-media-tile"><span class="tm-media-icon">📊</span>'
|
||||
var optionsHtml = '';
|
||||
if (m.options && m.options.length) {
|
||||
optionsHtml = '<ul class="tm-poll-options">';
|
||||
for (var k = 0; k < m.options.length; k++) {
|
||||
optionsHtml += '<li>' + tmEsc(m.options[k]) + '</li>';
|
||||
}
|
||||
optionsHtml += '</ul>';
|
||||
}
|
||||
return '<div class="tm-media-tile tm-media-poll"><span class="tm-media-icon">📊</span>'
|
||||
+ '<div class="tm-media-meta"><div class="tm-media-title">'
|
||||
+ tmEsc(m.title || tmI18n('telemirror_poll', 'Poll'))
|
||||
+ '</div><div class="tm-media-sub">' + tmEsc(m.subtitle || '') + '</div></div></div>';
|
||||
+ '</div><div class="tm-media-sub">' + tmEsc(m.subtitle || '') + '</div>'
|
||||
+ optionsHtml
|
||||
+ '</div></div>';
|
||||
}
|
||||
return '';
|
||||
}
|
||||
|
||||
// tmDownloadPhoto fetches the bytes and either hands them to the
|
||||
// Android bridge (saveMedia) or builds a blob-URL <a download> on
|
||||
// desktop. <a download> alone doesn't work on Android WebView for
|
||||
// cross-origin URLs.
|
||||
window.tmDownloadPhoto = function (anchor, ev) {
|
||||
if (ev) ev.stopPropagation();
|
||||
var url = anchor.getAttribute('href');
|
||||
var fname = anchor.getAttribute('data-fname') || 'photo.jpg';
|
||||
var bridge = (typeof window !== 'undefined' && window.Android) ? window.Android : null;
|
||||
|
||||
var doFetch = function () {
|
||||
return fetch(url, { referrerPolicy: 'no-referrer' }).then(function (r) {
|
||||
if (!r.ok) throw new Error('http ' + r.status);
|
||||
return r.blob();
|
||||
});
|
||||
};
|
||||
|
||||
if (bridge && typeof bridge.saveMedia === 'function') {
|
||||
doFetch().then(function (blob) {
|
||||
return tmBlobToBase64(blob).then(function (b64) {
|
||||
try { bridge.saveMedia(b64, blob.type || 'image/jpeg', fname); }
|
||||
catch (e) { tmFallbackOpen(url); }
|
||||
});
|
||||
}).catch(function () { tmFallbackOpen(url); });
|
||||
return false;
|
||||
}
|
||||
|
||||
doFetch().then(function (blob) {
|
||||
var objectUrl = URL.createObjectURL(blob);
|
||||
var a = document.createElement('a');
|
||||
a.href = objectUrl;
|
||||
a.download = fname;
|
||||
a.style.display = 'none';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
setTimeout(function () { URL.revokeObjectURL(objectUrl); a.remove(); }, 100);
|
||||
}).catch(function () { window.location.href = url; });
|
||||
return false;
|
||||
};
|
||||
|
||||
function tmBlobToBase64(blob) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
var fr = new FileReader();
|
||||
fr.onload = function () {
|
||||
var s = fr.result || '';
|
||||
var i = s.indexOf(',');
|
||||
resolve(i >= 0 ? s.substring(i + 1) : s);
|
||||
};
|
||||
fr.onerror = function () { reject(fr.error); };
|
||||
fr.readAsDataURL(blob);
|
||||
});
|
||||
}
|
||||
|
||||
function tmFallbackOpen(url) {
|
||||
try { window.open(url, '_blank'); } catch (e) { window.location.href = url; }
|
||||
}
|
||||
|
||||
window.tmCopyPost = function (btn) {
|
||||
var post = btn.closest ? btn.closest('.tm-post') : null;
|
||||
var pid = post ? post.getAttribute('data-pid') : '';
|
||||
|
||||
+71
-6
@@ -117,6 +117,10 @@ type ProfileList struct {
|
||||
// changes (each launch picks a fresh port → different localStorage
|
||||
// origin → flag was lost on every restart).
|
||||
ScanPromptOff bool `json:"scanPromptOff,omitempty"`
|
||||
|
||||
// ProfilePicsEnabled enables fetching avatars over DNS when the
|
||||
// GitHub relay can't serve them. Off by default.
|
||||
ProfilePicsEnabled bool `json:"profilePicsEnabled,omitempty"`
|
||||
}
|
||||
|
||||
// lastScanData is the on-disk structure for last_scan.json.
|
||||
@@ -216,6 +220,9 @@ type Server struct {
|
||||
|
||||
// Optional, removable backup feed (Telegram-via-Translate proxy).
|
||||
telemirror *telemirrorHub
|
||||
|
||||
// Optional per-channel profile pictures cache.
|
||||
profilePics *profilePicsHub
|
||||
}
|
||||
|
||||
// New creates a new web server.
|
||||
@@ -253,6 +260,7 @@ func New(dataDir string, port int, host string, password string) (*Server, error
|
||||
dlProgress: make(map[string]*mediaDLProgress),
|
||||
relayInfo: newRelayCache(),
|
||||
telemirror: newTelemirrorHub(dataDir),
|
||||
profilePics: newProfilePicsHub(dataDir),
|
||||
}
|
||||
|
||||
if mediaCache != nil {
|
||||
@@ -345,6 +353,11 @@ func (s *Server) Run() error {
|
||||
mux.HandleFunc("/api/telemirror/channels", s.telemirror.handleChannels)
|
||||
mux.HandleFunc("/api/telemirror/channel/", s.telemirror.handleChannel)
|
||||
mux.HandleFunc("/api/telemirror/img", s.telemirror.handleImg)
|
||||
// Profile-pics cache + control endpoints.
|
||||
mux.HandleFunc("/api/profile-pics/", s.profilePics.handleProfilePic)
|
||||
mux.HandleFunc("/api/profile-pics", s.handleProfilePicsList)
|
||||
mux.HandleFunc("/api/profile-pics/refresh", s.handleProfilePicsRefresh)
|
||||
mux.HandleFunc("/api/profile-pics/progress", s.handleProfilePicsProgress)
|
||||
mux.HandleFunc("/", s.handleIndex)
|
||||
|
||||
// Listen on the specified host (default 127.0.0.1)
|
||||
@@ -1319,6 +1332,42 @@ func (s *Server) refreshMetadataOnly() {
|
||||
if needsFetch {
|
||||
go s.ensureTitlesFetched(basectx)
|
||||
}
|
||||
|
||||
go s.maybeRefreshProfilePics(basectx)
|
||||
}
|
||||
|
||||
// maybeRefreshProfilePics fires a refresh when GitHub relay is up or
|
||||
// the user has opted into the DNS path. No-op otherwise; hub coalesces.
|
||||
func (s *Server) maybeRefreshProfilePics(parentCtx context.Context) {
|
||||
s.mu.RLock()
|
||||
hub := s.profilePics
|
||||
fetcher := s.fetcher
|
||||
rc := s.relayInfo
|
||||
s.mu.RUnlock()
|
||||
if hub == nil || fetcher == nil {
|
||||
return
|
||||
}
|
||||
dnsAllowed := s.profilePicsEnabled()
|
||||
githubLikelyUp := false
|
||||
if rc != nil {
|
||||
ctx, cancel := context.WithTimeout(parentCtx, 5*time.Second)
|
||||
info, err := rc.get(ctx, fetcher)
|
||||
cancel()
|
||||
if err == nil && info.GitHubRepo != "" {
|
||||
githubLikelyUp = true
|
||||
}
|
||||
}
|
||||
if !dnsAllowed && !githubLikelyUp {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
onStored := func(string) {
|
||||
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
|
||||
}
|
||||
if err := hub.refresh(ctx, fetcher, dnsAllowed, s.fetchFromGitHubRelayBytes, onStored); err == nil {
|
||||
s.broadcast("event: update\ndata: \"profile-pics\"\n\n")
|
||||
}
|
||||
}
|
||||
|
||||
// ensureTitlesFetched fetches channel display names from TitlesChannel in the background.
|
||||
@@ -2758,16 +2807,26 @@ func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) {
|
||||
if pl == nil {
|
||||
pl = &ProfileList{}
|
||||
}
|
||||
writeJSON(w, map[string]any{"fontSize": pl.FontSize, "debug": pl.Debug, "theme": pl.Theme, "lang": pl.Lang, "scanPromptOff": pl.ScanPromptOff, "version": version.Version, "commit": version.Commit})
|
||||
writeJSON(w, map[string]any{
|
||||
"fontSize": pl.FontSize,
|
||||
"debug": pl.Debug,
|
||||
"theme": pl.Theme,
|
||||
"lang": pl.Lang,
|
||||
"scanPromptOff": pl.ScanPromptOff,
|
||||
"profilePicsEnabled": pl.ProfilePicsEnabled,
|
||||
"version": version.Version,
|
||||
"commit": version.Commit,
|
||||
})
|
||||
|
||||
case http.MethodPost:
|
||||
// Optional pointers so partial requests don't reset other fields.
|
||||
var req struct {
|
||||
FontSize *int `json:"fontSize"`
|
||||
Debug *bool `json:"debug"`
|
||||
Theme *string `json:"theme"`
|
||||
Lang *string `json:"lang"`
|
||||
ScanPromptOff *bool `json:"scanPromptOff"`
|
||||
FontSize *int `json:"fontSize"`
|
||||
Debug *bool `json:"debug"`
|
||||
Theme *string `json:"theme"`
|
||||
Lang *string `json:"lang"`
|
||||
ScanPromptOff *bool `json:"scanPromptOff"`
|
||||
ProfilePicsEnabled *bool `json:"profilePicsEnabled"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
http.Error(w, "invalid JSON", 400)
|
||||
@@ -2805,6 +2864,9 @@ func (s *Server) handleSettings(w http.ResponseWriter, r *http.Request) {
|
||||
if req.ScanPromptOff != nil {
|
||||
pl.ScanPromptOff = *req.ScanPromptOff
|
||||
}
|
||||
if req.ProfilePicsEnabled != nil {
|
||||
pl.ProfilePicsEnabled = *req.ProfilePicsEnabled
|
||||
}
|
||||
if err := s.saveProfiles(pl); err != nil {
|
||||
http.Error(w, fmt.Sprintf("save: %v", err), 500)
|
||||
return
|
||||
@@ -3585,6 +3647,9 @@ func (s *Server) handleClearCache(w http.ResponseWriter, r *http.Request) {
|
||||
if s.telemirror != nil {
|
||||
s.telemirror.ClearCache()
|
||||
}
|
||||
if s.profilePics != nil {
|
||||
s.profilePics.Clear()
|
||||
}
|
||||
mediaDeleted := 0
|
||||
if s.mediaCache != nil {
|
||||
mediaDeleted = s.mediaCache.Clear()
|
||||
|
||||
Reference in New Issue
Block a user