feat: media download with DNS query

This commit is contained in:
Sarto
2026-04-29 01:45:27 +03:30
parent 11946c0147
commit b4e9cd8714
34 changed files with 6303 additions and 137 deletions
+229
View File
@@ -0,0 +1,229 @@
package protocol
import (
"encoding/hex"
"fmt"
"hash/fnv"
"strconv"
"strings"
)
// MediaMeta describes a downloadable media blob attached to a feed message.
//
// Wire format embedded in a message's text body (immediately after the media
// tag, before any caption):
//
// [IMAGE]<size>:<dl>:<ch>:<blk>:<crc32hex>[:<filename>]
// caption goes here on the next line(s)
//
// The filename field is optional; when present it carries an OS-friendly
// suggested filename (server-sanitised: no newlines, no path separators, no
// control characters, length-capped). Old clients that split on ':' and
// only read parts[0..4] keep working — they just ignore the trailing field.
type MediaMeta struct {
Tag string // e.g. MediaImage, MediaVideo, MediaFile
Size int64
Downloadable bool
Channel uint16
Blocks uint16
CRC32 uint32
Filename string
}
// String renders the metadata in the wire format documented above, including
// the leading tag and trailing newline that separates the metadata row from
// any caption.
func (m MediaMeta) String() string {
dl := 0
if m.Downloadable {
dl = 1
}
if fn := SanitiseMediaFilename(m.Filename); fn != "" {
return fmt.Sprintf("%s%d:%d:%d:%d:%08x:%s\n",
m.Tag, m.Size, dl, m.Channel, m.Blocks, m.CRC32, fn)
}
return fmt.Sprintf("%s%d:%d:%d:%d:%08x\n",
m.Tag, m.Size, dl, m.Channel, m.Blocks, m.CRC32)
}
// SanitiseMediaFilename returns a filename safe to embed in the wire
// metadata line. The output uses a restricted alphabet ([A-Za-z0-9._-]) so
// no path separator, colon, newline, or control char can ever survive.
// When the input is too long the base name is replaced with a short
// hash-derived id but the extension is preserved so other OSes still
// recognise the file type.
func SanitiseMediaFilename(s string) string {
s = strings.TrimSpace(s)
if s == "" {
return ""
}
if i := strings.LastIndexAny(s, `/\`); i >= 0 {
s = s[i+1:]
}
cleaned := filterFilenameRunes(s)
if cleaned == "" || cleaned == "." || cleaned == ".." {
return ""
}
const maxBase = 24
const maxExt = 8
base, ext := splitFilenameExt(cleaned)
if len(ext) > maxExt {
ext = ext[:maxExt]
}
if len(base) > maxBase {
h := fnv.New64a()
_, _ = h.Write([]byte(cleaned))
base = "media-" + hex.EncodeToString(h.Sum(nil))[:8]
}
if base == "" || base == "." {
base = "media"
}
if ext != "" {
return base + "." + ext
}
return base
}
func filterFilenameRunes(s string) string {
var b strings.Builder
for _, r := range s {
switch {
case r >= '0' && r <= '9',
r >= 'A' && r <= 'Z',
r >= 'a' && r <= 'z',
r == '.', r == '_', r == '-':
b.WriteRune(r)
}
}
return b.String()
}
func splitFilenameExt(s string) (base, ext string) {
if i := strings.LastIndexByte(s, '.'); i >= 0 && i < len(s)-1 {
return s[:i], s[i+1:]
}
return s, ""
}
// EncodeMediaText prepends the metadata line to an optional caption and
// returns the combined message text. A nil/empty caption yields just the tag
// + metadata + trailing newline-less string (the caption split is by the
// metadata line's trailing \n, so an empty caption simply has no extra body).
func EncodeMediaText(meta MediaMeta, caption string) string {
header := meta.String()
if caption == "" {
// Drop the trailing newline so the message text doesn't end with a
// blank line for caption-less media.
return strings.TrimSuffix(header, "\n")
}
return header + caption
}
// ParseMediaText parses a message body that begins with a known media tag.
// On success it returns the metadata and the remaining caption (which may be
// empty). When the body uses the legacy "[TAG]\ncaption" form (no metadata
// suffix), ParseMediaText returns ok=true with Downloadable=false and
// Channel=0 — the caller can treat it as a non-downloadable placeholder
// exactly like before.
//
// Unknown tags return ok=false. Malformed metadata for a known tag also
// returns ok=false so the caller falls back to legacy display.
func ParseMediaText(body string) (meta MediaMeta, caption string, ok bool) {
tag, rest, found := splitKnownMediaTag(body)
if !found {
return MediaMeta{}, body, false
}
meta.Tag = tag
// The bit between the tag and the first newline is the metadata payload.
nl := strings.IndexByte(rest, '\n')
var metaLine string
if nl < 0 {
metaLine = rest
caption = ""
} else {
metaLine = rest[:nl]
caption = rest[nl+1:]
}
metaLine = strings.TrimSpace(metaLine)
if metaLine == "" {
// Legacy [TAG]\ncaption — no per-file metadata. Treat as not-downloadable.
return MediaMeta{Tag: tag}, caption, true
}
parts := strings.Split(metaLine, ":")
if len(parts) < 5 {
// Looks like a caption line that happens to start with this tag (e.g.
// "[IMAGE]nice photo"). Don't claim a structured parse — return the
// whole `rest` as caption so the message still renders.
return MediaMeta{Tag: tag}, rest, true
}
size, err := strconv.ParseInt(parts[0], 10, 64)
if err != nil || size < 0 {
return MediaMeta{Tag: tag}, rest, true
}
dl, err := strconv.Atoi(parts[1])
if err != nil || (dl != 0 && dl != 1) {
return MediaMeta{Tag: tag}, rest, true
}
ch, err := strconv.ParseUint(parts[2], 10, 16)
if err != nil {
return MediaMeta{Tag: tag}, rest, true
}
blk, err := strconv.ParseUint(parts[3], 10, 16)
if err != nil {
return MediaMeta{Tag: tag}, rest, true
}
crc, err := strconv.ParseUint(parts[4], 16, 32)
if err != nil {
return MediaMeta{Tag: tag}, rest, true
}
// Reject any channel claimed inside a parseable metadata line that falls
// outside the reserved media range — that can only be a malformed message
// or a tampering attempt; refuse to surface it as downloadable.
channel := uint16(ch)
downloadable := dl == 1
if downloadable && (!IsMediaChannel(channel) || blk == 0) {
downloadable = false
}
meta.Size = size
meta.Downloadable = downloadable
meta.Channel = channel
meta.Blocks = uint16(blk)
meta.CRC32 = uint32(crc)
if len(parts) >= 6 {
// SanitiseMediaFilename strips the field separator, so we can't
// reach this point with a colon inside the filename. Take parts[5]
// directly and re-sanitise defensively.
meta.Filename = SanitiseMediaFilename(parts[5])
}
return meta, caption, true
}
// knownMediaTags are the message text prefixes that mark a downloadable media
// attachment. Order matters only for prefix matching; longer/more-specific
// tags are not currently aliased so the order is alphabetical for clarity.
var knownMediaTags = []string{
MediaAudio,
MediaFile,
MediaGIF,
MediaImage,
MediaSticker,
MediaVideo,
}
// splitKnownMediaTag returns the matched tag and the remainder of the body
// when body starts with one of knownMediaTags.
func splitKnownMediaTag(body string) (tag, rest string, ok bool) {
for _, t := range knownMediaTags {
if strings.HasPrefix(body, t) {
return t, body[len(t):], true
}
}
return "", body, false
}
+102
View File
@@ -0,0 +1,102 @@
package protocol
import (
"encoding/binary"
"fmt"
)
// MediaCompression names a compression method applied to a cached media
// file's bytes before they're split into DNS blocks.
type MediaCompression byte
const (
MediaCompressionNone MediaCompression = 0
MediaCompressionGzip MediaCompression = 1
MediaCompressionDeflate MediaCompression = 2
)
// MediaHeaderVersion is the current header version. Bumped when the layout
// changes incompatibly; until then, the reserved bytes carry future fields.
const MediaHeaderVersion uint8 = 1
// MediaBlockHeaderLen is the fixed length of the metadata prefix that the
// server prepends to a cached media file's bytes before splitting into
// blocks. Block 0 of every media channel begins with these bytes.
//
// Layout (big-endian where multi-byte):
// [0:4] CRC32(IEEE) of the DECOMPRESSED file content
// [4] header version (currently 1)
// [5] compression byte (MediaCompression*)
// [6:16] reserved (zero) — room for future protocol fields without
// bumping the version byte
const MediaBlockHeaderLen = 16
// MediaBlockHeader is the parsed form of a media-channel block-0 header.
type MediaBlockHeader struct {
CRC32 uint32
Version uint8
Compression MediaCompression
}
// EncodeMediaBlockHeader writes the binary header into a fresh slice of
// length MediaBlockHeaderLen. Reserved bytes are zero-padded.
func EncodeMediaBlockHeader(h MediaBlockHeader) []byte {
buf := make([]byte, MediaBlockHeaderLen)
binary.BigEndian.PutUint32(buf[0:4], h.CRC32)
if h.Version == 0 {
h.Version = MediaHeaderVersion
}
buf[4] = h.Version
buf[5] = byte(h.Compression)
return buf
}
// DecodeMediaBlockHeader parses the first MediaBlockHeaderLen bytes of a
// media block. Errors on truncation or unknown header version.
func DecodeMediaBlockHeader(b []byte) (MediaBlockHeader, error) {
if len(b) < MediaBlockHeaderLen {
return MediaBlockHeader{}, fmt.Errorf("media block header truncated: have %d bytes, need %d", len(b), MediaBlockHeaderLen)
}
h := MediaBlockHeader{
CRC32: binary.BigEndian.Uint32(b[0:4]),
Version: b[4],
Compression: MediaCompression(b[5]),
}
if h.Version != MediaHeaderVersion {
return MediaBlockHeader{}, fmt.Errorf("media block header version %d not supported (want %d)", h.Version, MediaHeaderVersion)
}
switch h.Compression {
case MediaCompressionNone, MediaCompressionGzip, MediaCompressionDeflate:
default:
return MediaBlockHeader{}, fmt.Errorf("media block header: unknown compression %d", h.Compression)
}
return h, nil
}
// ParseMediaCompressionName returns the MediaCompression matching one of
// "none", "gzip", "deflate" (case-insensitive). Used by the CLI flag to
// translate user input.
func ParseMediaCompressionName(s string) (MediaCompression, error) {
switch s {
case "", "none":
return MediaCompressionNone, nil
case "gzip":
return MediaCompressionGzip, nil
case "deflate":
return MediaCompressionDeflate, nil
}
return 0, fmt.Errorf("unknown media compression %q", s)
}
// String returns the canonical name of the compression value.
func (c MediaCompression) String() string {
switch c {
case MediaCompressionNone:
return "none"
case MediaCompressionGzip:
return "gzip"
case MediaCompressionDeflate:
return "deflate"
}
return fmt.Sprintf("unknown(%d)", byte(c))
}
+78
View File
@@ -0,0 +1,78 @@
package protocol
import (
"bytes"
"testing"
)
func TestEncodeDecodeMediaBlockHeader(t *testing.T) {
cases := []MediaBlockHeader{
{CRC32: 0x01020304, Version: MediaHeaderVersion, Compression: MediaCompressionNone},
{CRC32: 0xdeadbeef, Version: MediaHeaderVersion, Compression: MediaCompressionGzip},
{CRC32: 0, Version: MediaHeaderVersion, Compression: MediaCompressionDeflate},
}
for _, h := range cases {
buf := EncodeMediaBlockHeader(h)
if len(buf) != MediaBlockHeaderLen {
t.Fatalf("encoded length = %d, want %d", len(buf), MediaBlockHeaderLen)
}
// Reserved bytes must be zero for forward compatibility.
if !bytes.Equal(buf[6:], make([]byte, MediaBlockHeaderLen-6)) {
t.Fatalf("reserved bytes not zero: %x", buf[6:])
}
got, err := DecodeMediaBlockHeader(buf)
if err != nil {
t.Fatalf("Decode: %v", err)
}
if got != h {
t.Fatalf("round-trip: got %+v, want %+v", got, h)
}
}
}
func TestDecodeMediaBlockHeaderRejectsBadVersion(t *testing.T) {
buf := EncodeMediaBlockHeader(MediaBlockHeader{CRC32: 1, Version: MediaHeaderVersion, Compression: MediaCompressionNone})
buf[4] = 9 // bogus version
_, err := DecodeMediaBlockHeader(buf)
if err == nil {
t.Fatal("expected error for unknown version")
}
}
func TestDecodeMediaBlockHeaderRejectsBadCompression(t *testing.T) {
buf := EncodeMediaBlockHeader(MediaBlockHeader{Version: MediaHeaderVersion})
buf[5] = 99
_, err := DecodeMediaBlockHeader(buf)
if err == nil {
t.Fatal("expected error for unknown compression")
}
}
func TestDecodeMediaBlockHeaderRejectsTruncated(t *testing.T) {
_, err := DecodeMediaBlockHeader(make([]byte, MediaBlockHeaderLen-1))
if err == nil {
t.Fatal("expected error for truncated header")
}
}
func TestParseMediaCompressionName(t *testing.T) {
cases := map[string]MediaCompression{
"": MediaCompressionNone,
"none": MediaCompressionNone,
"gzip": MediaCompressionGzip,
"deflate": MediaCompressionDeflate,
}
for in, want := range cases {
got, err := ParseMediaCompressionName(in)
if err != nil {
t.Errorf("ParseMediaCompressionName(%q): %v", in, err)
continue
}
if got != want {
t.Errorf("ParseMediaCompressionName(%q) = %v, want %v", in, got, want)
}
}
if _, err := ParseMediaCompressionName("brotli"); err == nil {
t.Fatal("expected error for unknown compression name")
}
}
+232
View File
@@ -0,0 +1,232 @@
package protocol
import (
"strings"
"testing"
)
func TestEncodeMediaTextRoundTrip(t *testing.T) {
cases := []struct {
name string
meta MediaMeta
caption string
}{
{
name: "image with caption",
meta: MediaMeta{
Tag: MediaImage,
Size: 123456,
Downloadable: true,
Channel: 12345,
Blocks: 42,
CRC32: 0xabcdef01,
},
caption: "hello world\nmulti-line",
},
{
name: "file with filename",
meta: MediaMeta{
Tag: MediaFile,
Size: 800,
Downloadable: true,
Channel: MediaChannelStart,
Blocks: 2,
CRC32: 0,
Filename: "report.zip",
},
caption: "",
},
{
name: "filename strips path traversal",
meta: MediaMeta{
Tag: MediaFile,
Size: 100,
Downloadable: true,
Channel: MediaChannelStart + 1,
Blocks: 1,
CRC32: 0xdeadbeef,
// Server-side sanitisation strips dirs, control chars, and ":"
// before the metadata reaches the wire — so a parsed filename
// is never going to contain any of those.
Filename: "/tmp/../etc/passwd:bad\nname",
},
caption: "",
},
{
name: "non-downloadable image",
meta: MediaMeta{
Tag: MediaImage,
Size: 50_000_000,
Downloadable: false,
Channel: 0,
Blocks: 0,
CRC32: 0xdeadbeef,
},
caption: "too big",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
body := EncodeMediaText(tc.meta, tc.caption)
meta, caption, ok := ParseMediaText(body)
if !ok {
t.Fatalf("ParseMediaText returned ok=false for body %q", body)
}
if caption != tc.caption {
t.Fatalf("caption = %q, want %q", caption, tc.caption)
}
if meta.Tag != tc.meta.Tag {
t.Fatalf("Tag = %q, want %q", meta.Tag, tc.meta.Tag)
}
if meta.Size != tc.meta.Size {
t.Fatalf("Size = %d, want %d", meta.Size, tc.meta.Size)
}
if meta.Downloadable != tc.meta.Downloadable {
t.Fatalf("Downloadable = %v, want %v", meta.Downloadable, tc.meta.Downloadable)
}
if meta.Channel != tc.meta.Channel {
t.Fatalf("Channel = %d, want %d", meta.Channel, tc.meta.Channel)
}
if meta.Blocks != tc.meta.Blocks {
t.Fatalf("Blocks = %d, want %d", meta.Blocks, tc.meta.Blocks)
}
if meta.CRC32 != tc.meta.CRC32 {
t.Fatalf("CRC32 = %x, want %x", meta.CRC32, tc.meta.CRC32)
}
wantFilename := SanitiseMediaFilename(tc.meta.Filename)
if meta.Filename != wantFilename {
t.Fatalf("Filename = %q, want %q", meta.Filename, wantFilename)
}
})
}
}
func TestSanitiseMediaFilename(t *testing.T) {
cases := map[string]string{
"": "",
"report.zip": "report.zip",
"path/to/report.zip": "report.zip",
"..": "",
"a:b\nc.txt": "abc.txt",
"hello": "hello",
"WeIrD-Name_v2.tar.gz": "WeIrD-Name_v2.tar.gz",
"\xff\xfe.txt": "media.txt",
"\u062d\u0645\u0644\u0647.zip": "media.zip",
}
for in, want := range cases {
if got := SanitiseMediaFilename(in); got != want {
t.Errorf("SanitiseMediaFilename(%q) = %q, want %q", in, got, want)
}
}
}
func TestSanitiseMediaFilenameLongName(t *testing.T) {
long := strings.Repeat("abc", 50) + ".zip"
got := SanitiseMediaFilename(long)
if !strings.HasPrefix(got, "media-") || !strings.HasSuffix(got, ".zip") {
t.Fatalf("long filename = %q, want media-<hash>.zip", got)
}
if len(got) > 6+8+1+3 {
t.Fatalf("long filename too long: %q", got)
}
if again := SanitiseMediaFilename(long); again != got {
t.Fatalf("non-deterministic: %q vs %q", got, again)
}
}
// Backward compat: legacy "[IMAGE]\ncaption" must still parse cleanly with
// caption preserved and Downloadable=false.
func TestParseMediaTextLegacy(t *testing.T) {
body := "[IMAGE]\nlook at this"
meta, caption, ok := ParseMediaText(body)
if !ok {
t.Fatalf("ParseMediaText ok=false on legacy body")
}
if meta.Tag != MediaImage {
t.Fatalf("Tag = %q, want %q", meta.Tag, MediaImage)
}
if meta.Downloadable {
t.Fatalf("Downloadable should be false on legacy body")
}
if caption != "look at this" {
t.Fatalf("caption = %q, want %q", caption, "look at this")
}
}
// Backward compat: legacy [IMAGE] with no caption.
func TestParseMediaTextLegacyNoCaption(t *testing.T) {
for _, body := range []string{"[IMAGE]", "[IMAGE]\n"} {
meta, caption, ok := ParseMediaText(body)
if !ok {
t.Fatalf("ok=false on %q", body)
}
if meta.Tag != MediaImage {
t.Fatalf("Tag = %q, want [IMAGE]", meta.Tag)
}
if meta.Downloadable {
t.Fatalf("legacy body should not be downloadable")
}
if caption != "" {
t.Fatalf("caption = %q, want empty", caption)
}
}
}
// A normal caption that happens to lead with a media tag should not be
// misparsed as downloadable metadata.
func TestParseMediaTextHumanCaption(t *testing.T) {
body := "[IMAGE]nice picture\nrest of post"
meta, caption, ok := ParseMediaText(body)
if !ok {
t.Fatalf("ok=false on caption-leading body")
}
if meta.Downloadable {
t.Fatalf("downloadable should be false for a human caption")
}
if meta.Channel != 0 {
t.Fatalf("channel should be 0 for non-metadata body, got %d", meta.Channel)
}
want := "nice picture\nrest of post"
if caption != want {
t.Fatalf("caption = %q, want %q", caption, want)
}
}
// Unknown tag → ok=false.
func TestParseMediaTextUnknownTag(t *testing.T) {
_, _, ok := ParseMediaText("not a tag")
if ok {
t.Fatalf("ok=true for non-tag body")
}
}
// A metadata line that names a channel outside the media range must NOT be
// surfaced as downloadable.
func TestParseMediaTextRejectsOutOfRangeChannel(t *testing.T) {
body := "[IMAGE]100:1:5:200:00000000\ncaption"
meta, _, ok := ParseMediaText(body)
if !ok {
t.Fatalf("ok=false on otherwise-valid metadata")
}
if meta.Downloadable {
t.Fatalf("Downloadable should be false for channel %d outside media range", meta.Channel)
}
}
func TestIsMediaChannel(t *testing.T) {
checks := map[uint16]bool{
0: false,
1: false,
MediaChannelStart - 1: false,
MediaChannelStart: true,
MediaChannelStart + 100: true,
MediaChannelEnd: true,
MediaChannelEnd + 1: false,
65535: false,
}
for ch, want := range checks {
if got := IsMediaChannel(ch); got != want {
t.Errorf("IsMediaChannel(%d) = %v, want %v", ch, got, want)
}
}
}
+22
View File
@@ -20,6 +20,12 @@ const (
// DefaultBlockPayload is kept for compatibility; equals MaxBlockPayload.
DefaultBlockPayload = MaxBlockPayload
// MediaBlockPayload is the fixed payload size used for media (image/file)
// blocks. Media blocks are raw binary, and using a fixed size simplifies
// both server-side block boundaries and client-side range/resume math.
// Tuned for safe DNS UDP response after AES-GCM + base64 + padding.
MediaBlockPayload = MaxBlockPayload
// DefaultMaxPadding is the default random padding added to responses to vary DNS response size.
DefaultMaxPadding = 32
@@ -29,6 +35,14 @@ const (
// MetadataChannel is the special channel number for server metadata.
MetadataChannel = 0
// MediaChannelStart and MediaChannelEnd bound the channel-number range
// reserved for cached binary media (images, files, ...). Each cached file
// occupies one channel; bytes are split into raw blocks served via the
// usual DNS TXT path. The range is well above typical feed channel counts
// and well below the special control channels at the top of uint16 space.
MediaChannelStart uint16 = 10000
MediaChannelEnd uint16 = 60000 // inclusive
// MarkerSize is the random marker in metadata to verify data freshness.
MarkerSize = 3
@@ -46,6 +60,14 @@ const (
MsgContentHashSize = 4
)
// IsMediaChannel reports whether ch falls inside the reserved media-blob
// channel range. Media channels are not enumerated in Metadata; the client
// learns each (channel, blocks, hash) tuple from the corresponding feed
// message text via [TAG]<size>:<dl>:<ch>:<blk>:<crc32hex>.
func IsMediaChannel(ch uint16) bool {
return ch >= MediaChannelStart && ch <= MediaChannelEnd
}
// Media placeholder strings for non-text content.
const (
MediaImage = "[IMAGE]"