package backend import ( "bytes" "encoding/base64" "fmt" "html" "io" "mime" "mime/multipart" "mime/quotedprintable" "net/mail" "os" "path/filepath" "sort" "strings" "time" "github.com/klauspost/compress/zstd" ) // mbox ist das lokale Cold-Backup-Format: eine Datei pro IMAP-Ordner, der // Ordnerbaum 1:1 auf der Platte. eM Client / Thunderbird importieren mbox // direkt. (Kein PST -- das laesst sich in Go nicht lean schreiben.) // MboxWriter haengt Nachrichten an die mbox-Datei eines Ordners an. type MboxWriter struct { root string } type MboxAppendInfo struct { Path string FileOffset int64 FrameLen int64 InnerOffset int64 InnerLen int64 Subject string From string Date string } // NewMboxWriter legt // an und spiegelt den Ordnerbaum. func NewMboxWriter(dir string) (*MboxWriter, error) { if err := os.MkdirAll(dir, 0o700); err != nil { return nil, err } return &MboxWriter{root: dir}, nil } // Append schreibt eine Mail im mbox-Format ("From "-Trennzeile, >From-Quoting) // in .mbox. func (w *MboxWriter) Append(folder string, m RawMessage) (MboxAppendInfo, error) { record := mboxRecord(m) info := mboxAppendInfoFromMessage(m, record) path := filepath.Join(w.root, safeMboxName(folder)+mboxFileExt()) info.Path = path if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { return info, err } f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) if err != nil { return info, err } defer f.Close() offset, err := f.Seek(0, io.SeekEnd) if err != nil { return info, err } info.FileOffset = offset var written int if Cfg.MboxCompression == "zstd" { enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault)) if err != nil { return info, err } compressed := enc.EncodeAll(record, nil) written, err = f.Write(compressed) info.FrameLen = int64(len(compressed)) } else { written, err = f.Write(record) info.FrameLen = int64(written) } if err != nil { return info, err } if written != int(info.FrameLen) { return info, io.ErrShortWrite } return info, nil } func mboxFileExt() string { if Cfg.MboxCompression == "zstd" { return ".mbox.zst" } return ".mbox" } func mboxRecord(m RawMessage) []byte { var b bytes.Buffer fromDate := m.InternalDate if fromDate.IsZero() { fromDate = time.Now() } fmt.Fprintf(&b, "From MAILER-DAEMON %s\r\n", fromDate.Format("Mon Jan _2 15:04:05 2006")) body := bytes.ReplaceAll(m.Body, []byte("\r\nFrom "), []byte("\r\n>From ")) body = bytes.ReplaceAll(body, []byte("\nFrom "), []byte("\n>From ")) _, _ = b.Write(body) if !bytes.HasSuffix(body, []byte("\n")) { _, _ = b.WriteString("\r\n") } _, _ = b.WriteString("\r\n") return b.Bytes() } func mboxAppendInfoFromMessage(m RawMessage, record []byte) MboxAppendInfo { info := MboxAppendInfo{InnerLen: int64(len(record))} parts := splitMboxRecordsWithOffsets(record) if len(parts) > 0 { info.InnerOffset = parts[0].Offset info.InnerLen = int64(len(parts[0].Message)) } msg, err := mail.ReadMessage(bytes.NewReader(m.Body)) if err != nil { return info } info.Subject = decodeHeader(msg.Header.Get("Subject")) info.From = decodeHeader(msg.Header.Get("From")) info.Date = decodeHeader(msg.Header.Get("Date")) return info } // --- Viewer-Seite: mbox wieder lesen fuers Browser-Betrachten (08-viewer.go) --- // MboxEntry ist eine Nachricht wie im Listen-Pane angezeigt. type MboxEntry struct { Index int From string Subject string Date string } // ReadMboxList parst die Kopfzeilen aller Mails einer mbox-Datei (fuer die // Nachrichtenliste). ReadMboxMessage liefert eine einzelne Mail als Rohtext. func ReadMboxList(path string) ([]MboxEntry, error) { if err := EnsureMboxIndexed(path); err != nil { return nil, err } if entries, ok := readMboxListFromIndex(path); ok { return entries, nil } msgs, err := readMboxMessages(path) if err != nil { return nil, err } out := make([]MboxEntry, 0, len(msgs)) for i, raw := range msgs { msg, err := mail.ReadMessage(bytes.NewReader(raw)) if err != nil { out = append(out, MboxEntry{Index: i, From: "(unlesbar)", Subject: "Unlesbare Nachricht"}) continue } out = append(out, MboxEntry{ Index: i, From: decodeHeader(msg.Header.Get("From")), Subject: decodeHeader(msg.Header.Get("Subject")), Date: decodeHeader(msg.Header.Get("Date")), }) } sortMboxEntriesByDateDesc(out) return out, nil } func ReadMboxMessage(path string, index int) ([]byte, error) { if err := EnsureMboxIndexed(path); err != nil { return nil, err } if raw, ok, err := readMboxMessageFromIndex(path, index); ok || err != nil { return raw, err } msgs, err := readMboxMessages(path) if err != nil { return nil, err } if index < 0 || index >= len(msgs) { return nil, fmt.Errorf("message index out of range") } return msgs[index], nil } func readMboxMessages(path string) ([][]byte, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } if isZstdMbox(path, b) { dec, err := zstd.NewReader(nil) if err != nil { return nil, err } defer dec.Close() b, err = dec.DecodeAll(b, nil) if err != nil { return nil, err } } return readMboxMessagesBytes(b), nil } func EnsureMboxIndexed(path string) error { accountID, folder, ok := mboxIndexContext(path) if !ok { return nil } info, err := os.Stat(path) if err != nil { return err } indexed, err := MboxIndexedBytes(accountID, folder) if err != nil { return err } if indexed == info.Size() { return nil } return ReindexMboxFile(accountID, folder, path) } func ReindexMboxFile(accountID int64, folder, path string) error { info, err := os.Stat(path) if err != nil { return err } var entries []MboxIndexEntry if strings.HasSuffix(strings.ToLower(path), ".zst") { entries, err = reindexZstdMbox(path) } else { entries, err = reindexPlainMbox(path) } if err != nil { return err } return ReplaceMboxIndex(accountID, folder, entries, info.Size()) } func ReindexArchives(name string) error { name = strings.TrimSpace(name) if name == "" { return fmt.Errorf("reindex ziel fehlt") } accounts, err := ListAccounts() if err != nil { return err } matched := 0 for _, account := range accounts { archive := accountMboxDir(account) if archive == "" { continue } if name != "all" && !strings.EqualFold(name, account.Name) && !strings.EqualFold(name, archive) { continue } if err := reindexAccountArchive(account); err != nil { return err } matched++ } if matched == 0 { return fmt.Errorf("kein Konto/Archiv fuer reindex %q gefunden", name) } return nil } func reindexAccountArchive(account Account) error { dir := filepath.Join(Cfg.MboxRoot, accountMboxDir(account)) for _, path := range archiveMboxPaths(dir) { folder := folderNameFromMboxPath(path) if folder == "" { continue } if err := ReindexMboxFile(account.ID, folder, path); err != nil { return fmt.Errorf("%s %s: %w", account.Name, folder, err) } } return nil } func reindexPlainMbox(path string) ([]MboxIndexEntry, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } var out []MboxIndexEntry for _, part := range splitMboxRecordsWithOffsets(b) { entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length)) if entry.MessageID != "" { out = append(out, entry) } } return out, nil } func reindexZstdMbox(path string) ([]MboxIndexEntry, error) { b, err := os.ReadFile(path) if err != nil { return nil, err } dec, err := zstd.NewReader(nil) if err != nil { return nil, err } defer dec.Close() var out []MboxIndexEntry for _, frame := range splitZstdFrames(b) { record, err := dec.DecodeAll(frame.Data, nil) if err != nil { return nil, err } for _, part := range splitMboxRecordsWithOffsets(record) { entry := indexEntryFromMboxRecord(part.Message, frame.Offset, int64(frame.Length), part.Offset, int64(len(part.Message))) if entry.MessageID != "" { out = append(out, entry) } } } return out, nil } type mboxRecordPart struct { Offset int64 Length int Message []byte } func splitMboxRecordsWithOffsets(b []byte) []mboxRecordPart { normalized := bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) lines := bytes.SplitAfter(normalized, []byte("\n")) var out []mboxRecordPart var cur bytes.Buffer var curStart int64 var pos int64 inMsg := false for _, lineWithNL := range lines { line := bytes.TrimSuffix(lineWithNL, []byte("\n")) if bytes.HasPrefix(line, []byte("From ")) { if inMsg && cur.Len() > 0 { msg := bytes.TrimRight(cur.Bytes(), "\n") out = append(out, mboxRecordPart{Offset: curStart, Length: int(pos - curStart), Message: append([]byte(nil), msg...)}) cur.Reset() } inMsg = true pos += int64(len(lineWithNL)) curStart = pos continue } if inMsg { if bytes.HasPrefix(line, []byte(">From ")) { line = line[1:] } _, _ = cur.Write(line) _ = cur.WriteByte('\n') } pos += int64(len(lineWithNL)) } if inMsg && cur.Len() > 0 { msg := bytes.TrimRight(cur.Bytes(), "\n") out = append(out, mboxRecordPart{Offset: curStart, Length: int(pos - curStart), Message: append([]byte(nil), msg...)}) } return out } type zstdFramePart struct { Offset int64 Length int Data []byte } func splitZstdFrames(b []byte) []zstdFramePart { magic := []byte{0x28, 0xb5, 0x2f, 0xfd} var starts []int for i := 0; i <= len(b)-len(magic); i++ { if bytes.Equal(b[i:i+len(magic)], magic) { starts = append(starts, i) } } if len(starts) == 0 { return nil } out := make([]zstdFramePart, 0, len(starts)) for i, start := range starts { end := len(b) if i+1 < len(starts) { end = starts[i+1] } out = append(out, zstdFramePart{Offset: int64(start), Length: end - start, Data: b[start:end]}) } return out } func indexEntryFromMboxRecord(raw []byte, fileOffset, frameLen, innerOffset, innerLen int64) MboxIndexEntry { entry := MboxIndexEntry{FileOffset: fileOffset, FrameLen: frameLen, InnerOffset: innerOffset, InnerLen: innerLen} msg, err := mail.ReadMessage(bytes.NewReader(raw)) if err != nil { return entry } entry.MessageID = normalizeMessageID(msg.Header.Get("Message-ID")) if entry.MessageID == "" { entry.MessageID = messageID(raw) } entry.Subject = decodeHeader(msg.Header.Get("Subject")) entry.From = decodeHeader(msg.Header.Get("From")) entry.Date = decodeHeader(msg.Header.Get("Date")) return entry } func unescapeMboxMessage(raw []byte) []byte { raw = bytes.ReplaceAll(raw, []byte("\r\n"), []byte("\n")) lines := bytes.Split(raw, []byte("\n")) var b bytes.Buffer for i, line := range lines { if i > 0 { _ = b.WriteByte('\n') } if bytes.HasPrefix(line, []byte(">From ")) { line = line[1:] } _, _ = b.Write(line) } return bytes.TrimRight(b.Bytes(), "\n") } func readMboxMessagesBytes(b []byte) [][]byte { b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n")) lines := bytes.Split(b, []byte("\n")) var msgs [][]byte var cur bytes.Buffer inMsg := false for _, line := range lines { if bytes.HasPrefix(line, []byte("From ")) { if inMsg && cur.Len() > 0 { msgs = append(msgs, cloneTrimmedMessage(cur.Bytes())) cur.Reset() } inMsg = true continue } if !inMsg { continue } if bytes.HasPrefix(line, []byte(">From ")) { line = line[1:] } _, _ = cur.Write(line) _ = cur.WriteByte('\n') } if inMsg && cur.Len() > 0 { msgs = append(msgs, cloneTrimmedMessage(cur.Bytes())) } return msgs } func cloneTrimmedMessage(b []byte) []byte { trimmed := bytes.TrimRight(b, "\n") return append([]byte(nil), trimmed...) } func readMboxListFromIndex(path string) ([]MboxEntry, bool) { accountID, folder, ok := mboxIndexContext(path) if !ok { return nil, false } index, err := ListMboxIndex(accountID, folder) if err != nil || len(index) == 0 { return nil, false } sort.SliceStable(index, func(i, j int) bool { di := parseMboxIndexDate(index[i].Date) dj := parseMboxIndexDate(index[j].Date) if !di.Equal(dj) { return di.After(dj) } return index[i].Seq > index[j].Seq }) out := make([]MboxEntry, 0, len(index)) for _, entry := range index { out = append(out, MboxEntry{ Index: entry.Seq, From: entry.From, Subject: entry.Subject, Date: entry.Date, }) } return out, true } func sortMboxEntriesByDateDesc(entries []MboxEntry) { sort.SliceStable(entries, func(i, j int) bool { di := parseMboxIndexDate(entries[i].Date) dj := parseMboxIndexDate(entries[j].Date) if !di.Equal(dj) { return di.After(dj) } return entries[i].Index > entries[j].Index }) } func parseMboxIndexDate(value string) time.Time { t, err := mail.ParseDate(strings.TrimSpace(value)) if err == nil { return t } return time.Time{} } func readMboxMessageFromIndex(path string, index int) ([]byte, bool, error) { accountID, folder, ok := mboxIndexContext(path) if !ok { return nil, false, nil } entry, err := GetMboxIndex(accountID, folder, index) if err != nil { return nil, false, nil } if !strings.HasSuffix(strings.ToLower(path), ".zst") { return nil, false, nil } f, err := os.Open(path) if err != nil { return nil, true, err } defer f.Close() if _, err := f.Seek(entry.FileOffset, io.SeekStart); err != nil { return nil, true, err } frame := make([]byte, entry.FrameLen) if _, err := io.ReadFull(f, frame); err != nil { return nil, true, err } dec, err := zstd.NewReader(nil) if err != nil { return nil, true, err } defer dec.Close() record, err := dec.DecodeAll(frame, nil) if err != nil { return nil, true, err } start := entry.InnerOffset end := entry.InnerOffset + entry.InnerLen if start < 0 || end > int64(len(record)) || start >= end { return nil, true, fmt.Errorf("message index out of range") } return unescapeMboxMessage(record[start:end]), true, nil } func mboxIndexContext(path string) (int64, string, bool) { if DB == nil || Cfg.MboxRoot == "" { return 0, "", false } root, err := filepath.Abs(Cfg.MboxRoot) if err != nil { return 0, "", false } abs, err := filepath.Abs(path) if err != nil { return 0, "", false } rel, err := filepath.Rel(root, abs) if err != nil || strings.HasPrefix(rel, "..") { return 0, "", false } parts := strings.Split(rel, string(os.PathSeparator)) if len(parts) < 2 { return 0, "", false } account, ok := accountByMboxDir(parts[0]) if !ok || account.ID == 0 { return 0, "", false } folder := folderNameFromMboxPath(path) if folder == "" { return 0, "", false } return account.ID, folder, true } func isZstdMbox(path string, b []byte) bool { if strings.HasSuffix(strings.ToLower(path), ".zst") { return true } return len(b) >= 4 && b[0] == 0x28 && b[1] == 0xb5 && b[2] == 0x2f && b[3] == 0xfd } func decodeHeader(v string) string { v = strings.TrimSpace(v) if v == "" { return "" } decoded, err := new(mime.WordDecoder).DecodeHeader(v) if err != nil { return v } return decoded } func messageBody(raw []byte) string { msg, err := mail.ReadMessage(bytes.NewReader(raw)) if err != nil { return string(raw) } body, err := io.ReadAll(msg.Body) if err != nil { return string(raw) } return extractTextBody(msg.Header, body) } type headerGetter interface { Get(string) string } func extractTextBody(h headerGetter, body []byte) string { mediaType, params, _ := mime.ParseMediaType(h.Get("Content-Type")) decoded := decodeTransfer(body, h.Get("Content-Transfer-Encoding")) if strings.HasPrefix(mediaType, "multipart/") { boundary := params["boundary"] if boundary == "" { return string(decoded) } mr := multipart.NewReader(bytes.NewReader(decoded), boundary) var htmlBody string for { part, err := mr.NextPart() if err == io.EOF { break } if err != nil { break } partBody, _ := io.ReadAll(part) text := extractTextBody(part.Header, partBody) partType, _, _ := mime.ParseMediaType(part.Header.Get("Content-Type")) if strings.HasPrefix(partType, "text/plain") && strings.TrimSpace(text) != "" { return text } if strings.HasPrefix(partType, "text/html") && htmlBody == "" { htmlBody = text } } if htmlBody != "" { return htmlBody } return "" } if strings.HasPrefix(mediaType, "text/html") || looksLikeHTML(decoded) { return htmlToText(string(decoded)) } return string(decoded) } func looksLikeHTML(body []byte) bool { s := strings.ToLower(strings.TrimSpace(string(body))) return strings.HasPrefix(s, "': inTag = false default: if inTag { continue } if r == '\n' { b.WriteByte('\n') lastSpace = true continue } if r == '\r' || r == '\t' || r == ' ' { if !lastSpace { b.WriteByte(' ') lastSpace = true } continue } b.WriteRune(r) lastSpace = false } } text := html.UnescapeString(b.String()) lines := strings.Split(text, "\n") out := make([]string, 0, len(lines)) for _, line := range lines { line = strings.TrimSpace(line) if line != "" { out = append(out, line) } } if len(out) == 0 { return strings.TrimSpace(text) } return strings.Join(out, "\n") } func markHTMLBreaks(s string) string { repls := []struct { Needle string With string }{ {"
", "\n"}, {"
", "\n"}, {"
", "\n"}, {"

", "\n"}, {"", "\n"}, {"", "\n"}, {"", " "}, {"", " "}, {"", "\n"}, } for _, repl := range repls { s = replaceCaseInsensitive(s, repl.Needle, repl.With) } return s } func replaceCaseInsensitive(s, old, new string) string { var b strings.Builder lower := strings.ToLower(s) needle := strings.ToLower(old) for { i := strings.Index(lower, needle) if i < 0 { b.WriteString(s) return b.String() } b.WriteString(s[:i]) b.WriteString(new) cut := i + len(old) s = s[cut:] lower = lower[cut:] } } func stripHTMLBlock(s, tag string) string { lower := strings.ToLower(s) open := "<" + tag close := "" for { start := strings.Index(lower, open) if start < 0 { return s } end := strings.Index(lower[start:], close) if end < 0 { return s[:start] } end += start + len(close) s = s[:start] + " " + s[end:] lower = strings.ToLower(s) } } func decodeTransfer(body []byte, enc string) []byte { switch strings.ToLower(strings.TrimSpace(enc)) { case "quoted-printable": b, err := io.ReadAll(quotedprintable.NewReader(bytes.NewReader(body))) if err == nil { return b } case "base64": cleaned := strings.NewReplacer("\r", "", "\n", "", " ", "", "\t", "").Replace(string(body)) b, err := base64.StdEncoding.DecodeString(cleaned) if err == nil { return b } } return body } func safeMboxName(folder string) string { folder = strings.TrimSpace(folder) folder = strings.ReplaceAll(folder, "\\", "_") folder = strings.ReplaceAll(folder, "/", "_") folder = strings.ReplaceAll(folder, ":", "_") if folder == "" { return "INBOX" } return folder } func archiveMboxPaths(dir string) []string { plain, _ := filepath.Glob(filepath.Join(dir, "*.mbox")) compressed, _ := filepath.Glob(filepath.Join(dir, "*.mbox.zst")) seen := map[string]bool{} out := make([]string, 0, len(plain)+len(compressed)) for _, path := range plain { seen[folderNameFromMboxPath(path)] = true out = append(out, path) } for _, path := range compressed { if seen[folderNameFromMboxPath(path)] { continue } out = append(out, path) } sortStrings(out) return out } func folderNameFromMboxPath(path string) string { name := filepath.Base(path) name = strings.TrimSuffix(name, ".zst") name = strings.TrimSuffix(name, ".mbox") return name }