560 lines
13 KiB
Go
560 lines
13 KiB
Go
package backend
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/base64"
|
|
"fmt"
|
|
"html"
|
|
"io"
|
|
"mime"
|
|
"mime/multipart"
|
|
"mime/quotedprintable"
|
|
"net/mail"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// mbox ist das lokale Cold-Backup-Format: eine Datei pro IMAP-Ordner, der
|
|
// Ordnerbaum 1:1 auf der Platte. eM Client / Thunderbird importieren mbox
|
|
// direkt. (Kein PST -- das laesst sich in Go nicht lean schreiben.)
|
|
|
|
// MboxWriter haengt Nachrichten an die mbox-Datei eines Ordners an.
|
|
type MboxWriter struct {
|
|
root string
|
|
}
|
|
|
|
type MboxAppendInfo struct {
|
|
Path string
|
|
FileOffset int64
|
|
FrameLen int64
|
|
InnerOffset int64
|
|
InnerLen int64
|
|
Subject string
|
|
From string
|
|
Date string
|
|
}
|
|
|
|
// NewMboxWriter legt <mboxRoot>/<account>/ an und spiegelt den Ordnerbaum.
|
|
func NewMboxWriter(dir string) (*MboxWriter, error) {
|
|
if err := os.MkdirAll(dir, 0o700); err != nil {
|
|
return nil, err
|
|
}
|
|
return &MboxWriter{root: dir}, nil
|
|
}
|
|
|
|
// Append schreibt eine Mail im mbox-Format ("From "-Trennzeile, >From-Quoting)
|
|
// in <ordner>.mbox.
|
|
func (w *MboxWriter) Append(folder string, m RawMessage) (MboxAppendInfo, error) {
|
|
record := mboxRecord(m)
|
|
info := mboxAppendInfoFromMessage(m, record)
|
|
path := filepath.Join(w.root, safeMboxName(folder)+mboxFileExt())
|
|
info.Path = path
|
|
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
|
return info, err
|
|
}
|
|
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
defer f.Close()
|
|
offset, err := f.Seek(0, io.SeekEnd)
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
info.FileOffset = offset
|
|
var written int
|
|
if Cfg.MboxCompression == "zstd" {
|
|
enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
compressed := enc.EncodeAll(record, nil)
|
|
written, err = f.Write(compressed)
|
|
info.FrameLen = int64(len(compressed))
|
|
} else {
|
|
written, err = f.Write(record)
|
|
info.FrameLen = int64(written)
|
|
}
|
|
if err != nil {
|
|
return info, err
|
|
}
|
|
if written != int(info.FrameLen) {
|
|
return info, io.ErrShortWrite
|
|
}
|
|
return info, nil
|
|
}
|
|
|
|
func mboxFileExt() string {
|
|
if Cfg.MboxCompression == "zstd" {
|
|
return ".mbox.zst"
|
|
}
|
|
return ".mbox"
|
|
}
|
|
|
|
func mboxRecord(m RawMessage) []byte {
|
|
var b bytes.Buffer
|
|
fromDate := m.InternalDate
|
|
if fromDate.IsZero() {
|
|
fromDate = time.Now()
|
|
}
|
|
fmt.Fprintf(&b, "From MAILER-DAEMON %s\r\n", fromDate.Format("Mon Jan _2 15:04:05 2006"))
|
|
body := bytes.ReplaceAll(m.Body, []byte("\r\nFrom "), []byte("\r\n>From "))
|
|
body = bytes.ReplaceAll(body, []byte("\nFrom "), []byte("\n>From "))
|
|
_, _ = b.Write(body)
|
|
if !bytes.HasSuffix(body, []byte("\n")) {
|
|
_, _ = b.WriteString("\r\n")
|
|
}
|
|
_, _ = b.WriteString("\r\n")
|
|
return b.Bytes()
|
|
}
|
|
|
|
func mboxAppendInfoFromMessage(m RawMessage, record []byte) MboxAppendInfo {
|
|
info := MboxAppendInfo{InnerLen: int64(len(record))}
|
|
msg, err := mail.ReadMessage(bytes.NewReader(m.Body))
|
|
if err != nil {
|
|
return info
|
|
}
|
|
info.Subject = decodeHeader(msg.Header.Get("Subject"))
|
|
info.From = decodeHeader(msg.Header.Get("From"))
|
|
info.Date = decodeHeader(msg.Header.Get("Date"))
|
|
return info
|
|
}
|
|
|
|
// --- Viewer-Seite: mbox wieder lesen fuers Browser-Betrachten (08-viewer.go) ---
|
|
|
|
// MboxEntry ist eine Nachricht wie im Listen-Pane angezeigt.
|
|
type MboxEntry struct {
|
|
Index int
|
|
From string
|
|
Subject string
|
|
Date string
|
|
}
|
|
|
|
// ReadMboxList parst die Kopfzeilen aller Mails einer mbox-Datei (fuer die
|
|
// Nachrichtenliste). ReadMboxMessage liefert eine einzelne Mail als Rohtext.
|
|
func ReadMboxList(path string) ([]MboxEntry, error) {
|
|
if entries, ok := readMboxListFromIndex(path); ok {
|
|
return entries, nil
|
|
}
|
|
msgs, err := readMboxMessages(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
out := make([]MboxEntry, 0, len(msgs))
|
|
for i, raw := range msgs {
|
|
msg, err := mail.ReadMessage(bytes.NewReader(raw))
|
|
if err != nil {
|
|
out = append(out, MboxEntry{Index: i, From: "(unlesbar)", Subject: "Unlesbare Nachricht"})
|
|
continue
|
|
}
|
|
out = append(out, MboxEntry{
|
|
Index: i,
|
|
From: decodeHeader(msg.Header.Get("From")),
|
|
Subject: decodeHeader(msg.Header.Get("Subject")),
|
|
Date: decodeHeader(msg.Header.Get("Date")),
|
|
})
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func ReadMboxMessage(path string, index int) ([]byte, error) {
|
|
if raw, ok, err := readMboxMessageFromIndex(path, index); ok || err != nil {
|
|
return raw, err
|
|
}
|
|
msgs, err := readMboxMessages(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if index < 0 || index >= len(msgs) {
|
|
return nil, fmt.Errorf("message index out of range")
|
|
}
|
|
return msgs[index], nil
|
|
}
|
|
|
|
func readMboxMessages(path string) ([][]byte, error) {
|
|
b, err := os.ReadFile(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if isZstdMbox(path, b) {
|
|
dec, err := zstd.NewReader(nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer dec.Close()
|
|
b, err = dec.DecodeAll(b, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
return readMboxMessagesBytes(b), nil
|
|
}
|
|
|
|
func readMboxMessagesBytes(b []byte) [][]byte {
|
|
b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
|
|
lines := bytes.Split(b, []byte("\n"))
|
|
var msgs [][]byte
|
|
var cur bytes.Buffer
|
|
inMsg := false
|
|
for _, line := range lines {
|
|
if bytes.HasPrefix(line, []byte("From ")) {
|
|
if inMsg && cur.Len() > 0 {
|
|
msgs = append(msgs, bytes.TrimRight(cur.Bytes(), "\n"))
|
|
cur.Reset()
|
|
}
|
|
inMsg = true
|
|
continue
|
|
}
|
|
if !inMsg {
|
|
continue
|
|
}
|
|
if bytes.HasPrefix(line, []byte(">From ")) {
|
|
line = line[1:]
|
|
}
|
|
_, _ = cur.Write(line)
|
|
_ = cur.WriteByte('\n')
|
|
}
|
|
if inMsg && cur.Len() > 0 {
|
|
msgs = append(msgs, bytes.TrimRight(cur.Bytes(), "\n"))
|
|
}
|
|
return msgs
|
|
}
|
|
|
|
func readMboxListFromIndex(path string) ([]MboxEntry, bool) {
|
|
accountID, folder, ok := mboxIndexContext(path)
|
|
if !ok {
|
|
return nil, false
|
|
}
|
|
index, err := ListMboxIndex(accountID, folder)
|
|
if err != nil || len(index) == 0 {
|
|
return nil, false
|
|
}
|
|
out := make([]MboxEntry, 0, len(index))
|
|
for _, entry := range index {
|
|
out = append(out, MboxEntry{
|
|
Index: entry.Seq,
|
|
From: entry.From,
|
|
Subject: entry.Subject,
|
|
Date: entry.Date,
|
|
})
|
|
}
|
|
return out, true
|
|
}
|
|
|
|
func readMboxMessageFromIndex(path string, index int) ([]byte, bool, error) {
|
|
accountID, folder, ok := mboxIndexContext(path)
|
|
if !ok {
|
|
return nil, false, nil
|
|
}
|
|
entry, err := GetMboxIndex(accountID, folder, index)
|
|
if err != nil {
|
|
return nil, false, nil
|
|
}
|
|
if !strings.HasSuffix(strings.ToLower(path), ".zst") {
|
|
return nil, false, nil
|
|
}
|
|
f, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, true, err
|
|
}
|
|
defer f.Close()
|
|
if _, err := f.Seek(entry.FileOffset, io.SeekStart); err != nil {
|
|
return nil, true, err
|
|
}
|
|
frame := make([]byte, entry.FrameLen)
|
|
if _, err := io.ReadFull(f, frame); err != nil {
|
|
return nil, true, err
|
|
}
|
|
dec, err := zstd.NewReader(nil)
|
|
if err != nil {
|
|
return nil, true, err
|
|
}
|
|
defer dec.Close()
|
|
record, err := dec.DecodeAll(frame, nil)
|
|
if err != nil {
|
|
return nil, true, err
|
|
}
|
|
msgs := readMboxMessagesBytes(record)
|
|
if len(msgs) == 0 {
|
|
return nil, true, fmt.Errorf("message index out of range")
|
|
}
|
|
return msgs[0], true, nil
|
|
}
|
|
|
|
func mboxIndexContext(path string) (int64, string, bool) {
|
|
if DB == nil || Cfg.MboxRoot == "" {
|
|
return 0, "", false
|
|
}
|
|
root, err := filepath.Abs(Cfg.MboxRoot)
|
|
if err != nil {
|
|
return 0, "", false
|
|
}
|
|
abs, err := filepath.Abs(path)
|
|
if err != nil {
|
|
return 0, "", false
|
|
}
|
|
rel, err := filepath.Rel(root, abs)
|
|
if err != nil || strings.HasPrefix(rel, "..") {
|
|
return 0, "", false
|
|
}
|
|
parts := strings.Split(rel, string(os.PathSeparator))
|
|
if len(parts) < 2 {
|
|
return 0, "", false
|
|
}
|
|
account, ok := accountByMboxDir(parts[0])
|
|
if !ok || account.ID == 0 {
|
|
return 0, "", false
|
|
}
|
|
folder := folderNameFromMboxPath(path)
|
|
if folder == "" {
|
|
return 0, "", false
|
|
}
|
|
return account.ID, folder, true
|
|
}
|
|
|
|
func isZstdMbox(path string, b []byte) bool {
|
|
if strings.HasSuffix(strings.ToLower(path), ".zst") {
|
|
return true
|
|
}
|
|
return len(b) >= 4 && b[0] == 0x28 && b[1] == 0xb5 && b[2] == 0x2f && b[3] == 0xfd
|
|
}
|
|
|
|
func decodeHeader(v string) string {
|
|
v = strings.TrimSpace(v)
|
|
if v == "" {
|
|
return ""
|
|
}
|
|
decoded, err := new(mime.WordDecoder).DecodeHeader(v)
|
|
if err != nil {
|
|
return v
|
|
}
|
|
return decoded
|
|
}
|
|
|
|
func messageBody(raw []byte) string {
|
|
msg, err := mail.ReadMessage(bytes.NewReader(raw))
|
|
if err != nil {
|
|
return string(raw)
|
|
}
|
|
body, err := io.ReadAll(msg.Body)
|
|
if err != nil {
|
|
return string(raw)
|
|
}
|
|
return extractTextBody(msg.Header, body)
|
|
}
|
|
|
|
type headerGetter interface {
|
|
Get(string) string
|
|
}
|
|
|
|
func extractTextBody(h headerGetter, body []byte) string {
|
|
mediaType, params, _ := mime.ParseMediaType(h.Get("Content-Type"))
|
|
decoded := decodeTransfer(body, h.Get("Content-Transfer-Encoding"))
|
|
if strings.HasPrefix(mediaType, "multipart/") {
|
|
boundary := params["boundary"]
|
|
if boundary == "" {
|
|
return string(decoded)
|
|
}
|
|
mr := multipart.NewReader(bytes.NewReader(decoded), boundary)
|
|
var htmlBody string
|
|
for {
|
|
part, err := mr.NextPart()
|
|
if err == io.EOF {
|
|
break
|
|
}
|
|
if err != nil {
|
|
break
|
|
}
|
|
partBody, _ := io.ReadAll(part)
|
|
text := extractTextBody(part.Header, partBody)
|
|
partType, _, _ := mime.ParseMediaType(part.Header.Get("Content-Type"))
|
|
if strings.HasPrefix(partType, "text/plain") && strings.TrimSpace(text) != "" {
|
|
return text
|
|
}
|
|
if strings.HasPrefix(partType, "text/html") && htmlBody == "" {
|
|
htmlBody = text
|
|
}
|
|
}
|
|
if htmlBody != "" {
|
|
return htmlBody
|
|
}
|
|
return ""
|
|
}
|
|
if strings.HasPrefix(mediaType, "text/html") || looksLikeHTML(decoded) {
|
|
return htmlToText(string(decoded))
|
|
}
|
|
return string(decoded)
|
|
}
|
|
|
|
func looksLikeHTML(body []byte) bool {
|
|
s := strings.ToLower(strings.TrimSpace(string(body)))
|
|
return strings.HasPrefix(s, "<!doctype html") || strings.HasPrefix(s, "<html") || strings.Contains(s, "<body")
|
|
}
|
|
|
|
func htmlToText(s string) string {
|
|
s = stripHTMLBlock(s, "script")
|
|
s = stripHTMLBlock(s, "style")
|
|
s = markHTMLBreaks(s)
|
|
var b strings.Builder
|
|
inTag := false
|
|
lastSpace := false
|
|
for _, r := range s {
|
|
switch r {
|
|
case '<':
|
|
inTag = true
|
|
if !lastSpace {
|
|
b.WriteByte(' ')
|
|
lastSpace = true
|
|
}
|
|
case '>':
|
|
inTag = false
|
|
default:
|
|
if inTag {
|
|
continue
|
|
}
|
|
if r == '\n' {
|
|
b.WriteByte('\n')
|
|
lastSpace = true
|
|
continue
|
|
}
|
|
if r == '\r' || r == '\t' || r == ' ' {
|
|
if !lastSpace {
|
|
b.WriteByte(' ')
|
|
lastSpace = true
|
|
}
|
|
continue
|
|
}
|
|
b.WriteRune(r)
|
|
lastSpace = false
|
|
}
|
|
}
|
|
text := html.UnescapeString(b.String())
|
|
lines := strings.Split(text, "\n")
|
|
out := make([]string, 0, len(lines))
|
|
for _, line := range lines {
|
|
line = strings.TrimSpace(line)
|
|
if line != "" {
|
|
out = append(out, line)
|
|
}
|
|
}
|
|
if len(out) == 0 {
|
|
return strings.TrimSpace(text)
|
|
}
|
|
return strings.Join(out, "\n")
|
|
}
|
|
|
|
func markHTMLBreaks(s string) string {
|
|
repls := []struct {
|
|
Needle string
|
|
With string
|
|
}{
|
|
{"<br>", "\n"},
|
|
{"<br/>", "\n"},
|
|
{"<br />", "\n"},
|
|
{"</p>", "\n"},
|
|
{"</div>", "\n"},
|
|
{"</tr>", "\n"},
|
|
{"</td>", " "},
|
|
{"</th>", " "},
|
|
{"</li>", "\n"},
|
|
}
|
|
for _, repl := range repls {
|
|
s = replaceCaseInsensitive(s, repl.Needle, repl.With)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func replaceCaseInsensitive(s, old, new string) string {
|
|
var b strings.Builder
|
|
lower := strings.ToLower(s)
|
|
needle := strings.ToLower(old)
|
|
for {
|
|
i := strings.Index(lower, needle)
|
|
if i < 0 {
|
|
b.WriteString(s)
|
|
return b.String()
|
|
}
|
|
b.WriteString(s[:i])
|
|
b.WriteString(new)
|
|
cut := i + len(old)
|
|
s = s[cut:]
|
|
lower = lower[cut:]
|
|
}
|
|
}
|
|
|
|
func stripHTMLBlock(s, tag string) string {
|
|
lower := strings.ToLower(s)
|
|
open := "<" + tag
|
|
close := "</" + tag + ">"
|
|
for {
|
|
start := strings.Index(lower, open)
|
|
if start < 0 {
|
|
return s
|
|
}
|
|
end := strings.Index(lower[start:], close)
|
|
if end < 0 {
|
|
return s[:start]
|
|
}
|
|
end += start + len(close)
|
|
s = s[:start] + " " + s[end:]
|
|
lower = strings.ToLower(s)
|
|
}
|
|
}
|
|
|
|
func decodeTransfer(body []byte, enc string) []byte {
|
|
switch strings.ToLower(strings.TrimSpace(enc)) {
|
|
case "quoted-printable":
|
|
b, err := io.ReadAll(quotedprintable.NewReader(bytes.NewReader(body)))
|
|
if err == nil {
|
|
return b
|
|
}
|
|
case "base64":
|
|
cleaned := strings.NewReplacer("\r", "", "\n", "", " ", "", "\t", "").Replace(string(body))
|
|
b, err := base64.StdEncoding.DecodeString(cleaned)
|
|
if err == nil {
|
|
return b
|
|
}
|
|
}
|
|
return body
|
|
}
|
|
|
|
func safeMboxName(folder string) string {
|
|
folder = strings.TrimSpace(folder)
|
|
folder = strings.ReplaceAll(folder, "\\", "_")
|
|
folder = strings.ReplaceAll(folder, "/", "_")
|
|
folder = strings.ReplaceAll(folder, ":", "_")
|
|
if folder == "" {
|
|
return "INBOX"
|
|
}
|
|
return folder
|
|
}
|
|
|
|
func archiveMboxPaths(dir string) []string {
|
|
plain, _ := filepath.Glob(filepath.Join(dir, "*.mbox"))
|
|
compressed, _ := filepath.Glob(filepath.Join(dir, "*.mbox.zst"))
|
|
seen := map[string]bool{}
|
|
out := make([]string, 0, len(plain)+len(compressed))
|
|
for _, path := range plain {
|
|
seen[folderNameFromMboxPath(path)] = true
|
|
out = append(out, path)
|
|
}
|
|
for _, path := range compressed {
|
|
if seen[folderNameFromMboxPath(path)] {
|
|
continue
|
|
}
|
|
out = append(out, path)
|
|
}
|
|
sortStrings(out)
|
|
return out
|
|
}
|
|
|
|
func folderNameFromMboxPath(path string) string {
|
|
name := filepath.Base(path)
|
|
name = strings.TrimSuffix(name, ".zst")
|
|
name = strings.TrimSuffix(name, ".mbox")
|
|
return name
|
|
}
|