Add optional zstd mbox archives
This commit is contained in:
DonVoo
parent
21844688ac
commit
e612896e06
14 changed files with 620 additions and 39 deletions
|
|
@ -14,6 +14,8 @@ import (
|
|||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
// mbox ist das lokale Cold-Backup-Format: eine Datei pro IMAP-Ordner, der
|
||||
|
|
@ -25,6 +27,17 @@ type MboxWriter struct {
|
|||
root string
|
||||
}
|
||||
|
||||
type MboxAppendInfo struct {
|
||||
Path string
|
||||
FileOffset int64
|
||||
FrameLen int64
|
||||
InnerOffset int64
|
||||
InnerLen int64
|
||||
Subject string
|
||||
From string
|
||||
Date string
|
||||
}
|
||||
|
||||
// NewMboxWriter legt <mboxRoot>/<account>/ an und spiegelt den Ordnerbaum.
|
||||
func NewMboxWriter(dir string) (*MboxWriter, error) {
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
|
|
@ -35,35 +48,80 @@ func NewMboxWriter(dir string) (*MboxWriter, error) {
|
|||
|
||||
// Append schreibt eine Mail im mbox-Format ("From "-Trennzeile, >From-Quoting)
|
||||
// in <ordner>.mbox.
|
||||
func (w *MboxWriter) Append(folder string, m RawMessage) error {
|
||||
path := filepath.Join(w.root, safeMboxName(folder)+".mbox")
|
||||
func (w *MboxWriter) Append(folder string, m RawMessage) (MboxAppendInfo, error) {
|
||||
record := mboxRecord(m)
|
||||
info := mboxAppendInfoFromMessage(m, record)
|
||||
path := filepath.Join(w.root, safeMboxName(folder)+mboxFileExt())
|
||||
info.Path = path
|
||||
if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
|
||||
return err
|
||||
return info, err
|
||||
}
|
||||
f, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
return info, err
|
||||
}
|
||||
defer f.Close()
|
||||
offset, err := f.Seek(0, io.SeekEnd)
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
info.FileOffset = offset
|
||||
var written int
|
||||
if Cfg.MboxCompression == "zstd" {
|
||||
enc, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.SpeedDefault))
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
compressed := enc.EncodeAll(record, nil)
|
||||
written, err = f.Write(compressed)
|
||||
info.FrameLen = int64(len(compressed))
|
||||
} else {
|
||||
written, err = f.Write(record)
|
||||
info.FrameLen = int64(written)
|
||||
}
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
if written != int(info.FrameLen) {
|
||||
return info, io.ErrShortWrite
|
||||
}
|
||||
return info, nil
|
||||
}
|
||||
|
||||
func mboxFileExt() string {
|
||||
if Cfg.MboxCompression == "zstd" {
|
||||
return ".mbox.zst"
|
||||
}
|
||||
return ".mbox"
|
||||
}
|
||||
|
||||
func mboxRecord(m RawMessage) []byte {
|
||||
var b bytes.Buffer
|
||||
fromDate := m.InternalDate
|
||||
if fromDate.IsZero() {
|
||||
fromDate = time.Now()
|
||||
}
|
||||
if _, err := fmt.Fprintf(f, "From MAILER-DAEMON %s\r\n", fromDate.Format("Mon Jan _2 15:04:05 2006")); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(&b, "From MAILER-DAEMON %s\r\n", fromDate.Format("Mon Jan _2 15:04:05 2006"))
|
||||
body := bytes.ReplaceAll(m.Body, []byte("\r\nFrom "), []byte("\r\n>From "))
|
||||
body = bytes.ReplaceAll(body, []byte("\nFrom "), []byte("\n>From "))
|
||||
if _, err := f.Write(body); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = b.Write(body)
|
||||
if !bytes.HasSuffix(body, []byte("\n")) {
|
||||
if _, err := f.WriteString("\r\n"); err != nil {
|
||||
return err
|
||||
}
|
||||
_, _ = b.WriteString("\r\n")
|
||||
}
|
||||
_, err = f.WriteString("\r\n")
|
||||
return err
|
||||
_, _ = b.WriteString("\r\n")
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
func mboxAppendInfoFromMessage(m RawMessage, record []byte) MboxAppendInfo {
|
||||
info := MboxAppendInfo{InnerLen: int64(len(record))}
|
||||
msg, err := mail.ReadMessage(bytes.NewReader(m.Body))
|
||||
if err != nil {
|
||||
return info
|
||||
}
|
||||
info.Subject = decodeHeader(msg.Header.Get("Subject"))
|
||||
info.From = decodeHeader(msg.Header.Get("From"))
|
||||
info.Date = decodeHeader(msg.Header.Get("Date"))
|
||||
return info
|
||||
}
|
||||
|
||||
// --- Viewer-Seite: mbox wieder lesen fuers Browser-Betrachten (08-viewer.go) ---
|
||||
|
|
@ -79,6 +137,9 @@ type MboxEntry struct {
|
|||
// ReadMboxList parst die Kopfzeilen aller Mails einer mbox-Datei (fuer die
|
||||
// Nachrichtenliste). ReadMboxMessage liefert eine einzelne Mail als Rohtext.
|
||||
func ReadMboxList(path string) ([]MboxEntry, error) {
|
||||
if entries, ok := readMboxListFromIndex(path); ok {
|
||||
return entries, nil
|
||||
}
|
||||
msgs, err := readMboxMessages(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -101,6 +162,9 @@ func ReadMboxList(path string) ([]MboxEntry, error) {
|
|||
}
|
||||
|
||||
func ReadMboxMessage(path string, index int) ([]byte, error) {
|
||||
if raw, ok, err := readMboxMessageFromIndex(path, index); ok || err != nil {
|
||||
return raw, err
|
||||
}
|
||||
msgs, err := readMboxMessages(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -116,6 +180,21 @@ func readMboxMessages(path string) ([][]byte, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if isZstdMbox(path, b) {
|
||||
dec, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer dec.Close()
|
||||
b, err = dec.DecodeAll(b, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return readMboxMessagesBytes(b), nil
|
||||
}
|
||||
|
||||
func readMboxMessagesBytes(b []byte) [][]byte {
|
||||
b = bytes.ReplaceAll(b, []byte("\r\n"), []byte("\n"))
|
||||
lines := bytes.Split(b, []byte("\n"))
|
||||
var msgs [][]byte
|
||||
|
|
@ -142,7 +221,106 @@ func readMboxMessages(path string) ([][]byte, error) {
|
|||
if inMsg && cur.Len() > 0 {
|
||||
msgs = append(msgs, bytes.TrimRight(cur.Bytes(), "\n"))
|
||||
}
|
||||
return msgs, nil
|
||||
return msgs
|
||||
}
|
||||
|
||||
func readMboxListFromIndex(path string) ([]MboxEntry, bool) {
|
||||
accountID, folder, ok := mboxIndexContext(path)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
index, err := ListMboxIndex(accountID, folder)
|
||||
if err != nil || len(index) == 0 {
|
||||
return nil, false
|
||||
}
|
||||
out := make([]MboxEntry, 0, len(index))
|
||||
for _, entry := range index {
|
||||
out = append(out, MboxEntry{
|
||||
Index: entry.Seq,
|
||||
From: entry.From,
|
||||
Subject: entry.Subject,
|
||||
Date: entry.Date,
|
||||
})
|
||||
}
|
||||
return out, true
|
||||
}
|
||||
|
||||
func readMboxMessageFromIndex(path string, index int) ([]byte, bool, error) {
|
||||
accountID, folder, ok := mboxIndexContext(path)
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
entry, err := GetMboxIndex(accountID, folder, index)
|
||||
if err != nil {
|
||||
return nil, false, nil
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(path), ".zst") {
|
||||
return nil, false, nil
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := f.Seek(entry.FileOffset, io.SeekStart); err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
frame := make([]byte, entry.FrameLen)
|
||||
if _, err := io.ReadFull(f, frame); err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
dec, err := zstd.NewReader(nil)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
defer dec.Close()
|
||||
record, err := dec.DecodeAll(frame, nil)
|
||||
if err != nil {
|
||||
return nil, true, err
|
||||
}
|
||||
msgs := readMboxMessagesBytes(record)
|
||||
if len(msgs) == 0 {
|
||||
return nil, true, fmt.Errorf("message index out of range")
|
||||
}
|
||||
return msgs[0], true, nil
|
||||
}
|
||||
|
||||
func mboxIndexContext(path string) (int64, string, bool) {
|
||||
if DB == nil || Cfg.MboxRoot == "" {
|
||||
return 0, "", false
|
||||
}
|
||||
root, err := filepath.Abs(Cfg.MboxRoot)
|
||||
if err != nil {
|
||||
return 0, "", false
|
||||
}
|
||||
abs, err := filepath.Abs(path)
|
||||
if err != nil {
|
||||
return 0, "", false
|
||||
}
|
||||
rel, err := filepath.Rel(root, abs)
|
||||
if err != nil || strings.HasPrefix(rel, "..") {
|
||||
return 0, "", false
|
||||
}
|
||||
parts := strings.Split(rel, string(os.PathSeparator))
|
||||
if len(parts) < 2 {
|
||||
return 0, "", false
|
||||
}
|
||||
account, ok := accountByMboxDir(parts[0])
|
||||
if !ok || account.ID == 0 {
|
||||
return 0, "", false
|
||||
}
|
||||
folder := folderNameFromMboxPath(path)
|
||||
if folder == "" {
|
||||
return 0, "", false
|
||||
}
|
||||
return account.ID, folder, true
|
||||
}
|
||||
|
||||
func isZstdMbox(path string, b []byte) bool {
|
||||
if strings.HasSuffix(strings.ToLower(path), ".zst") {
|
||||
return true
|
||||
}
|
||||
return len(b) >= 4 && b[0] == 0x28 && b[1] == 0xb5 && b[2] == 0x2f && b[3] == 0xfd
|
||||
}
|
||||
|
||||
func decodeHeader(v string) string {
|
||||
|
|
@ -354,3 +532,29 @@ func safeMboxName(folder string) string {
|
|||
}
|
||||
return folder
|
||||
}
|
||||
|
||||
func archiveMboxPaths(dir string) []string {
|
||||
plain, _ := filepath.Glob(filepath.Join(dir, "*.mbox"))
|
||||
compressed, _ := filepath.Glob(filepath.Join(dir, "*.mbox.zst"))
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(plain)+len(compressed))
|
||||
for _, path := range plain {
|
||||
seen[folderNameFromMboxPath(path)] = true
|
||||
out = append(out, path)
|
||||
}
|
||||
for _, path := range compressed {
|
||||
if seen[folderNameFromMboxPath(path)] {
|
||||
continue
|
||||
}
|
||||
out = append(out, path)
|
||||
}
|
||||
sortStrings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func folderNameFromMboxPath(path string) string {
|
||||
name := filepath.Base(path)
|
||||
name = strings.TrimSuffix(name, ".zst")
|
||||
name = strings.TrimSuffix(name, ".mbox")
|
||||
return name
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue