Add verified snapshots and harden archive identity

backend/06-mbox.go

@@ -1,6 +1,7 @@
 package backend
 
 import (
+	"bufio"
 	"bytes"
 	"encoding/base64"
 	"fmt"
@@ -290,17 +291,55 @@ func reindexAccountArchive(account Account) error {
 }
 
 func reindexPlainMbox(path string) ([]MboxIndexEntry, error) {
-	b, err := os.ReadFile(path)
+	f, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}
+	defer f.Close()
+	reader := bufio.NewReaderSize(f, 256*1024)
 	var out []MboxIndexEntry
-	for _, part := range splitMboxRecordsWithOffsets(b) {
-		entry := indexEntryFromMboxRecord(part.Message, part.Offset, int64(part.Length), 0, int64(part.Length))
+	var record bytes.Buffer
+	var offset int64
+	var messageStart int64 = -1
+	appendRecord := func(end int64) {
+		if messageStart < 0 || end <= messageStart {
+			return
+		}
+		message := unescapeMboxMessage(record.Bytes())
+		if len(message) == 0 {
+			return
+		}
+		entry := indexEntryFromMboxRecord(message, messageStart, end-messageStart, 0, end-messageStart)
 		if entry.MessageID != "" || entry.BodySHA256 != "" {
 			out = append(out, entry)
 		}
 	}
+	for {
+		lineStart := offset
+		line, readErr := reader.ReadBytes('\n')
+		offset += int64(len(line))
+		content := line
+		if len(content) > 0 && content[len(content)-1] == '\n' {
+			content = content[:len(content)-1]
+		}
+		if len(content) > 0 && content[len(content)-1] == '\r' {
+			content = content[:len(content)-1]
+		}
+		if bytes.HasPrefix(content, []byte("From ")) {
+			appendRecord(lineStart)
+			record = bytes.Buffer{}
+			messageStart = offset
+		} else if messageStart >= 0 && len(line) > 0 {
+			_, _ = record.Write(line)
+		}
+		if readErr != nil {
+			if readErr != io.EOF {
+				return nil, readErr
+			}
+			break
+		}
+	}
+	appendRecord(offset)
 	return out, nil
 }