diff --git a/REVIEW-FIXES.md b/REVIEW-FIXES.md index c04844a..68f2709 100644 --- a/REVIEW-FIXES.md +++ b/REVIEW-FIXES.md @@ -11,6 +11,7 @@ Der Migrationskern soll auf `emersion/go-imap/v2` umgestellt werden. Der aktuell 1. `simpleIMAP` durch `emersion/go-imap/v2` ersetzen. - Betrifft `backend/04-imap-source.go` und `backend/05-imap-target.go`. - Ziel: IMAP UTF-7 korrekt dekodieren, `FETCH` robust parsen, Flags korrekt normalisieren, APPEND RFC-konform senden. + - Detailbrief: `go-imap-migration.md`. 2. IMAP UTF-7 Ordnernamen korrekt behandeln. - Aktuelles Risiko: Rollen-Erkennung fuer Papierkorb/Geloescht/Gesendet scheitert bei Alt-Providern. diff --git a/backend/04-imap-source.go b/backend/04-imap-source.go index 34dbf66..8732257 100644 --- a/backend/04-imap-source.go +++ b/backend/04-imap-source.go @@ -1,72 +1,64 @@ package backend import ( - "bufio" "crypto/sha256" "crypto/tls" + "errors" "fmt" - "io" "net" "net/mail" "strconv" "strings" "time" + + "github.com/emersion/go-imap/v2" + "github.com/emersion/go-imap/v2/imapclient" ) // RawMessage ist eine 1:1 aus der Quelle geholte Nachricht: der Rohkoerper // plus die Metadaten, die den sauberen Umzug ausmachen. type RawMessage struct { - MessageID string // aus Header, Schluessel fuer Idempotenz + MessageID string // aus Envelope/Header, Schluessel fuer Idempotenz Body []byte // kompletter RFC822-Rohtext, wird NICHT umgeschrieben Flags []string // \Seen \Answered \Flagged ... 1:1 uebernehmen InternalDate time.Time // Original-Zeit, per APPEND erhalten } // Folder ist ein Ordner der Quelle MIT den Infos, die 11-folders.go fuer die -// Rollen-Zuordnung braucht: dekodierter Name (Umlaute als Klartext, NICHT -// modified-UTF-7), SPECIAL-USE-Attribute und der Hierarchie-Trenner. +// Rollen-Zuordnung braucht: dekodierter Name, SPECIAL-USE-Attribute und der +// Hierarchie-Trenner. type Folder struct { - Name string // dekodiert, z.B. "Gelöschte Objekte" oder "INBOX.Sent" + Name string // dekodiert, z.B. "Geloeschte Objekte" oder "INBOX.Sent" Attrs []string // z.B. ["\\Sent"] aus SPECIAL-USE - Delim string // "." oder "/" — servereigener Trenner + Delim string // "." oder "/" - servereigener Trenner } // SourceMailbox abstrahiert Quelle (IMAP primaer, POP3 Fallback), damit // 07-migrate.go gegen ein Interface arbeitet. type SourceMailbox interface { Folders() ([]Folder, error) // rekursiver Ordnerbaum - Fetch(folder string) ([]RawMessage, error) // Stream: BODY[] FLAGS INTERNALDATE + Fetch(folder string) ([]RawMessage, error) // BODY[] FLAGS INTERNALDATE Close() error } -// OpenIMAPSource verbindet die Quelle per go-imap/v2. -// -// Verschluesselung nach a.SrcSecurity: -// - "tls": imapclient.DialTLS (impliziter TLS, Port 993) -// - "starttls": imapclient.DialStartTLS auf Klartext-Port 143 -// - "none": imapclient.Dial (REINER KLARTEXT, alte Provider!) -// -// a.SrcInsecure -> tls.Config{InsecureSkipVerify:true} fuer kaputte/selbst- -// signierte Zerts. go-imap dekodiert modified-UTF-7-Ordnernamen selbst; wir -// geben in Folder.Name den Klartext weiter. func OpenIMAPSource(a Account) (SourceMailbox, error) { - c, err := dialIMAP(a.SrcHost, a.SrcPort, a.SrcSecurity, a.SrcInsecure) + c, err := dialIMAPClient(a.SrcHost, a.SrcPort, a.SrcSecurity, a.SrcInsecure) if err != nil { return nil, err } - if err := c.login(a.SrcUser, a.SrcPass); err != nil { + if err := c.Login(a.SrcUser, a.SrcPass).Wait(); err != nil { _ = c.Close() return nil, err } - return &imapSource{c: c}, nil + return &imapSource{mailbox: &imapClientMailbox{c: c}}, nil } type imapSource struct { - c *simpleIMAP + mailbox *imapClientMailbox } func (s *imapSource) Folders() ([]Folder, error) { - boxes, err := s.c.listMailboxes() + boxes, err := s.mailbox.listMailboxes() if err != nil { return nil, err } @@ -84,132 +76,15 @@ func (s *imapSource) Folders() ([]Folder, error) { } func (s *imapSource) Fetch(folder string) ([]RawMessage, error) { - if err := s.c.selectMailbox(folder); err != nil { - return nil, err - } - return s.c.fetchAll() + return s.mailbox.fetchAll(folder) } func (s *imapSource) Close() error { - return s.c.Close() + return s.mailbox.Close() } -type simpleIMAP struct { - conn net.Conn - r *bufio.Reader - w *bufio.Writer - tag int -} - -func dialIMAP(host string, port int, security string, insecure bool) (*simpleIMAP, error) { - addr := net.JoinHostPort(host, strconv.Itoa(port)) - tlsCfg := &tls.Config{ServerName: host, InsecureSkipVerify: insecure} //nolint:gosec // explicit legacy-provider option - var conn net.Conn - var err error - if security == "tls" { - conn, err = tls.Dial("tcp", addr, tlsCfg) - } else { - conn, err = net.DialTimeout("tcp", addr, 30*time.Second) - } - if err != nil { - return nil, err - } - c := &simpleIMAP{conn: conn, r: bufio.NewReader(conn), w: bufio.NewWriter(conn)} - if _, err := c.r.ReadString('\n'); err != nil { - _ = conn.Close() - return nil, err - } - if security == "starttls" { - if err := c.simple("STARTTLS"); err != nil { - _ = conn.Close() - return nil, err - } - tlsConn := tls.Client(conn, tlsCfg) - if err := tlsConn.Handshake(); err != nil { - _ = conn.Close() - return nil, err - } - c.conn = tlsConn - c.r = bufio.NewReader(tlsConn) - c.w = bufio.NewWriter(tlsConn) - } - return c, nil -} - -func (c *simpleIMAP) login(user, pass string) error { - return c.simpleLabel("LOGIN "+imapQuote(user)+" "+imapQuote(pass), "LOGIN "+imapQuote(user)+" ") -} - -func (c *simpleIMAP) selectMailbox(name string) error { - return c.simple("SELECT " + imapQuote(name)) -} - -func (c *simpleIMAP) simple(cmd string) error { - return c.simpleLabel(cmd, cmd) -} - -func (c *simpleIMAP) simpleLabel(cmd, label string) error { - tag := c.nextTag() - if _, err := fmt.Fprintf(c.w, "%s %s\r\n", tag, cmd); err != nil { - return err - } - if err := c.w.Flush(); err != nil { - return err - } - for { - line, err := c.r.ReadString('\n') - if err != nil { - return err - } - if strings.HasPrefix(line, tag+" ") { - if strings.Contains(line, " OK") { - return nil - } - return fmt.Errorf("imap %s failed: %s", label, strings.TrimSpace(line)) - } - } -} - -func (c *simpleIMAP) fetchAll() ([]RawMessage, error) { - tag := c.nextTag() - if _, err := fmt.Fprintf(c.w, "%s UID FETCH 1:* (UID FLAGS INTERNALDATE BODY.PEEK[])\r\n", tag); err != nil { - return nil, err - } - if err := c.w.Flush(); err != nil { - return nil, err - } - var out []RawMessage - for { - line, err := c.r.ReadString('\n') - if err != nil { - return nil, err - } - if strings.HasPrefix(line, tag+" ") { - if strings.Contains(line, " OK") { - return out, nil - } - return out, fmt.Errorf("imap fetch failed: %s", strings.TrimSpace(line)) - } - if !strings.HasPrefix(line, "* ") || !strings.Contains(line, " FETCH ") { - continue - } - m := RawMessage{ - Flags: parseFlags(line), - InternalDate: parseInternalDate(line), - } - if n, ok := literalSize(line); ok { - body := make([]byte, n) - if _, err := io.ReadFull(c.r, body); err != nil { - return out, err - } - m.Body = body - m.MessageID = messageID(body) - if rest, err := c.r.ReadString('\n'); err == nil { - _ = rest - } - out = append(out, m) - } - } +type imapClientMailbox struct { + c *imapclient.Client } type imapListMailbox struct { @@ -218,93 +93,100 @@ type imapListMailbox struct { Delim string } -func (c *simpleIMAP) listMailboxes() ([]imapListMailbox, error) { - tag := c.nextTag() - if _, err := fmt.Fprintf(c.w, "%s LIST \"\" \"*\"\r\n", tag); err != nil { - return nil, err +func dialIMAPClient(host string, port int, security string, insecure bool) (*imapclient.Client, error) { + addr := net.JoinHostPort(host, strconv.Itoa(port)) + opts := &imapclient.Options{ + TLSConfig: &tls.Config{ + ServerName: host, + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: insecure, //nolint:gosec // explicit legacy-provider option + }, } - if err := c.w.Flush(); err != nil { - return nil, err + switch strings.ToLower(strings.TrimSpace(security)) { + case "tls", "": + return imapclient.DialTLS(addr, opts) + case "starttls": + return imapclient.DialStartTLS(addr, opts) + case "none": + return imapclient.DialInsecure(addr, opts) + default: + return nil, fmt.Errorf("unbekannter IMAP-Security-Modus %q", security) } +} + +func (m *imapClientMailbox) listMailboxes() ([]imapListMailbox, error) { + cmd := m.c.List("", "*", &imap.ListOptions{ReturnSpecialUse: true}) + defer cmd.Close() + var out []imapListMailbox for { - line, err := c.r.ReadString('\n') + data := cmd.Next() + if data == nil { + break + } + out = append(out, imapListMailbox{ + Name: data.Mailbox, + Attrs: mailboxAttrsToStrings(data.Attrs), + Delim: delimString(data.Delim), + }) + } + if err := cmd.Close(); err != nil { + return out, err + } + return out, nil +} + +func (m *imapClientMailbox) fetchAll(folder string) ([]RawMessage, error) { + if _, err := m.c.Select(folder, &imap.SelectOptions{ReadOnly: true}).Wait(); err != nil { + return nil, err + } + + section := &imap.FetchItemBodySection{Peek: true} + seqSet := imap.SeqSet{} + seqSet.AddRange(1, 0) + cmd := m.c.Fetch(seqSet, &imap.FetchOptions{ + Envelope: true, + Flags: true, + InternalDate: true, + BodySection: []*imap.FetchItemBodySection{section}, + }) + defer cmd.Close() + + var out []RawMessage + for { + data := cmd.Next() + if data == nil { + break + } + buf, err := data.Collect() if err != nil { - return nil, err + return out, err } - if strings.HasPrefix(line, tag+" ") { - if strings.Contains(line, " OK") { - return out, nil - } - return out, fmt.Errorf("imap list failed: %s", strings.TrimSpace(line)) - } - if !strings.HasPrefix(line, "* LIST ") { - continue - } - if box, ok := parseListMailbox(strings.TrimSpace(line)); ok { - out = append(out, box) + body := buf.FindBodySection(section) + if body == nil { + body = []byte{} } + out = append(out, RawMessage{ + MessageID: messageIDFromFetch(buf, body), + Body: body, + Flags: flagsToStrings(sanitizeIMAPFlags(buf.Flags)), + InternalDate: buf.InternalDate, + }) } + if err := cmd.Close(); err != nil { + return out, err + } + return out, nil } -func parseListMailbox(line string) (imapListMailbox, bool) { - rest := strings.TrimSpace(strings.TrimPrefix(line, "* LIST ")) - if !strings.HasPrefix(rest, "(") { - return imapListMailbox{}, false +func (m *imapClientMailbox) Close() error { + if m == nil || m.c == nil { + return nil } - endAttrs := strings.Index(rest, ")") - if endAttrs < 0 { - return imapListMailbox{}, false + if err := m.c.Logout().Wait(); err != nil { + return m.c.Close() } - attrs := strings.Fields(rest[1:endAttrs]) - rest = strings.TrimSpace(rest[endAttrs+1:]) - delim, next, ok := parseIMAPAtomOrQuoted(rest) - if !ok { - return imapListMailbox{}, false - } - if strings.EqualFold(delim, "NIL") { - delim = "/" - } - rest = strings.TrimSpace(rest[next:]) - name, _, ok := parseIMAPAtomOrQuoted(rest) - if !ok || name == "" { - return imapListMailbox{}, false - } - return imapListMailbox{Name: name, Attrs: attrs, Delim: delim}, true -} - -func parseIMAPAtomOrQuoted(s string) (string, int, bool) { - s = strings.TrimLeft(s, " \t") - trimmed := len(s) - if s == "" { - return "", 0, false - } - if s[0] != '"' { - end := strings.IndexAny(s, " \t\r\n") - if end < 0 { - end = len(s) - } - return s[:end], trimmed - len(s) + end, true - } - var b strings.Builder - escaped := false - for i := 1; i < len(s); i++ { - ch := s[i] - if escaped { - b.WriteByte(ch) - escaped = false - continue - } - if ch == '\\' { - escaped = true - continue - } - if ch == '"' { - return b.String(), trimmed - len(s) + i + 1, true - } - b.WriteByte(ch) - } - return "", 0, false + return m.c.Close() } func hasIMAPAttr(attrs []string, want string) bool { @@ -316,111 +198,60 @@ func hasIMAPAttr(attrs []string, want string) bool { return false } -func (c *simpleIMAP) appendMessage(folder string, m RawMessage) error { - tag := c.nextTag() - flags := "" - if len(m.Flags) > 0 { - flags = " (" + strings.Join(m.Flags, " ") + ")" +func mailboxAttrsToStrings(attrs []imap.MailboxAttr) []string { + out := make([]string, 0, len(attrs)) + for _, attr := range attrs { + out = append(out, string(attr)) } - date := "" - if !m.InternalDate.IsZero() { - date = " " + imapQuote(m.InternalDate.Format("02-Jan-2006 15:04:05 -0700")) + return out +} + +func delimString(delim rune) string { + if delim == 0 { + return "/" } - if _, err := fmt.Fprintf(c.w, "%s APPEND %s%s%s {%d}\r\n", tag, imapQuote(folder), flags, date, len(m.Body)); err != nil { - return err - } - if err := c.w.Flush(); err != nil { - return err - } - line, err := c.r.ReadString('\n') - if err != nil { - return err - } - if !strings.HasPrefix(line, "+") { - return fmt.Errorf("imap append literal rejected: %s", strings.TrimSpace(line)) - } - if _, err := c.w.Write(m.Body); err != nil { - return err - } - if _, err := c.w.WriteString("\r\n"); err != nil { - return err - } - if err := c.w.Flush(); err != nil { - return err - } - for { - line, err := c.r.ReadString('\n') - if err != nil { - return err + return string(delim) +} + +func sanitizeIMAPFlags(in []imap.Flag) []imap.Flag { + out := make([]imap.Flag, 0, len(in)) + for _, flag := range in { + switch strings.ToLower(string(flag)) { + case `\recent`, `\*`: + continue + default: + out = append(out, flag) } - if strings.HasPrefix(line, tag+" ") { - if strings.Contains(line, " OK") { - return nil + } + return out +} + +func flagsToStrings(flags []imap.Flag) []string { + out := make([]string, 0, len(flags)) + for _, flag := range flags { + out = append(out, string(flag)) + } + return out +} + +func stringsToFlags(flags []string) []imap.Flag { + return sanitizeIMAPFlags(func() []imap.Flag { + out := make([]imap.Flag, 0, len(flags)) + for _, flag := range flags { + flag = strings.TrimSpace(flag) + if flag != "" { + out = append(out, imap.Flag(flag)) } - return fmt.Errorf("imap append failed: %s", strings.TrimSpace(line)) } - } + return out + }()) } -func (c *simpleIMAP) nextTag() string { - c.tag++ - return fmt.Sprintf("A%04d", c.tag) -} - -func (c *simpleIMAP) Close() error { - _ = c.simple("LOGOUT") - return c.conn.Close() -} - -func imapQuote(s string) string { - s = strings.ReplaceAll(s, `\`, `\\`) - s = strings.ReplaceAll(s, `"`, `\"`) - return `"` + s + `"` -} - -func literalSize(line string) (int, bool) { - end := strings.LastIndex(line, "}") - start := strings.LastIndex(line[:end+1], "{") - if start < 0 || end < 0 || end <= start+1 { - return 0, false +func messageIDFromFetch(buf *imapclient.FetchMessageBuffer, body []byte) string { + if buf != nil && buf.Envelope != nil && strings.TrimSpace(buf.Envelope.MessageID) != "" { + return strings.TrimSpace(buf.Envelope.MessageID) } - n, err := strconv.Atoi(line[start+1 : end]) - return n, err == nil -} - -func parseFlags(line string) []string { - i := strings.Index(line, "FLAGS (") - if i < 0 { - return nil - } - i += len("FLAGS (") - j := strings.Index(line[i:], ")") - if j < 0 { - return nil - } - fields := strings.Fields(line[i : i+j]) - return fields -} - -func parseInternalDate(line string) time.Time { - i := strings.Index(line, "INTERNALDATE ") - if i < 0 { - return time.Now() - } - rest := line[i+len("INTERNALDATE "):] - if !strings.HasPrefix(rest, `"`) { - return time.Now() - } - rest = rest[1:] - j := strings.Index(rest, `"`) - if j < 0 { - return time.Now() - } - t, err := time.Parse("02-Jan-2006 15:04:05 -0700", rest[:j]) - if err != nil { - return time.Now() - } - return t + return messageID(body) } func messageID(body []byte) string { @@ -430,10 +261,15 @@ func messageID(body []byte) string { return id } } - return fmt.Sprintf("sha256:%x", sha256Bytes(body)) + return fmt.Sprintf("sha256:%x", sha256.Sum256(body)) } -func sha256Bytes(b []byte) []byte { - h := sha256.Sum256(b) - return h[:] +func isAlreadyExistsError(err error) bool { + var imapErr *imap.Error + if errors.As(err, &imapErr) { + text := strings.ToLower(imapErr.Text) + return strings.Contains(text, "exist") || strings.Contains(text, "already") + } + text := strings.ToLower(err.Error()) + return strings.Contains(text, "exist") || strings.Contains(text, "already") } diff --git a/backend/04-imap-source_test.go b/backend/04-imap-source_test.go new file mode 100644 index 0000000..1d05415 --- /dev/null +++ b/backend/04-imap-source_test.go @@ -0,0 +1,26 @@ +package backend + +import ( + "strings" + "testing" + + "github.com/emersion/go-imap/v2" +) + +func TestSanitizeIMAPFlagsDropsRecentAndWildcard(t *testing.T) { + got := sanitizeIMAPFlags([]imap.Flag{imap.FlagSeen, imap.Flag(`\Recent`), imap.FlagWildcard, imap.FlagAnswered}) + if len(got) != 2 || got[0] != imap.FlagSeen || got[1] != imap.FlagAnswered { + t.Fatalf("sanitizeIMAPFlags returned %#v", got) + } +} + +func TestMessageIDFallsBackToBodyHash(t *testing.T) { + body := []byte("From: a@example.com\r\nSubject: Test\r\n\r\nBody") + id := messageID(body) + if !strings.HasPrefix(id, "sha256:") { + t.Fatalf("expected sha256 fallback, got %q", id) + } + if id != messageID(body) { + t.Fatal("messageID hash fallback is not stable") + } +} diff --git a/backend/05-imap-target.go b/backend/05-imap-target.go index 735ff90..1c04173 100644 --- a/backend/05-imap-target.go +++ b/backend/05-imap-target.go @@ -1,41 +1,44 @@ package backend -import "strings" +import ( + "fmt" + "strings" + + "github.com/emersion/go-imap/v2" +) // TargetMailbox = das Ziel-Postfach (neuer Host). Der Umzug schreibt hier per // IMAP APPEND hinein -- NICHT per SMTP. APPEND erhaelt Ordner, Flags und // Original-Datum; SMTP wuerde alles als "neu/ungelesen" mit falschem Datum // zustellen und die Ordnerstruktur zerstoeren. type TargetMailbox interface { - Folders() ([]TargetFolder, error) // vorhandene Ziel-Ordner (fuer Rollen-Zuordnung) - Delim() string // Hierarchie-Trenner des Ziels - EnsureFolder(name string) error // anlegen + subscriben, falls fehlt - Append(folder string, m RawMessage) error // mit m.Flags und m.InternalDate + Folders() ([]TargetFolder, error) // vorhandene Ziel-Ordner (fuer Rollen-Zuordnung) + Delim() string // Hierarchie-Trenner des Ziels + EnsureFolder(name string) error // anlegen + subscriben, falls fehlt + Append(folder string, m RawMessage) error // mit m.Flags und m.InternalDate Fetch(folder string) ([]RawMessage, error) // Ziel-Audit: geschriebene Mail wiederfinden Close() error } -// OpenIMAPTarget verbindet das Ziel per go-imap/v2 (Security wie Quelle, siehe -// a.DstSecurity/a.DstInsecure; meist "tls"). func OpenIMAPTarget(a Account) (TargetMailbox, error) { - c, err := dialIMAP(a.DstHost, a.DstPort, a.DstSecurity, a.DstInsecure) + c, err := dialIMAPClient(a.DstHost, a.DstPort, a.DstSecurity, a.DstInsecure) if err != nil { return nil, err } - if err := c.login(a.DstUser, a.DstPass); err != nil { + if err := c.Login(a.DstUser, a.DstPass).Wait(); err != nil { _ = c.Close() return nil, err } - return &imapTarget{c: c}, nil + return &imapTarget{mailbox: &imapClientMailbox{c: c}}, nil } type imapTarget struct { - c *simpleIMAP - delim string + mailbox *imapClientMailbox + delim string } func (t *imapTarget) Folders() ([]TargetFolder, error) { - boxes, err := t.c.listMailboxes() + boxes, err := t.mailbox.listMailboxes() if err != nil { return nil, err } @@ -63,29 +66,64 @@ func (t *imapTarget) Delim() string { } func (t *imapTarget) EnsureFolder(name string) error { - if name == "INBOX" { + name = strings.TrimSpace(name) + if name == "" || strings.EqualFold(name, "INBOX") { return nil } - if err := t.c.simple("CREATE " + imapQuote(name)); err != nil { - if !strings.Contains(strings.ToLower(err.Error()), "exist") && !strings.Contains(strings.ToLower(err.Error()), "already") { - return err - } + if err := t.ensureParentFolders(name); err != nil { + return err } - _ = t.c.simple("SUBSCRIBE " + imapQuote(name)) + if err := t.createFolder(name); err != nil { + return err + } + _ = t.mailbox.c.Subscribe(name).Wait() return nil } +func (t *imapTarget) ensureParentFolders(name string) error { + delim := t.Delim() + if delim == "" || !strings.Contains(name, delim) { + return nil + } + parts := strings.Split(name, delim) + for i := 1; i < len(parts); i++ { + parent := strings.Join(parts[:i], delim) + if strings.TrimSpace(parent) == "" || strings.EqualFold(parent, "INBOX") { + continue + } + if err := t.createFolder(parent); err != nil { + return err + } + } + return nil +} + +func (t *imapTarget) createFolder(name string) error { + err := t.mailbox.c.Create(name, nil).Wait() + if err == nil || isAlreadyExistsError(err) { + return nil + } + return fmt.Errorf("imap create %q failed: %w", name, err) +} + func (t *imapTarget) Append(folder string, m RawMessage) error { - return t.c.appendMessage(folder, m) + opts := &imap.AppendOptions{Flags: stringsToFlags(m.Flags), Time: m.InternalDate} + cmd := t.mailbox.c.Append(folder, int64(len(m.Body)), opts) + if _, err := cmd.Write(m.Body); err != nil { + _ = cmd.Close() + return err + } + if err := cmd.Close(); err != nil { + return err + } + _, err := cmd.Wait() + return err } func (t *imapTarget) Fetch(folder string) ([]RawMessage, error) { - if err := t.c.selectMailbox(folder); err != nil { - return nil, err - } - return t.c.fetchAll() + return t.mailbox.fetchAll(folder) } func (t *imapTarget) Close() error { - return t.c.Close() + return t.mailbox.Close() } diff --git a/go-imap-migration.md b/go-imap-migration.md new file mode 100644 index 0000000..f000d67 --- /dev/null +++ b/go-imap-migration.md @@ -0,0 +1,160 @@ +# Codex-Brief — Migrationskern auf emersion/go-imap/v2 + +Ziel: den handgerollten IMAP-Parser (`simpleIMAP` in `04-imap-source.go`, plus +`05-imap-target.go`) durch **`github.com/emersion/go-imap/v2`** ersetzen. Das +behebt in einem Zug die vier Integritäts-Bugs aus dem Review (siehe +`REVIEW-FIXES.md`), die alle Symptome desselben Problems sind: eigener Parser +statt Bibliothek. + +**Warum:** Bei einem Beweis-/Compliance-Archiv ist Vollständigkeit der ganze +Zweck. go-imap ist eine gepflegte pure-Go-Lib (keine Dependency-Lawine) und +erledigt genau das, was der handgerollte Parser falsch macht. + +## Bleibt UNVERÄNDERT (nicht anfassen) + +- Die Interfaces und Typen: `SourceMailbox`, `TargetMailbox`, `RawMessage`, + `Folder{Name,Attrs,Delim}`, `TargetFolder{Name,Attrs}`. Nur die *Implementierung* + dahinter wird getauscht. +- `06-mbox.go` (mbox-Writer), `11-folders.go` (Rollen-Mapping), + `02-database.go`, die ganze Web-/UI-Schicht. +- Die **Idempotenz-Reihenfolge** in `07-migrate.go`: prüfen → `dst.Append` → + `mbox.Append` → **erst dann** `MarkCopied`. Nicht verändern. +- Die drei Security-Modi `tls`/`starttls`/`none` + `insecure` (Semantik bleibt, + nur die API dahinter wird go-imap). + +## Neu bauen: `04-imap-source.go` + `05-imap-target.go` + +Alles Handgerollte raus: `simpleIMAP`, `dialIMAP`, `parseListMailbox`, +`fetchAll`, `parseFlags`, `parseInternalDate`, `appendMessage`. Und die +irreführenden Kommentare („go-imap dekodiert…") stimmen danach endlich. + +> `go-imap/v2` ist **Beta** — die exakten Signaturen gegen die gepinnte Version +> prüfen (`go get github.com/emersion/go-imap/v2@latest`, dann die aufgelöste +> Version in `go.mod` festnageln). Unten steht die *Form*, nicht das Evangelium. + +### 1. Verbinden nach Security-Modus (`imapclient`) +```go +opts := &imapclient.Options{ TLSConfig: &tls.Config{ + ServerName: host, MinVersion: tls.VersionTLS12, + InsecureSkipVerify: a.SrcInsecure } } +switch a.SrcSecurity { +case "tls": c, err = imapclient.DialTLS(addr, opts) // implizit, 993 +case "starttls": c, err = imapclient.DialStartTLS(addr, opts) // 143 + Upgrade +case "none": c, err = imapclient.DialInsecure(addr, opts) // reiner Klartext +} +c.Login(a.SrcUser, a.SrcPass).Wait() +``` +`none` (`DialInsecure`) ist der Alt-Provider-Fall — muss ohne TLS funktionieren. + +### 2. `Folders()` — LIST mit SPECIAL-USE + Delimiter +```go +cmd := c.List("", "*", &imap.ListOptions{ReturnSpecialUse: true}) +for { d := cmd.Next(); if d == nil { break } + attrs := make([]string, len(d.Attrs)) + for i, at := range d.Attrs { attrs[i] = string(at) } // z.B. "\\Sent" + folders = append(folders, Folder{ + Name: d.Mailbox, // **bereits UTF-8-dekodiert** -> Bug 1 weg + Attrs: attrs, // -> 11-folders RoleFromAttrs funktioniert + Delim: string(d.Delim) }) // -> Trenner-Übersetzung stimmt +} +``` +go-imap gibt `d.Mailbox` als dekodierten Klartext zurück → `Gelöschte Objekte` +statt `Gel&APY-schte…`. Damit greift `RoleFromName`/`RoleFromAttrs` und die +Standardordner werden **nicht mehr gedoppelt**. + +### 3. `Fetch()` — FLAGS + INTERNALDATE + Body, ein Message pro Iteration +```go +c.Select(folder, nil).Wait() +seq := imap.SeqSetRange(1, 0) // 1:* (alle) +fo := &imap.FetchOptions{ + Flags: true, InternalDate: true, Envelope: true, + BodySection: []*imap.FetchItemBodySection{{Peek: true}} } // BODY.PEEK[] -> setzt kein \Seen +fcmd := c.Fetch(seq, fo) +for { + msg := fcmd.Next(); if msg == nil { break } + buf, err := msg.Collect() // puffert GENAU EINE Mail -> kein OOM mehr + raw := RawMessage{ + MessageID: messageID(buf), // s.u. + Body: buf.FindBodySection(&imap.FetchItemBodySection{Peek:true}), + Flags: sanitizeFlags(buf.Flags), // s.u. — \Recent raus + InternalDate: buf.InternalDate } // go-imap liefert echte time.Time -> Bug 3 weg + // ... an Callback/Slice geben +} +``` +`buf.Flags`/`buf.InternalDate`/`buf.Envelope` sind strukturiert und +**reihenfolge-unabhängig** geparst → Bug 4 weg. `InternalDate` als `time.Time` +direkt aus der Lib → Bug 3 (Leerzeichen-Tag) weg. + +### 4. `sanitizeFlags` — `\Recent`/`\*` beim APPEND filtern (Bug 2) +```go +func sanitizeFlags(in []imap.Flag) []imap.Flag { + var out []imap.Flag + for _, f := range in { + if f == imap.FlagRecent || f == imap.FlagWildcard { continue } + out = append(out, f) + } + return out +} +``` +`\Recent` per APPEND ist RFC-verboten → Server antwortet `NO` → Mail ginge sonst +verloren. Muss raus. + +### 5. Message-ID (Dedup-Schlüssel) — aus Envelope, mit Hash-Fallback +```go +func messageID(buf *imapclient.FetchMessageBuffer) string { + if buf.Envelope != nil && buf.Envelope.MessageID != "" { return buf.Envelope.MessageID } + return "sha256:" + hex(sha256(body)) // Mails ohne Message-ID trotzdem deduplizieren +} +``` +Verbessert das aktuelle Verhalten (Mails ohne Message-ID wurden bisher bei jedem +Lauf erneut kopiert → Dubletten). + +### 6. Ziel (`05-imap-target.go`) +- `Folders()` / `Delim()`: gleiche LIST-Logik wie oben, in `[]TargetFolder`. +- `EnsureFolder(name)`: `c.Create(name, nil).Wait()`; „already exists" schlucken; + danach `c.Subscribe(name).Wait()`. **Verschachtelte Ordner:** wenn `name` + Trenner enthält, Eltern-Pfade zuerst anlegen (manche Server legen sie nicht + automatisch an — Review-Bug 6). +- `Append(folder, m)`: +```go +opts := &imap.AppendOptions{ Flags: sanitizeFlags(m.Flags), Time: m.InternalDate } +ac := c.Append(folder, int64(len(m.Body)), opts) +ac.Write(m.Body); ac.Close() +_, err := ac.Wait() +``` +Flags + Originaldatum bleiben erhalten (Regel 1). Nie SMTP. + +## Streaming (Schritt 2 — nach dem Kern-Swap, separat verifizieren) + +Damit ein GB-Postfach nicht in den RAM läuft, `Fetch` von „Slice zurückgeben" +auf **Callback pro Mail** umstellen: +```go +// Interface: +Fetch(folder string, fn func(RawMessage) error) error +``` +In `07-migrate.go` wandert die Pro-Mail-Schleife (dedup → Append → mbox → +MarkCopied) **in den Callback** — die Reihenfolge bleibt exakt gleich, nur der +Rahmen ändert sich. So ist immer nur eine Mail im Speicher. + +## go.mod +`github.com/emersion/go-imap/v2` hinzufügen, `go mod tidy`. (Zieht go-sasl/ +go-message transitiv; kein cgo — bleibt single-binary.) + +## Abnahme (mit unserem echten Testkorpus) +Nutze die **9 Test-Mails in `vdevop-02@golddata.eu`** (Text, Anhänge einzeln/ +mehrfach, ZIP, HTML-Body, Umlaut-Betreff) als Quelle: +1. `go test ./...` grün (inkl. `sanitizeFlags`- und `messageID`-Unit-Tests). +2. Migration **zweimal** laufen → zweiter Lauf kopiert **0** (Idempotenz). +3. Ziel-`INTERNALDATE` = Originaldatum der Quelle (nicht „heute") — inkl. einer + Mail mit einstelligem Tag. +4. Quell-Ordner `Gesendete Objekte`/`Papierkorb` landen im **vorhandenen** + Rollen-Ordner des Ziels, **keine Dublette**. +5. Eine Mail mit `\Recent` in der Quelle wird migriert (nicht mit `NO` abgelehnt). +6. Anhänge (PDF/PNG/ZIP) und Umlaut-Betreff kommen **byte-identisch** an + (Quelle-Body == Ziel-Body == mbox-Body). + +## NICHT in diesem Schritt +- POP3-Fallback verdrahten (`src_proto=pop3` → `OpenPOP3Source` in + `migrateAccount`) — eigener kleiner Folgeschritt. +- Nicht-ASCII-Betreff-Encoding im Forward (RFC 2047) — gehört zum späteren + kontrollierten Forward-Endpunkt. diff --git a/go.mod b/go.mod index 699097d..300ecb6 100644 --- a/go.mod +++ b/go.mod @@ -2,10 +2,15 @@ module mail-graveyard go 1.25 -require modernc.org/sqlite v1.40.1 +require ( + github.com/emersion/go-imap/v2 v2.0.0-beta.8 + modernc.org/sqlite v1.40.1 +) require ( github.com/dustin/go-humanize v1.0.1 // indirect + github.com/emersion/go-message v0.18.2 // indirect + github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect diff --git a/go.sum b/go.sum index 86155cd..a1888fe 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,11 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug= +github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48= +github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= +github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA= +github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk= +github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -10,17 +16,48 @@ github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdh github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b h1:M2rDM6z3Fhozi9O7NWsxAkg/yqS/lQJ6PmkyIV3YP+o= golang.org/x/exp v0.0.0-20250620022241-b7579e27df2b/go.mod h1:3//PLf8L/X+8b4vuAfHzxeRUl04Adcb341+IGKfnqS8= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.27.0 h1:kb+q2PyFnEADO2IEF935ehFUXlWiNjJWtRNgBLSfbxQ= golang.org/x/mod v0.27.0/go.mod h1:rWI627Fq0DEoudcK+MBkNkCe0EetEaDSwJJkCcjpazc= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k= golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.36.0 h1:kWS0uv/zsvHEle1LbV5LE8QujrxB3wfQyxHfhOk0Qkg= golang.org/x/tools v0.36.0/go.mod h1:WBDiHKJK8YgLHlcQPYQzNCkUxUypCaa5ZegCVutKm+s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A=