Sync all IMAP folders

backend/04-imap-source.go

@@ -66,7 +66,21 @@ type imapSource struct {
 }
 
 func (s *imapSource) Folders() ([]Folder, error) {
-	return []Folder{{Name: "INBOX", Delim: "/"}}, nil
+	boxes, err := s.c.listMailboxes()
+	if err != nil {
+		return nil, err
+	}
+	folders := make([]Folder, 0, len(boxes))
+	for _, box := range boxes {
+		if hasIMAPAttr(box.Attrs, `\Noselect`) {
+			continue
+		}
+		folders = append(folders, Folder{Name: box.Name, Attrs: box.Attrs, Delim: box.Delim})
+	}
+	if len(folders) == 0 {
+		folders = append(folders, Folder{Name: "INBOX", Delim: "/"})
+	}
+	return folders, nil
 }
 
 func (s *imapSource) Fetch(folder string) ([]RawMessage, error) {
@@ -198,6 +212,110 @@ func (c *simpleIMAP) fetchAll() ([]RawMessage, error) {
 	}
 }
 
+type imapListMailbox struct {
+	Name  string
+	Attrs []string
+	Delim string
+}
+
+func (c *simpleIMAP) listMailboxes() ([]imapListMailbox, error) {
+	tag := c.nextTag()
+	if _, err := fmt.Fprintf(c.w, "%s LIST \"\" \"*\"\r\n", tag); err != nil {
+		return nil, err
+	}
+	if err := c.w.Flush(); err != nil {
+		return nil, err
+	}
+	var out []imapListMailbox
+	for {
+		line, err := c.r.ReadString('\n')
+		if err != nil {
+			return nil, err
+		}
+		if strings.HasPrefix(line, tag+" ") {
+			if strings.Contains(line, " OK") {
+				return out, nil
+			}
+			return out, fmt.Errorf("imap list failed: %s", strings.TrimSpace(line))
+		}
+		if !strings.HasPrefix(line, "* LIST ") {
+			continue
+		}
+		if box, ok := parseListMailbox(strings.TrimSpace(line)); ok {
+			out = append(out, box)
+		}
+	}
+}
+
+func parseListMailbox(line string) (imapListMailbox, bool) {
+	rest := strings.TrimSpace(strings.TrimPrefix(line, "* LIST "))
+	if !strings.HasPrefix(rest, "(") {
+		return imapListMailbox{}, false
+	}
+	endAttrs := strings.Index(rest, ")")
+	if endAttrs < 0 {
+		return imapListMailbox{}, false
+	}
+	attrs := strings.Fields(rest[1:endAttrs])
+	rest = strings.TrimSpace(rest[endAttrs+1:])
+	delim, next, ok := parseIMAPAtomOrQuoted(rest)
+	if !ok {
+		return imapListMailbox{}, false
+	}
+	if strings.EqualFold(delim, "NIL") {
+		delim = "/"
+	}
+	rest = strings.TrimSpace(rest[next:])
+	name, _, ok := parseIMAPAtomOrQuoted(rest)
+	if !ok || name == "" {
+		return imapListMailbox{}, false
+	}
+	return imapListMailbox{Name: name, Attrs: attrs, Delim: delim}, true
+}
+
+func parseIMAPAtomOrQuoted(s string) (string, int, bool) {
+	s = strings.TrimLeft(s, " \t")
+	trimmed := len(s)
+	if s == "" {
+		return "", 0, false
+	}
+	if s[0] != '"' {
+		end := strings.IndexAny(s, " \t\r\n")
+		if end < 0 {
+			end = len(s)
+		}
+		return s[:end], trimmed - len(s) + end, true
+	}
+	var b strings.Builder
+	escaped := false
+	for i := 1; i < len(s); i++ {
+		ch := s[i]
+		if escaped {
+			b.WriteByte(ch)
+			escaped = false
+			continue
+		}
+		if ch == '\\' {
+			escaped = true
+			continue
+		}
+		if ch == '"' {
+			return b.String(), trimmed - len(s) + i + 1, true
+		}
+		b.WriteByte(ch)
+	}
+	return "", 0, false
+}
+
+func hasIMAPAttr(attrs []string, want string) bool {
+	for _, attr := range attrs {
+		if strings.EqualFold(attr, want) {
+			return true
+		}
+	}
+	return false
+}
+
 func (c *simpleIMAP) appendMessage(folder string, m RawMessage) error {
 	tag := c.nextTag()
 	flags := ""

backend/05-imap-target.go

@@ -30,13 +30,34 @@ func OpenIMAPTarget(a Account) (TargetMailbox, error) {
 
 type imapTarget struct {
 	c     *simpleIMAP
+	delim string
 }
 
 func (t *imapTarget) Folders() ([]TargetFolder, error) {
-	return []TargetFolder{{Name: "INBOX"}}, nil
+	boxes, err := t.c.listMailboxes()
+	if err != nil {
+		return nil, err
+	}
+	folders := make([]TargetFolder, 0, len(boxes))
+	for _, box := range boxes {
+		if t.delim == "" && box.Delim != "" {
+			t.delim = box.Delim
+		}
+		if hasIMAPAttr(box.Attrs, `\Noselect`) {
+			continue
+		}
+		folders = append(folders, TargetFolder{Name: box.Name, Attrs: box.Attrs})
+	}
+	if len(folders) == 0 {
+		folders = append(folders, TargetFolder{Name: "INBOX"})
+	}
+	return folders, nil
 }
 
 func (t *imapTarget) Delim() string {
+	if t.delim != "" {
+		return t.delim
+	}
 	return "/"
 }
 

backend/07-migrate.go

@@ -86,49 +86,75 @@ func migrateAccount(a Account) error {
 		return err
 	}
 	jobID, _ := startJob(a.ID)
-	msgs, err := src.Fetch("INBOX")
+	folders, err := src.Folders()
 	if err != nil {
 		_ = finishJob(jobID, 0, 0, 1, "error")
 		return err
 	}
-	if err := dst.EnsureFolder("INBOX"); err != nil {
+	targets, err := dst.Folders()
-		_ = finishJob(jobID, len(msgs), 0, 1, "error")
+	if err != nil {
+		_ = finishJob(jobID, 0, 0, 1, "error")
 		return err
 	}
-	done, errs := 0, 0
+	total, done, errs := 0, 0, 0
-	for _, m := range msgs {
+	for _, folder := range folders {
-		already, err := AlreadyCopied(a.ID, "INBOX", m.MessageID)
+		dstFolder := MapSourceToTarget(folder.Name, folder.Attrs, folder.Delim, dst.Delim(), targets, "")
+		if dstFolder == "" {
+			dstFolder = folder.Name
+		}
+		msgs, err := src.Fetch(folder.Name)
 		if err != nil {
 			errs++
-			log.Printf("%s INBOX dedup error %s: %v", a.Name, m.MessageID, err)
+			log.Printf("%s %s fetch error: %v", a.Name, folder.Name, err)
+			continue
+		}
+		total += len(msgs)
+		if err := dst.EnsureFolder(dstFolder); err != nil {
+			errs++
+			log.Printf("%s %s ensure target %s error: %v", a.Name, folder.Name, dstFolder, err)
+			continue
+		}
+		folderDone, folderErrs := 0, 0
+		for _, m := range msgs {
+			already, err := AlreadyCopied(a.ID, folder.Name, m.MessageID)
+			if err != nil {
+				errs++
+				folderErrs++
+				log.Printf("%s %s dedup error %s: %v", a.Name, folder.Name, m.MessageID, err)
 				continue
 			}
 			if already {
 				continue
 			}
-		if err := dst.Append("INBOX", m); err != nil {
+			if err := dst.Append(dstFolder, m); err != nil {
 				errs++
-			log.Printf("%s INBOX append error %s: %v", a.Name, m.MessageID, err)
+				folderErrs++
+				log.Printf("%s %s append to %s error %s: %v", a.Name, folder.Name, dstFolder, m.MessageID, err)
 				continue
 			}
-		if err := mbox.Append("INBOX", m); err != nil {
+			if err := mbox.Append(folder.Name, m); err != nil {
 				errs++
-			log.Printf("%s INBOX mbox error %s: %v", a.Name, m.MessageID, err)
+				folderErrs++
+				log.Printf("%s %s mbox error %s: %v", a.Name, folder.Name, m.MessageID, err)
 				continue
 			}
-		if err := MarkCopied(a.ID, "INBOX", m.MessageID); err != nil {
+			if err := MarkCopied(a.ID, folder.Name, m.MessageID); err != nil {
 				errs++
-			log.Printf("%s INBOX mark error %s: %v", a.Name, m.MessageID, err)
+				folderErrs++
+				log.Printf("%s %s mark error %s: %v", a.Name, folder.Name, m.MessageID, err)
 				continue
 			}
 			done++
+			folderDone++
+		}
+		log.Printf("migration %s %s -> %s: total=%d copied=%d errors=%d", a.Name, folder.Name, dstFolder, len(msgs), folderDone, folderErrs)
 	}
 	state := "success"
 	if errs > 0 {
 		state = "partial"
 	}
-	_ = finishJob(jobID, len(msgs), done, errs, state)
+	_ = finishJob(jobID, total, done, errs, state)
-	log.Printf("migration %s INBOX: total=%d copied=%d errors=%d", a.Name, len(msgs), done, errs)
+	log.Printf("migration %s all folders: total=%d copied=%d errors=%d", a.Name, total, done, errs)
 	if errs > 0 {
 		return fmt.Errorf("migration %s finished with %d errors", a.Name, errs)
 	}
@@ -145,16 +171,25 @@ func CheckAccount(name string) error {
 		return fmt.Errorf("open source %s: %w", a.Name, err)
 	}
 	defer src.Close()
-	msgs, err := src.Fetch("INBOX")
+	folders, err := src.Folders()
 	if err != nil {
-		return fmt.Errorf("fetch source INBOX %s: %w", a.Name, err)
+		return fmt.Errorf("list source folders %s: %w", a.Name, err)
+	}
+	total := 0
+	for _, folder := range folders {
+		msgs, err := src.Fetch(folder.Name)
+		if err != nil {
+			return fmt.Errorf("fetch source %s %s: %w", a.Name, folder.Name, err)
+		}
+		log.Printf("account %s source folder %s messages=%d", a.Name, folder.Name, len(msgs))
+		total += len(msgs)
 	}
 	dst, err := OpenIMAPTarget(a)
 	if err != nil {
 		return fmt.Errorf("open target %s: %w", a.Name, err)
 	}
 	defer dst.Close()
-	log.Printf("account %s ok: source INBOX messages=%d, target login ok", a.Name, len(msgs))
+	log.Printf("account %s ok: source folders=%d messages=%d, target login ok", a.Name, len(folders), total)
 	return nil
 }
 

backend/11-folders.go

@@ -107,6 +107,28 @@ type TargetFolder struct {
 // Antwort). override kommt aus folder_map.
 func MapSourceToTarget(srcName string, srcAttrs []string, srcDelim, dstDelim string,
 	targets []TargetFolder, override string) string {
-	// TODO Codex.
+	if strings.TrimSpace(override) != "" {
-	return ""
+		return override
+	}
+	srcRole := Role(srcAttrs, srcName, srcDelim)
+	if srcRole == RoleInbox {
+		return "INBOX"
+	}
+	if srcRole != RoleNone {
+		for _, target := range targets {
+			if Role(target.Attrs, target.Name, dstDelim) == srcRole {
+				return target.Name
+			}
+		}
+	}
+	if srcDelim == "" {
+		srcDelim = "/"
+	}
+	if dstDelim == "" {
+		dstDelim = "/"
+	}
+	if srcDelim != dstDelim {
+		return strings.ReplaceAll(srcName, srcDelim, dstDelim)
+	}
+	return srcName
 }