import { createHash } from 'node:crypto'; import { opendir, readFile, stat } from 'node:fs/promises'; import path from 'node:path'; import { parseInboundMessage } from './inbound-mail.js'; import { decodeModifiedUtf7 } from './imap-utf7.js'; export { decodeModifiedUtf7 }; const standardFolders = new Map([ ['inbox', 'INBOX'], ['sent', 'Sent'], ['sent items', 'Sent'], ['draft', 'Drafts'], ['drafts', 'Drafts'], ['trash', 'Trash'], ['deleted messages', 'Trash'], ['junk', 'Junk'], ['junk e-mail', 'Junk'], ['spam', 'Junk'], ['archive', 'Archive'] ]); const standardMaildirFlags = new Map([ ['D', '\\Draft'], ['F', '\\Flagged'], ['P', '\\Passed'], ['R', '\\Answered'], ['S', '\\Seen'], ['T', '\\Deleted'] ]); /** * Imports a filesystem snapshot without coupling the reader to MailHub's DB. * * A writable adapter implements ensureDomain, ensureMailbox, hasMessage and * createMessage. hasMessage receives the stable sourceKey and makes reruns and * crash recovery idempotent. dryRun never invokes adapter methods. */ export async function importVestaSnapshot({ root, adapter = null, dryRun = false, vestaUser = '', userDataRoot = '', homeRoot = '', resumeAfterSourceKey = '', onCheckpoint = null, signal = null, parseMessage = parseInboundMessage } = {}) { if (!dryRun) validateAdapter(adapter); const snapshot = await readVestaSnapshotMetadata({ root, vestaUser, userDataRoot, homeRoot }); const report = { dryRun: Boolean(dryRun), domains: snapshot.domains.length, mailboxes: snapshot.mailboxes.length, messages: 0, bytes: 0, plannedMessages: 0, importedMessages: 0, skippedMessages: 0, resumeSkippedMessages: 0, lastSourceKey: '', warnings: [...snapshot.warnings], warningCount: snapshot.warnings.length }; const domainReferences = new Map(); const mailboxReferences = new Map(); const seenSourceKeys = new Set(); let waitingForCheckpoint = Boolean(resumeAfterSourceKey); for (const domain of snapshot.domains) { abortIfNeeded(signal); const reference = dryRun ? domain : await adapter.ensureDomain(domain); domainReferences.set(domainKey(domain), reference ?? domain); } for (const mailbox of snapshot.mailboxes) { abortIfNeeded(signal); const domainReference = domainReferences.get(domainKey(mailbox)); const mailboxReference = dryRun ? mailbox : await adapter.ensureMailbox(mailbox, { domain: domainReference }); mailboxReferences.set(mailbox.address, mailboxReference ?? mailbox); if (!dryRun && typeof adapter.ensureFolder === 'function') { for (const folder of mailbox.folders || []) { await adapter.ensureFolder(mailboxReference ?? mailbox, folder, { sourceMailbox: mailbox }); } } for await (const sourceMessage of iterateVestaMaildirMessages(mailbox, { signal })) { abortIfNeeded(signal); report.messages += 1; report.bytes += sourceMessage.rawMessageBytes.length; if (waitingForCheckpoint) { report.resumeSkippedMessages += 1; seenSourceKeys.add(sourceMessage.sourceKey); if (sourceMessage.sourceKey === resumeAfterSourceKey) { waitingForCheckpoint = false; report.lastSourceKey = sourceMessage.sourceKey; } continue; } if (seenSourceKeys.has(sourceMessage.sourceKey)) { report.skippedMessages += 1; continue; } seenSourceKeys.add(sourceMessage.sourceKey); const context = { domain: domainReference, mailbox: mailboxReferences.get(mailbox.address), sourceMailbox: mailbox }; if (!dryRun && await adapter.hasMessage(sourceMessage.sourceKey, context)) { report.skippedMessages += 1; report.lastSourceKey = sourceMessage.sourceKey; await onCheckpoint?.(sourceMessage.sourceKey, { ...context, message: sourceMessage }); continue; } let parsed = {}; if (parseMessage) { try { parsed = await parseMessage(sourceMessage.rawMessageBytes, []); } catch { // Preserve the original message even when a malformed MIME structure // cannot be indexed. Protocol clients can still retrieve it verbatim. report.warningCount += 1; } } const message = { ...parsed, ...sourceMessage, recipients: parsed.recipients?.length ? parsed.recipients : [mailbox.address], receivedAt: sourceMessage.modifiedAt, read: sourceMessage.flags.includes('\\Seen') }; if (dryRun) report.plannedMessages += 1; else { const result = await adapter.createMessage(message, context); if (result?.created === false) report.skippedMessages += 1; else report.importedMessages += 1; } report.lastSourceKey = sourceMessage.sourceKey; if (!dryRun) await onCheckpoint?.(sourceMessage.sourceKey, { ...context, message }); } } if (waitingForCheckpoint) { throw new Error(`Vesta 导入断点不存在:${resumeAfterSourceKey}`); } return report; } export async function readVestaSnapshotMetadata({ root, vestaUser = '', userDataRoot = '', homeRoot = '' } = {}) { const snapshotRoot = requireSnapshotRoot(root); const userLocations = await findVestaUserLocations(snapshotRoot, { vestaUser, userDataRoot }); if (!userLocations.length) throw new Error('Vesta 快照中未找到 mail.conf。'); const domains = []; const mailboxes = []; const warnings = []; for (const location of userLocations) { const domainRecords = await readConfigRecords(path.join(location.userDataDir, 'mail.conf')); const home = await resolveVestaHome(snapshotRoot, location.vestaUser, homeRoot); for (const config of domainRecords) { if (!config.DOMAIN) continue; const name = String(config.DOMAIN).trim().toLowerCase(); const catchAll = normalizeAddress(config.CATCHALL, name); const domain = { source: 'vesta', vestaUser: location.vestaUser, name, domain: name, catchAll, catchAllAddress: catchAll, suspended: isYes(config.SUSPENDED), status: isYes(config.SUSPENDED) ? 'suspended' : 'active', config, sourceFile: path.join(location.userDataDir, 'mail.conf') }; domains.push(domain); const accountFile = path.join(location.userDataDir, 'mail', `${name}.conf`); const accountRecords = await readConfigRecords(accountFile, { optional: true }); const passwdFile = home.confMailRoot ? path.join(home.confMailRoot, name, 'passwd') : ''; const passwdRecords = passwdFile ? await readVestaPasswd(passwdFile, { optional: true }) : []; const accounts = mergeAccountRecords(accountRecords, passwdRecords); if (!accounts.length) warnings.push(`域名 ${name} 没有邮箱账户元数据。`); for (const account of accounts) { const localPart = String(account.ACCOUNT || account.account || '').trim().toLowerCase(); if (!localPart) continue; const passwordHash = String(account.passwordHash || account.MD5 || '').trim(); const maildirPath = home.mailRoot ? path.join(home.mailRoot, name, localPart) : ''; const address = `${localPart}@${name}`; const maildirAvailable = Boolean(maildirPath) && await isDirectory(maildirPath); const folders = maildirAvailable ? (await listMaildirFolders(maildirPath)).map((folder) => folder.name) : []; if (!passwordHash) warnings.push(`邮箱 ${address} 没有可迁移密码哈希。`); if (!maildirAvailable) warnings.push(`邮箱 ${address} 没有 Maildir,仍会导入账户。`); mailboxes.push({ source: 'vesta', vestaUser: location.vestaUser, domain: name, localPart, address, displayName: '', aliases: normalizeAddressList(account.ALIAS, name), forwardTo: normalizeAddressList(account.FWD), forwardOnly: isYes(account.FWD_ONLY), keepForwarded: !isYes(account.FWD_ONLY), quotaMb: normalizeQuota(account.QUOTA ?? account.quota), suspended: isYes(account.SUSPENDED), status: isYes(account.SUSPENDED) ? 'suspended' : 'active', legacyPasswordHash: passwordHash, passwordHash, passwordScheme: passwordScheme(passwordHash), createdAt: vestaTimestamp(account.DATE, account.TIME), maildirPath, maildirAvailable, folders, config: account, sourceFile: accountFile, passwordSourceFile: account.passwordSourceFile || '' }); } } } domains.sort((left, right) => compareText(domainKey(left), domainKey(right))); mailboxes.sort((left, right) => compareText(left.address, right.address)); return { root: snapshotRoot, domains, mailboxes, warnings }; } export async function* iterateVestaMaildirMessages(mailbox, { signal = null } = {}) { if (!mailbox?.maildirPath || !await isDirectory(mailbox.maildirPath)) return; const folders = await listMaildirFolders(mailbox.maildirPath); for (const folder of folders) { const keywordMap = await readDovecotKeywords(folder.path, mailbox.maildirPath); for (const bucket of ['new', 'cur']) { const directory = path.join(folder.path, bucket); const entries = await readDirectoryEntries(directory); for (const entry of entries.filter((item) => item.isFile()).sort(compareDirents)) { abortIfNeeded(signal); const filePath = path.join(directory, entry.name); const [rawMessageBytes, fileStat] = await Promise.all([readFile(filePath), stat(filePath)]); const flagInfo = parseMaildirFlags(entry.name, keywordMap); const contentSha256 = createHash('sha256').update(rawMessageBytes).digest('hex'); const sourceKey = createVestaMessageSourceKey({ address: mailbox.address, folder: folder.name, fileName: entry.name, contentSha256 }); yield { source: 'vesta-maildir', sourceKey, sourcePath: path.relative(mailbox.maildirPath, filePath).split(path.sep).join('/'), fileName: entry.name, folder: folder.name, rawMessageBytes, contentSha256, modifiedAt: fileStat.mtime.toISOString(), size: fileStat.size, flags: flagInfo.flags, keywords: flagInfo.keywords, maildirFlags: flagInfo.raw }; } } } } export function createVestaMessageSourceKey({ address, folder, fileName, contentSha256 }) { const stableFileName = String(fileName || '').replace(/:2,[^/]*$/, ''); const source = [ 'vesta-maildir-v1', String(address || '').trim().toLowerCase(), normalizeFolder(folder), stableFileName, String(contentSha256 || '').toLowerCase() ].join('\0'); return `vesta-maildir-v1:${createHash('sha256').update(source).digest('hex')}`; } export function parseVestaConfigLine(line) { const source = String(line || ''); const output = {}; let index = 0; while (index < source.length) { while (/\s/.test(source[index] || '')) index += 1; if (!source[index] || source[index] === '#') break; const keyMatch = source.slice(index).match(/^([A-Za-z_][A-Za-z0-9_]*)=/); if (!keyMatch) { while (source[index] && !/\s/.test(source[index])) index += 1; continue; } const key = keyMatch[1]; index += keyMatch[0].length; let value = ''; const quote = source[index] === "'" || source[index] === '"' ? source[index++] : ''; if (quote) { while (index < source.length && source[index] !== quote) { if (quote === '"' && source[index] === '\\' && index + 1 < source.length) index += 1; value += source[index++]; } if (source[index] === quote) index += 1; } else { while (source[index] && !/\s/.test(source[index])) value += source[index++]; } output[key] = value; } return output; } export async function readVestaPasswd(filePath, { optional = false } = {}) { const content = await readTextFile(filePath, { optional }); if (content === null) return []; return content.split(/\r?\n/).map((line) => { const clean = line.trim(); if (!clean || clean.startsWith('#')) return null; const fields = clean.split(':'); if (!fields[0] || !fields[1]) return null; return { account: fields[0].trim().toLowerCase(), passwordHash: fields[1].trim(), user: fields[2] || '', group: fields[3] || '', home: fields[5] || '', quota: fields[6] || '', passwordSourceFile: filePath }; }).filter(Boolean); } function validateAdapter(adapter) { for (const method of ['ensureDomain', 'ensureMailbox', 'hasMessage', 'createMessage']) { if (typeof adapter?.[method] !== 'function') throw new TypeError(`Vesta 导入 adapter 缺少 ${method}()。`); } } async function findVestaUserLocations(root, { vestaUser, userDataRoot }) { const locations = []; const explicitRoot = userDataRoot ? path.resolve(userDataRoot) : ''; const bases = explicitRoot ? [explicitRoot] : [ path.join(root, 'usr/local/vesta/data/users'), path.join(root, 'usr/local/hestia/data/users'), path.join(root, 'vesta/data/users'), path.join(root, 'data/users'), path.join(root, 'users') ]; if (await isFile(path.join(root, 'mail.conf'))) { locations.push({ vestaUser: vestaUser || path.basename(root), userDataDir: root }); } for (const base of bases) { if (await isFile(path.join(base, 'mail.conf'))) { const user = vestaUser || path.basename(base); locations.push({ vestaUser: user, userDataDir: base }); continue; } for (const entry of await readDirectoryEntries(base)) { if (!entry.isDirectory() || (vestaUser && entry.name !== vestaUser)) continue; const userDataDir = path.join(base, entry.name); if (await isFile(path.join(userDataDir, 'mail.conf'))) { locations.push({ vestaUser: entry.name, userDataDir }); } } } const unique = new Map(locations.map((location) => [path.resolve(location.userDataDir), location])); return [...unique.values()].sort((left, right) => compareText(left.vestaUser, right.vestaUser)); } async function resolveVestaHome(root, vestaUser, homeRoot) { const explicitRoot = homeRoot ? path.resolve(homeRoot) : ''; const homeCandidates = explicitRoot ? [path.join(explicitRoot, vestaUser), explicitRoot] : [path.join(root, 'home', vestaUser), path.join(root, vestaUser)]; for (const homeDirectory of homeCandidates) { if (!await isDirectory(homeDirectory)) continue; return { homeDirectory, mailRoot: await isDirectory(path.join(homeDirectory, 'mail')) ? path.join(homeDirectory, 'mail') : '', confMailRoot: await isDirectory(path.join(homeDirectory, 'conf/mail')) ? path.join(homeDirectory, 'conf/mail') : '' }; } return { homeDirectory: '', mailRoot: '', confMailRoot: '' }; } async function readConfigRecords(filePath, { optional = false } = {}) { const content = await readTextFile(filePath, { optional }); if (content === null) return []; return content.split(/\r?\n/) .map(parseVestaConfigLine) .filter((record) => Object.keys(record).length); } function mergeAccountRecords(accountRecords, passwdRecords) { const records = new Map(); for (const config of accountRecords) { const account = String(config.ACCOUNT || '').trim().toLowerCase(); if (account) records.set(account, { ...config, ACCOUNT: account }); } for (const passwd of passwdRecords) { const previous = records.get(passwd.account) || { ACCOUNT: passwd.account }; records.set(passwd.account, { ...previous, passwordHash: passwd.passwordHash || previous.MD5 || '', passwordSourceFile: passwd.passwordSourceFile, quota: passwd.quota }); } return [...records.values()].sort((left, right) => compareText(left.ACCOUNT, right.ACCOUNT)); } async function listMaildirFolders(maildirPath) { const folders = [{ name: 'INBOX', path: maildirPath }]; for (const entry of (await readDirectoryEntries(maildirPath)).sort(compareDirents)) { if (!entry.isDirectory() || !entry.name.startsWith('.') || entry.name === '.') continue; const folderPath = path.join(maildirPath, entry.name); if (!await hasMaildirMessagesDirectory(folderPath)) continue; const segments = entry.name.slice(1).split('.').filter(Boolean).map(decodeModifiedUtf7); if (segments[0]?.toLowerCase() === 'inbox') segments.shift(); const name = normalizeFolder(segments.join('/')); if (name && name !== 'INBOX') folders.push({ name, path: folderPath }); } return folders.sort((left, right) => compareText(left.name, right.name)); } async function hasMaildirMessagesDirectory(directory) { return await isDirectory(path.join(directory, 'cur')) || await isDirectory(path.join(directory, 'new')); } async function readDovecotKeywords(folderPath, maildirPath) { const mapping = new Map(); for (const filePath of [...new Set([ path.join(maildirPath, 'dovecot-keywords'), path.join(folderPath, 'dovecot-keywords') ])]) { const content = await readTextFile(filePath, { optional: true }); if (content === null) continue; for (const line of content.split(/\r?\n/)) { const match = line.match(/^\s*(\d+)\s+(.+?)\s*$/); const index = Number(match?.[1]); if (!match || !Number.isInteger(index) || index < 0 || index > 25) continue; mapping.set(String.fromCharCode(97 + index), match[2]); } } return mapping; } function parseMaildirFlags(fileName, keywordMap) { const raw = String(fileName || '').match(/:2,([^/]*)$/)?.[1] || ''; const flags = []; const keywords = []; for (const flag of raw) { if (standardMaildirFlags.has(flag)) flags.push(standardMaildirFlags.get(flag)); else if (/[a-z]/.test(flag)) keywords.push(keywordMap.get(flag) || flag); else flags.push(flag); } return { raw, flags: [...new Set(flags)], keywords: [...new Set(keywords)] }; } function normalizeFolder(value) { const clean = String(value || '').trim().replace(/^\/+|\/+$/g, ''); const standard = standardFolders.get(clean.toLowerCase()); return standard || clean || 'INBOX'; } function normalizeAddress(value, domain = '') { const clean = String(value || '').trim().toLowerCase(); if (!clean || ['no', 'none', 'reject', ':fail:'].includes(clean)) return ''; if (['blackhole', ':blackhole:'].includes(clean)) return '/dev/null'; return clean.includes('@') || !domain ? clean : `${clean}@${domain}`; } function normalizeAddressList(value, domain = '') { return [...new Set(String(value || '') .split(/[\s,;]+/) .map((item) => normalizeAddress(item, domain)) .filter(Boolean))]; } function normalizeQuota(value) { const clean = String(value ?? '').trim().toLowerCase(); if (!clean || clean === '0' || clean === 'unlimited') return null; const quota = Number(clean); return Number.isFinite(quota) && quota >= 0 ? quota : null; } function passwordScheme(value) { const clean = String(value || '').trim(); if (/^(?:\{(?:MD5|MD5-CRYPT)\})?\$1\$/i.test(clean)) return 'md5-crypt'; if (/^\{[^}]+\}/.test(clean)) return clean.slice(1, clean.indexOf('}')).toLowerCase(); return clean ? 'unknown' : ''; } function vestaTimestamp(date, time) { const cleanDate = String(date || '').trim(); const cleanTime = String(time || '').trim(); return cleanDate ? `${cleanDate}${cleanTime ? `T${cleanTime}` : ''}` : null; } function isYes(value) { return ['yes', 'true', '1', 'on'].includes(String(value || '').trim().toLowerCase()); } function domainKey(value) { return `${value.vestaUser || ''}\0${value.domain || value.name || ''}`; } function requireSnapshotRoot(root) { const clean = String(root || '').trim(); if (!clean) throw new TypeError('Vesta 快照 root 不能为空。'); return path.resolve(clean); } async function readTextFile(filePath, { optional = false } = {}) { try { return await readFile(filePath, 'utf8'); } catch (error) { if (optional && error?.code === 'ENOENT') return null; throw error; } } async function readDirectoryEntries(directory) { try { const entries = []; const handle = await opendir(directory); for await (const entry of handle) entries.push(entry); return entries; } catch (error) { if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return []; throw error; } } async function isFile(filePath) { try { return (await stat(filePath)).isFile(); } catch (error) { if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return false; throw error; } } async function isDirectory(directory) { try { return (await stat(directory)).isDirectory(); } catch (error) { if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return false; throw error; } } function compareDirents(left, right) { return compareText(left.name, right.name); } function compareText(left, right) { return left < right ? -1 : left > right ? 1 : 0; } function abortIfNeeded(signal) { if (signal?.aborted) throw signal.reason || new Error('Vesta 导入已取消。'); }