MailSend/src/vesta-maildir-import.js

import { createHash } from 'node:crypto';
import { opendir, readFile, stat } from 'node:fs/promises';
import path from 'node:path';
import { parseInboundMessage } from './inbound-mail.js';
import { decodeModifiedUtf7 } from './imap-utf7.js';

export { decodeModifiedUtf7 };

const standardFolders = new Map([
  ['inbox', 'INBOX'],
  ['sent', 'Sent'],
  ['sent items', 'Sent'],
  ['draft', 'Drafts'],
  ['drafts', 'Drafts'],
  ['trash', 'Trash'],
  ['deleted messages', 'Trash'],
  ['junk', 'Junk'],
  ['junk e-mail', 'Junk'],
  ['spam', 'Junk'],
  ['archive', 'Archive']
]);

const standardMaildirFlags = new Map([
  ['D', '\\Draft'],
  ['F', '\\Flagged'],
  ['P', '\\Passed'],
  ['R', '\\Answered'],
  ['S', '\\Seen'],
  ['T', '\\Deleted']
]);

/**
 * Imports a filesystem snapshot without coupling the reader to MailHub's DB.
 *
 * A writable adapter implements ensureDomain, ensureMailbox, hasMessage and
 * createMessage. hasMessage receives the stable sourceKey and makes reruns and
 * crash recovery idempotent. dryRun never invokes adapter methods.
 */
export async function importVestaSnapshot({
  root,
  adapter = null,
  dryRun = false,
  vestaUser = '',
  userDataRoot = '',
  homeRoot = '',
  resumeAfterSourceKey = '',
  onCheckpoint = null,
  signal = null,
  parseMessage = parseInboundMessage
} = {}) {
  if (!dryRun) validateAdapter(adapter);
  const snapshot = await readVestaSnapshotMetadata({ root, vestaUser, userDataRoot, homeRoot });
  const report = {
    dryRun: Boolean(dryRun),
    domains: snapshot.domains.length,
    mailboxes: snapshot.mailboxes.length,
    messages: 0,
    bytes: 0,
    plannedMessages: 0,
    importedMessages: 0,
    skippedMessages: 0,
    resumeSkippedMessages: 0,
    lastSourceKey: '',
    warnings: [...snapshot.warnings],
    warningCount: snapshot.warnings.length
  };
  const domainReferences = new Map();
  const mailboxReferences = new Map();
  const seenSourceKeys = new Set();
  let waitingForCheckpoint = Boolean(resumeAfterSourceKey);

  for (const domain of snapshot.domains) {
    abortIfNeeded(signal);
    const reference = dryRun ? domain : await adapter.ensureDomain(domain);
    domainReferences.set(domainKey(domain), reference ?? domain);
  }

  for (const mailbox of snapshot.mailboxes) {
    abortIfNeeded(signal);
    const domainReference = domainReferences.get(domainKey(mailbox));
    const mailboxReference = dryRun
      ? mailbox
      : await adapter.ensureMailbox(mailbox, { domain: domainReference });
    mailboxReferences.set(mailbox.address, mailboxReference ?? mailbox);
    if (!dryRun && typeof adapter.ensureFolder === 'function') {
      for (const folder of mailbox.folders || []) {
        await adapter.ensureFolder(mailboxReference ?? mailbox, folder, { sourceMailbox: mailbox });
      }
    }

    for await (const sourceMessage of iterateVestaMaildirMessages(mailbox, { signal })) {
      abortIfNeeded(signal);
      report.messages += 1;
      report.bytes += sourceMessage.rawMessageBytes.length;

      if (waitingForCheckpoint) {
        report.resumeSkippedMessages += 1;
        seenSourceKeys.add(sourceMessage.sourceKey);
        if (sourceMessage.sourceKey === resumeAfterSourceKey) {
          waitingForCheckpoint = false;
          report.lastSourceKey = sourceMessage.sourceKey;
        }
        continue;
      }

      if (seenSourceKeys.has(sourceMessage.sourceKey)) {
        report.skippedMessages += 1;
        continue;
      }
      seenSourceKeys.add(sourceMessage.sourceKey);

      const context = {
        domain: domainReference,
        mailbox: mailboxReferences.get(mailbox.address),
        sourceMailbox: mailbox
      };
      if (!dryRun && await adapter.hasMessage(sourceMessage.sourceKey, context)) {
        report.skippedMessages += 1;
        report.lastSourceKey = sourceMessage.sourceKey;
        await onCheckpoint?.(sourceMessage.sourceKey, { ...context, message: sourceMessage });
        continue;
      }

      let parsed = {};
      if (parseMessage) {
        try {
          parsed = await parseMessage(sourceMessage.rawMessageBytes, []);
        } catch {
          // Preserve the original message even when a malformed MIME structure
          // cannot be indexed. Protocol clients can still retrieve it verbatim.
          report.warningCount += 1;
        }
      }
      const message = {
        ...parsed,
        ...sourceMessage,
        recipients: parsed.recipients?.length ? parsed.recipients : [mailbox.address],
        receivedAt: sourceMessage.modifiedAt,
        read: sourceMessage.flags.includes('\\Seen')
      };
      if (dryRun) report.plannedMessages += 1;
      else {
        const result = await adapter.createMessage(message, context);
        if (result?.created === false) report.skippedMessages += 1;
        else report.importedMessages += 1;
      }
      report.lastSourceKey = sourceMessage.sourceKey;
      if (!dryRun) await onCheckpoint?.(sourceMessage.sourceKey, { ...context, message });
    }
  }

  if (waitingForCheckpoint) {
    throw new Error(`Vesta 导入断点不存在：${resumeAfterSourceKey}`);
  }
  return report;
}

export async function readVestaSnapshotMetadata({
  root,
  vestaUser = '',
  userDataRoot = '',
  homeRoot = ''
} = {}) {
  const snapshotRoot = requireSnapshotRoot(root);
  const userLocations = await findVestaUserLocations(snapshotRoot, { vestaUser, userDataRoot });
  if (!userLocations.length) throw new Error('Vesta 快照中未找到 mail.conf。');

  const domains = [];
  const mailboxes = [];
  const warnings = [];
  for (const location of userLocations) {
    const domainRecords = await readConfigRecords(path.join(location.userDataDir, 'mail.conf'));
    const home = await resolveVestaHome(snapshotRoot, location.vestaUser, homeRoot);
    for (const config of domainRecords) {
      if (!config.DOMAIN) continue;
      const name = String(config.DOMAIN).trim().toLowerCase();
      const catchAll = normalizeAddress(config.CATCHALL, name);
      const domain = {
        source: 'vesta',
        vestaUser: location.vestaUser,
        name,
        domain: name,
        catchAll,
        catchAllAddress: catchAll,
        suspended: isYes(config.SUSPENDED),
        status: isYes(config.SUSPENDED) ? 'suspended' : 'active',
        config,
        sourceFile: path.join(location.userDataDir, 'mail.conf')
      };
      domains.push(domain);

      const accountFile = path.join(location.userDataDir, 'mail', `${name}.conf`);
      const accountRecords = await readConfigRecords(accountFile, { optional: true });
      const passwdFile = home.confMailRoot ? path.join(home.confMailRoot, name, 'passwd') : '';
      const passwdRecords = passwdFile
        ? await readVestaPasswd(passwdFile, { optional: true })
        : [];
      const accounts = mergeAccountRecords(accountRecords, passwdRecords);
      if (!accounts.length) warnings.push(`域名 ${name} 没有邮箱账户元数据。`);

      for (const account of accounts) {
        const localPart = String(account.ACCOUNT || account.account || '').trim().toLowerCase();
        if (!localPart) continue;
        const passwordHash = String(account.passwordHash || account.MD5 || '').trim();
        const maildirPath = home.mailRoot
          ? path.join(home.mailRoot, name, localPart)
          : '';
        const address = `${localPart}@${name}`;
        const maildirAvailable = Boolean(maildirPath) && await isDirectory(maildirPath);
        const folders = maildirAvailable
          ? (await listMaildirFolders(maildirPath)).map((folder) => folder.name)
          : [];
        if (!passwordHash) warnings.push(`邮箱 ${address} 没有可迁移密码哈希。`);
        if (!maildirAvailable) warnings.push(`邮箱 ${address} 没有 Maildir，仍会导入账户。`);
        mailboxes.push({
          source: 'vesta',
          vestaUser: location.vestaUser,
          domain: name,
          localPart,
          address,
          displayName: '',
          aliases: normalizeAddressList(account.ALIAS, name),
          forwardTo: normalizeAddressList(account.FWD),
          forwardOnly: isYes(account.FWD_ONLY),
          keepForwarded: !isYes(account.FWD_ONLY),
          quotaMb: normalizeQuota(account.QUOTA ?? account.quota),
          suspended: isYes(account.SUSPENDED),
          status: isYes(account.SUSPENDED) ? 'suspended' : 'active',
          legacyPasswordHash: passwordHash,
          passwordHash,
          passwordScheme: passwordScheme(passwordHash),
          createdAt: vestaTimestamp(account.DATE, account.TIME),
          maildirPath,
          maildirAvailable,
          folders,
          config: account,
          sourceFile: accountFile,
          passwordSourceFile: account.passwordSourceFile || ''
        });
      }
    }
  }

  domains.sort((left, right) => compareText(domainKey(left), domainKey(right)));
  mailboxes.sort((left, right) => compareText(left.address, right.address));
  return { root: snapshotRoot, domains, mailboxes, warnings };
}

export async function* iterateVestaMaildirMessages(mailbox, { signal = null } = {}) {
  if (!mailbox?.maildirPath || !await isDirectory(mailbox.maildirPath)) return;
  const folders = await listMaildirFolders(mailbox.maildirPath);
  for (const folder of folders) {
    const keywordMap = await readDovecotKeywords(folder.path, mailbox.maildirPath);
    for (const bucket of ['new', 'cur']) {
      const directory = path.join(folder.path, bucket);
      const entries = await readDirectoryEntries(directory);
      for (const entry of entries.filter((item) => item.isFile()).sort(compareDirents)) {
        abortIfNeeded(signal);
        const filePath = path.join(directory, entry.name);
        const [rawMessageBytes, fileStat] = await Promise.all([readFile(filePath), stat(filePath)]);
        const flagInfo = parseMaildirFlags(entry.name, keywordMap);
        const contentSha256 = createHash('sha256').update(rawMessageBytes).digest('hex');
        const sourceKey = createVestaMessageSourceKey({
          address: mailbox.address,
          folder: folder.name,
          fileName: entry.name,
          contentSha256
        });
        yield {
          source: 'vesta-maildir',
          sourceKey,
          sourcePath: path.relative(mailbox.maildirPath, filePath).split(path.sep).join('/'),
          fileName: entry.name,
          folder: folder.name,
          rawMessageBytes,
          contentSha256,
          modifiedAt: fileStat.mtime.toISOString(),
          size: fileStat.size,
          flags: flagInfo.flags,
          keywords: flagInfo.keywords,
          maildirFlags: flagInfo.raw
        };
      }
    }
  }
}

export function createVestaMessageSourceKey({ address, folder, fileName, contentSha256 }) {
  const stableFileName = String(fileName || '').replace(/:2,[^/]*$/, '');
  const source = [
    'vesta-maildir-v1',
    String(address || '').trim().toLowerCase(),
    normalizeFolder(folder),
    stableFileName,
    String(contentSha256 || '').toLowerCase()
  ].join('\0');
  return `vesta-maildir-v1:${createHash('sha256').update(source).digest('hex')}`;
}

export function parseVestaConfigLine(line) {
  const source = String(line || '');
  const output = {};
  let index = 0;
  while (index < source.length) {
    while (/\s/.test(source[index] || '')) index += 1;
    if (!source[index] || source[index] === '#') break;
    const keyMatch = source.slice(index).match(/^([A-Za-z_][A-Za-z0-9_]*)=/);
    if (!keyMatch) {
      while (source[index] && !/\s/.test(source[index])) index += 1;
      continue;
    }
    const key = keyMatch[1];
    index += keyMatch[0].length;
    let value = '';
    const quote = source[index] === "'" || source[index] === '"' ? source[index++] : '';
    if (quote) {
      while (index < source.length && source[index] !== quote) {
        if (quote === '"' && source[index] === '\\' && index + 1 < source.length) index += 1;
        value += source[index++];
      }
      if (source[index] === quote) index += 1;
    } else {
      while (source[index] && !/\s/.test(source[index])) value += source[index++];
    }
    output[key] = value;
  }
  return output;
}

export async function readVestaPasswd(filePath, { optional = false } = {}) {
  const content = await readTextFile(filePath, { optional });
  if (content === null) return [];
  return content.split(/\r?\n/).map((line) => {
    const clean = line.trim();
    if (!clean || clean.startsWith('#')) return null;
    const fields = clean.split(':');
    if (!fields[0] || !fields[1]) return null;
    return {
      account: fields[0].trim().toLowerCase(),
      passwordHash: fields[1].trim(),
      user: fields[2] || '',
      group: fields[3] || '',
      home: fields[5] || '',
      quota: fields[6] || '',
      passwordSourceFile: filePath
    };
  }).filter(Boolean);
}

function validateAdapter(adapter) {
  for (const method of ['ensureDomain', 'ensureMailbox', 'hasMessage', 'createMessage']) {
    if (typeof adapter?.[method] !== 'function') throw new TypeError(`Vesta 导入 adapter 缺少 ${method}()。`);
  }
}

async function findVestaUserLocations(root, { vestaUser, userDataRoot }) {
  const locations = [];
  const explicitRoot = userDataRoot ? path.resolve(userDataRoot) : '';
  const bases = explicitRoot
    ? [explicitRoot]
    : [
        path.join(root, 'usr/local/vesta/data/users'),
        path.join(root, 'usr/local/hestia/data/users'),
        path.join(root, 'vesta/data/users'),
        path.join(root, 'data/users'),
        path.join(root, 'users')
      ];

  if (await isFile(path.join(root, 'mail.conf'))) {
    locations.push({ vestaUser: vestaUser || path.basename(root), userDataDir: root });
  }
  for (const base of bases) {
    if (await isFile(path.join(base, 'mail.conf'))) {
      const user = vestaUser || path.basename(base);
      locations.push({ vestaUser: user, userDataDir: base });
      continue;
    }
    for (const entry of await readDirectoryEntries(base)) {
      if (!entry.isDirectory() || (vestaUser && entry.name !== vestaUser)) continue;
      const userDataDir = path.join(base, entry.name);
      if (await isFile(path.join(userDataDir, 'mail.conf'))) {
        locations.push({ vestaUser: entry.name, userDataDir });
      }
    }
  }
  const unique = new Map(locations.map((location) => [path.resolve(location.userDataDir), location]));
  return [...unique.values()].sort((left, right) => compareText(left.vestaUser, right.vestaUser));
}

async function resolveVestaHome(root, vestaUser, homeRoot) {
  const explicitRoot = homeRoot ? path.resolve(homeRoot) : '';
  const homeCandidates = explicitRoot
    ? [path.join(explicitRoot, vestaUser), explicitRoot]
    : [path.join(root, 'home', vestaUser), path.join(root, vestaUser)];
  for (const homeDirectory of homeCandidates) {
    if (!await isDirectory(homeDirectory)) continue;
    return {
      homeDirectory,
      mailRoot: await isDirectory(path.join(homeDirectory, 'mail')) ? path.join(homeDirectory, 'mail') : '',
      confMailRoot: await isDirectory(path.join(homeDirectory, 'conf/mail')) ? path.join(homeDirectory, 'conf/mail') : ''
    };
  }
  return { homeDirectory: '', mailRoot: '', confMailRoot: '' };
}

async function readConfigRecords(filePath, { optional = false } = {}) {
  const content = await readTextFile(filePath, { optional });
  if (content === null) return [];
  return content.split(/\r?\n/)
    .map(parseVestaConfigLine)
    .filter((record) => Object.keys(record).length);
}

function mergeAccountRecords(accountRecords, passwdRecords) {
  const records = new Map();
  for (const config of accountRecords) {
    const account = String(config.ACCOUNT || '').trim().toLowerCase();
    if (account) records.set(account, { ...config, ACCOUNT: account });
  }
  for (const passwd of passwdRecords) {
    const previous = records.get(passwd.account) || { ACCOUNT: passwd.account };
    records.set(passwd.account, {
      ...previous,
      passwordHash: passwd.passwordHash || previous.MD5 || '',
      passwordSourceFile: passwd.passwordSourceFile,
      quota: passwd.quota
    });
  }
  return [...records.values()].sort((left, right) => compareText(left.ACCOUNT, right.ACCOUNT));
}

async function listMaildirFolders(maildirPath) {
  const folders = [{ name: 'INBOX', path: maildirPath }];
  for (const entry of (await readDirectoryEntries(maildirPath)).sort(compareDirents)) {
    if (!entry.isDirectory() || !entry.name.startsWith('.') || entry.name === '.') continue;
    const folderPath = path.join(maildirPath, entry.name);
    if (!await hasMaildirMessagesDirectory(folderPath)) continue;
    const segments = entry.name.slice(1).split('.').filter(Boolean).map(decodeModifiedUtf7);
    if (segments[0]?.toLowerCase() === 'inbox') segments.shift();
    const name = normalizeFolder(segments.join('/'));
    if (name && name !== 'INBOX') folders.push({ name, path: folderPath });
  }
  return folders.sort((left, right) => compareText(left.name, right.name));
}

async function hasMaildirMessagesDirectory(directory) {
  return await isDirectory(path.join(directory, 'cur')) || await isDirectory(path.join(directory, 'new'));
}

async function readDovecotKeywords(folderPath, maildirPath) {
  const mapping = new Map();
  for (const filePath of [...new Set([
    path.join(maildirPath, 'dovecot-keywords'),
    path.join(folderPath, 'dovecot-keywords')
  ])]) {
    const content = await readTextFile(filePath, { optional: true });
    if (content === null) continue;
    for (const line of content.split(/\r?\n/)) {
      const match = line.match(/^\s*(\d+)\s+(.+?)\s*$/);
      const index = Number(match?.[1]);
      if (!match || !Number.isInteger(index) || index < 0 || index > 25) continue;
      mapping.set(String.fromCharCode(97 + index), match[2]);
    }
  }
  return mapping;
}

function parseMaildirFlags(fileName, keywordMap) {
  const raw = String(fileName || '').match(/:2,([^/]*)$/)?.[1] || '';
  const flags = [];
  const keywords = [];
  for (const flag of raw) {
    if (standardMaildirFlags.has(flag)) flags.push(standardMaildirFlags.get(flag));
    else if (/[a-z]/.test(flag)) keywords.push(keywordMap.get(flag) || flag);
    else flags.push(flag);
  }
  return { raw, flags: [...new Set(flags)], keywords: [...new Set(keywords)] };
}

function normalizeFolder(value) {
  const clean = String(value || '').trim().replace(/^\/+|\/+$/g, '');
  const standard = standardFolders.get(clean.toLowerCase());
  return standard || clean || 'INBOX';
}

function normalizeAddress(value, domain = '') {
  const clean = String(value || '').trim().toLowerCase();
  if (!clean || ['no', 'none', 'reject', ':fail:'].includes(clean)) return '';
  if (['blackhole', ':blackhole:'].includes(clean)) return '/dev/null';
  return clean.includes('@') || !domain ? clean : `${clean}@${domain}`;
}

function normalizeAddressList(value, domain = '') {
  return [...new Set(String(value || '')
    .split(/[\s,;]+/)
    .map((item) => normalizeAddress(item, domain))
    .filter(Boolean))];
}

function normalizeQuota(value) {
  const clean = String(value ?? '').trim().toLowerCase();
  if (!clean || clean === '0' || clean === 'unlimited') return null;
  const quota = Number(clean);
  return Number.isFinite(quota) && quota >= 0 ? quota : null;
}

function passwordScheme(value) {
  const clean = String(value || '').trim();
  if (/^(?:\{(?:MD5|MD5-CRYPT)\})?\$1\$/i.test(clean)) return 'md5-crypt';
  if (/^\{[^}]+\}/.test(clean)) return clean.slice(1, clean.indexOf('}')).toLowerCase();
  return clean ? 'unknown' : '';
}

function vestaTimestamp(date, time) {
  const cleanDate = String(date || '').trim();
  const cleanTime = String(time || '').trim();
  return cleanDate ? `${cleanDate}${cleanTime ? `T${cleanTime}` : ''}` : null;
}

function isYes(value) {
  return ['yes', 'true', '1', 'on'].includes(String(value || '').trim().toLowerCase());
}

function domainKey(value) {
  return `${value.vestaUser || ''}\0${value.domain || value.name || ''}`;
}

function requireSnapshotRoot(root) {
  const clean = String(root || '').trim();
  if (!clean) throw new TypeError('Vesta 快照 root 不能为空。');
  return path.resolve(clean);
}

async function readTextFile(filePath, { optional = false } = {}) {
  try {
    return await readFile(filePath, 'utf8');
  } catch (error) {
    if (optional && error?.code === 'ENOENT') return null;
    throw error;
  }
}

async function readDirectoryEntries(directory) {
  try {
    const entries = [];
    const handle = await opendir(directory);
    for await (const entry of handle) entries.push(entry);
    return entries;
  } catch (error) {
    if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return [];
    throw error;
  }
}

async function isFile(filePath) {
  try {
    return (await stat(filePath)).isFile();
  } catch (error) {
    if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return false;
    throw error;
  }
}

async function isDirectory(directory) {
  try {
    return (await stat(directory)).isDirectory();
  } catch (error) {
    if (error?.code === 'ENOENT' || error?.code === 'ENOTDIR') return false;
    throw error;
  }
}

function compareDirents(left, right) {
  return compareText(left.name, right.name);
}

function compareText(left, right) {
  return left < right ? -1 : left > right ? 1 : 0;
}

function abortIfNeeded(signal) {
  if (signal?.aborted) throw signal.reason || new Error('Vesta 导入已取消。');
}