sync-tls-certificate.sh 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393
  1. #!/usr/bin/env bash
  2. set -euo pipefail
  3. script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
  4. project_dir="${MAILHUB_CERT_PROJECT_DIR:-$(cd "${script_dir}/.." && pwd -P)}"
  5. project_dir="$(cd "${project_dir}" && pwd -P)"
  6. env_file="${MAILHUB_CERT_ENV_FILE:-${project_dir}/.env}"
  7. source_dir="${MAILHUB_CERT_SOURCE_DIR:-}"
  8. restart_app="${MAILHUB_CERT_RESTART:-0}"
  9. verify_host="${MAILHUB_CERT_VERIFY_HOST:-}"
  10. verify_endpoints="${MAILHUB_CERT_VERIFY_ENDPOINTS:-465 993}"
  11. certs_dir="${project_dir}/certs"
  12. lock_file="${MAILHUB_CERT_LOCK_FILE:-${certs_dir}/.sync-tls-certificate.lock}"
  13. work_dir=""
  14. target_cert=""
  15. target_key=""
  16. backup_cert_exists=0
  17. backup_key_exists=0
  18. rollback_required=0
  19. target_owner_uid=""
  20. target_reader_gid=""
  21. fail() {
  22. echo "Certificate sync failed: $*" >&2
  23. exit 1
  24. }
  25. read_env_value() {
  26. local key="$1"
  27. local line value
  28. line="$(grep -E "^${key}=" "${env_file}" 2>/dev/null | tail -n 1 || true)"
  29. value="${line#*=}"
  30. value="${value%$'\r'}"
  31. if [[ "${value}" == \"*\" && "${value}" == *\" ]]; then
  32. value="${value:1:${#value}-2}"
  33. elif [[ "${value}" == \'*\' && "${value}" == *\' ]]; then
  34. value="${value:1:${#value}-2}"
  35. fi
  36. printf '%s' "${value}"
  37. }
  38. resolve_target() {
  39. local container_path="$1"
  40. local label="$2"
  41. local relative
  42. [[ "${container_path}" == /certs/* ]] || fail "${label} must point to a direct file under /certs."
  43. relative="${container_path#/certs/}"
  44. [[ -n "${relative}" && "${relative}" != */* && "${relative}" != "." && "${relative}" != ".." ]] \
  45. || fail "${label} must point to a direct file under /certs."
  46. printf '%s/%s' "${certs_dir}" "${relative}"
  47. }
  48. validate_pair() {
  49. local certificate="$1"
  50. local private_key="$2"
  51. local label="$3"
  52. local cert_public_key="${work_dir}/${label}-cert-public.der"
  53. local key_public_key="${work_dir}/${label}-key-public.der"
  54. openssl x509 -in "${certificate}" -noout >/dev/null 2>&1 \
  55. || fail "${label} certificate is invalid."
  56. openssl x509 -in "${certificate}" -checkend 86400 -noout >/dev/null 2>&1 \
  57. || fail "${label} certificate expires within 24 hours."
  58. openssl x509 -in "${certificate}" -checkhost "${mail_hostname}" -noout >/dev/null 2>&1 \
  59. || fail "${label} certificate does not cover ${mail_hostname}."
  60. openssl pkey -in "${private_key}" -check -noout >/dev/null 2>&1 \
  61. || fail "${label} private key is invalid."
  62. openssl x509 -in "${certificate}" -pubkey -noout \
  63. | openssl pkey -pubin -outform DER >"${cert_public_key}" 2>/dev/null
  64. openssl pkey -in "${private_key}" -pubout -outform DER >"${key_public_key}" 2>/dev/null
  65. cmp -s "${cert_public_key}" "${key_public_key}" \
  66. || fail "${label} certificate and private key do not match."
  67. }
  68. resolve_target_ownership() {
  69. local detected_gid=""
  70. if [[ "$(id -u)" == "0" ]]; then
  71. target_owner_uid="${MAILHUB_CERT_OWNER_UID:-0}"
  72. if [[ -n "${MAILHUB_CERT_READER_GID:-}" ]]; then
  73. target_reader_gid="${MAILHUB_CERT_READER_GID}"
  74. else
  75. command -v docker >/dev/null 2>&1 \
  76. || fail "docker is required to detect the MailHub app group id."
  77. detected_gid="$(cd "${project_dir}" && docker compose exec -T app node -e \
  78. 'process.stdout.write(String(process.getgid()))' 2>/dev/null || true)"
  79. [[ -n "${detected_gid}" ]] \
  80. || fail "unable to determine the MailHub app group id; set MAILHUB_CERT_READER_GID explicitly."
  81. target_reader_gid="${detected_gid}"
  82. fi
  83. [[ "${target_owner_uid}" =~ ^[0-9]+$ ]] || fail "MAILHUB_CERT_OWNER_UID must be numeric."
  84. [[ "${target_reader_gid}" =~ ^[0-9]+$ ]] || fail "MAILHUB_CERT_READER_GID must be numeric."
  85. elif [[ -n "${MAILHUB_CERT_OWNER_UID:-}" || -n "${MAILHUB_CERT_READER_GID:-}" ]]; then
  86. fail "certificate ownership can only be changed when the sync runs as root."
  87. fi
  88. }
  89. set_file_metadata() {
  90. local file="$1"
  91. local mode="$2"
  92. chmod "${mode}" "${file}"
  93. if [[ -n "${target_owner_uid}" ]]; then
  94. chown "${target_owner_uid}:${target_reader_gid}" "${file}"
  95. fi
  96. }
  97. set_target_metadata() {
  98. set_file_metadata "$1" 0644
  99. set_file_metadata "$2" 0640
  100. }
  101. wait_for_app_health() {
  102. local attempt
  103. for attempt in $(seq 1 45); do
  104. if docker compose exec -T app node -e \
  105. "fetch('http://127.0.0.1:3000/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" \
  106. >/dev/null 2>&1; then
  107. return 0
  108. fi
  109. sleep 2
  110. done
  111. return 1
  112. }
  113. wait_for_dovecot_health() {
  114. local attempt
  115. for attempt in $(seq 1 45); do
  116. if docker compose exec -T dovecot doveconf -n >/dev/null 2>&1; then
  117. return 0
  118. fi
  119. sleep 2
  120. done
  121. return 1
  122. }
  123. restart_tls_services() {
  124. docker compose restart app dovecot
  125. wait_for_app_health || fail "MailHub app did not become healthy after the certificate restart."
  126. wait_for_dovecot_health || fail "Dovecot did not become healthy after the certificate restart."
  127. }
  128. verify_container_access() {
  129. docker compose exec -T app node -e \
  130. "const fs=require('node:fs');const tls=require('node:tls');tls.createSecureContext({cert:fs.readFileSync(process.argv[1]),key:fs.readFileSync(process.argv[2])});" \
  131. "${container_cert}" "${container_key}" >/dev/null 2>&1 \
  132. || fail "MailHub app cannot read or parse the synchronized certificate pair."
  133. docker compose exec -T dovecot sh -c \
  134. 'test -r "$1" && test -r "$2"' -- "${container_cert}" "${container_key}" >/dev/null 2>&1 \
  135. || fail "Dovecot cannot read the synchronized certificate pair."
  136. }
  137. tls_endpoint_matches() {
  138. local port="$1" service="app" container_port="$1"
  139. local mapping container_id mapped_host mapped_port connect_host connection output presented
  140. local expected_fingerprint actual_fingerprint
  141. local -a protocol_args=()
  142. [[ "${port}" =~ ^[0-9]+$ ]] || return 1
  143. case "${port}" in
  144. 25|587|2525) protocol_args=(-starttls smtp) ;;
  145. 110) service="dovecot"; container_port="31110"; protocol_args=(-starttls pop3) ;;
  146. 143) service="dovecot"; container_port="31143"; protocol_args=(-starttls imap) ;;
  147. 993) service="dovecot"; container_port="31993" ;;
  148. 995) service="dovecot"; container_port="31995" ;;
  149. esac
  150. if ! mapping="$(docker compose port "${service}" "${container_port}" 2>/dev/null | head -n 1)" \
  151. || [[ -z "${mapping}" ]]; then
  152. container_id="$(docker compose ps -q "${service}" 2>/dev/null | head -n 1 || true)"
  153. if [[ -n "${container_id}" ]]; then
  154. mapping="$(docker port "${container_id}" "${container_port}/tcp" 2>/dev/null | head -n 1 || true)"
  155. else
  156. mapping=""
  157. fi
  158. fi
  159. [[ -n "${mapping}" ]] || return 1
  160. mapped_port="${mapping##*:}"
  161. if [[ "${mapping}" == \[*\]:* ]]; then
  162. mapped_host="${mapping#\[}"
  163. mapped_host="${mapped_host%%\]*}"
  164. else
  165. mapped_host="${mapping%:*}"
  166. fi
  167. connect_host="${verify_host:-${mapped_host}}"
  168. [[ "${connect_host}" != "0.0.0.0" ]] || connect_host="127.0.0.1"
  169. [[ "${connect_host}" != "::" ]] || connect_host="::1"
  170. if [[ "${connect_host}" == *:* && "${connect_host}" != \[*\] ]]; then
  171. connection="[${connect_host}]:${mapped_port}"
  172. else
  173. connection="${connect_host}:${mapped_port}"
  174. fi
  175. output="${work_dir}/tls-${port}.txt"
  176. presented="${work_dir}/tls-${port}.pem"
  177. if command -v timeout >/dev/null 2>&1; then
  178. if ! timeout 15 openssl s_client "${protocol_args[@]}" -connect "${connection}" -servername "${mail_hostname}" -showcerts \
  179. -verify_hostname "${mail_hostname}" -verify_return_error \
  180. < /dev/null >"${output}" 2>/dev/null; then return 1; fi
  181. elif ! openssl s_client "${protocol_args[@]}" -connect "${connection}" -servername "${mail_hostname}" -showcerts \
  182. -verify_hostname "${mail_hostname}" -verify_return_error \
  183. < /dev/null >"${output}" 2>/dev/null; then
  184. return 1
  185. fi
  186. openssl x509 -in "${output}" -out "${presented}" >/dev/null 2>&1 \
  187. || return 1
  188. openssl x509 -in "${presented}" -checkhost "${mail_hostname}" -noout >/dev/null 2>&1 \
  189. || return 1
  190. expected_fingerprint="$(openssl x509 -in "${target_cert}" -noout -fingerprint -sha256)"
  191. actual_fingerprint="$(openssl x509 -in "${presented}" -noout -fingerprint -sha256)"
  192. [[ -n "${expected_fingerprint}" && "${expected_fingerprint}" == "${actual_fingerprint}" ]] \
  193. || return 1
  194. }
  195. verify_tls_endpoint() {
  196. tls_endpoint_matches "$1" \
  197. || fail "MailHub TLS endpoint $1 is not serving a valid synchronized certificate."
  198. }
  199. restore_previous_pair() {
  200. local rollback_failed=0
  201. set +e
  202. if [[ "${backup_cert_exists}" == "1" ]]; then
  203. cp -a "${work_dir}/backup-cert.pem" "${target_cert}"
  204. else
  205. rm -f -- "${target_cert}"
  206. fi
  207. [[ "$?" == "0" ]] || rollback_failed=1
  208. if [[ "${backup_key_exists}" == "1" ]]; then
  209. cp -a "${work_dir}/backup-key.pem" "${target_key}"
  210. else
  211. rm -f -- "${target_key}"
  212. fi
  213. [[ "$?" == "0" ]] || rollback_failed=1
  214. set -e
  215. return "${rollback_failed}"
  216. }
  217. on_exit() {
  218. local status=$?
  219. trap - EXIT
  220. if [[ "${status}" != "0" && "${rollback_required}" == "1" ]]; then
  221. echo "Certificate sync failed after promotion; restoring the previous certificate pair." >&2
  222. if restore_previous_pair; then
  223. if [[ "${restart_app}" == "1" ]]; then
  224. set +e
  225. docker compose restart app dovecot >/dev/null 2>&1
  226. wait_for_app_health >/dev/null 2>&1
  227. wait_for_dovecot_health >/dev/null 2>&1
  228. set -e
  229. fi
  230. echo "Previous MailHub certificate pair restored." >&2
  231. else
  232. echo "Certificate rollback failed; inspect ${target_cert} and ${target_key} immediately." >&2
  233. fi
  234. fi
  235. if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then
  236. rm -rf -- "${work_dir}"
  237. fi
  238. exit "${status}"
  239. }
  240. trap on_exit EXIT
  241. [[ -f "${env_file}" ]] || fail "environment file not found: ${env_file}"
  242. [[ "${restart_app}" == "0" || "${restart_app}" == "1" ]] \
  243. || fail "MAILHUB_CERT_RESTART must be 0 or 1."
  244. command -v openssl >/dev/null 2>&1 || fail "openssl is required."
  245. [[ ! -L "${certs_dir}" ]] || fail "certificate directory must not be a symbolic link: ${certs_dir}"
  246. mkdir -p "${certs_dir}"
  247. canonical_certs_dir="$(cd "${certs_dir}" && pwd -P)"
  248. [[ "${canonical_certs_dir}" == "${certs_dir}" ]] \
  249. || fail "certificate directory resolves outside the project: ${certs_dir}"
  250. if command -v flock >/dev/null 2>&1; then
  251. if [[ -L "${lock_file}" ]]; then
  252. fail "certificate lock file must not be a symbolic link: ${lock_file}"
  253. fi
  254. exec 9>"${lock_file}"
  255. if ! flock -n 9; then
  256. echo "Another MailHub certificate sync is already running; skipping."
  257. exit 0
  258. fi
  259. fi
  260. if [[ -z "${source_dir}" ]]; then
  261. source_dir="$(read_env_value MAILHUB_CERT_SOURCE_DIR)"
  262. fi
  263. if [[ -z "${source_dir}" ]]; then
  264. echo "Certificate sync is not configured; skipping."
  265. exit 0
  266. fi
  267. source_cert="${source_dir%/}/fullchain.pem"
  268. source_key="${source_dir%/}/privkey.pem"
  269. [[ -r "${source_cert}" ]] || fail "source certificate is not readable: ${source_cert}"
  270. [[ -r "${source_key}" ]] || fail "source private key is not readable: ${source_key}"
  271. [[ ! "${source_cert}" -ef "${source_key}" ]] || fail "source certificate and private key must be different files."
  272. container_cert="$(read_env_value SUBMISSION_TLS_CERT)"
  273. container_key="$(read_env_value SUBMISSION_TLS_KEY)"
  274. mail_hostname="$(read_env_value MAIL_HOSTNAME)"
  275. [[ -n "${container_cert}" ]] || fail "SUBMISSION_TLS_CERT is not configured."
  276. [[ -n "${container_key}" ]] || fail "SUBMISSION_TLS_KEY is not configured."
  277. [[ -n "${mail_hostname}" ]] || fail "MAIL_HOSTNAME is not configured."
  278. target_cert="$(resolve_target "${container_cert}" SUBMISSION_TLS_CERT)"
  279. target_key="$(resolve_target "${container_key}" SUBMISSION_TLS_KEY)"
  280. [[ "${target_cert}" != "${target_key}" ]] \
  281. || fail "SUBMISSION_TLS_CERT and SUBMISSION_TLS_KEY must point to different files."
  282. [[ ! -L "${target_cert}" ]] || fail "target certificate must not be a symbolic link."
  283. [[ ! -L "${target_key}" ]] || fail "target private key must not be a symbolic link."
  284. [[ ! -L "${target_cert}.previous" ]] || fail "previous certificate backup must not be a symbolic link."
  285. [[ ! -L "${target_key}.previous" ]] || fail "previous private key backup must not be a symbolic link."
  286. [[ ! -e "${target_cert}.previous" || -f "${target_cert}.previous" ]] \
  287. || fail "previous certificate backup must be a regular file."
  288. [[ ! -e "${target_key}.previous" || -f "${target_key}.previous" ]] \
  289. || fail "previous private key backup must be a regular file."
  290. resolve_target_ownership
  291. work_dir="$(mktemp -d "${certs_dir}/.cert-sync.XXXXXX")"
  292. chmod 0700 "${work_dir}"
  293. staged_cert="${work_dir}/next-cert.pem"
  294. staged_key="${work_dir}/next-key.pem"
  295. cp -- "${source_cert}" "${staged_cert}"
  296. cp -- "${source_key}" "${staged_key}"
  297. set_target_metadata "${staged_cert}" "${staged_key}"
  298. validate_pair "${staged_cert}" "${staged_key}" source
  299. if [[ -f "${target_cert}" && -f "${target_key}" ]] \
  300. && cmp -s "${staged_cert}" "${target_cert}" \
  301. && cmp -s "${staged_key}" "${target_key}"; then
  302. set_target_metadata "${target_cert}" "${target_key}"
  303. if [[ "${restart_app}" == "1" ]]; then
  304. cd "${project_dir}"
  305. verify_container_access
  306. live_certificate_current=1
  307. for port in ${verify_endpoints}; do
  308. if ! tls_endpoint_matches "${port}"; then
  309. live_certificate_current=0
  310. break
  311. fi
  312. done
  313. if [[ "${live_certificate_current}" == "0" ]]; then
  314. restart_tls_services
  315. for port in ${verify_endpoints}; do
  316. verify_tls_endpoint "${port}"
  317. done
  318. echo "MailHub TLS files were current; the app was restarted to load and verify them."
  319. exit 0
  320. fi
  321. fi
  322. echo "MailHub TLS certificate is already up to date."
  323. exit 0
  324. fi
  325. if [[ -e "${target_cert}" ]]; then
  326. [[ -f "${target_cert}" ]] || fail "target certificate is not a regular file."
  327. cp -a -- "${target_cert}" "${work_dir}/backup-cert.pem"
  328. backup_cert_exists=1
  329. fi
  330. if [[ -e "${target_key}" ]]; then
  331. [[ -f "${target_key}" ]] || fail "target private key is not a regular file."
  332. cp -a -- "${target_key}" "${work_dir}/backup-key.pem"
  333. backup_key_exists=1
  334. fi
  335. rollback_required=1
  336. mv -f -- "${staged_cert}" "${target_cert}"
  337. mv -f -- "${staged_key}" "${target_key}"
  338. validate_pair "${target_cert}" "${target_key}" synchronized
  339. if [[ "${restart_app}" == "1" ]]; then
  340. cd "${project_dir}"
  341. verify_container_access
  342. restart_tls_services
  343. for port in ${verify_endpoints}; do
  344. verify_tls_endpoint "${port}"
  345. done
  346. fi
  347. rollback_required=0
  348. if [[ "${backup_cert_exists}" == "1" ]]; then
  349. set_file_metadata "${work_dir}/backup-cert.pem" 0644
  350. mv -f -- "${work_dir}/backup-cert.pem" "${target_cert}.previous"
  351. fi
  352. if [[ "${backup_key_exists}" == "1" ]]; then
  353. set_file_metadata "${work_dir}/backup-key.pem" 0640
  354. mv -f -- "${work_dir}/backup-key.pem" "${target_key}.previous"
  355. fi
  356. echo "MailHub TLS certificate synchronized for ${mail_hostname}."
  357. if [[ "${restart_app}" == "1" ]]; then
  358. echo "MailHub app and Dovecot restarted and verified on TLS endpoints: ${verify_endpoints}."
  359. fi