sync-tls-certificate.sh 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385
  1. #!/usr/bin/env bash
  2. set -euo pipefail
  3. script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
  4. project_dir="${MAILHUB_CERT_PROJECT_DIR:-$(cd "${script_dir}/.." && pwd -P)}"
  5. project_dir="$(cd "${project_dir}" && pwd -P)"
  6. env_file="${MAILHUB_CERT_ENV_FILE:-${project_dir}/.env}"
  7. source_dir="${MAILHUB_CERT_SOURCE_DIR:-}"
  8. restart_app="${MAILHUB_CERT_RESTART:-0}"
  9. verify_host="${MAILHUB_CERT_VERIFY_HOST:-}"
  10. verify_endpoints="${MAILHUB_CERT_VERIFY_ENDPOINTS:-465 993}"
  11. certs_dir="${project_dir}/certs"
  12. lock_file="${MAILHUB_CERT_LOCK_FILE:-${certs_dir}/.sync-tls-certificate.lock}"
  13. work_dir=""
  14. target_cert=""
  15. target_key=""
  16. backup_cert_exists=0
  17. backup_key_exists=0
  18. rollback_required=0
  19. target_owner_uid=""
  20. target_reader_gid=""
  21. fail() {
  22. echo "Certificate sync failed: $*" >&2
  23. exit 1
  24. }
  25. read_env_value() {
  26. local key="$1"
  27. local line value
  28. line="$(grep -E "^${key}=" "${env_file}" 2>/dev/null | tail -n 1 || true)"
  29. value="${line#*=}"
  30. value="${value%$'\r'}"
  31. if [[ "${value}" == \"*\" && "${value}" == *\" ]]; then
  32. value="${value:1:${#value}-2}"
  33. elif [[ "${value}" == \'*\' && "${value}" == *\' ]]; then
  34. value="${value:1:${#value}-2}"
  35. fi
  36. printf '%s' "${value}"
  37. }
  38. resolve_target() {
  39. local container_path="$1"
  40. local label="$2"
  41. local relative
  42. [[ "${container_path}" == /certs/* ]] || fail "${label} must point to a direct file under /certs."
  43. relative="${container_path#/certs/}"
  44. [[ -n "${relative}" && "${relative}" != */* && "${relative}" != "." && "${relative}" != ".." ]] \
  45. || fail "${label} must point to a direct file under /certs."
  46. printf '%s/%s' "${certs_dir}" "${relative}"
  47. }
  48. validate_pair() {
  49. local certificate="$1"
  50. local private_key="$2"
  51. local label="$3"
  52. local cert_public_key="${work_dir}/${label}-cert-public.der"
  53. local key_public_key="${work_dir}/${label}-key-public.der"
  54. openssl x509 -in "${certificate}" -noout >/dev/null 2>&1 \
  55. || fail "${label} certificate is invalid."
  56. openssl x509 -in "${certificate}" -checkend 86400 -noout >/dev/null 2>&1 \
  57. || fail "${label} certificate expires within 24 hours."
  58. openssl x509 -in "${certificate}" -checkhost "${mail_hostname}" -noout >/dev/null 2>&1 \
  59. || fail "${label} certificate does not cover ${mail_hostname}."
  60. openssl pkey -in "${private_key}" -check -noout >/dev/null 2>&1 \
  61. || fail "${label} private key is invalid."
  62. openssl x509 -in "${certificate}" -pubkey -noout \
  63. | openssl pkey -pubin -outform DER >"${cert_public_key}" 2>/dev/null
  64. openssl pkey -in "${private_key}" -pubout -outform DER >"${key_public_key}" 2>/dev/null
  65. cmp -s "${cert_public_key}" "${key_public_key}" \
  66. || fail "${label} certificate and private key do not match."
  67. }
  68. resolve_target_ownership() {
  69. local detected_gid=""
  70. if [[ "$(id -u)" == "0" ]]; then
  71. target_owner_uid="${MAILHUB_CERT_OWNER_UID:-0}"
  72. if [[ -n "${MAILHUB_CERT_READER_GID:-}" ]]; then
  73. target_reader_gid="${MAILHUB_CERT_READER_GID}"
  74. else
  75. command -v docker >/dev/null 2>&1 \
  76. || fail "docker is required to detect the MailHub app group id."
  77. detected_gid="$(cd "${project_dir}" && docker compose exec -T app node -e \
  78. 'process.stdout.write(String(process.getgid()))' 2>/dev/null || true)"
  79. [[ -n "${detected_gid}" ]] \
  80. || fail "unable to determine the MailHub app group id; set MAILHUB_CERT_READER_GID explicitly."
  81. target_reader_gid="${detected_gid}"
  82. fi
  83. [[ "${target_owner_uid}" =~ ^[0-9]+$ ]] || fail "MAILHUB_CERT_OWNER_UID must be numeric."
  84. [[ "${target_reader_gid}" =~ ^[0-9]+$ ]] || fail "MAILHUB_CERT_READER_GID must be numeric."
  85. elif [[ -n "${MAILHUB_CERT_OWNER_UID:-}" || -n "${MAILHUB_CERT_READER_GID:-}" ]]; then
  86. fail "certificate ownership can only be changed when the sync runs as root."
  87. fi
  88. }
  89. set_file_metadata() {
  90. local file="$1"
  91. local mode="$2"
  92. chmod "${mode}" "${file}"
  93. if [[ -n "${target_owner_uid}" ]]; then
  94. chown "${target_owner_uid}:${target_reader_gid}" "${file}"
  95. fi
  96. }
  97. set_target_metadata() {
  98. set_file_metadata "$1" 0644
  99. set_file_metadata "$2" 0640
  100. }
  101. wait_for_app_health() {
  102. local attempt
  103. for attempt in $(seq 1 45); do
  104. if docker compose exec -T app node -e \
  105. "fetch('http://127.0.0.1:3000/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" \
  106. >/dev/null 2>&1; then
  107. return 0
  108. fi
  109. sleep 2
  110. done
  111. return 1
  112. }
  113. wait_for_dovecot_health() {
  114. local attempt
  115. for attempt in $(seq 1 45); do
  116. if docker compose exec -T dovecot doveconf -n >/dev/null 2>&1; then
  117. return 0
  118. fi
  119. sleep 2
  120. done
  121. return 1
  122. }
  123. restart_tls_services() {
  124. docker compose restart app dovecot
  125. wait_for_app_health || fail "MailHub app did not become healthy after the certificate restart."
  126. wait_for_dovecot_health || fail "Dovecot did not become healthy after the certificate restart."
  127. }
  128. verify_container_access() {
  129. docker compose exec -T app node -e \
  130. "const fs=require('node:fs');const tls=require('node:tls');tls.createSecureContext({cert:fs.readFileSync(process.argv[1]),key:fs.readFileSync(process.argv[2])});" \
  131. "${container_cert}" "${container_key}" >/dev/null 2>&1 \
  132. || fail "MailHub app cannot read or parse the synchronized certificate pair."
  133. docker compose exec -T dovecot sh -c \
  134. 'test -r "$1" && test -r "$2"' -- "${container_cert}" "${container_key}" >/dev/null 2>&1 \
  135. || fail "Dovecot cannot read the synchronized certificate pair."
  136. }
  137. tls_endpoint_matches() {
  138. local port="$1" service="app" container_port="$1"
  139. local mapping mapped_host mapped_port connect_host connection output presented
  140. local expected_fingerprint actual_fingerprint
  141. local -a protocol_args=()
  142. [[ "${port}" =~ ^[0-9]+$ ]] || return 1
  143. case "${port}" in
  144. 25|587|2525) protocol_args=(-starttls smtp) ;;
  145. 110) service="dovecot"; container_port="31110"; protocol_args=(-starttls pop3) ;;
  146. 143) service="dovecot"; container_port="31143"; protocol_args=(-starttls imap) ;;
  147. 993) service="dovecot"; container_port="31993" ;;
  148. 995) service="dovecot"; container_port="31995" ;;
  149. esac
  150. mapping="$(docker compose port "${service}" "${container_port}" 2>/dev/null | head -n 1 || true)"
  151. [[ -n "${mapping}" ]] || return 1
  152. mapped_port="${mapping##*:}"
  153. if [[ "${mapping}" == \[*\]:* ]]; then
  154. mapped_host="${mapping#\[}"
  155. mapped_host="${mapped_host%%\]*}"
  156. else
  157. mapped_host="${mapping%:*}"
  158. fi
  159. connect_host="${verify_host:-${mapped_host}}"
  160. [[ "${connect_host}" != "0.0.0.0" ]] || connect_host="127.0.0.1"
  161. [[ "${connect_host}" != "::" ]] || connect_host="::1"
  162. if [[ "${connect_host}" == *:* && "${connect_host}" != \[*\] ]]; then
  163. connection="[${connect_host}]:${mapped_port}"
  164. else
  165. connection="${connect_host}:${mapped_port}"
  166. fi
  167. output="${work_dir}/tls-${port}.txt"
  168. presented="${work_dir}/tls-${port}.pem"
  169. if command -v timeout >/dev/null 2>&1; then
  170. if ! timeout 15 openssl s_client "${protocol_args[@]}" -connect "${connection}" -servername "${mail_hostname}" -showcerts \
  171. -verify_hostname "${mail_hostname}" -verify_return_error \
  172. < /dev/null >"${output}" 2>/dev/null; then return 1; fi
  173. elif ! openssl s_client "${protocol_args[@]}" -connect "${connection}" -servername "${mail_hostname}" -showcerts \
  174. -verify_hostname "${mail_hostname}" -verify_return_error \
  175. < /dev/null >"${output}" 2>/dev/null; then
  176. return 1
  177. fi
  178. openssl x509 -in "${output}" -out "${presented}" >/dev/null 2>&1 \
  179. || return 1
  180. openssl x509 -in "${presented}" -checkhost "${mail_hostname}" -noout >/dev/null 2>&1 \
  181. || return 1
  182. expected_fingerprint="$(openssl x509 -in "${target_cert}" -noout -fingerprint -sha256)"
  183. actual_fingerprint="$(openssl x509 -in "${presented}" -noout -fingerprint -sha256)"
  184. [[ -n "${expected_fingerprint}" && "${expected_fingerprint}" == "${actual_fingerprint}" ]] \
  185. || return 1
  186. }
  187. verify_tls_endpoint() {
  188. tls_endpoint_matches "$1" \
  189. || fail "MailHub TLS endpoint $1 is not serving a valid synchronized certificate."
  190. }
  191. restore_previous_pair() {
  192. local rollback_failed=0
  193. set +e
  194. if [[ "${backup_cert_exists}" == "1" ]]; then
  195. cp -a "${work_dir}/backup-cert.pem" "${target_cert}"
  196. else
  197. rm -f -- "${target_cert}"
  198. fi
  199. [[ "$?" == "0" ]] || rollback_failed=1
  200. if [[ "${backup_key_exists}" == "1" ]]; then
  201. cp -a "${work_dir}/backup-key.pem" "${target_key}"
  202. else
  203. rm -f -- "${target_key}"
  204. fi
  205. [[ "$?" == "0" ]] || rollback_failed=1
  206. set -e
  207. return "${rollback_failed}"
  208. }
  209. on_exit() {
  210. local status=$?
  211. trap - EXIT
  212. if [[ "${status}" != "0" && "${rollback_required}" == "1" ]]; then
  213. echo "Certificate sync failed after promotion; restoring the previous certificate pair." >&2
  214. if restore_previous_pair; then
  215. if [[ "${restart_app}" == "1" ]]; then
  216. set +e
  217. docker compose restart app dovecot >/dev/null 2>&1
  218. wait_for_app_health >/dev/null 2>&1
  219. wait_for_dovecot_health >/dev/null 2>&1
  220. set -e
  221. fi
  222. echo "Previous MailHub certificate pair restored." >&2
  223. else
  224. echo "Certificate rollback failed; inspect ${target_cert} and ${target_key} immediately." >&2
  225. fi
  226. fi
  227. if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then
  228. rm -rf -- "${work_dir}"
  229. fi
  230. exit "${status}"
  231. }
  232. trap on_exit EXIT
  233. [[ -f "${env_file}" ]] || fail "environment file not found: ${env_file}"
  234. [[ "${restart_app}" == "0" || "${restart_app}" == "1" ]] \
  235. || fail "MAILHUB_CERT_RESTART must be 0 or 1."
  236. command -v openssl >/dev/null 2>&1 || fail "openssl is required."
  237. [[ ! -L "${certs_dir}" ]] || fail "certificate directory must not be a symbolic link: ${certs_dir}"
  238. mkdir -p "${certs_dir}"
  239. canonical_certs_dir="$(cd "${certs_dir}" && pwd -P)"
  240. [[ "${canonical_certs_dir}" == "${certs_dir}" ]] \
  241. || fail "certificate directory resolves outside the project: ${certs_dir}"
  242. if command -v flock >/dev/null 2>&1; then
  243. if [[ -L "${lock_file}" ]]; then
  244. fail "certificate lock file must not be a symbolic link: ${lock_file}"
  245. fi
  246. exec 9>"${lock_file}"
  247. if ! flock -n 9; then
  248. echo "Another MailHub certificate sync is already running; skipping."
  249. exit 0
  250. fi
  251. fi
  252. if [[ -z "${source_dir}" ]]; then
  253. source_dir="$(read_env_value MAILHUB_CERT_SOURCE_DIR)"
  254. fi
  255. if [[ -z "${source_dir}" ]]; then
  256. echo "Certificate sync is not configured; skipping."
  257. exit 0
  258. fi
  259. source_cert="${source_dir%/}/fullchain.pem"
  260. source_key="${source_dir%/}/privkey.pem"
  261. [[ -r "${source_cert}" ]] || fail "source certificate is not readable: ${source_cert}"
  262. [[ -r "${source_key}" ]] || fail "source private key is not readable: ${source_key}"
  263. [[ ! "${source_cert}" -ef "${source_key}" ]] || fail "source certificate and private key must be different files."
  264. container_cert="$(read_env_value SUBMISSION_TLS_CERT)"
  265. container_key="$(read_env_value SUBMISSION_TLS_KEY)"
  266. mail_hostname="$(read_env_value MAIL_HOSTNAME)"
  267. [[ -n "${container_cert}" ]] || fail "SUBMISSION_TLS_CERT is not configured."
  268. [[ -n "${container_key}" ]] || fail "SUBMISSION_TLS_KEY is not configured."
  269. [[ -n "${mail_hostname}" ]] || fail "MAIL_HOSTNAME is not configured."
  270. target_cert="$(resolve_target "${container_cert}" SUBMISSION_TLS_CERT)"
  271. target_key="$(resolve_target "${container_key}" SUBMISSION_TLS_KEY)"
  272. [[ "${target_cert}" != "${target_key}" ]] \
  273. || fail "SUBMISSION_TLS_CERT and SUBMISSION_TLS_KEY must point to different files."
  274. [[ ! -L "${target_cert}" ]] || fail "target certificate must not be a symbolic link."
  275. [[ ! -L "${target_key}" ]] || fail "target private key must not be a symbolic link."
  276. [[ ! -L "${target_cert}.previous" ]] || fail "previous certificate backup must not be a symbolic link."
  277. [[ ! -L "${target_key}.previous" ]] || fail "previous private key backup must not be a symbolic link."
  278. [[ ! -e "${target_cert}.previous" || -f "${target_cert}.previous" ]] \
  279. || fail "previous certificate backup must be a regular file."
  280. [[ ! -e "${target_key}.previous" || -f "${target_key}.previous" ]] \
  281. || fail "previous private key backup must be a regular file."
  282. resolve_target_ownership
  283. work_dir="$(mktemp -d "${certs_dir}/.cert-sync.XXXXXX")"
  284. chmod 0700 "${work_dir}"
  285. staged_cert="${work_dir}/next-cert.pem"
  286. staged_key="${work_dir}/next-key.pem"
  287. cp -- "${source_cert}" "${staged_cert}"
  288. cp -- "${source_key}" "${staged_key}"
  289. set_target_metadata "${staged_cert}" "${staged_key}"
  290. validate_pair "${staged_cert}" "${staged_key}" source
  291. if [[ -f "${target_cert}" && -f "${target_key}" ]] \
  292. && cmp -s "${staged_cert}" "${target_cert}" \
  293. && cmp -s "${staged_key}" "${target_key}"; then
  294. set_target_metadata "${target_cert}" "${target_key}"
  295. if [[ "${restart_app}" == "1" ]]; then
  296. cd "${project_dir}"
  297. verify_container_access
  298. live_certificate_current=1
  299. for port in ${verify_endpoints}; do
  300. if ! tls_endpoint_matches "${port}"; then
  301. live_certificate_current=0
  302. break
  303. fi
  304. done
  305. if [[ "${live_certificate_current}" == "0" ]]; then
  306. restart_tls_services
  307. for port in ${verify_endpoints}; do
  308. verify_tls_endpoint "${port}"
  309. done
  310. echo "MailHub TLS files were current; the app was restarted to load and verify them."
  311. exit 0
  312. fi
  313. fi
  314. echo "MailHub TLS certificate is already up to date."
  315. exit 0
  316. fi
  317. if [[ -e "${target_cert}" ]]; then
  318. [[ -f "${target_cert}" ]] || fail "target certificate is not a regular file."
  319. cp -a -- "${target_cert}" "${work_dir}/backup-cert.pem"
  320. backup_cert_exists=1
  321. fi
  322. if [[ -e "${target_key}" ]]; then
  323. [[ -f "${target_key}" ]] || fail "target private key is not a regular file."
  324. cp -a -- "${target_key}" "${work_dir}/backup-key.pem"
  325. backup_key_exists=1
  326. fi
  327. rollback_required=1
  328. mv -f -- "${staged_cert}" "${target_cert}"
  329. mv -f -- "${staged_key}" "${target_key}"
  330. validate_pair "${target_cert}" "${target_key}" synchronized
  331. if [[ "${restart_app}" == "1" ]]; then
  332. cd "${project_dir}"
  333. verify_container_access
  334. restart_tls_services
  335. for port in ${verify_endpoints}; do
  336. verify_tls_endpoint "${port}"
  337. done
  338. fi
  339. rollback_required=0
  340. if [[ "${backup_cert_exists}" == "1" ]]; then
  341. set_file_metadata "${work_dir}/backup-cert.pem" 0644
  342. mv -f -- "${work_dir}/backup-cert.pem" "${target_cert}.previous"
  343. fi
  344. if [[ "${backup_key_exists}" == "1" ]]; then
  345. set_file_metadata "${work_dir}/backup-key.pem" 0640
  346. mv -f -- "${work_dir}/backup-key.pem" "${target_key}.previous"
  347. fi
  348. echo "MailHub TLS certificate synchronized for ${mail_hostname}."
  349. if [[ "${restart_app}" == "1" ]]; then
  350. echo "MailHub app and Dovecot restarted and verified on TLS endpoints: ${verify_endpoints}."
  351. fi