sync-tls-certificate.sh 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. #!/usr/bin/env bash
  2. set -euo pipefail
  3. script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
  4. project_dir="${MAILHUB_CERT_PROJECT_DIR:-$(cd "${script_dir}/.." && pwd -P)}"
  5. project_dir="$(cd "${project_dir}" && pwd -P)"
  6. env_file="${MAILHUB_CERT_ENV_FILE:-${project_dir}/.env}"
  7. source_dir="${MAILHUB_CERT_SOURCE_DIR:-}"
  8. restart_app="${MAILHUB_CERT_RESTART:-0}"
  9. verify_host="${MAILHUB_CERT_VERIFY_HOST:-}"
  10. verify_endpoints="${MAILHUB_CERT_VERIFY_ENDPOINTS:-465 993}"
  11. certs_dir="${project_dir}/certs"
  12. lock_file="${MAILHUB_CERT_LOCK_FILE:-${certs_dir}/.sync-tls-certificate.lock}"
  13. work_dir=""
  14. target_cert=""
  15. target_key=""
  16. backup_cert_exists=0
  17. backup_key_exists=0
  18. rollback_required=0
  19. target_owner_uid=""
  20. target_reader_gid=""
  21. fail() {
  22. echo "Certificate sync failed: $*" >&2
  23. exit 1
  24. }
  25. read_env_value() {
  26. local key="$1"
  27. local line value
  28. line="$(grep -E "^${key}=" "${env_file}" 2>/dev/null | tail -n 1 || true)"
  29. value="${line#*=}"
  30. value="${value%$'\r'}"
  31. if [[ "${value}" == \"*\" && "${value}" == *\" ]]; then
  32. value="${value:1:${#value}-2}"
  33. elif [[ "${value}" == \'*\' && "${value}" == *\' ]]; then
  34. value="${value:1:${#value}-2}"
  35. fi
  36. printf '%s' "${value}"
  37. }
  38. resolve_target() {
  39. local container_path="$1"
  40. local label="$2"
  41. local relative
  42. [[ "${container_path}" == /certs/* ]] || fail "${label} must point to a direct file under /certs."
  43. relative="${container_path#/certs/}"
  44. [[ -n "${relative}" && "${relative}" != */* && "${relative}" != "." && "${relative}" != ".." ]] \
  45. || fail "${label} must point to a direct file under /certs."
  46. printf '%s/%s' "${certs_dir}" "${relative}"
  47. }
  48. validate_pair() {
  49. local certificate="$1"
  50. local private_key="$2"
  51. local label="$3"
  52. local cert_public_key="${work_dir}/${label}-cert-public.der"
  53. local key_public_key="${work_dir}/${label}-key-public.der"
  54. openssl x509 -in "${certificate}" -noout >/dev/null 2>&1 \
  55. || fail "${label} certificate is invalid."
  56. openssl x509 -in "${certificate}" -checkend 86400 -noout >/dev/null 2>&1 \
  57. || fail "${label} certificate expires within 24 hours."
  58. openssl x509 -in "${certificate}" -checkhost "${mail_hostname}" -noout >/dev/null 2>&1 \
  59. || fail "${label} certificate does not cover ${mail_hostname}."
  60. openssl pkey -in "${private_key}" -check -noout >/dev/null 2>&1 \
  61. || fail "${label} private key is invalid."
  62. openssl x509 -in "${certificate}" -pubkey -noout \
  63. | openssl pkey -pubin -outform DER >"${cert_public_key}" 2>/dev/null
  64. openssl pkey -in "${private_key}" -pubout -outform DER >"${key_public_key}" 2>/dev/null
  65. cmp -s "${cert_public_key}" "${key_public_key}" \
  66. || fail "${label} certificate and private key do not match."
  67. }
  68. resolve_target_ownership() {
  69. local detected_gid=""
  70. if [[ "$(id -u)" == "0" ]]; then
  71. target_owner_uid="${MAILHUB_CERT_OWNER_UID:-0}"
  72. if [[ -n "${MAILHUB_CERT_READER_GID:-}" ]]; then
  73. target_reader_gid="${MAILHUB_CERT_READER_GID}"
  74. else
  75. command -v docker >/dev/null 2>&1 \
  76. || fail "docker is required to detect the MailHub app group id."
  77. detected_gid="$(cd "${project_dir}" && docker compose exec -T app node -e \
  78. 'process.stdout.write(String(process.getgid()))' 2>/dev/null || true)"
  79. [[ -n "${detected_gid}" ]] \
  80. || fail "unable to determine the MailHub app group id; set MAILHUB_CERT_READER_GID explicitly."
  81. target_reader_gid="${detected_gid}"
  82. fi
  83. [[ "${target_owner_uid}" =~ ^[0-9]+$ ]] || fail "MAILHUB_CERT_OWNER_UID must be numeric."
  84. [[ "${target_reader_gid}" =~ ^[0-9]+$ ]] || fail "MAILHUB_CERT_READER_GID must be numeric."
  85. elif [[ -n "${MAILHUB_CERT_OWNER_UID:-}" || -n "${MAILHUB_CERT_READER_GID:-}" ]]; then
  86. fail "certificate ownership can only be changed when the sync runs as root."
  87. fi
  88. }
  89. set_file_metadata() {
  90. local file="$1"
  91. local mode="$2"
  92. chmod "${mode}" "${file}"
  93. if [[ -n "${target_owner_uid}" ]]; then
  94. chown "${target_owner_uid}:${target_reader_gid}" "${file}"
  95. fi
  96. }
  97. set_target_metadata() {
  98. set_file_metadata "$1" 0644
  99. set_file_metadata "$2" 0640
  100. }
  101. wait_for_app_health() {
  102. local attempt
  103. for attempt in $(seq 1 45); do
  104. if docker compose exec -T app node -e \
  105. "fetch('http://127.0.0.1:3000/healthz').then(r=>process.exit(r.ok?0:1)).catch(()=>process.exit(1))" \
  106. >/dev/null 2>&1; then
  107. return 0
  108. fi
  109. sleep 2
  110. done
  111. return 1
  112. }
  113. verify_container_access() {
  114. docker compose exec -T app node -e \
  115. "const fs=require('node:fs');const tls=require('node:tls');tls.createSecureContext({cert:fs.readFileSync(process.argv[1]),key:fs.readFileSync(process.argv[2])});" \
  116. "${container_cert}" "${container_key}" >/dev/null 2>&1 \
  117. || fail "MailHub app cannot read or parse the synchronized certificate pair."
  118. }
  119. tls_endpoint_matches() {
  120. local port="$1"
  121. local mapping mapped_host mapped_port connect_host connection output presented
  122. local expected_fingerprint actual_fingerprint
  123. [[ "${port}" =~ ^[0-9]+$ ]] || return 1
  124. mapping="$(docker compose port app "${port}" 2>/dev/null | head -n 1 || true)"
  125. [[ -n "${mapping}" ]] || return 1
  126. mapped_port="${mapping##*:}"
  127. if [[ "${mapping}" == \[*\]:* ]]; then
  128. mapped_host="${mapping#\[}"
  129. mapped_host="${mapped_host%%\]*}"
  130. else
  131. mapped_host="${mapping%:*}"
  132. fi
  133. connect_host="${verify_host:-${mapped_host}}"
  134. [[ "${connect_host}" != "0.0.0.0" ]] || connect_host="127.0.0.1"
  135. [[ "${connect_host}" != "::" ]] || connect_host="::1"
  136. if [[ "${connect_host}" == *:* && "${connect_host}" != \[*\] ]]; then
  137. connection="[${connect_host}]:${mapped_port}"
  138. else
  139. connection="${connect_host}:${mapped_port}"
  140. fi
  141. output="${work_dir}/tls-${port}.txt"
  142. presented="${work_dir}/tls-${port}.pem"
  143. if command -v timeout >/dev/null 2>&1; then
  144. if ! timeout 15 openssl s_client -connect "${connection}" -servername "${mail_hostname}" -showcerts \
  145. -verify_hostname "${mail_hostname}" -verify_return_error \
  146. < /dev/null >"${output}" 2>/dev/null; then return 1; fi
  147. elif ! openssl s_client -connect "${connection}" -servername "${mail_hostname}" -showcerts \
  148. -verify_hostname "${mail_hostname}" -verify_return_error \
  149. < /dev/null >"${output}" 2>/dev/null; then
  150. return 1
  151. fi
  152. openssl x509 -in "${output}" -out "${presented}" >/dev/null 2>&1 \
  153. || return 1
  154. openssl x509 -in "${presented}" -checkhost "${mail_hostname}" -noout >/dev/null 2>&1 \
  155. || return 1
  156. expected_fingerprint="$(openssl x509 -in "${target_cert}" -noout -fingerprint -sha256)"
  157. actual_fingerprint="$(openssl x509 -in "${presented}" -noout -fingerprint -sha256)"
  158. [[ -n "${expected_fingerprint}" && "${expected_fingerprint}" == "${actual_fingerprint}" ]] \
  159. || return 1
  160. }
  161. verify_tls_endpoint() {
  162. tls_endpoint_matches "$1" \
  163. || fail "MailHub TLS endpoint $1 is not serving a valid synchronized certificate."
  164. }
  165. restore_previous_pair() {
  166. local rollback_failed=0
  167. set +e
  168. if [[ "${backup_cert_exists}" == "1" ]]; then
  169. cp -a "${work_dir}/backup-cert.pem" "${target_cert}"
  170. else
  171. rm -f -- "${target_cert}"
  172. fi
  173. [[ "$?" == "0" ]] || rollback_failed=1
  174. if [[ "${backup_key_exists}" == "1" ]]; then
  175. cp -a "${work_dir}/backup-key.pem" "${target_key}"
  176. else
  177. rm -f -- "${target_key}"
  178. fi
  179. [[ "$?" == "0" ]] || rollback_failed=1
  180. set -e
  181. return "${rollback_failed}"
  182. }
  183. on_exit() {
  184. local status=$?
  185. trap - EXIT
  186. if [[ "${status}" != "0" && "${rollback_required}" == "1" ]]; then
  187. echo "Certificate sync failed after promotion; restoring the previous certificate pair." >&2
  188. if restore_previous_pair; then
  189. if [[ "${restart_app}" == "1" ]]; then
  190. set +e
  191. docker compose restart app >/dev/null 2>&1
  192. wait_for_app_health >/dev/null 2>&1
  193. set -e
  194. fi
  195. echo "Previous MailHub certificate pair restored." >&2
  196. else
  197. echo "Certificate rollback failed; inspect ${target_cert} and ${target_key} immediately." >&2
  198. fi
  199. fi
  200. if [[ -n "${work_dir}" && -d "${work_dir}" ]]; then
  201. rm -rf -- "${work_dir}"
  202. fi
  203. exit "${status}"
  204. }
  205. trap on_exit EXIT
  206. [[ -f "${env_file}" ]] || fail "environment file not found: ${env_file}"
  207. [[ "${restart_app}" == "0" || "${restart_app}" == "1" ]] \
  208. || fail "MAILHUB_CERT_RESTART must be 0 or 1."
  209. command -v openssl >/dev/null 2>&1 || fail "openssl is required."
  210. [[ ! -L "${certs_dir}" ]] || fail "certificate directory must not be a symbolic link: ${certs_dir}"
  211. mkdir -p "${certs_dir}"
  212. canonical_certs_dir="$(cd "${certs_dir}" && pwd -P)"
  213. [[ "${canonical_certs_dir}" == "${certs_dir}" ]] \
  214. || fail "certificate directory resolves outside the project: ${certs_dir}"
  215. if command -v flock >/dev/null 2>&1; then
  216. if [[ -L "${lock_file}" ]]; then
  217. fail "certificate lock file must not be a symbolic link: ${lock_file}"
  218. fi
  219. exec 9>"${lock_file}"
  220. if ! flock -n 9; then
  221. echo "Another MailHub certificate sync is already running; skipping."
  222. exit 0
  223. fi
  224. fi
  225. if [[ -z "${source_dir}" ]]; then
  226. source_dir="$(read_env_value MAILHUB_CERT_SOURCE_DIR)"
  227. fi
  228. if [[ -z "${source_dir}" ]]; then
  229. echo "Certificate sync is not configured; skipping."
  230. exit 0
  231. fi
  232. source_cert="${source_dir%/}/fullchain.pem"
  233. source_key="${source_dir%/}/privkey.pem"
  234. [[ -r "${source_cert}" ]] || fail "source certificate is not readable: ${source_cert}"
  235. [[ -r "${source_key}" ]] || fail "source private key is not readable: ${source_key}"
  236. [[ ! "${source_cert}" -ef "${source_key}" ]] || fail "source certificate and private key must be different files."
  237. container_cert="$(read_env_value SUBMISSION_TLS_CERT)"
  238. container_key="$(read_env_value SUBMISSION_TLS_KEY)"
  239. mail_hostname="$(read_env_value MAIL_HOSTNAME)"
  240. [[ -n "${container_cert}" ]] || fail "SUBMISSION_TLS_CERT is not configured."
  241. [[ -n "${container_key}" ]] || fail "SUBMISSION_TLS_KEY is not configured."
  242. [[ -n "${mail_hostname}" ]] || fail "MAIL_HOSTNAME is not configured."
  243. target_cert="$(resolve_target "${container_cert}" SUBMISSION_TLS_CERT)"
  244. target_key="$(resolve_target "${container_key}" SUBMISSION_TLS_KEY)"
  245. [[ "${target_cert}" != "${target_key}" ]] \
  246. || fail "SUBMISSION_TLS_CERT and SUBMISSION_TLS_KEY must point to different files."
  247. [[ ! -L "${target_cert}" ]] || fail "target certificate must not be a symbolic link."
  248. [[ ! -L "${target_key}" ]] || fail "target private key must not be a symbolic link."
  249. [[ ! -L "${target_cert}.previous" ]] || fail "previous certificate backup must not be a symbolic link."
  250. [[ ! -L "${target_key}.previous" ]] || fail "previous private key backup must not be a symbolic link."
  251. [[ ! -e "${target_cert}.previous" || -f "${target_cert}.previous" ]] \
  252. || fail "previous certificate backup must be a regular file."
  253. [[ ! -e "${target_key}.previous" || -f "${target_key}.previous" ]] \
  254. || fail "previous private key backup must be a regular file."
  255. resolve_target_ownership
  256. work_dir="$(mktemp -d "${certs_dir}/.cert-sync.XXXXXX")"
  257. chmod 0700 "${work_dir}"
  258. staged_cert="${work_dir}/next-cert.pem"
  259. staged_key="${work_dir}/next-key.pem"
  260. cp -- "${source_cert}" "${staged_cert}"
  261. cp -- "${source_key}" "${staged_key}"
  262. set_target_metadata "${staged_cert}" "${staged_key}"
  263. validate_pair "${staged_cert}" "${staged_key}" source
  264. if [[ -f "${target_cert}" && -f "${target_key}" ]] \
  265. && cmp -s "${staged_cert}" "${target_cert}" \
  266. && cmp -s "${staged_key}" "${target_key}"; then
  267. set_target_metadata "${target_cert}" "${target_key}"
  268. if [[ "${restart_app}" == "1" ]]; then
  269. cd "${project_dir}"
  270. verify_container_access
  271. live_certificate_current=1
  272. for port in ${verify_endpoints}; do
  273. if ! tls_endpoint_matches "${port}"; then
  274. live_certificate_current=0
  275. break
  276. fi
  277. done
  278. if [[ "${live_certificate_current}" == "0" ]]; then
  279. docker compose restart app
  280. wait_for_app_health || fail "MailHub app did not become healthy after the certificate restart."
  281. for port in ${verify_endpoints}; do
  282. verify_tls_endpoint "${port}"
  283. done
  284. echo "MailHub TLS files were current; the app was restarted to load and verify them."
  285. exit 0
  286. fi
  287. fi
  288. echo "MailHub TLS certificate is already up to date."
  289. exit 0
  290. fi
  291. if [[ -e "${target_cert}" ]]; then
  292. [[ -f "${target_cert}" ]] || fail "target certificate is not a regular file."
  293. cp -a -- "${target_cert}" "${work_dir}/backup-cert.pem"
  294. backup_cert_exists=1
  295. fi
  296. if [[ -e "${target_key}" ]]; then
  297. [[ -f "${target_key}" ]] || fail "target private key is not a regular file."
  298. cp -a -- "${target_key}" "${work_dir}/backup-key.pem"
  299. backup_key_exists=1
  300. fi
  301. rollback_required=1
  302. mv -f -- "${staged_cert}" "${target_cert}"
  303. mv -f -- "${staged_key}" "${target_key}"
  304. validate_pair "${target_cert}" "${target_key}" synchronized
  305. if [[ "${restart_app}" == "1" ]]; then
  306. cd "${project_dir}"
  307. verify_container_access
  308. docker compose restart app
  309. wait_for_app_health || fail "MailHub app did not become healthy after the certificate restart."
  310. for port in ${verify_endpoints}; do
  311. verify_tls_endpoint "${port}"
  312. done
  313. fi
  314. rollback_required=0
  315. if [[ "${backup_cert_exists}" == "1" ]]; then
  316. set_file_metadata "${work_dir}/backup-cert.pem" 0644
  317. mv -f -- "${work_dir}/backup-cert.pem" "${target_cert}.previous"
  318. fi
  319. if [[ "${backup_key_exists}" == "1" ]]; then
  320. set_file_metadata "${work_dir}/backup-key.pem" 0640
  321. mv -f -- "${work_dir}/backup-key.pem" "${target_key}.previous"
  322. fi
  323. echo "MailHub TLS certificate synchronized for ${mail_hostname}."
  324. if [[ "${restart_app}" == "1" ]]; then
  325. echo "MailHub app restarted and verified on TLS endpoints: ${verify_endpoints}."
  326. fi