diff --git a/scripts/release-portal-canary-prod.sh b/scripts/release-portal-canary-prod.sh index 3afa553..fca5731 100755 --- a/scripts/release-portal-canary-prod.sh +++ b/scripts/release-portal-canary-prod.sh @@ -225,6 +225,69 @@ say "Upload the verified candidate bundle" ssh -o BatchMode=yes "${HOST}" "mkdir -p '${INCOMING_DIR}' '${BACKUP_DIR}' '${CANDIDATE_BASE}'" scp -q "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}" "${HOST}:${INCOMING_DIR}/" +say "Create and verify the 103 stable backups before candidate install" +ssh -o BatchMode=yes "${HOST}" \ + "RELEASE_ID='${RELEASE_ID}' \ + STABLE_DIR='${STABLE_DIR}' \ + BACKUP_DIR='${BACKUP_DIR}' \ + /bin/bash" <<'REMOTE_BACKUP_SCRIPT' +set -euo pipefail + +FULL_BACKUP_TAR="${BACKUP_DIR}/memind-full-${RELEASE_ID}-before-canary.tar.gz" +PERSIST_BACKUP_TAR="${BACKUP_DIR}/memind-persisted-${RELEASE_ID}-before-canary.tar.gz" + +say() { + printf '\n[remote-backup %s] %s\n' "$(date +%H:%M:%S)" "$*" +} + +verify_full_backup() { + test -f "${FULL_BACKUP_TAR}" + test -f "${FULL_BACKUP_TAR}.sha256" + shasum -a 256 -c "${FULL_BACKUP_TAR}.sha256" + gzip -t "${FULL_BACKUP_TAR}" + tar -tzf "${FULL_BACKUP_TAR}" "Memind/.env" >/dev/null +} + +verify_persisted_backup() { + test -f "${PERSIST_BACKUP_TAR}" + test -f "${PERSIST_BACKUP_TAR}.sha256" + shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256" + gzip -t "${PERSIST_BACKUP_TAR}" + tar -tzf "${PERSIST_BACKUP_TAR}" ".env" >/dev/null \ + || tar -tzf "${PERSIST_BACKUP_TAR}" "./.env" >/dev/null +} + +say "Create and verify the full stable backup" +COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \ + -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${STABLE_DIR}")" "$(basename "${STABLE_DIR}")" +shasum -a 256 "${FULL_BACKUP_TAR}" > "${FULL_BACKUP_TAR}.sha256" +verify_full_backup + +say "Create and verify the persisted-data backup" +persist_tmp="$(mktemp -d "${TMPDIR:-/tmp}/memind-canary-persist.XXXXXX")" +for item in .env MindSpace data users .tailscale public/plaza-covers logs; do + if [[ -e "${STABLE_DIR}/${item}" ]]; then + mkdir -p "${persist_tmp}/$(dirname "${item}")" + if [[ "${item}" == ".tailscale" ]]; then + mkdir -p "${persist_tmp}/${item}" + ( + cd "${STABLE_DIR}/${item}" + COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - . + ) | ( + cd "${persist_tmp}/${item}" + COPYFILE_DISABLE=1 tar -xf - + ) + else + cp -a "${STABLE_DIR}/${item}" "${persist_tmp}/$(dirname "${item}")/" + fi + fi +done +COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${persist_tmp}" . +rm -rf "${persist_tmp}" +shasum -a 256 "${PERSIST_BACKUP_TAR}" > "${PERSIST_BACKUP_TAR}.sha256" +verify_persisted_backup +REMOTE_BACKUP_SCRIPT + say "Install the isolated candidate and identity router on 103" ssh -o BatchMode=yes "${HOST}" \ "RELEASE_ID='${RELEASE_ID}' \ @@ -390,39 +453,20 @@ trap rollback ERR cd "${INCOMING_DIR}" shasum -a 256 -c "$(basename "${SHA_FILE}")" -say "Create and verify the full stable backup" -COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \ - -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${STABLE_DIR}")" "$(basename "${STABLE_DIR}")" -shasum -a 256 "${FULL_BACKUP_TAR}" > "${FULL_BACKUP_TAR}.sha256" +say "Verify the pre-created full stable backup" +test -f "${FULL_BACKUP_TAR}" +test -f "${FULL_BACKUP_TAR}.sha256" shasum -a 256 -c "${FULL_BACKUP_TAR}.sha256" gzip -t "${FULL_BACKUP_TAR}" -tar -tzf "${FULL_BACKUP_TAR}" | grep '^Memind/.env$' >/dev/null +tar -tzf "${FULL_BACKUP_TAR}" "Memind/.env" >/dev/null -say "Create and verify the persisted-data backup" -persist_tmp="$(mktemp -d "${TMPDIR:-/tmp}/memind-canary-persist.XXXXXX")" -for item in .env MindSpace data users .tailscale public/plaza-covers logs; do - if [[ -e "${STABLE_DIR}/${item}" ]]; then - mkdir -p "${persist_tmp}/$(dirname "${item}")" - if [[ "${item}" == ".tailscale" ]]; then - mkdir -p "${persist_tmp}/${item}" - ( - cd "${STABLE_DIR}/${item}" - COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - . - ) | ( - cd "${persist_tmp}/${item}" - COPYFILE_DISABLE=1 tar -xf - - ) - else - cp -a "${STABLE_DIR}/${item}" "${persist_tmp}/$(dirname "${item}")/" - fi - fi -done -COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${persist_tmp}" . -rm -rf "${persist_tmp}" -shasum -a 256 "${PERSIST_BACKUP_TAR}" > "${PERSIST_BACKUP_TAR}.sha256" +say "Verify the pre-created persisted-data backup" +test -f "${PERSIST_BACKUP_TAR}" +test -f "${PERSIST_BACKUP_TAR}.sha256" shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256" gzip -t "${PERSIST_BACKUP_TAR}" -tar -tzf "${PERSIST_BACKUP_TAR}" | grep -E '^(\./)?\.env$' >/dev/null +tar -tzf "${PERSIST_BACKUP_TAR}" ".env" >/dev/null \ + || tar -tzf "${PERSIST_BACKUP_TAR}" "./.env" >/dev/null say "Create and verify the active 105 nginx routing backup" edge_ssh \