fix: split portal canary backup preflight

scripts/release-portal-canary-prod.sh

@@ -225,6 +225,69 @@ say "Upload the verified candidate bundle"
 ssh -o BatchMode=yes "${HOST}" "mkdir -p '${INCOMING_DIR}' '${BACKUP_DIR}' '${CANDIDATE_BASE}'"
 scp -q "${BUNDLE_PATH}" "${MANIFEST_PATH}" "${SHA_PATH}" "${HOST}:${INCOMING_DIR}/"
 
+say "Create and verify the 103 stable backups before candidate install"
+ssh -o BatchMode=yes "${HOST}" \
+  "RELEASE_ID='${RELEASE_ID}' \
+   STABLE_DIR='${STABLE_DIR}' \
+   BACKUP_DIR='${BACKUP_DIR}' \
+   /bin/bash" <<'REMOTE_BACKUP_SCRIPT'
+set -euo pipefail
+
+FULL_BACKUP_TAR="${BACKUP_DIR}/memind-full-${RELEASE_ID}-before-canary.tar.gz"
+PERSIST_BACKUP_TAR="${BACKUP_DIR}/memind-persisted-${RELEASE_ID}-before-canary.tar.gz"
+
+say() {
+  printf '\n[remote-backup %s] %s\n' "$(date +%H:%M:%S)" "$*"
+}
+
+verify_full_backup() {
+  test -f "${FULL_BACKUP_TAR}"
+  test -f "${FULL_BACKUP_TAR}.sha256"
+  shasum -a 256 -c "${FULL_BACKUP_TAR}.sha256"
+  gzip -t "${FULL_BACKUP_TAR}"
+  tar -tzf "${FULL_BACKUP_TAR}" "Memind/.env" >/dev/null
+}
+
+verify_persisted_backup() {
+  test -f "${PERSIST_BACKUP_TAR}"
+  test -f "${PERSIST_BACKUP_TAR}.sha256"
+  shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256"
+  gzip -t "${PERSIST_BACKUP_TAR}"
+  tar -tzf "${PERSIST_BACKUP_TAR}" ".env" >/dev/null \
+    || tar -tzf "${PERSIST_BACKUP_TAR}" "./.env" >/dev/null
+}
+
+say "Create and verify the full stable backup"
+COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
+  -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${STABLE_DIR}")" "$(basename "${STABLE_DIR}")"
+shasum -a 256 "${FULL_BACKUP_TAR}" > "${FULL_BACKUP_TAR}.sha256"
+verify_full_backup
+
+say "Create and verify the persisted-data backup"
+persist_tmp="$(mktemp -d "${TMPDIR:-/tmp}/memind-canary-persist.XXXXXX")"
+for item in .env MindSpace data users .tailscale public/plaza-covers logs; do
+  if [[ -e "${STABLE_DIR}/${item}" ]]; then
+    mkdir -p "${persist_tmp}/$(dirname "${item}")"
+    if [[ "${item}" == ".tailscale" ]]; then
+      mkdir -p "${persist_tmp}/${item}"
+      (
+        cd "${STABLE_DIR}/${item}"
+        COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - .
+      ) | (
+        cd "${persist_tmp}/${item}"
+        COPYFILE_DISABLE=1 tar -xf -
+      )
+    else
+      cp -a "${STABLE_DIR}/${item}" "${persist_tmp}/$(dirname "${item}")/"
+    fi
+  fi
+done
+COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${persist_tmp}" .
+rm -rf "${persist_tmp}"
+shasum -a 256 "${PERSIST_BACKUP_TAR}" > "${PERSIST_BACKUP_TAR}.sha256"
+verify_persisted_backup
+REMOTE_BACKUP_SCRIPT
+
 say "Install the isolated candidate and identity router on 103"
 ssh -o BatchMode=yes "${HOST}" \
   "RELEASE_ID='${RELEASE_ID}' \
@@ -390,39 +453,20 @@ trap rollback ERR
 cd "${INCOMING_DIR}"
 shasum -a 256 -c "$(basename "${SHA_FILE}")"
 
-say "Create and verify the full stable backup"
+say "Verify the pre-created full stable backup"
-COPYFILE_DISABLE=1 tar --exclude='Memind/.tailscale/*.sock' \
+test -f "${FULL_BACKUP_TAR}"
-  -czf "${FULL_BACKUP_TAR}" -C "$(dirname "${STABLE_DIR}")" "$(basename "${STABLE_DIR}")"
+test -f "${FULL_BACKUP_TAR}.sha256"
-shasum -a 256 "${FULL_BACKUP_TAR}" > "${FULL_BACKUP_TAR}.sha256"
 shasum -a 256 -c "${FULL_BACKUP_TAR}.sha256"
 gzip -t "${FULL_BACKUP_TAR}"
-tar -tzf "${FULL_BACKUP_TAR}" | grep '^Memind/.env$' >/dev/null
+tar -tzf "${FULL_BACKUP_TAR}" "Memind/.env" >/dev/null
 
-say "Create and verify the persisted-data backup"
+say "Verify the pre-created persisted-data backup"
-persist_tmp="$(mktemp -d "${TMPDIR:-/tmp}/memind-canary-persist.XXXXXX")"
+test -f "${PERSIST_BACKUP_TAR}"
-for item in .env MindSpace data users .tailscale public/plaza-covers logs; do
+test -f "${PERSIST_BACKUP_TAR}.sha256"
-  if [[ -e "${STABLE_DIR}/${item}" ]]; then
-    mkdir -p "${persist_tmp}/$(dirname "${item}")"
-    if [[ "${item}" == ".tailscale" ]]; then
-      mkdir -p "${persist_tmp}/${item}"
-      (
-        cd "${STABLE_DIR}/${item}"
-        COPYFILE_DISABLE=1 tar --exclude='*.sock' -cf - .
-      ) | (
-        cd "${persist_tmp}/${item}"
-        COPYFILE_DISABLE=1 tar -xf -
-      )
-    else
-      cp -a "${STABLE_DIR}/${item}" "${persist_tmp}/$(dirname "${item}")/"
-    fi
-  fi
-done
-COPYFILE_DISABLE=1 tar -czf "${PERSIST_BACKUP_TAR}" -C "${persist_tmp}" .
-rm -rf "${persist_tmp}"
-shasum -a 256 "${PERSIST_BACKUP_TAR}" > "${PERSIST_BACKUP_TAR}.sha256"
 shasum -a 256 -c "${PERSIST_BACKUP_TAR}.sha256"
 gzip -t "${PERSIST_BACKUP_TAR}"
-tar -tzf "${PERSIST_BACKUP_TAR}" | grep -E '^(\./)?\.env$' >/dev/null
+tar -tzf "${PERSIST_BACKUP_TAR}" ".env" >/dev/null \
+  || tar -tzf "${PERSIST_BACKUP_TAR}" "./.env" >/dev/null
 
 say "Create and verify the active 105 nginx routing backup"
 edge_ssh \