#!/usr/bin/env bash # ============================================================================= # SurfSense — Database Migration Script # # Extracts data from the legacy all-in-one surfsense-data volume (PostgreSQL 14) # and saves it as a SQL dump + SECRET_KEY file ready for install.sh to restore. # # Usage: # bash migrate-database.sh [options] # # Options: # --db-user USER Old PostgreSQL username (default: surfsense) # --db-password PASS Old PostgreSQL password (default: surfsense) # --db-name NAME Old PostgreSQL database (default: surfsense) # --yes / -y Skip all confirmation prompts # --help / -h Show this help # # Prerequisites: # - Docker installed and running # - The legacy surfsense-data volume must exist # - ~500 MB free disk space for the dump file # # What this script does: # 1. Stops any container using surfsense-data (to prevent corruption) # 2. Starts a temporary PG14 container against the old volume # 3. Dumps the database to ./surfsense_migration_backup.sql # 4. Recovers the SECRET_KEY to ./surfsense_migration_secret.key # 5. Exits — leaving installation to install.sh # # What this script does NOT do: # - Delete the original surfsense-data volume (do this manually after verifying) # - Install the new SurfSense stack (install.sh handles that automatically) # # Note: # install.sh downloads and runs this script automatically when it detects the # legacy surfsense-data volume. You only need to run this script manually if # you have custom database credentials (--db-user / --db-password / --db-name) # or if the automatic migration inside install.sh fails at the extraction step. # ============================================================================= set -euo pipefail # ── Colours ────────────────────────────────────────────────────────────────── CYAN='\033[1;36m' YELLOW='\033[1;33m' GREEN='\033[0;32m' RED='\033[0;31m' BOLD='\033[1m' NC='\033[0m' # ── Logging — tee everything to a log file ─────────────────────────────────── LOG_FILE="./surfsense-migration.log" exec > >(tee -a "${LOG_FILE}") 2>&1 # ── Output helpers ──────────────────────────────────────────────────────────── info() { printf "${CYAN}[SurfSense]${NC} %s\n" "$1"; } success() { printf "${GREEN}[SurfSense]${NC} %s\n" "$1"; } warn() { printf "${YELLOW}[SurfSense]${NC} %s\n" "$1"; } error() { printf "${RED}[SurfSense]${NC} ERROR: %s\n" "$1" >&2; exit 1; } step() { printf "\n${BOLD}${CYAN}── Step %s: %s${NC}\n" "$1" "$2"; } # ── Constants ───────────────────────────────────────────────────────────────── OLD_VOLUME="surfsense-data" TEMP_CONTAINER="surfsense-pg14-migration" DUMP_FILE="./surfsense_migration_backup.sql" KEY_FILE="./surfsense_migration_secret.key" PG14_IMAGE="pgvector/pgvector:pg14" # ── Defaults ────────────────────────────────────────────────────────────────── OLD_DB_USER="surfsense" OLD_DB_PASSWORD="surfsense" OLD_DB_NAME="surfsense" AUTO_YES=false # ── Argument parsing ────────────────────────────────────────────────────────── while [[ $# -gt 0 ]]; do case "$1" in --db-user) OLD_DB_USER="$2"; shift 2 ;; --db-password) OLD_DB_PASSWORD="$2"; shift 2 ;; --db-name) OLD_DB_NAME="$2"; shift 2 ;; --yes|-y) AUTO_YES=true; shift ;; --help|-h) grep '^#' "$0" | grep -v '^#!/' | sed 's/^# \{0,1\}//' exit 0 ;; *) error "Unknown option: $1 — run with --help for usage." ;; esac done # ── Confirmation helper ─────────────────────────────────────────────────────── confirm() { if $AUTO_YES; then return 0; fi printf "${YELLOW}[SurfSense]${NC} %s [y/N] " "$1" read -r reply [[ "$reply" =~ ^[Yy]$ ]] || { warn "Aborted."; exit 0; } } # ── Cleanup trap — always remove the temp container ────────────────────────── cleanup() { local exit_code=$? if docker ps -a --format '{{.Names}}' 2>/dev/null < /dev/null | grep -q "^${TEMP_CONTAINER}$"; then info "Cleaning up temporary container '${TEMP_CONTAINER}'..." docker stop "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true docker rm "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true fi if [[ $exit_code -ne 0 ]]; then printf "\n${RED}[SurfSense]${NC} Migration data extraction failed (exit code %s).\n" "${exit_code}" >&2 printf "${RED}[SurfSense]${NC} Full log: %s\n" "${LOG_FILE}" >&2 printf "${YELLOW}[SurfSense]${NC} Your original data in '${OLD_VOLUME}' is untouched.\n" >&2 fi } trap cleanup EXIT # ── Wait-for-postgres helper ────────────────────────────────────────────────── wait_for_pg() { local container="$1" local user="$2" local label="${3:-PostgreSQL}" local max_attempts=45 local attempt=0 info "Waiting for ${label} to accept connections..." until docker exec "${container}" pg_isready -U "${user}" -q 2>/dev/null < /dev/null; do attempt=$((attempt + 1)) if [[ $attempt -ge $max_attempts ]]; then error "${label} did not become ready after $((max_attempts * 2)) seconds. Check: docker logs ${container}" fi printf "." sleep 2 done printf "\n" success "${label} is ready." } info "Migrating data from legacy database (PostgreSQL 14 → 17)" # ── Step 0: Pre-flight checks ───────────────────────────────────────────────── step "0" "Pre-flight checks" # Docker CLI command -v docker >/dev/null 2>&1 \ || error "Docker is not installed. Install it at: https://docs.docker.com/get-docker/" # Docker daemon docker info >/dev/null 2>&1 < /dev/null \ || error "Docker daemon is not running. Please start Docker and try again." # Old volume must exist docker volume ls --format '{{.Name}}' < /dev/null | grep -q "^${OLD_VOLUME}$" \ || error "Legacy volume '${OLD_VOLUME}' not found.\n Are you sure you ran the old all-in-one SurfSense container?" success "Found legacy volume: ${OLD_VOLUME}" # Detect and stop any container currently using the old volume # (mounting a live PG volume into a second container causes the new container's # entrypoint to chown the data files, breaking the running container's access) OLD_CONTAINER=$(docker ps --filter "volume=${OLD_VOLUME}" --format '{{.Names}}' < /dev/null | head -n1 || true) if [[ -n "${OLD_CONTAINER}" ]]; then warn "Container '${OLD_CONTAINER}' is running and using the '${OLD_VOLUME}' volume." warn "It must be stopped before migration to prevent data file corruption." confirm "Stop '${OLD_CONTAINER}' now and proceed with data extraction?" docker stop "${OLD_CONTAINER}" >/dev/null 2>&1 < /dev/null \ || error "Failed to stop '${OLD_CONTAINER}'. Try: docker stop ${OLD_CONTAINER}" success "Container '${OLD_CONTAINER}' stopped." fi # Bail out if a dump already exists — don't overwrite a previous successful run if [[ -f "${DUMP_FILE}" ]]; then warn "Dump file '${DUMP_FILE}' already exists." warn "If a previous extraction succeeded, just run install.sh now." warn "To re-extract, remove the file first: rm ${DUMP_FILE}" error "Aborting to avoid overwriting an existing dump." fi # Clean up any stale temp container from a previous failed run if docker ps -a --format '{{.Names}}' < /dev/null | grep -q "^${TEMP_CONTAINER}$"; then warn "Stale migration container '${TEMP_CONTAINER}' found — removing it." docker stop "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true docker rm "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true fi # Disk space (warn if < 500 MB free) if command -v df >/dev/null 2>&1; then FREE_KB=$(df -k . | awk 'NR==2 {print $4}') FREE_MB=$(( FREE_KB / 1024 )) if [[ $FREE_MB -lt 500 ]]; then warn "Low disk space: ${FREE_MB} MB free. At least 500 MB recommended for the dump." confirm "Continue anyway?" else success "Disk space: ${FREE_MB} MB free." fi fi success "All pre-flight checks passed." # ── Confirmation prompt ─────────────────────────────────────────────────────── printf "\n${BOLD}Extraction plan:${NC}\n" printf " Source volume : ${YELLOW}%s${NC} (PG14 data at /data/postgres)\n" "${OLD_VOLUME}" printf " Old credentials : user=${YELLOW}%s${NC} db=${YELLOW}%s${NC}\n" "${OLD_DB_USER}" "${OLD_DB_NAME}" printf " Dump saved to : ${YELLOW}%s${NC}\n" "${DUMP_FILE}" printf " SECRET_KEY to : ${YELLOW}%s${NC}\n" "${KEY_FILE}" printf " Log file : ${YELLOW}%s${NC}\n\n" "${LOG_FILE}" confirm "Start data extraction? (Your original data will not be deleted or modified.)" # ── Step 1: Start temporary PostgreSQL 14 container ────────────────────────── step "1" "Starting temporary PostgreSQL 14 container" info "Pulling ${PG14_IMAGE}..." docker pull "${PG14_IMAGE}" >/dev/null 2>&1 < /dev/null \ || warn "Could not pull ${PG14_IMAGE} — using cached image if available." # Detect the UID that owns the existing data files and run the temp container # as that user. This prevents the official postgres image entrypoint from # running as root and doing `chown -R postgres /data/postgres`, which would # re-own the files to UID 999 and break any subsequent access by the original # container's postgres process (which may run as a different UID). DATA_UID=$(docker run --rm -v "${OLD_VOLUME}:/data" alpine \ stat -c '%u' /data/postgres 2>/dev/null < /dev/null || echo "") if [[ -z "${DATA_UID}" || "${DATA_UID}" == "0" ]]; then warn "Could not detect data directory UID — falling back to default (may chown files)." USER_FLAG="" else info "Data directory owned by UID ${DATA_UID} — starting temp container as that user." USER_FLAG="--user ${DATA_UID}" fi docker run -d \ --name "${TEMP_CONTAINER}" \ -v "${OLD_VOLUME}:/data" \ -e PGDATA=/data/postgres \ -e POSTGRES_USER="${OLD_DB_USER}" \ -e POSTGRES_PASSWORD="${OLD_DB_PASSWORD}" \ -e POSTGRES_DB="${OLD_DB_NAME}" \ ${USER_FLAG} \ "${PG14_IMAGE}" >/dev/null < /dev/null success "Temporary container '${TEMP_CONTAINER}' started." wait_for_pg "${TEMP_CONTAINER}" "${OLD_DB_USER}" "PostgreSQL 14" # ── Step 2: Dump the database ───────────────────────────────────────────────── step "2" "Dumping PostgreSQL 14 database" info "Running pg_dump — this may take a while for large databases..." if ! docker exec \ -e PGPASSWORD="${OLD_DB_PASSWORD}" \ "${TEMP_CONTAINER}" \ pg_dump -U "${OLD_DB_USER}" --no-password "${OLD_DB_NAME}" \ > "${DUMP_FILE}" 2>/tmp/surfsense_pgdump_err < /dev/null; then cat /tmp/surfsense_pgdump_err >&2 error "pg_dump failed. See above for details." fi # Validate: non-empty file [[ -s "${DUMP_FILE}" ]] \ || error "Dump file '${DUMP_FILE}' is empty. Something went wrong with pg_dump." # Validate: looks like a real PG dump grep -q "PostgreSQL database dump" "${DUMP_FILE}" \ || error "Dump file does not contain a valid PostgreSQL dump header — the file may be corrupt." # Validate: sanity-check line count DUMP_LINES=$(wc -l < "${DUMP_FILE}" | tr -d ' ') [[ $DUMP_LINES -ge 10 ]] \ || error "Dump has only ${DUMP_LINES} lines — suspiciously small. Aborting." DUMP_SIZE=$(du -sh "${DUMP_FILE}" 2>/dev/null | cut -f1) success "Dump complete: ${DUMP_SIZE} (${DUMP_LINES} lines) → ${DUMP_FILE}" # Stop the temp container (trap will also handle it on unexpected exit) info "Stopping temporary PostgreSQL 14 container..." docker stop "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true docker rm "${TEMP_CONTAINER}" >/dev/null 2>&1 < /dev/null || true success "Temporary container removed." # ── Step 3: Recover SECRET_KEY ──────────────────────────────────────────────── step "3" "Recovering SECRET_KEY" RECOVERED_KEY="" if docker run --rm -v "${OLD_VOLUME}:/data" alpine \ sh -c 'test -f /data/.secret_key && cat /data/.secret_key' \ 2>/dev/null < /dev/null | grep -q .; then RECOVERED_KEY=$( docker run --rm -v "${OLD_VOLUME}:/data" alpine \ cat /data/.secret_key 2>/dev/null < /dev/null | tr -d '[:space:]' ) success "Recovered SECRET_KEY from '${OLD_VOLUME}'." else warn "No SECRET_KEY file found at /data/.secret_key in '${OLD_VOLUME}'." warn "This means the all-in-one container was launched with SECRET_KEY set as an explicit env var." if $AUTO_YES; then # Non-interactive (called from install.sh) — auto-generate rather than hanging on read RECOVERED_KEY=$(openssl rand -base64 32 2>/dev/null \ || head -c 32 /dev/urandom | base64 | tr -d '\n') warn "Non-interactive mode: generated a new SECRET_KEY automatically." warn "All active browser sessions will be logged out after migration." warn "To restore your original key, update SECRET_KEY in ./surfsense/.env afterwards." else printf "${YELLOW}[SurfSense]${NC} Enter the SECRET_KEY from your old container's environment\n" printf "${YELLOW}[SurfSense]${NC} (press Enter to generate a new one — existing sessions will be invalidated): " read -r RECOVERED_KEY if [[ -z "${RECOVERED_KEY}" ]]; then RECOVERED_KEY=$(openssl rand -base64 32 2>/dev/null \ || head -c 32 /dev/urandom | base64 | tr -d '\n') warn "Generated a new SECRET_KEY. All active browser sessions will be logged out after migration." fi fi fi # Save SECRET_KEY to a file for install.sh to pick up printf '%s' "${RECOVERED_KEY}" > "${KEY_FILE}" success "SECRET_KEY saved to ${KEY_FILE}" # ── Done ────────────────────────────────────────────────────────────────────── printf "\n${GREEN}${BOLD}" printf "══════════════════════════════════════════════════════════════\n" printf " Data extraction complete!\n" printf "══════════════════════════════════════════════════════════════\n" printf "${NC}\n" success "Dump file : ${DUMP_FILE} (${DUMP_SIZE})" success "Secret key: ${KEY_FILE}" printf "\n" info "Next step — run install.sh from this same directory:" printf "\n" printf "${CYAN} curl -fsSL https://raw.githubusercontent.com/MODSetter/SurfSense/main/docker/scripts/install.sh | bash${NC}\n" printf "\n" info "install.sh will detect the dump, restore your data into PostgreSQL 17," info "and start the full SurfSense stack automatically." printf "\n" warn "Keep both files until you have verified the migration:" warn " ${DUMP_FILE}" warn " ${KEY_FILE}" warn "Full log saved to: ${LOG_FILE}" printf "\n"