instance-setup-stuff/template/example_server-backup

#!/bin/bash
#
# Backup script for example service
# Stops the service, creates a backup, uploads to remote storage, and restarts the service
#

set -euo pipefail

# shellcheck source=example_server-env
. "${HOME}"/"${USER}"-env

#======================================
# Constants
#======================================
readonly SERVICE_NAME="Example"
readonly PRIORITY_INFO=2
readonly PRIORITY_WARNING=3
readonly PRIORITY_ERROR=4
readonly EXIT_SUCCESS=0
readonly EXIT_VALIDATION_ERROR=1
readonly EXIT_DOCKER_ERROR=2
readonly EXIT_BACKUP_ERROR=3
readonly EXIT_UPLOAD_ERROR=4

#======================================
# Global Variables
#======================================
backupDir="/tmp/${USER}-backup"
composeFile="${HOME}/${USER}-compose.yaml"
logFile="${HOME}/backup_logs/$(date +%y_%m).log"
serviceRestartFailed=false

#======================================
# Helper Functions
#======================================

# Log a message with timestamp
# Output to stderr to avoid buffering issues with stdout
log() {
  local level="$1"
  shift
  local message="$*"
  echo "$(date '+%Y-%m-%d %H:%M:%S') - [${level}] ${message}" >&2
}

# Log info message
log_info() {
  log "INFO" "$@"
}

# Log warning message
log_warn() {
  log "WARN" "$@"
}

# Log error message
log_error() {
  log "ERROR" "$@"
}

# Cleanup function - ensures temporary files are removed
# shellcheck disable=SC2329
cleanup() {
  local exit_code=$?

  if [[ -d "${backupDir}" ]]; then
    log_info "Cleaning up backup directory"
    rm -rf "${backupDir}"
  fi

  if [[ ${exit_code} -ne 0 ]]; then
    log_error "Backup failed with exit code ${exit_code}"
  fi
}

# Set trap to ensure cleanup on exit
trap cleanup EXIT

# Send ntfy notification via curl
# Arguments: priority, tags, message
send_notification() {
  local priority="$1"
  local tags="$2"
  local message="$3"

  if ! curl -Ss \
    -H "Title: ${SERVICE_NAME}" \
    -H "Priority: ${priority}" \
    -H "Tags: ${tags}" \
    -d "${message}" \
    "${NOTIF_URL}"; then
    log_warn "Failed to send notification"
    return 1
  fi

  return 0
}

# Validate required environment variables
validate_environment() {
  log_info "Validating environment variables"

  : "${USER:?USER environment variable is required}"
  : "${VOLUME_PATH:?VOLUME_PATH environment variable is required}"
  : "${BUCKET_PATH:?BUCKET_PATH environment variable is required}"
  : "${NOTIF_URL:?NOTIF_URL environment variable is required}"
}

# Validate required dependencies and paths
validate_dependencies() {
  log_info "Validating dependencies and paths"

  # Check for rclone
  if ! command -v rclone &>/dev/null; then
    log_error "rclone command not found"
    send_notification "${PRIORITY_ERROR}" "x,backup" "Backup failed: rclone not found"
    return "${EXIT_VALIDATION_ERROR}"
  fi

  # Check docker compose file
  if [[ ! -f "${composeFile}" ]]; then
    log_error "Docker compose file not found: ${composeFile}"
    send_notification "${PRIORITY_ERROR}" "x,backup" "Backup failed: compose file not found"
    return "${EXIT_VALIDATION_ERROR}"
  fi

  # Check volume path
  if [[ ! -d "${VOLUME_PATH}" ]]; then
    log_error "Volume path not found: ${VOLUME_PATH}"
    send_notification "${PRIORITY_ERROR}" "x,backup" "Backup failed: volume path not found"
    return "${EXIT_VALIDATION_ERROR}"
  fi

  return "${EXIT_SUCCESS}"
}

# Stop docker compose services
stop_services() {
  log_info "Stopping docker compose services"

  if ! sudo docker compose -f "${composeFile}" stop; then
    log_error "Failed to stop docker compose services"
    send_notification "${PRIORITY_ERROR}" "x,backup" "Backup failed: could not stop services"
    return "${EXIT_DOCKER_ERROR}"
  fi

  return "${EXIT_SUCCESS}"
}

# Start docker compose services
start_services() {
  log_info "Starting docker compose services"

  if ! sudo docker compose -f "${composeFile}" start; then
    log_warn "Failed to start docker compose services"
    serviceRestartFailed=true
    return 1
  fi

  return "${EXIT_SUCCESS}"
}

# Create backup of volume data
create_backup() {
  log_info "Creating backup directory"
  mkdir -p "${backupDir}"

  log_info "Copying files from ${VOLUME_PATH}"
  if ! cp -pr "${VOLUME_PATH}"/* "${backupDir}"/; then
    log_error "Failed to copy files to backup directory"
    return "${EXIT_BACKUP_ERROR}"
  fi

  # Verify backup has content
  if [[ -z "$(ls -A "${backupDir}")" ]]; then
    log_error "Backup directory is empty - no files to backup"
    return "${EXIT_BACKUP_ERROR}"
  fi

  log_info "Backup created successfully"
  return "${EXIT_SUCCESS}"
}

# Upload backup to remote storage
upload_backup() {
  local backup_size

  log_info "Uploading backup to ${BUCKET_PATH}"

  if ! rclone copy "${backupDir}" "${BUCKET_PATH}" -v --retries 3; then
    log_error "Failed to upload backup to remote storage"
    return "${EXIT_UPLOAD_ERROR}"
  fi

  # Calculate and report backup size
  backup_size=$(du -sh "${backupDir}" | cut -f1)
  log_info "Backup uploaded successfully (${backup_size})"

  echo "${backup_size}"
  return "${EXIT_SUCCESS}"
}

#======================================
# Main Execution
#======================================
main() {
  log_info "Starting ${SERVICE_NAME} backup"

  # Validate environment
  validate_environment || exit "${EXIT_VALIDATION_ERROR}"

  # Validate dependencies and paths
  validate_dependencies || exit "$?"

  # Stop services to ensure data consistency
  stop_services || exit "$?"

  # Create backup
  if ! create_backup; then
    start_services # Attempt to restart services even if backup failed
    send_notification "${PRIORITY_ERROR}" "x,backup" "Backup failed: could not copy files"
    exit "${EXIT_BACKUP_ERROR}"
  fi

  # Restart services immediately to minimize downtime
  start_services

  # Upload backup to remote storage
  local backup_size
  if ! backup_size=$(upload_backup); then
    if [[ "${serviceRestartFailed}" == "true" ]]; then
      send_notification "${PRIORITY_ERROR}" "x,backup" "Backup upload failed AND services failed to restart"
    else
      send_notification "${PRIORITY_WARNING}" "warning,backup" "Backup not completed: upload failed"
    fi
    exit "${EXIT_UPLOAD_ERROR}"
  fi

  # Send success notification
  if [[ "${serviceRestartFailed}" == "true" ]]; then
    send_notification "${PRIORITY_WARNING}" "warning,backup" "Backup completed (${backup_size}) but services failed to restart"
    log_warn "Backup completed but services failed to restart"
    exit "${EXIT_DOCKER_ERROR}"
  else
    send_notification "${PRIORITY_INFO}" "heavy_check_mark,backup" "Backup completed (${backup_size})"
    log_info "Backup completed successfully"
    exit "${EXIT_SUCCESS}"
  fi
}

# Setup logging directory
mkdir -p "${HOME}"/backup_logs

# Execute main function with output redirected to log file
{
  main
} &>>"${logFile}"