From 070fe1b416e331c3ba9fbbbd24623b96540458b1 Mon Sep 17 00:00:00 2001 From: nyne Date: Sun, 23 Nov 2025 19:32:22 +0800 Subject: [PATCH] feat: backup --- .env.example | 32 ++++++++ .gitignore | 4 +- Dockerfile.backup | 28 +++++++ backup-entrypoint.sh | 44 ++++++++++ backup.sh | 190 +++++++++++++++++++++++++++++++++++++++++++ docker-compose.yml | 39 +++++---- 6 files changed, 322 insertions(+), 15 deletions(-) create mode 100644 .env.example create mode 100644 Dockerfile.backup create mode 100644 backup-entrypoint.sh create mode 100644 backup.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..d8e9497 --- /dev/null +++ b/.env.example @@ -0,0 +1,32 @@ +# Database Configuration +DB_HOST=db +DB_PORT=3306 +DB_USER=nysoure +DB_PASSWORD=your_secure_password_here +DB_NAME=nysoure +MYSQL_ROOT_PASSWORD=your_secure_root_password_here +MYSQL_DATABASE=nysoure +MYSQL_USER=nysoure +MYSQL_PASSWORD=your_secure_password_here + +# Redis Configuration +REDIS_HOST=redis +REDIS_PORT=6379 + +# Application Configuration +BANNED_REDIRECT_DOMAINS=example.com,example.org +ALLOWED_URL_REGEXPS= + +# Backup Configuration +# Object storage configuration (S3-compatible) +S3_ENDPOINT=https://s3.amazonaws.com +S3_BUCKET=nysoure-backups +S3_ACCESS_KEY=your_access_key_here +S3_SECRET_KEY=your_secret_key_here +S3_REGION=us-east-1 + +# Backup schedule (cron format) - default: daily at 2 AM +BACKUP_SCHEDULE=0 2 * * * + +# Retention policy (days) +BACKUP_RETENTION_DAYS=30 diff --git a/.gitignore b/.gitignore index 1ee2d97..b79d285 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,6 @@ *.iml test.db .idea/ -build/ \ No newline at end of file +build/ +temp/ +.env \ No newline at end of file diff --git a/Dockerfile.backup b/Dockerfile.backup new file mode 100644 index 0000000..a33f990 --- /dev/null +++ b/Dockerfile.backup @@ -0,0 +1,28 @@ +FROM alpine:latest + +# Install required packages +RUN apk add --no-cache \ + bash \ + mysql-client \ + rclone \ + tzdata \ + supercronic \ + && rm -rf /var/cache/apk/* + +# Set timezone (optional, adjust as needed) +ENV TZ=UTC + +# Create backup directory +RUN mkdir -p /backup/local + +# Copy backup script +COPY backup.sh /usr/local/bin/backup.sh +RUN chmod +x /usr/local/bin/backup.sh + +# Copy entrypoint script +COPY backup-entrypoint.sh /usr/local/bin/backup-entrypoint.sh +RUN chmod +x /usr/local/bin/backup-entrypoint.sh + +WORKDIR /backup + +ENTRYPOINT ["/usr/local/bin/backup-entrypoint.sh"] diff --git a/backup-entrypoint.sh b/backup-entrypoint.sh new file mode 100644 index 0000000..96a96aa --- /dev/null +++ b/backup-entrypoint.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +set -e + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +# Default schedule: daily at 2 AM +BACKUP_SCHEDULE="${BACKUP_SCHEDULE:-0 2 * * *}" + +log "Backup container starting..." +log "Backup schedule: ${BACKUP_SCHEDULE}" + +# Wait for database to be ready +log "Waiting for database to be ready..." +for i in {1..30}; do + if mysql -h ${DB_HOST} -P ${DB_PORT} -u ${DB_USER} -p${DB_PASSWORD} -e "SELECT 1" > /dev/null 2>&1; then + log "Database is ready!" + break + fi + log "Waiting for database... (${i}/30)" + sleep 2 +done + +# Validate S3 configuration +if [ -z "${S3_BUCKET}" ] || [ -z "${S3_ACCESS_KEY}" ] || [ -z "${S3_SECRET_KEY}" ]; then + log "ERROR: S3 configuration is incomplete!" + log "Please set S3_BUCKET, S3_ACCESS_KEY, and S3_SECRET_KEY environment variables." + exit 1 +fi + +# Run initial backup +log "Running initial backup..." +/usr/local/bin/backup.sh + +# Create crontab +echo "${BACKUP_SCHEDULE} /usr/local/bin/backup.sh >> /var/log/backup.log 2>&1" > /tmp/crontab + +log "Starting scheduled backups with supercronic..." +log "Logs will be written to /var/log/backup.log" + +# Run supercronic with the crontab +exec supercronic /tmp/crontab diff --git a/backup.sh b/backup.sh new file mode 100644 index 0000000..4e4960a --- /dev/null +++ b/backup.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +set -e + +# Configuration +TIMESTAMP=$(date +%Y%m%d_%H%M%S) +BACKUP_DIR="/backup/local" +APP_DATA_DIR="/backup/app_data" +BACKUP_DATE=$(date +%Y-%m-%d) + +# S3 configuration +S3_ENDPOINT="${S3_ENDPOINT}" +S3_BUCKET="${S3_BUCKET}" +S3_ACCESS_KEY="${S3_ACCESS_KEY}" +S3_SECRET_KEY="${S3_SECRET_KEY}" +S3_REGION="${S3_REGION:-us-east-1}" + +# Retention +RETENTION_DAYS="${BACKUP_RETENTION_DAYS:-30}" + +# State file for incremental backups +STATE_FILE="${BACKUP_DIR}/last_backup_state.txt" + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" +} + +# Configure rclone for S3 +configure_rclone() { + mkdir -p ~/.config/rclone + cat > ~/.config/rclone/rclone.conf < ${DB_BACKUP_FILE} + + log "Database backup completed: ${DB_BACKUP_FILE}" + + # Upload to S3 + rclone copy ${DB_BACKUP_FILE} s3:${S3_BUCKET}/database/ --progress + + log "Database backup uploaded to S3" + + # Clean up local backup file after successful upload + rm -f ${DB_BACKUP_FILE} +} + +# Backup config.json +backup_config() { + log "Backing up config.json..." + + if [ -f "${APP_DATA_DIR}/config.json" ]; then + CONFIG_BACKUP="${BACKUP_DIR}/config_${TIMESTAMP}.json" + cp "${APP_DATA_DIR}/config.json" ${CONFIG_BACKUP} + + # Upload to S3 + rclone copy ${CONFIG_BACKUP} s3:${S3_BUCKET}/config/ --progress + + log "Config backup uploaded to S3" + rm -f ${CONFIG_BACKUP} + else + log "Warning: config.json not found" + fi +} + +# Incremental backup for images using rclone sync with checksums +backup_images() { + log "Starting incremental image backup..." + + # Backup images directory + if [ -d "${APP_DATA_DIR}/images" ]; then + log "Syncing images directory (incremental)..." + # Using rclone sync with --checksum for efficient incremental backup + # Only uploads new or modified files + rclone sync "${APP_DATA_DIR}/images" s3:${S3_BUCKET}/images \ + --checksum \ + --transfers 8 \ + --checkers 16 \ + --fast-list \ + --progress \ + --log-file="${BACKUP_DIR}/images_sync_${TIMESTAMP}.log" + + log "Images backup completed" + else + log "Warning: images directory not found" + fi +} + +# Backup avatars +backup_avatars() { + log "Starting avatar backup..." + + if [ -d "${APP_DATA_DIR}/avatar" ]; then + log "Syncing avatar directory..." + # Avatar directory is usually smaller, but still use incremental sync + rclone sync "${APP_DATA_DIR}/avatar" s3:${S3_BUCKET}/avatar \ + --checksum \ + --transfers 4 \ + --progress \ + --log-file="${BACKUP_DIR}/avatar_sync_${TIMESTAMP}.log" + + log "Avatar backup completed" + else + log "Warning: avatar directory not found" + fi +} + +# Clean up old database backups from S3 +cleanup_old_backups() { + log "Cleaning up backups older than ${RETENTION_DAYS} days..." + + # Delete old database backups + rclone delete s3:${S3_BUCKET}/database \ + --min-age ${RETENTION_DAYS}d \ + --progress || true + + # Delete old config backups + rclone delete s3:${S3_BUCKET}/config \ + --min-age ${RETENTION_DAYS}d \ + --progress || true + + # Clean up local logs + find ${BACKUP_DIR} -name "*.log" -mtime +7 -delete || true + + log "Cleanup completed" +} + +# Create backup state file +create_state_file() { + echo "LAST_BACKUP_DATE=${BACKUP_DATE}" > ${STATE_FILE} + echo "LAST_BACKUP_TIMESTAMP=${TIMESTAMP}" >> ${STATE_FILE} +} + +# Main backup function +run_backup() { + log "==========================================" + log "Starting backup process..." + log "==========================================" + + # Create backup directory + mkdir -p ${BACKUP_DIR} + + # Configure rclone + configure_rclone + + # Test S3 connection + log "Testing S3 connection..." + if ! rclone lsd s3:${S3_BUCKET} > /dev/null 2>&1; then + log "Error: Cannot connect to S3 bucket. Please check your credentials." + exit 1 + fi + + # Perform backups + backup_database + backup_config + backup_images + backup_avatars + + # Cleanup old backups + cleanup_old_backups + + # Update state file + create_state_file + + log "==========================================" + log "Backup process completed successfully!" + log "==========================================" +} + +# Run backup +run_backup diff --git a/docker-compose.yml b/docker-compose.yml index d19639d..dae7a11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,15 +10,8 @@ services: depends_on: - db - redis - environment: - - DB_HOST=db - - DB_PORT=3306 - - DB_USER=nysoure - - DB_PASSWORD=nysoure_password - - DB_NAME=nysoure - - REDIS_HOST=redis - - REDIS_PORT=6379 - - BANNED_REDIRECT_DOMAINS=example.com,example.org + env_file: + - .env restart: unless-stopped logging: driver: "json-file" @@ -30,11 +23,8 @@ services: image: mariadb:latest volumes: - db_data:/var/lib/mysql - environment: - - MYSQL_ROOT_PASSWORD=root_password - - MYSQL_DATABASE=nysoure - - MYSQL_USER=nysoure - - MYSQL_PASSWORD=nysoure_password + env_file: + - .env restart: unless-stopped logging: driver: "json-file" @@ -51,6 +41,27 @@ services: max-size: "10m" max-file: "3" + backup: + build: + context: . + dockerfile: Dockerfile.backup + volumes: + - app_data:/backup/app_data:ro + - db_data:/backup/db_data:ro + - backup_data:/backup/local + depends_on: + - db + - app + env_file: + - .env + restart: unless-stopped + logging: + driver: "json-file" + options: + max-size: "10m" + max-file: "3" + volumes: app_data: db_data: + backup_data: