handy_scripts/restore-mysql-daily

#!/bin/bash
#---   ---  ---   ---  ---   ---   ---  ---   ---  ---  ---   ---  ---   ---  ---#
## script was written with daily, dated, backups from holland backup             #
## mysqldump per table in directories with date/time like
## /backups/daily/20221104_65127/backup_data/database-name.sql.gz
##

### path to archive directory. e.g., /home/backupusr/backups/daily/
ARCHPATH='/path/to/backup/daily/'

### sudirectory from date. e.g., /backup_data/
SUBDIR='backup_data'

### databases to skip. e.g., mysql
SKIPDBS=('mysql')

### restore only changed databases or restore all 
FULLRESTORE=false

### database to use for master log position
SETPOSDB=db_for_master_log_position

### config file for mysql credentials
CFG='.my.cnf'

### dry-run will show actions for testing
DRY_RUN=false

### noprompt for skipping user input
NOPROMPT=false

### do not start will not restart replication when complete
DONOTSTART=false

### values for configuring replication master
### empty if replication already configured
MASTER_HOST=''
MASTER_USER=''
MASTER_PASSWORD=''

#                                                                                #
#---   ---  ---   ---  ---   ---   ---  ---   ---  ---  ---   ---  ---   ---  ---#

while [[ $# -gt 0 ]]; do
    case $1 in
    -d|--dry-run) 
        DRY_RUN=true 
        CMD=echo
        shift 1
        ;;
    -a|--archive)
        ARCHPATH=$2
        shift 2
        ;;
    -sd|--subdir)
        SUBDIR=$2
        shift 2
        ;;
    -sk|--skip)
        SKIPDBS=($2)
        shift 2
        ;;
    -f|--full)
        FULLRESTORE=true
        shift 1
        ;;
    -p|--pos)
        SETPOSDB=$2
        shift 2
        ;;
    -c|--cfg)
        CFG=$2
        shift 2
        ;;
    -ns|--nostart)
        DONOTSTART=true
        shift 1
        ;;
    -np|--noprompt)
        NOPROMPT=true
        shift 1
        ;;
    -mh|--masterhost)
        MASTER_HOST=$2
        shift 2
        ;;
    -mu|--masteruser)
        MASTER_USER=$2
        shift 2
        ;;
    -mp|--masterpass)
        MASTER_PASSWORD=$2
        shift 2
        ;;    
    -h|--help)
        echo -e "\n\n\t _____________________________/‾‾‾   MySQL restoration script options   ‾‾‾\____________________________________\n"
        echo -e "\t   |   arg         |                                                                |   type     |  default  |\n"
        echo -e "\t ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾"
        echo -e "\t -h|  --help         display this menu                                                [ flag   ]"
        echo -e "\t -d|  --dry-run      no permanent changes, show plan                                  [ flag   ]"
        echo -e "\t -a|  --archive      directory containing archive directories                         [ string ]"
        echo -e "\t-sd|  --subdir       directory containing actual compressed sql files                 [ string ]"
        echo -e "\t-sk|  --skip         databases to not restore (e.g. mysql would replace users)        [ array  ]      mysql"
        echo -e "\t -f|  --full         restore all tables not excluded (no comparison)                  [ array  ]      off"
        echo -e "\t -p|  --pos          datbase from which master log position will be used              [ string ]"
        echo -e "\t -c|  --cfg          path of config file for mysql credentials                        [ string ]      .my.cnf"
        echo -e "\t-ns|  --nostart      do not restart replication                                       [ flag   ]      off"
        echo -e "\t-np|  --noprompt     do not prompt user (Y/n)                                         [ flag   ]      off"
        echo -e "\t-mh|  --masterhost   replication master host address                                  [ string ]"
        echo -e "\t-mu|  --masteruser   user name for replication                                        [ string ]"
        echo -e "\t-mp|  --masterpass   password for replication                                         [ string ]"
        echo -e "\nEXAMPLE: $0 -p main_db -sd backup_data -a /home/backups/mysql/daily -sk 'mysql performance_stats' --dry-run" 
        exit
        ;;
    *) 
        echo 'error in command line parsing' >&2
        echo "try: '$0 --help"
        exit 1
    esac
done

ARCHPATH="${ARCHPATH%/}/"
SUBDIR="${SUBDIR%/}/"

echo "restore with options:"
echo -e "\tarchive path         = $ARCHPATH"
echo -e "\tsubdirectory         = $SUBDIR"
echo -e "\tDBs to skip          = ${SKIPDBS[@]}"
echo -e "\tfull restore         = $FULLRESTORE"
echo -e "\tDB for position      = $SETPOSDB"
echo -e "\tconfig file          = $CFG"
echo -e "\tdry run              = $DRY_RUN"
echo -e "\n"
echo -e "\treplication master   = $MASTER_HOST"
echo -e "\treplication user     = $MASTER_USER"
echo -e "\treplication password = $([[ $MASTER_PASSWORD ]] && echo true || echo false )"
echo -e "\n"

if [[ -f "$CFG" ]]; then
    if [[ ! $(grep user $CFG) ]]; then
        echo "ERROR: 'user' field not found in '$CFG'"
        exit 1
    fi
    if [[ ! $(grep password $CFG) ]]; then
        echo "ERROR: 'password' field not found in '$CFG'"
        exit 1
    fi
else
    echo "ERROR: configuration file '$CFG' not found"
    exit 1
fi

if [[ $NOPROMPT == false ]]; then
    read -n 1 -p "restore? [Y/n] " reply; 
    if [[ $reply == [Nn] ]]; then 
        echo -e "\naborting"
        exit 1
    fi
fi

echo -e "\nrestoring" 

START=$(date +%s)

echo -e "\n--START: $(date +%F\ %T)\nrestoring most recent backups from $ARCHPATH"
echo -e "\tstopping replication\n";
$CMD mysql -e "STOP SLAVE;"

NEWDATE=$(ls -1r "$ARCHPATH" | head -1)
NEWPATH="$ARCHPATH$NEWDATE/$SUBDIR"

if [[ $FULLRESTORE == false ]]; then
    OLDDATE=$(ls -1rt "$ARCHPATH" | head -1)
    OLDPATH="$ARCHPATH$OLDDATE/$SUBDIR"
    echo -e "compartive restoration selected:\n\tfiles from:\t'$NEWDATE'\n\tcompared to:\t'$OLDDATE'"
else
    echo "full restorion selected: comparisons will not be made, all non-excluded databases will be restored"
fi

echo -e "\nfinding files in '$NEWPATH*.gz'"
for FILE in $NEWPATH*.gz; do
    STARTLOOP=$(date +%s)
    FILE=$(basename $FILE)
    echo -e "\n$(date +%F\ %T) | $FILE:"
    
    if [[ "${SETPOSDB}" =~ ${FILE%%.*} ]]; then echo -e "\t-- recognized '$FILE' for master position and log file"
        SETPOSFILE=$NEWPATH$FILE
        SETPOSSTMT=$(zcat $SETPOSFILE | head -40 | grep MASTER | sed 's/-- //')
        echo -e "\t$SETPOSSTMT"
    fi

    if [[ "${SKIPDBS[*]}" =~ ${FILE%%.*} ]]; then echo -e "\t-- skipping $FILE";continue; fi; 

    if [[ FULLRESTORE == false ]]; then
        OLDSIZE=$(stat -c %s $OLDPATH$FILE)
    else
        OLDSIZE=0
    fi

    NEWSIZE=$(stat -c %s $NEWPATH$FILE)
    
    if [[ $OLDSIZE != $NEWSIZE ]]; then
        echo -e "\t$OLDDATE:$FILE:$OLDSIZE != $NEWDATE:$FILE:$NEWSIZE"
    	DB=$(echo $FILE | cut -f1 -d'.')
        echo -e "\tDROPPING/CREATING '$DB'
        pv $NEWPATH$FILE | gunzip | mysql $DB"
        $CMD mysql -e "DROP DATABASE IF EXISTS $DB"
        $CMD mysql -e "CREATE DATABASE $DB"
        if [[ -z $CMD ]]; then pv $NEWPATH$FILE | gunzip | mysql $DB; fi
    fi
    ENDLOOP=$(date +%s)
    EXCTIME=$(expr $ENDLOOP - $STARTLOOP)
    printf '\t%02dh:%02dm:%02ds\n' $((EXCTIME/3600)) $((EXCTIME%3600/60)) $((EXCTIME%60))        
done

if [[ $MASTER_HOST && $MASTER_USER && $MASTER_PASSWORD ]]; then
    echo -e "\napplying configuration for replication\n\thost: $MASTER_HOST\n\tuser: $MASTER_USER"
    $CMD mysql -e "CHANGE MASTER TO master_host='$MASTER_HOST',master_user='$MASTER_USER',master_password='$MASTER_PASSWORD';"
else
    echo -e "\nreplication credentials not configured"
fi

if [[ $SETPOSSTMT ]]; then
    $CMD mysql -e "$SETPOSSTMT"
else
    DONOTSTART=true
    echo "\nmaster position database undefined, listing suggestions"
    for FILE in $NEWPATH*.gz; do 
        echo $(basename $FILE)
        echo -e "\t$(zcat $FILE | head -40 | grep MASTER)"
    done
fi

if [[ $NOPROMPT == false ]]; then
    read -n 1 -p "restart replication? [Y/n] " reply; 
    if [[ $reply == [Nn] ]]; then 
        DONOTSTART=true
    fi
fi

if [[ $DONOTSTART != true ]]; then
    echo -e "\nrestarting replication"
    $CMD mysql -e "START SLAVE"
fi

END=$(date +%s)
EXCTIME=$(expr $END - $START)
printf '\ntotal runtime: %02dh:%02dm:%02ds\n' $((EXCTIME/3600)) $((EXCTIME%3600/60)) $((EXCTIME%60))
echo "\n--END: $(date +%F\ %T)\n\n"