From 7d52d88723259aded88b895fc1dac7f608ec2f5d Mon Sep 17 00:00:00 2001 From: george Date: Tue, 8 Nov 2022 07:46:27 -0500 Subject: [PATCH] 2022-11-08 07:46:27 --- addons.sh | 45 +++++++++++++ restore-mysql-daily | 155 +++++++++++++++++++++++++------------------- 2 files changed, 135 insertions(+), 65 deletions(-) create mode 100644 addons.sh diff --git a/addons.sh b/addons.sh new file mode 100644 index 0000000..73f01f6 --- /dev/null +++ b/addons.sh @@ -0,0 +1,45 @@ +# prompt no +prn(){ + read -n1 -p "$1 (y/N)" yn; if [[ $yn != [Yy] ]]; then printf '\naborted\n'; return 1; else return 0; fi +} + +# prompt yes +pry(){ + read -n1 -p "$1 (Y/n)" yn; if [[ $yn != [Nn] ]]; then return 0; else printf '\naborted\n'; return 1; fi +} + +# find project +fprj(){ + find /cygdrive/p/project_notes/. -type d -maxdepth 1 -iname "*$1*" | grep -oP '(?<=\/)([0-9]+).*' +} + +# search project +sprj(){ + if [[ -z $2 ]]; then grep -Ri $1 /cygdrive/p/project_notes/* --include="*.*" + else grep -Ri $1 /cygdrive/p/project_notes/* --include="*.$2" + fi +} + +# new project +function new(){ + prjroot="/cygdrive/p/project_notes/" + editor="/cygdrive/c/Program Files/Sublime\ Text/subl.exe" + if [[ -z $1 ]]; then echo 'i need the project name, bae'; exit 1; else prjname=$(echo $1|sed -e 's/ /-/g'); fi + lastnum=$(find $prjroot. -type d -maxdepth 1 -iname "*[0-9]*" | grep -oP '(?<=\/)([0-9]+)(?=\.)' | sort | tail -1) + prjnum=$(echo $lastnum | awk -F, '{printf("%04d",($lastnum".")+1)}') + prjname="$prjnum.$prjname" + mkdir -p $prjroot$prjname || echo "cannot mkdir '$prjroot$prjname'" && exit 1 + touch $prjroot$prjname/$prjname.txt && $editor $prjroot$prjname/$prjname.txt + if [[ -f .git ]]; then git branch -b $prjname; else echo "no '.git' found"; fi +} + +# git new branch +function gnb(){ + git branch -b $1 +} + +# git commit quick +function gcq(){ + if [[ -z $1 ]]; then msg="$(date +%F\ %T)"; else msg=$1; fi + git commit -am "$msg" && read -n1 -p "push? [Y/n] " reply; if [[ $reply != [Nn] ]]; then git push; fi +} diff --git a/restore-mysql-daily b/restore-mysql-daily index a43468e..6d8feff 100644 --- a/restore-mysql-daily +++ b/restore-mysql-daily @@ -32,20 +32,25 @@ NOPROMPT=false ### do not start will not restart replication when complete DONOTSTART=false +### file to log output +LOG_FILE="_$(basename $0).$(date +%F_%T).log" + ### values for configuring replication master ### empty if replication already configured MASTER_HOST='' MASTER_USER='' MASTER_PASSWORD='' + + # # #--- --- --- --- --- --- --- --- --- --- --- --- --- --- ---# + while [[ $# -gt 0 ]]; do case $1 in -d|--dry-run) - DRY_RUN=true - CMD="echo -e \t$(tput setaf 3)EXECUTE: $(tput sgr 0)"; echo + DRY_RUN=true shift 1 ;; -a|--archive) @@ -72,6 +77,14 @@ while [[ $# -gt 0 ]]; do CFG=$2 shift 2 ;; + -l|--log) + LOG_FILE=$2 + shift 2 + ;; + -nl|--nolog) + LOG_FILE='' + shift 2 + ;; -ns|--nostart) DONOTSTART=true shift 1 @@ -97,19 +110,21 @@ while [[ $# -gt 0 ]]; do echo -e "\t | arg | | type | default |\n" echo -e "\t ‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾‾" echo -e "\t -h| --help display this menu [ flag ]" - echo -e "\t -d| --dry-run no permanent changes, show plan [ flag ]" - echo -e "\t -a| --archive directory containing archive directories [ string ]" - echo -e "\t-sd| --subdir directory containing actual compressed sql files [ string ]" - echo -e "\t-sk| --skip databases to not restore (e.g. mysql would replace users) [ array ] mysql" - echo -e "\t -f| --full restore all tables not excluded (no comparison) [ array ] off" - echo -e "\t -p| --pos datbase from which master log position will be used [ string ]" - echo -e "\t -c| --cfg path of config file for mysql credentials [ string ] .my.cnf" - echo -e "\t-ns| --nostart do not restart replication [ flag ] off" - echo -e "\t-np| --noprompt do not prompt user (Y/n) [ flag ] off" - echo -e "\t-mh| --masterhost replication master host address [ string ]" - echo -e "\t-mu| --masteruser user name for replication [ string ]" - echo -e "\t-mp| --masterpass password for replication [ string ]" - echo -e "\nEXAMPLE: $0 -p main_db -sd backup_data -a /home/backups/mysql/daily -sk 'mysql performance_stats' --dry-run" + echo -e "\t -d| --dry-run no permanent changes, show plan [ flag ] $DRY_RUN" + echo -e "\t -a| --archive directory containing archive directories [ string ] $ARCHPATH" + echo -e "\t-sd| --subdir directory containing actual compressed sql files [ string ] $SUBDIR" + echo -e "\t-sk| --skip databases to not restore (e.g. mysql would replace users) [ array ] $SKIPDBS" + echo -e "\t -f| --full restore all tables not excluded (no comparison) [ array ] $FULLRESTORE" + echo -e "\t -p| --pos datbase from which master log position will be used [ string ] $SETPOSFILE" + echo -e "\t -c| --cfg path of config file for mysql credentials [ string ] $CFG" + echo -e "\t -l| --log path of log file [ string ] $LOG_FILE" + echo -e "\t-nl| --nolog disable logging [ string ] $([[ $LOG_FILE ]] && echo false || echo true)" + echo -e "\t-ns| --nostart do not restart replication [ flag ] $DONOTSTART" + echo -e "\t-np| --noprompt do not prompt user (Y/n) [ flag ] $NOPROMPT" + echo -e "\t-mh| --masterhost replication master host address [ string ] $MASTER_HOST" + echo -e "\t-mu| --masteruser user name for replication [ string ] $MASTER_USER" + echo -e "\t-mp| --masterpass password for replication [ string ] $([[ $MASTER_PASSWORD ]] && echo true || echo false)" + echo -e "\nEXAMPLE: $0 -p main_db -sd backup_data -a /home/backups/mysql/daily -sk 'mysql performance_stats' --dry-run -np" exit ;; *) @@ -119,52 +134,65 @@ while [[ $# -gt 0 ]]; do esac done +function hecko() { + if [[ $LOG_FILE ]]; then echo -e "$1" | tee -a $LOG_FILE + else echo -e "$1"; fi +} + +function xct() { + if [[ $DRY_RUN == true ]]; then hecko "\t$(tput setaf 3)EXEC PLAN: $(tput sgr 0)$1" + else hecko "\t$(tput setaf 5)EXECUTE: $(tput sgr 0)$1"; eval $1; fi +} + + ARCHPATH="${ARCHPATH%/}/" SUBDIR="${SUBDIR%/}/" -echo "restore with options:" -echo -e "\tarchive path = $ARCHPATH" -echo -e "\tsubdirectory = $SUBDIR" -echo -e "\tDBs to skip = ${SKIPDBS[@]}" -echo -e "\tfull restore = $FULLRESTORE" -echo -e "\tDB for position = $SETPOSDB" -echo -e "\tconfig file = $CFG" -echo -e "\tdry run = $DRY_RUN" -echo -e "\n" -echo -e "\treplication master = $MASTER_HOST" -echo -e "\treplication user = $MASTER_USER" -echo -e "\treplication password = $([[ $MASTER_PASSWORD ]] && echo true || echo false )" -echo -e "\n" +hecko "\n\n\t________/‾‾‾ MySQL restoration script options ‾‾‾\___________________\n" +hecko "\tarchive path = $ARCHPATH" +hecko "\tsubdirectory = $SUBDIR" +hecko "\tDBs to skip = ${SKIPDBS[@]}" +hecko "\tfull restore = $FULLRESTORE" +hecko "\tDB for position = $SETPOSDB" +hecko "\tconfig file = $CFG" +hecko "\tdry run = $DRY_RUN" +hecko "\n" +hecko "\treplication master = $MASTER_HOST" +hecko "\treplication user = $MASTER_USER" +hecko "\treplication password = $([[ $MASTER_PASSWORD ]] && echo true || echo false)" +hecko "\n" +hecko "\tlog file = $([[ $LOG_FILE ]] && echo $LOG_FILE || echo false)" +hecko "\t_____________________________________________________________________________\n" if [[ -f "$CFG" ]]; then if [[ ! $(grep user $CFG) ]]; then - echo "ERROR: 'user' field not found in '$CFG'" + hecko "ERROR: 'user' field not found in '$CFG'" exit 1 fi if [[ ! $(grep password $CFG) ]]; then - echo "ERROR: 'password' field not found in '$CFG'" + hecko "ERROR: 'password' field not found in '$CFG'" exit 1 fi else - echo "ERROR: configuration file '$CFG' not found" + hecko "ERROR: configuration file '$CFG' not found" exit 1 fi if [[ $NOPROMPT == false ]]; then read -n 1 -p "restore? [Y/n] " reply; if [[ $reply == [Nn] ]]; then - echo -e "\naborting" + hecko "\naborting" exit 1 fi fi -echo -e "\nrestoring" +hecko "\nrestoring" START=$(date +%s) -echo -e "\n--START: $(date +%F\ %T)\nrestoring most recent backups from $ARCHPATH" -echo -e "\tstopping replication"; -$CMD mysql -e "STOP SLAVE;" +hecko "\n--START: $(date +%F\ %T)\nrestoring most recent backups from $ARCHPATH" +hecko "\tstopping replication"; +xct "mysql -e 'STOP SLAVE;'" NEWDATE=$(ls -1r "$ARCHPATH" | head -1) NEWPATH="$ARCHPATH$NEWDATE/$SUBDIR" @@ -172,24 +200,24 @@ NEWPATH="$ARCHPATH$NEWDATE/$SUBDIR" if [[ $FULLRESTORE == false ]]; then OLDDATE=$(ls -1rt "$ARCHPATH" | head -1) OLDPATH="$ARCHPATH$OLDDATE/$SUBDIR" - echo -e "\n\ncompartive restoration selected:\n\tfiles from:\t'$NEWDATE'\n\tcompared to:\t'$OLDDATE'" + hecko "\n\ncompartive restoration selected:\n\tfiles from:\t'$NEWDATE'\n\tcompared to:\t'$OLDDATE'" else - echo "\n\nfull restorion selected: comparisons will not be made, all non-excluded databases will be restored" + hecko "\n\nfull restorion selected: comparisons will not be made, all non-excluded databases will be restored" fi -echo -e "\nfinding files in '$NEWPATH*.gz'" +hecko "\nfinding files in '$NEWPATH*.gz'" for FILE in $NEWPATH*.gz; do STARTLOOP=$(date +%s) FILE=$(basename $FILE) - echo -e "\n$(date +%F\ %T) | $FILE:" + hecko "\n$(date +%F\ %T) | $FILE:" - if [[ "${SETPOSDB}" =~ ${FILE%%.*} ]]; then echo -e "\t-- recognized '$FILE' for master position and log file" + if [[ "${SETPOSDB}" =~ ${FILE%%.*} ]]; then hecko "\t-- recognized '$FILE' for master position and log file" SETPOSFILE=$NEWPATH$FILE SETPOSSTMT=$(zcat $SETPOSFILE | head -40 | grep MASTER | sed 's/-- //') - echo -e "\t$SETPOSSTMT" + hecko "\t$SETPOSSTMT" fi - if [[ "${SKIPDBS[*]}" =~ ${FILE%%.*} ]]; then echo -e "\t-- skipping $FILE";continue; fi; + if [[ "${SKIPDBS[*]}" =~ ${FILE%%.*} ]]; then hecko "\t-- skipping $FILE";continue; fi; if [[ FULLRESTORE == false ]]; then OLDSIZE=$(stat -c %s $OLDPATH$FILE) @@ -200,54 +228,51 @@ for FILE in $NEWPATH*.gz; do NEWSIZE=$(stat -c %s $NEWPATH$FILE) if [[ $OLDSIZE != $NEWSIZE ]]; then - echo -e "\t$OLDDATE:$FILE:$OLDSIZE != $NEWDATE:$FILE:$NEWSIZE" + hecko "\t$OLDDATE:$FILE:$OLDSIZE != $NEWDATE:$FILE:$NEWSIZE" DB=$(echo $FILE | cut -f1 -d'.') - echo -e "\tDROPPING/CREATING '$DB' + hecko "\tDROPPING/CREATING '$DB' pv $NEWPATH$FILE | gunzip | mysql $DB" - $CMD mysql -e "DROP DATABASE IF EXISTS $DB" - $CMD mysql -e "CREATE DATABASE $DB" - if [[ $CMD ]]; then - $CMD "pv $NEWPATH$FILE | gunzip | mysql $DB;" - else - pv $NEWPATH$FILE | gunzip | mysql $DB; - fi + xct "mysql -e 'DROP DATABASE IF EXISTS $DB'" + xct "mysql -e 'CREATE DATABASE $DB'" + xct "pv $NEWPATH$FILE | gunzip | mysql $DB;" fi ENDLOOP=$(date +%s) EXCTIME=$(expr $ENDLOOP - $STARTLOOP) - printf '\t%02dh:%02dm:%02ds\n' $((EXCTIME/3600)) $((EXCTIME%3600/60)) $((EXCTIME%60)) + hecko $(printf '\t%02dh:%02dm:%02ds\n' $((EXCTIME/3600)) $((EXCTIME%3600/60)) $((EXCTIME%60))) done if [[ $MASTER_HOST && $MASTER_USER && $MASTER_PASSWORD ]]; then - echo -e "\napplying configuration for replication\n\thost: $MASTER_HOST\n\tuser: $MASTER_USER" - $CMD mysql -e "CHANGE MASTER TO master_host='$MASTER_HOST',master_user='$MASTER_USER',master_password='$MASTER_PASSWORD';" + hecko "\napplying configuration for replication\n\thost: $MASTER_HOST\n\tuser: $MASTER_USER" + xct "mysql -e 'CHANGE MASTER TO master_host=$MASTER_HOST,master_user=$MASTER_USER,master_password=$MASTER_PASSWORD;'" else - echo -e "\nreplication credentials not configured" + hecko "\nwarning: replication credentials not present; this might be bad if replication host is not already configured" fi if [[ $SETPOSSTMT ]]; then - $CMD mysql -e "$SETPOSSTMT" + xct "mysql -e '$SETPOSSTMT'" else DONOTSTART=true - echo "\nmaster position database undefined, listing suggestions" + hecko "\nmaster position database undefined, replication will not be started\n\tlisting suggestions:" for FILE in $NEWPATH*.gz; do - echo $(basename $FILE) - echo -e "\t$(zcat $FILE | head -40 | grep MASTER)" + hecko $(basename $FILE) + hecko "\t$(zcat $FILE | head -40 | grep MASTER)" done fi if [[ $NOPROMPT == false ]]; then - read -n 1 -p "restart replication? [Y/n] " reply; + read -n1 -p -t600 "restart replication? [Y/n] " reply; if [[ $reply == [Nn] ]]; then DONOTSTART=true fi fi if [[ $DONOTSTART != true ]]; then - echo -e "\nrestarting replication" - $CMD mysql -e "START SLAVE" + hecko "\nrestarting replication" + xct "mysql -e 'START SLAVE'" fi END=$(date +%s) EXCTIME=$(expr $END - $START) -printf '\ntotal runtime: %02dh:%02dm:%02ds\n' $((EXCTIME/3600)) $((EXCTIME%3600/60)) $((EXCTIME%60)) -echo -e "\n--END: $(date +%F\ %T)\n\n" +hecko $(printf '\t%02dh:%02dm:%02ds\n' $((EXCTIME/3600)) $((EXCTIME%3600/60)) $((EXCTIME%60))) +hecko "\n--END: $(date +%F\ %T)\n\n" +exit 0