From fefed1a5d454a63250b452e0fce4bdb2207b35cf Mon Sep 17 00:00:00 2001 From: awalon Date: Sat, 30 Jan 2016 10:02:51 +0100 Subject: [PATCH 1/4] Remove failed backups first --- ghettoVCB.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/ghettoVCB.sh b/ghettoVCB.sh index f641500..e42b152 100755 --- a/ghettoVCB.sh +++ b/ghettoVCB.sh @@ -581,6 +581,8 @@ indexedRotate() { done } +joinArray () { local IFS="$1"; shift; echo "$*"; } + checkVMBackupRotation() { local BACKUP_DIR_PATH=$1 local VM_TO_SEARCH_FOR=$2 @@ -589,17 +591,25 @@ checkVMBackupRotation() { if [[ -z ${VM_BACKUP_ROTATION_COUNT} ]]; then VM_BACKUP_ROTATION_COUNT=1 fi - + + logger "info" "Rotate backups of '${VM_TO_SEARCH_FOR}' while keeping ${VM_BACKUP_ROTATION_COUNT} valid versions" LIST_BACKUPS=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}") - BACKUPS_TO_KEEP=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" | head -"${VM_BACKUP_ROTATION_COUNT}") - + + BACKUPS_FAILED=$(find "${BACKUP_DIR_PATH}" -iname STATUS.error | grep -o "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}") + FILTER_FAILED=$(joinArray \| ${BACKUPS_FAILED}) + [[ -z ${FILTER_FAILED} ]] && FILTER_FAILED=###NOTHING### + logger "info" "Rotate backups of '${VM_TO_SEARCH_FOR}' and ignoring failed versions: '$(joinArray , ${BACKUPS_FAILED})'" + + BACKUPS_TO_KEEP=$(ls -t "${BACKUP_DIR_PATH}" | grep "${VM_TO_SEARCH_FOR}-[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}_[0-9]\{2\}-[0-9]\{2\}-[0-9]\{2\}" | grep -vE "(${FILTER_FAILED})" | head -"${VM_BACKUP_ROTATION_COUNT}") + logger "info" "Rotate backups of '${VM_TO_SEARCH_FOR}' and keeping: '$(joinArray , ${BACKUPS_TO_KEEP})'" + ORIG_IFS=${IFS} IFS=' ' for i in ${LIST_BACKUPS}; do FOUND=0 for j in ${BACKUPS_TO_KEEP}; do - [[ $i == $j ]] && FOUND=1 + [[ $i == $j ]] && FOUND=1 && break done if [[ $FOUND -eq 0 ]]; then From 2d876e14080393136fa17f6d8fff93e094a97bd9 Mon Sep 17 00:00:00 2001 From: awalon Date: Sat, 30 Jan 2016 10:17:46 +0100 Subject: [PATCH 2/4] Control ghettoVBC.sh to backup VMs on ESXi1 to NAS VM hosted on ESXi2, with IPMI controlled power on/off. --- esxi-backup-ghettovcb.sh | 174 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 esxi-backup-ghettovcb.sh diff --git a/esxi-backup-ghettovcb.sh b/esxi-backup-ghettovcb.sh new file mode 100644 index 0000000..78b9753 --- /dev/null +++ b/esxi-backup-ghettovcb.sh @@ -0,0 +1,174 @@ +#!/bin/sh + +# ESXi Server (VMs) +SOURCE_ESX_IP=192.168.1.14 +SOURCE_ESX_SSH_USR=root +SOURCE_ESX_SSH_KEY=~/.ssh/id_source_esxi + +# ESXi with NFS Server VM, having IPMI (for power on/off) +TARGET_IPMI_IP=192.168.1.11 +TARGET_IPMI_USR=root +# protected password for IPMI +TARGET_IPMI_PWD=~/ipmitool.pwd + +TARGET_ESX_IP=192.168.1.12 +TARGET_ESX_SSH_USR=root +TARGET_ESX_SSH_KEY=~/.ssh/id_target_esxi + +# ESXi hosted NFS Server (backup system) +TARGET_NAS_IP=192.168.1.13 +TARGET_NAS_SSH_USR=root +TARGET_NAS_SSH_KEY=~/.ssh/id_nas_nfs + + +###################################################################################### +PING_CMD="`which ping` -c1"; +PING_LIMIT=1; +SSH_OPT="-aTx -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5"; + +waitUntilStarted() { + local MAX_PING_COUNT=${PING_LIMIT}; + if [ -n "$1" ]; then + while ! ${PING_CMD} "$1">/dev/null 2>&1; do + [ ${MAX_PING_COUNT} -eq 0 ] && return 2; + MAX_PING_COUNT=`expr ${MAX_PING_COUNT} - 1`; + done; + return 0; + else + echo "waitUntilStarted: Server name or IP expected."; + return 1; + fi; +} + +waitForShutdown() { + local MAX_PING_COUNT=${PING_LIMIT}; + if [ -n "$1" ]; then + while ${PING_CMD} "$1">/dev/null 2>&1; do + [ ${MAX_PING_COUNT} -eq 0 ] && return 2; + MAX_PING_COUNT=`expr ${MAX_PING_COUNT} - 1`; + done; + return 0; + else + echo "waitForShutdown: Server name or IP expected."; + return 1; + fi; +} + +showErrorMessage() { + echo -n "+++ ERROR [`date +"%Y-%m-%d %H:%M:%S.%N"`] +++ "; + [ -n "$1" ] || exit 999 ; + case $1 in + 10 ) + echo "IPMI: Cannot show power status of ${TARGET_IPMI_IP}"; + ;; + 11 ) + echo "IPMI: Cannot power on ${TARGET_IPMI_IP}"; + ;; + 20 ) + echo "PING: Cannot reach ESXi server ${TARGET_ESX_IP}"; + ;; + 21 ) + echo "PING: Cannot reach NAS server ${TARGET_NAS_IP}"; + ;; + 30 ) + echo "SSH: Cannot execute backup task on ${TARGET_NAS_IP}"; + ;; + 40 ) + echo "SSH: Cannot execute shutdown command on ${TARGET_NAS_IP}"; + ;; + 41 ) + echo "PING: ${TARGET_NAS_IP} is still responding"; + ;; + 42 ) + echo "SSH: Cannot execute shutdown command on ${TARGET_ESX_IP}"; + ;; + 43 ) + echo "PING: ${TARGET_ESX_IP} is still responding"; + ;; + esac; + #exit $1; +} + +echo "=================================================================="; +echo " Start: `date +"%Y-%m-%d %H:%M:%S.%N"`"; +echo "=================================================================="; +echo ""; + +# start quad.goha.lan +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Power on backup system"; +echo "------------------------------------------------------------------"; +ipmitool -I lan -H ${TARGET_IPMI_IP} -U ${TARGET_IPMI_USR} -f ${TARGET_IPMI_PWD} power status || showErrorMessage 10 +ipmitool -I lan -H ${TARGET_IPMI_IP} -U ${TARGET_IPMI_USR} -f ${TARGET_IPMI_PWD} power on || showErrorMessage 11; +echo ""; + + +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Wait for ESXi and VM"; +echo "------------------------------------------------------------------"; +# wait for quad.goha.lan +waitUntilStarted ${TARGET_ESX_IP} || showErrorMessage 20; +# wait for omvesxibackup.goha.lan +waitUntilStarted ${TARGET_NAS_IP} || showErrorMessage 21; +sleep 20 +echo ""; + + +# start backup script +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Start backup script "; +echo "------------------------------------------------------------------"; +ssh ${SSH_OPT} -l ${SOURCE_ESX_SSH_USR} -i ${SOURCE_ESX_SSH_KEY} ${SOURCE_ESX_IP} << EOF +echo ""; + +# status of source system +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Status source system "; +echo "------------------------------------------------------------------"; +esxcli storage nfs list +df -h /vmfs/volumes/ | grep -vi vfat +echo ""; + +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Start ghettoVCB "; +echo "------------------------------------------------------------------"; +# run ghettoVCB script +sh /vmfs/volumes/datastore1/ghettoVCB/run.sh +# sync +sync +echo ""; + + +# create status +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Status source and target "; +echo "------------------------------------------------------------------"; +esxcli storage nfs list +df -h /vmfs/volumes/ | grep -vi vfat +find /vmfs/volumes/esxi_backup/ -iname status* +echo ""; + +# disconnect nfs storage +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Disconnect backup storage "; +echo "------------------------------------------------------------------"; +esxcli storage nfs remove -v esxi_backup +echo ""; +EOF +[ "$?" -eq "0" ] || showErrorMessage 30; + + +# shutdown omvesxibackup.goha.lan +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Shutdown VM"; +echo "------------------------------------------------------------------"; +ssh ${SSH_OPT} -l ${TARGET_NAS_SSH_USR} -i ${TARGET_NAS_SSH_KEY} ${TARGET_NAS_IP} "shutdown -h -y now" || showErrorMessage 40; +waitForShutdown ${TARGET_NAS_IP} || showErrorMessage 41; +echo ""; + +# shutdown quad.goha.lan +echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Shutdown ESXi"; +echo "------------------------------------------------------------------"; +ssh ${SSH_OPT} -l ${TARGET_ESX_SSH_USR} -i ${TARGET_ESX_SSH_KEY} ${TARGET_ESX_IP} "poweroff" || showErrorMessage 42; +waitForShutdown ${TARGET_ESX_IP} || showErrorMessage 43; +echo ""; + + +echo ""; +echo "=================================================================="; +echo " Finished: `date +"%Y-%m-%d %H:%M:%S.%N"`"; +echo "=================================================================="; + +return 0; \ No newline at end of file From 4b3336bf1317599665bf5ed1d7db86464e82e3e5 Mon Sep 17 00:00:00 2001 From: Ralf WeiK Date: Wed, 3 Feb 2016 00:54:32 +0100 Subject: [PATCH 3/4] Updated some comments --- esxi-backup-ghettovcb.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/esxi-backup-ghettovcb.sh b/esxi-backup-ghettovcb.sh index 78b9753..0d28ee4 100644 --- a/esxi-backup-ghettovcb.sh +++ b/esxi-backup-ghettovcb.sh @@ -94,7 +94,7 @@ echo " Start: `date +"%Y-%m-%d %H:%M:%S.%N"`"; echo "=================================================================="; echo ""; -# start quad.goha.lan +# start target ESXi server echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Power on backup system"; echo "------------------------------------------------------------------"; ipmitool -I lan -H ${TARGET_IPMI_IP} -U ${TARGET_IPMI_USR} -f ${TARGET_IPMI_PWD} power status || showErrorMessage 10 @@ -104,9 +104,9 @@ echo ""; echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Wait for ESXi and VM"; echo "------------------------------------------------------------------"; -# wait for quad.goha.lan +# wait for target ESXi server waitUntilStarted ${TARGET_ESX_IP} || showErrorMessage 20; -# wait for omvesxibackup.goha.lan +# wait for NAS server (VM hosted on target ESXi) waitUntilStarted ${TARGET_NAS_IP} || showErrorMessage 21; sleep 20 echo ""; @@ -151,14 +151,14 @@ EOF [ "$?" -eq "0" ] || showErrorMessage 30; -# shutdown omvesxibackup.goha.lan +# shutdown NAS server (VM hosted on target ESXi) echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Shutdown VM"; echo "------------------------------------------------------------------"; ssh ${SSH_OPT} -l ${TARGET_NAS_SSH_USR} -i ${TARGET_NAS_SSH_KEY} ${TARGET_NAS_IP} "shutdown -h -y now" || showErrorMessage 40; waitForShutdown ${TARGET_NAS_IP} || showErrorMessage 41; echo ""; -# shutdown quad.goha.lan +# shutdown target ESXi echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Shutdown ESXi"; echo "------------------------------------------------------------------"; ssh ${SSH_OPT} -l ${TARGET_ESX_SSH_USR} -i ${TARGET_ESX_SSH_KEY} ${TARGET_ESX_IP} "poweroff" || showErrorMessage 42; From 43beefd0cfa59865278a13701f39eff0165edb93 Mon Sep 17 00:00:00 2001 From: Awalon Date: Wed, 3 Feb 2016 01:05:32 +0100 Subject: [PATCH 4/4] Delete esxi-backup-ghettovcb.sh Moved out of ghettoVCB into ghettoVBC-tools --- esxi-backup-ghettovcb.sh | 174 --------------------------------------- 1 file changed, 174 deletions(-) delete mode 100644 esxi-backup-ghettovcb.sh diff --git a/esxi-backup-ghettovcb.sh b/esxi-backup-ghettovcb.sh deleted file mode 100644 index 0d28ee4..0000000 --- a/esxi-backup-ghettovcb.sh +++ /dev/null @@ -1,174 +0,0 @@ -#!/bin/sh - -# ESXi Server (VMs) -SOURCE_ESX_IP=192.168.1.14 -SOURCE_ESX_SSH_USR=root -SOURCE_ESX_SSH_KEY=~/.ssh/id_source_esxi - -# ESXi with NFS Server VM, having IPMI (for power on/off) -TARGET_IPMI_IP=192.168.1.11 -TARGET_IPMI_USR=root -# protected password for IPMI -TARGET_IPMI_PWD=~/ipmitool.pwd - -TARGET_ESX_IP=192.168.1.12 -TARGET_ESX_SSH_USR=root -TARGET_ESX_SSH_KEY=~/.ssh/id_target_esxi - -# ESXi hosted NFS Server (backup system) -TARGET_NAS_IP=192.168.1.13 -TARGET_NAS_SSH_USR=root -TARGET_NAS_SSH_KEY=~/.ssh/id_nas_nfs - - -###################################################################################### -PING_CMD="`which ping` -c1"; -PING_LIMIT=1; -SSH_OPT="-aTx -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=5"; - -waitUntilStarted() { - local MAX_PING_COUNT=${PING_LIMIT}; - if [ -n "$1" ]; then - while ! ${PING_CMD} "$1">/dev/null 2>&1; do - [ ${MAX_PING_COUNT} -eq 0 ] && return 2; - MAX_PING_COUNT=`expr ${MAX_PING_COUNT} - 1`; - done; - return 0; - else - echo "waitUntilStarted: Server name or IP expected."; - return 1; - fi; -} - -waitForShutdown() { - local MAX_PING_COUNT=${PING_LIMIT}; - if [ -n "$1" ]; then - while ${PING_CMD} "$1">/dev/null 2>&1; do - [ ${MAX_PING_COUNT} -eq 0 ] && return 2; - MAX_PING_COUNT=`expr ${MAX_PING_COUNT} - 1`; - done; - return 0; - else - echo "waitForShutdown: Server name or IP expected."; - return 1; - fi; -} - -showErrorMessage() { - echo -n "+++ ERROR [`date +"%Y-%m-%d %H:%M:%S.%N"`] +++ "; - [ -n "$1" ] || exit 999 ; - case $1 in - 10 ) - echo "IPMI: Cannot show power status of ${TARGET_IPMI_IP}"; - ;; - 11 ) - echo "IPMI: Cannot power on ${TARGET_IPMI_IP}"; - ;; - 20 ) - echo "PING: Cannot reach ESXi server ${TARGET_ESX_IP}"; - ;; - 21 ) - echo "PING: Cannot reach NAS server ${TARGET_NAS_IP}"; - ;; - 30 ) - echo "SSH: Cannot execute backup task on ${TARGET_NAS_IP}"; - ;; - 40 ) - echo "SSH: Cannot execute shutdown command on ${TARGET_NAS_IP}"; - ;; - 41 ) - echo "PING: ${TARGET_NAS_IP} is still responding"; - ;; - 42 ) - echo "SSH: Cannot execute shutdown command on ${TARGET_ESX_IP}"; - ;; - 43 ) - echo "PING: ${TARGET_ESX_IP} is still responding"; - ;; - esac; - #exit $1; -} - -echo "=================================================================="; -echo " Start: `date +"%Y-%m-%d %H:%M:%S.%N"`"; -echo "=================================================================="; -echo ""; - -# start target ESXi server -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Power on backup system"; -echo "------------------------------------------------------------------"; -ipmitool -I lan -H ${TARGET_IPMI_IP} -U ${TARGET_IPMI_USR} -f ${TARGET_IPMI_PWD} power status || showErrorMessage 10 -ipmitool -I lan -H ${TARGET_IPMI_IP} -U ${TARGET_IPMI_USR} -f ${TARGET_IPMI_PWD} power on || showErrorMessage 11; -echo ""; - - -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Wait for ESXi and VM"; -echo "------------------------------------------------------------------"; -# wait for target ESXi server -waitUntilStarted ${TARGET_ESX_IP} || showErrorMessage 20; -# wait for NAS server (VM hosted on target ESXi) -waitUntilStarted ${TARGET_NAS_IP} || showErrorMessage 21; -sleep 20 -echo ""; - - -# start backup script -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Start backup script "; -echo "------------------------------------------------------------------"; -ssh ${SSH_OPT} -l ${SOURCE_ESX_SSH_USR} -i ${SOURCE_ESX_SSH_KEY} ${SOURCE_ESX_IP} << EOF -echo ""; - -# status of source system -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Status source system "; -echo "------------------------------------------------------------------"; -esxcli storage nfs list -df -h /vmfs/volumes/ | grep -vi vfat -echo ""; - -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Start ghettoVCB "; -echo "------------------------------------------------------------------"; -# run ghettoVCB script -sh /vmfs/volumes/datastore1/ghettoVCB/run.sh -# sync -sync -echo ""; - - -# create status -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Status source and target "; -echo "------------------------------------------------------------------"; -esxcli storage nfs list -df -h /vmfs/volumes/ | grep -vi vfat -find /vmfs/volumes/esxi_backup/ -iname status* -echo ""; - -# disconnect nfs storage -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Disconnect backup storage "; -echo "------------------------------------------------------------------"; -esxcli storage nfs remove -v esxi_backup -echo ""; -EOF -[ "$?" -eq "0" ] || showErrorMessage 30; - - -# shutdown NAS server (VM hosted on target ESXi) -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Shutdown VM"; -echo "------------------------------------------------------------------"; -ssh ${SSH_OPT} -l ${TARGET_NAS_SSH_USR} -i ${TARGET_NAS_SSH_KEY} ${TARGET_NAS_IP} "shutdown -h -y now" || showErrorMessage 40; -waitForShutdown ${TARGET_NAS_IP} || showErrorMessage 41; -echo ""; - -# shutdown target ESXi -echo " `date +"%Y-%m-%d %H:%M:%S.%N"` - Shutdown ESXi"; -echo "------------------------------------------------------------------"; -ssh ${SSH_OPT} -l ${TARGET_ESX_SSH_USR} -i ${TARGET_ESX_SSH_KEY} ${TARGET_ESX_IP} "poweroff" || showErrorMessage 42; -waitForShutdown ${TARGET_ESX_IP} || showErrorMessage 43; -echo ""; - - -echo ""; -echo "=================================================================="; -echo " Finished: `date +"%Y-%m-%d %H:%M:%S.%N"`"; -echo "=================================================================="; - -return 0; \ No newline at end of file