mirror of
https://github.com/OpenNebula/one.git
synced 2026-11-22 20:54:49 +09:00
F #4985: Add recovery snapshots and cache monitor
Co-authored-by: Ruben S. Montero <rsmontero@opennebula.io>
This commit is contained in:
1 parent
c0dcd8c850
commit
c84fabd56c
23 files changed
+402
-69
No files matched your search
+2
-1
@@ -1722,7 +1722,8 @@ TM_SSH_FILES="src/tm_mad/ssh/clone \
|
||||
src/tm_mad/ssh/monitor_ds \
|
||||
src/tm_mad/ssh/cpds \
|
||||
src/tm_mad/ssh/resize \
|
||||
src/tm_mad/ssh/ssh_utils.sh"
|
||||
src/tm_mad/ssh/ssh_utils.sh \
|
||||
src/tm_mad/ssh/recovery_snap_create_live"
|
||||
|
||||
TM_SSH_ETC_FILES="src/tm_mad/ssh/sshrc"
|
||||
|
||||
|
||||
@@ -48,7 +48,8 @@ class DSMonitor
|
||||
# Skip if datastore is not marked for local monitoring
|
||||
next unless File.exist? "#{@ds_location}/#{ds_id}/.monitor"
|
||||
|
||||
puts ds_usage(ds_id)
|
||||
tm = File.read("#{@ds_location}/#{ds_id}/.monitor").strip
|
||||
puts ds_usage(ds_id, tm)
|
||||
end
|
||||
end
|
||||
|
||||
@@ -64,16 +65,33 @@ class DSMonitor
|
||||
EOS
|
||||
end
|
||||
|
||||
def ds_usage(ds_id)
|
||||
def replica_usage(path)
|
||||
rs = 0
|
||||
o, _e, s = Open3.capture3("du -sh #{path}")
|
||||
|
||||
if s.exitstatus == 0 && !o.empty?
|
||||
du_a = o.split
|
||||
rs = du_a.first if du_a
|
||||
end
|
||||
|
||||
'REPLICA_CACHE = "YES",' <<
|
||||
"REPLICA_CACHE_SIZE = #{rs}," <<
|
||||
"REPLICA_IMAGES = #{num_images(path)},"
|
||||
end
|
||||
|
||||
def ds_usage(ds_id, tm)
|
||||
sizes = path_usage("#{@ds_location}/#{ds_id}")
|
||||
|
||||
unindent(<<-EOS)
|
||||
DS = [ ID = #{ds_id},
|
||||
usage = "DS = [ ID = #{ds_id},"
|
||||
usage += replica_usage("#{@ds_location}/#{ds_id}") if tm == "replica"
|
||||
usage << <<-EOS
|
||||
USED_MB = #{sizes[0]},
|
||||
TOTAL_MB = #{sizes[1]},
|
||||
FREE_MB = #{sizes[2]}
|
||||
]
|
||||
EOS
|
||||
|
||||
unindent(usage)
|
||||
end
|
||||
|
||||
def path_usage(path)
|
||||
@@ -87,6 +105,15 @@ class DSMonitor
|
||||
[metrics[2], metrics[1], metrics[3]]
|
||||
end
|
||||
|
||||
def num_images(path)
|
||||
o, _e, s = Open3.capture3(
|
||||
"find #{path} -maxdepth 1 -mindepth 1 -type f " <<
|
||||
"-regex '.*/[-a-f0-9]*' | wc -l 2>/dev/null")
|
||||
|
||||
return "unknown" if s.exitstatus != 0 || o.empty?
|
||||
o.strip
|
||||
end
|
||||
|
||||
def unindent(str)
|
||||
mrc = str.match(/^(\s*)/)
|
||||
spaces = mrc[1].size
|
||||
|
||||
@@ -57,6 +57,8 @@ module DriverExecHelper
|
||||
# actions is remote or local. If the local actions has defined an special
|
||||
# script name this is used, otherwise the action name in downcase is
|
||||
# used as the script name.
|
||||
# When action is a String starting with '/' it's considered alreay full
|
||||
# path command and no modification is performed apart from adding params.
|
||||
#
|
||||
# @param [String, Symbol] action name of the action
|
||||
# @param [String] parameters arguments for the script
|
||||
@@ -64,7 +66,10 @@ module DriverExecHelper
|
||||
# @param [String, ''] directory to append to the scripts path for actions
|
||||
# @return [String] command line needed to execute the action
|
||||
def action_command_line(action, parameters, default_name=nil, directory='')
|
||||
if action_is_local? action
|
||||
if action.is_a?(String) && action[0] == '/'
|
||||
return action + " " + parameters if parameters
|
||||
return action
|
||||
elsif action_is_local? action
|
||||
script_path=File.join(@local_scripts_path, directory)
|
||||
else
|
||||
script_path=File.join(@remote_scripts_path, directory)
|
||||
|
||||
@@ -90,10 +90,12 @@ class OpenNebulaDriver < ActionManager
|
||||
:respond => true,
|
||||
:ssh_stream => nil,
|
||||
:base64 => false,
|
||||
:zip => false
|
||||
:zip => false,
|
||||
:no_extra_params => false
|
||||
}.merge(ops)
|
||||
|
||||
params = parameters + " #{id} #{host}"
|
||||
params = parameters
|
||||
params = "#{params} #{id} #{host}" unless options[:no_extra_params]
|
||||
command = action_command_line(aname, params, options[:script_name])
|
||||
|
||||
if action_is_local?(aname)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
# -------------------------------------------------------------------------- #
|
||||
e -------------------------------------------------------------------------- #
|
||||
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
||||
@@ -42,6 +42,7 @@ MKSWAP=${MKSWAP:-mkswap}
|
||||
QEMU_IMG=${QEMU_IMG:-qemu-img}
|
||||
RADOS=${RADOS:-rados}
|
||||
RBD=${RBD:-rbd}
|
||||
RDIFF=${RDIFF:-rdiff}
|
||||
READLINK=${READLINK:-readlink}
|
||||
RM=${RM:-rm}
|
||||
CP=${CP:-cp}
|
||||
@@ -162,6 +163,38 @@ function exclusive
|
||||
return $EXEC_RC
|
||||
}
|
||||
|
||||
# Retries if command fails and STDERR matches
|
||||
function retry_if
|
||||
{
|
||||
MATCH=$1
|
||||
TRIES=$2
|
||||
SLEEP=$3
|
||||
shift 3
|
||||
|
||||
unset TSTD TERR RC
|
||||
|
||||
while [ $TRIES -gt 0 ]; do
|
||||
TRIES=$(( TRIES - 1 ))
|
||||
|
||||
eval "$( ("$@" ) \
|
||||
2> >(TERR=$(cat); typeset -p TERR) \
|
||||
> >(TSTD=$(cat); typeset -p TSTD); RC=$?; typeset -p RC )"
|
||||
|
||||
[ $RC -eq 0 ] && break
|
||||
|
||||
if echo "$TERR" | grep -q "$MATCH"; then
|
||||
sleep $SLEEP;
|
||||
continue
|
||||
fi
|
||||
|
||||
break
|
||||
done
|
||||
|
||||
echo $TERR >&2
|
||||
echo $TSTD
|
||||
return $RC
|
||||
}
|
||||
|
||||
# Executes a command, if it fails returns error message and exits
|
||||
# If a second parameter is present it is used as the error message when
|
||||
# the command fails
|
||||
@@ -1154,4 +1187,4 @@ function send_to_monitor {
|
||||
# Send message
|
||||
echo "$msg_type $msg_result $msg_oid $payload_b64" |
|
||||
nc -u -w1 $mon_address $mon_port
|
||||
}
|
||||
}
|
||||
@@ -39,6 +39,7 @@ else
|
||||
fi
|
||||
|
||||
. $TMCOMMON
|
||||
. $SSH_UTILS
|
||||
|
||||
DRIVER_PATH=$(dirname $0)
|
||||
XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
|
||||
|
||||
@@ -76,10 +76,12 @@ while IFS= read -r -d '' element; do
|
||||
done < <(onevm show -x $VMID| $XPATH \
|
||||
/VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \
|
||||
/VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \
|
||||
/VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/RECOVERY_SNAPSHOT_FREQ \
|
||||
/VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID)
|
||||
|
||||
SIZE="${XPATH_ELEMENTS[j++]}"
|
||||
ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}"
|
||||
RECOVERY_SNAPSHOT_FREQ="${XPATH_ELEMENTS[j++]}"
|
||||
SYS_DS_ID="${XPATH_ELEMENTS[j++]}"
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
@@ -109,12 +111,11 @@ REPLICA_STORAGE_IP=$(awk 'gsub(/[\0]/, x)' \
|
||||
# Synchronize Image Datastore in the Replica Host. Use a recovery snapshot
|
||||
# if present in the RECOVERY_SNAPS_DIR
|
||||
# ------------------------------------------------------------------------------
|
||||
if ssh $REPLICA_HOST \
|
||||
"test -e ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID/${DST_FILE}.replica_snap"; then
|
||||
|
||||
# point to [disk].replica_snap files
|
||||
if recovery_snap_exists $REPLICA_HOST $VMID/$DST_FILE; then
|
||||
# point to [disk].recovery_snaphost files
|
||||
SRC_DIR=${REPLICA_RECOVERY_SNAPS_DIR}/$VMID
|
||||
SRC_FILE="${DST_FILE}.replica_snap"
|
||||
SRC_FILE="${DST_FILE}.recovery_snapshot"
|
||||
SRC_PATH="$SRC_DIR/$SRC_FILE"
|
||||
else
|
||||
rsync_img_to_replica "$SRC_PATH" "$REPLICA_HOST"
|
||||
|
||||
@@ -156,9 +157,38 @@ fi
|
||||
ssh_forward ssh_exec_and_log $REPLICA_HOST "$CLONE_CMD" \
|
||||
"Error copying $SRC to $DST"
|
||||
|
||||
# Create uncopressed base image copy on REPLICA for recovery snapshots
|
||||
# and upload it's signature to the DST host
|
||||
|
||||
if [ -n "$RECOVERY_SNAPSHOT_FREQ" ]; then
|
||||
SNAP_DIR="$REPLICA_RECOVERY_SNAPS_DIR/$VMID"
|
||||
|
||||
BASE_SIGN_CMD=$(cat <<EOF
|
||||
set -e -o pipefail
|
||||
|
||||
mkdir -p $SNAP_DIR
|
||||
cd $SNAP_DIR
|
||||
|
||||
if file $SRC_PATH | grep -q QCOW; then
|
||||
qemu-img convert -O qcow2 $SRC_PATH ${DST_FILE}.base
|
||||
else
|
||||
cp $SRC_PATH ${DST_FILE}.base
|
||||
fi
|
||||
|
||||
rm -f ${DST_FILE}.sign
|
||||
$RDIFF signature ${DST_FILE}.base ${DST_FILE}.sign
|
||||
|
||||
tar -cSf - ${DST_FILE}.sign | \
|
||||
ssh $REPLICA_SSH_OPTS ${REPLICA_STORAGE_IP:-$DST_HOST} "tar xSf - -C $DST_DIR"
|
||||
EOF
|
||||
)
|
||||
|
||||
ssh_forward ssh_exec_and_log "$REPLICA_HOST" "$BASE_SIGN_CMD" \
|
||||
"Error computing or uploading base image or signature"
|
||||
fi
|
||||
|
||||
|
||||
if [ -n "$RESIZE_CMD" ]; then
|
||||
ssh_exec_and_log "$DST_HOST" "$RESIZE_CMD" \
|
||||
"Error resizing image $DST"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
+4
-2
@@ -85,9 +85,10 @@ if [ "${LCM_STATE}" = '26' ] && [ "${SNAP_ID}" = '-1' ]; then
|
||||
log "VM is running, trying blockcopy, fsfreeze, suspend"
|
||||
|
||||
CPDS_CMD_EXPORT=$(cat <<EOF
|
||||
touch ${SRC_TEMP_PATH}
|
||||
$(type retry_if| grep -v 'is a function')
|
||||
|
||||
if ! virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PATH} --wait --finish; then
|
||||
touch ${SRC_TEMP_PATH}
|
||||
if ! retry_if "active block job" 3 5 virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PATH} --wait --finish; then
|
||||
set -e -o pipefail
|
||||
|
||||
if virsh -c ${LIBVIRT_URI} domfsfreeze ${DEPLOY_ID}; then
|
||||
@@ -101,6 +102,7 @@ if ! virsh -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} ${DISK_TARGET} ${SRC_TEMP_PA
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
EOF
|
||||
)
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
../common/delete
|
||||
Executable
+91
@@ -0,0 +1,91 @@
|
||||
#!/bin/bash
|
||||
|
||||
# -------------------------------------------------------------------------- #
|
||||
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
||||
# not use this file except in compliance with the License. You may obtain #
|
||||
# a copy of the License at #
|
||||
# #
|
||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software #
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
||||
# See the License for the specific language governing permissions and #
|
||||
# limitations under the License. #
|
||||
#--------------------------------------------------------------------------- #
|
||||
|
||||
# DELETE <host:remote_system_ds/disk.i|host:remote_system_ds/> vmid dsid
|
||||
# - host is the target host to deploy the VM
|
||||
# - remote_system_ds is the path for the system datastore in the host
|
||||
# - vmid is the id of the VM
|
||||
# - dsid is the target datastore (0 is the system datastore)
|
||||
|
||||
DST=$1
|
||||
|
||||
VMID=$2
|
||||
DSID=$3
|
||||
|
||||
if [ -z "${ONE_LOCATION}" ]; then
|
||||
TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
|
||||
SSH_RC=/var/lib/one/remotes/etc/tm/ssh/sshrc
|
||||
SSH_UTILS=/var/lib/one/remotes/tm/ssh/ssh_utils.sh
|
||||
else
|
||||
TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
|
||||
SSH_RC=$ONE_LOCATION/var/remotes/etc/tm/ssh/sshrc
|
||||
SSH_UTILS=$ONE_LOCATION/var/remotes/tm/ssh/ssh_utils.sh
|
||||
fi
|
||||
|
||||
. $TMCOMMON
|
||||
. $SSH_RC
|
||||
. $SSH_UTILS
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Delete recovery snapshots from replica_host, skip it for ACTION=14 (RECREATE)
|
||||
#-------------------------------------------------------------------------------
|
||||
REPLICA_HOST=$(get_replica_host $DSID)
|
||||
if [ -n "$REPLICA_HOST" ]; then
|
||||
XPATH="$(dirname $0)/../../datastore/xpath.rb"
|
||||
HIST='/VM/HISTORY_RECORDS/HISTORY[last()]/ACTION'
|
||||
ACTION="$(awk 'gsub(/[\0]/, x)' <(onevm show $VMID -x|$XPATH $HIST))"
|
||||
|
||||
if [ $ACTION != "14" ]; then
|
||||
ssh_exec_and_log "$REPLICA_HOST" \
|
||||
"rm -rf ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID" \
|
||||
"Error deleting ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID"
|
||||
fi
|
||||
fi
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
# Return if deleting a disk, we will delete them when removing the
|
||||
# remote_system_ds directory for the VM (remotely)
|
||||
#-------------------------------------------------------------------------------
|
||||
DST_PATH=`arg_path $DST`
|
||||
DST_HOST=`arg_host $DST`
|
||||
|
||||
log "Deleting $DST_PATH"
|
||||
|
||||
delete_file=$(cat <<EOF
|
||||
[ -e "$DST_PATH" ] || exit 0
|
||||
|
||||
times=10
|
||||
function="rm -rf $DST_PATH"
|
||||
|
||||
count=1
|
||||
|
||||
ret=\$(\$function)
|
||||
error=\$?
|
||||
|
||||
while [ \$count -lt \$times -a "\$error" != "0" ]; do
|
||||
sleep 1
|
||||
count=\$(( \$count + 1 ))
|
||||
ret=\$(\$function)
|
||||
error=\$?
|
||||
done
|
||||
|
||||
[ "x\$error" = "x0" ]
|
||||
EOF
|
||||
)
|
||||
|
||||
ssh_exec_and_log $DST_HOST "$delete_file" "Error deleting $DST_PATH"
|
||||
@@ -19,6 +19,8 @@
|
||||
dir="$1"
|
||||
|
||||
vms=$(ls "$dir" | grep '^[0-9]\+$')
|
||||
one_dir=$(dirname $0 | sed -e 's/tm\/ssh//')
|
||||
xpath="$one_dir/datastore/xpath.rb --stdin"
|
||||
|
||||
for vm in $vms; do
|
||||
vmdir="${dir}/${vm}"
|
||||
@@ -50,6 +52,28 @@ for vm in $vms; do
|
||||
"[ ID=${snap}, DISK_ID=${disk_id}, SIZE=${snap_size}]"
|
||||
done
|
||||
fi
|
||||
|
||||
# create disk recovery snapshot if required
|
||||
if [ -f "${dir}/${vm}/vm.xml" ] && [ -f "${dir}/${vm}/ds.xml" ] ; then
|
||||
freq=$(awk 'gsub(/[\0]/, x)' <(cat "${dir}/${vm}/vm.xml"| $xpath \
|
||||
/VM/TEMPLATE/DISK[DISK_ID=$disk_id]/RECOVERY_SNAPSHOT_FREQ))
|
||||
|
||||
replica_host=$(awk 'gsub(/[\0]/, x)' <(cat "${dir}/${vm}/ds.xml"| $xpath \
|
||||
/DATASTORE/TEMPLATE/REPLICA_HOST))
|
||||
|
||||
if [ -n "$freq" ] && [ -n "$replica_host" ]; then
|
||||
|
||||
snap_ts=$($one_dir/tm/ssh/recovery_snap_create_live \
|
||||
$vm $vmdir/$disk $freq $replica_host)
|
||||
rc=$?
|
||||
|
||||
if [ -n "$snap_ts" ] && [ "$rc" = "0" ]; then
|
||||
vm_monitor="${vm_monitor}\nDISK_RECOVERY_SNAPSHOT = [ ID=${disk_id}, TIMESTAMP=${snap_ts}]"
|
||||
else
|
||||
vm_monitor="${vm_monitor}\nDISK_RECOVERY_SNAPSHOT = [ ID=${disk_id}, MSG=\"ERROR $rc\"]"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "$vm_monitor" | base64 -w 0 -
|
||||
|
||||
Executable
+78
@@ -0,0 +1,78 @@
|
||||
#!/bin/bash
|
||||
|
||||
# -------------------------------------------------------------------------- #
|
||||
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
|
||||
# #
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
||||
# not use this file except in compliance with the License. You may obtain #
|
||||
# a copy of the License at #
|
||||
# #
|
||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
||||
# #
|
||||
# Unless required by applicable law or agreed to in writing, software #
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
||||
# See the License for the specific language governing permissions and #
|
||||
# limitations under the License. #
|
||||
#--------------------------------------------------------------------------- #
|
||||
# snap_create host:parent_image snap_id vmid ds_id
|
||||
|
||||
set -e -o pipefail
|
||||
|
||||
VMID=$1
|
||||
DISK_PATH=$2
|
||||
FREQ=$3
|
||||
REPLICA_HOST=$4
|
||||
|
||||
|
||||
DRIVER_PATH=$(dirname $0)
|
||||
source ${DRIVER_PATH}/../../etc/vmm/kvm/kvmrc
|
||||
source ${DRIVER_PATH}/../../etc/tm/ssh/sshrc
|
||||
source ${DRIVER_PATH}/../../scripts_common.sh
|
||||
|
||||
mkdir -p "${DISK_PATH}.snap"
|
||||
SNAP_PATH="${DISK_PATH}.snap/recovery_snapshot"
|
||||
DISK_NAME="$(basename $DISK_PATH)"
|
||||
VM_DIR="$(dirname $DISK_PATH)"
|
||||
|
||||
if [ -f $SNAP_PATH ]; then
|
||||
SNAP_OLD=$(stat -c "%Y" $SNAP_PATH)
|
||||
|
||||
if [ "$(( $(date +%s) - SNAP_OLD ))" -lt "$FREQ" ]; then
|
||||
echo "$SNAP_OLD"
|
||||
exit 0
|
||||
else
|
||||
rm $SNAP_PATH
|
||||
fi
|
||||
fi
|
||||
|
||||
touch $SNAP_PATH
|
||||
|
||||
retry_if "active block job" 3 5\
|
||||
virsh -q -c ${LIBVIRT_URI} blockcopy one-${VMID} \
|
||||
--path ${DISK_PATH} --dest $SNAP_PATH --wait --finish
|
||||
|
||||
cd $VM_DIR
|
||||
|
||||
if [ ! -f "$DISK_NAME.sign" ]; then
|
||||
tar --transform="flags=r;s|${DISK_NAME}.snap/recovery_snapshot|${DISK_NAME}.recovery_snapshot|" \
|
||||
-cSf - ${DISK_NAME}.snap/recovery_snapshot | \
|
||||
ssh $REPLICA_SSH_OPTS ${REPLICA_HOST} \
|
||||
"tar xSf - -C $REPLICA_RECOVERY_SNAPS_DIR/$VMID"
|
||||
RC=$?
|
||||
else
|
||||
rm -f "$DISK_NAME.snap/recovery_snapshot.rdiff"
|
||||
|
||||
$RDIFF delta "$DISK_NAME.sign" "$DISK_NAME.snap/recovery_snapshot" \
|
||||
"$DISK_NAME.snap/recovery_snapshot.rdiff"
|
||||
|
||||
tar --transform="flags=r;s|${DISK_NAME}.snap/recovery_snapshot.rdiff|${DISK_NAME}.rdiff|" \
|
||||
-cSf - ${DISK_NAME}.snap/recovery_snapshot.rdiff | \
|
||||
ssh $REPLICA_SSH_OPTS ${REPLICA_HOST} \
|
||||
"tar xSf - -C $REPLICA_RECOVERY_SNAPS_DIR/$VMID"
|
||||
RC=$?
|
||||
fi
|
||||
|
||||
if [ "$RC" = "0" ]; then
|
||||
stat -c "%c" $DISK_NAME.snap/recovery_snapshot
|
||||
fi
|
||||
@@ -67,11 +67,12 @@ REPLICA_HOST=$(get_replica_host $DS_ID)
|
||||
|
||||
CMD=$(cat <<EOT
|
||||
set -ex -o pipefail
|
||||
$(type retry_if| grep -v 'is a function')
|
||||
|
||||
mkdir -p "${SNAP_DIR}"
|
||||
touch $SNAP_PATH
|
||||
|
||||
virsh -q -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} \
|
||||
retry_if "active block job" 3 5 virsh -q -c ${LIBVIRT_URI} blockcopy ${DEPLOY_ID} \
|
||||
--path ${SRC_PATH} --dest $SNAP_PATH --wait --finish
|
||||
|
||||
if [ -n "$REPLICA_HOST" ]; then
|
||||
|
||||
+49
-37
@@ -43,20 +43,27 @@ function get_replica_host {
|
||||
# ------------------------------------------------------------------------------
|
||||
function md5sum_with_snaps {
|
||||
local IMG=$1
|
||||
local DIR=${2:-.snap}
|
||||
|
||||
SNAPS="$(ls ${IMG}${DIR} 2>/dev/null)"
|
||||
if [ -f "${IMG}.md5sum" ]; then
|
||||
cat "${IMG}.md5sum"
|
||||
return
|
||||
fi
|
||||
|
||||
if [ ! -f "${IMG}" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
SNAPS="$(ls ${IMG}.snap 2>/dev/null)"
|
||||
if [ -z "$SNAPS" ]; then
|
||||
md5sum $IMG | awk '{print $1}'
|
||||
md5sum $IMG | awk '{print $1}' | tee ${IMG}.md5sum
|
||||
else
|
||||
CMD="md5sum $IMG"
|
||||
|
||||
for S in $SNAPS; do
|
||||
CMD+=" ${IMG}${DIR}/${S}"
|
||||
CMD+=" ${IMG}$.snap/${S}"
|
||||
done
|
||||
|
||||
$CMD | md5sum | awk '{print $1}'
|
||||
$CMD | md5sum | awk '{print $1}' | tee ${IMG}.md5sum
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -67,37 +74,14 @@ function repl_img_outdated {
|
||||
local IMG_PATH="$1"
|
||||
local REPLICA_HOST="$2"
|
||||
|
||||
if [ ! -f "${IMG_PATH}.md5sum" ]; then
|
||||
md5sum_with_snaps "$IMG_PATH" > "${IMG_PATH}.md5sum"
|
||||
fi
|
||||
SRC_MD5SUM=$(md5sum_with_snaps "$IMG_PATH")
|
||||
|
||||
SRC_MD5SUM=$(cat "${IMG_PATH}.md5sum")
|
||||
|
||||
# replica can not reuse md5sum_with_snaps, duplicate it
|
||||
SCRIPT=$(cat <<EOF
|
||||
if [ -f ${IMG_PATH}.md5sum ]; then
|
||||
cat ${IMG_PATH}.md5sum
|
||||
else
|
||||
SNAPS="\$(ls ${IMG_PATH}.snap 2>/dev/null)"
|
||||
|
||||
if [ -z "\$SNAPS" ]; then
|
||||
md5sum $IMG_PATH 2>/dev/null | cut -d " " -f 1 > ${IMG_PATH}.md5sum
|
||||
else
|
||||
CMD="md5sum $IMG_PATH"
|
||||
|
||||
for S in \$SNAPS; do
|
||||
CMD+=" ${IMG_PATH}.snap/${S}"
|
||||
done
|
||||
|
||||
\$CMD | md5sum | cut -d " " -f 1 > ${IMG_PATH}.md5sum
|
||||
fi
|
||||
|
||||
cat ${IMG_PATH}.md5sum
|
||||
fi
|
||||
EOF
|
||||
)
|
||||
|
||||
DST_MD5SUM=$(ssh "$REPLICA_HOST" "$SCRIPT")
|
||||
DST_MD5SUM=`ssh $REPLICA_HOST bash -s <<EOF
|
||||
export LANG=C
|
||||
export LC_ALL=C
|
||||
$(type md5sum_with_snaps| grep -v 'is a function')
|
||||
md5sum_with_snaps $IMG_PATH
|
||||
EOF`
|
||||
|
||||
[ "$SRC_MD5SUM" != "$DST_MD5SUM" ]
|
||||
}
|
||||
@@ -108,10 +92,11 @@ EOF
|
||||
function rsync_img_to_replica {
|
||||
local IMG_PATH="$1"
|
||||
local REPLICA_HOST="$2"
|
||||
|
||||
local LOCK_TIMEOUT="${REPLICA_COPY_LOCK_TIMEOUT:-600}"
|
||||
local DST_DIR=$(dirname $IMG_PATH)
|
||||
|
||||
ssh_make_path $REPLICA_HOST $DST_DIR
|
||||
ssh_make_path $REPLICA_HOST $DST_DIR "replica"
|
||||
|
||||
# sync to replica, include .md5sum and .snap dir
|
||||
LOCK="$REPLICA_HOST-${IMG_PATH//\//-}"
|
||||
@@ -131,7 +116,6 @@ EOF
|
||||
"clean_cache \"$(dirname $IMG_PATH)\"" \
|
||||
"$ONE_LOCAL_VAR/remotes/tm/ssh/ssh_utils.sh" \
|
||||
"Failed to run clean_cache on $REPLICA_HOST"
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -186,3 +170,31 @@ function clean_cache() {
|
||||
done
|
||||
)
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------------------
|
||||
# Checks if recovery snapshot exists for given VM/DISK,
|
||||
# If there is a newer image base and snap.rdiff it restores snapshot
|
||||
# ------------------------------------------------------------------------------
|
||||
function recovery_snap_exists() {
|
||||
REPLICA_RECOVERY_SNAPS_DIR="/var/lib/one/datastores/replica_snaps"
|
||||
local REPLICA_HOST=$1
|
||||
local DISK=$2
|
||||
|
||||
SNAP_PATH="${REPLICA_RECOVERY_SNAPS_DIR}/$DISK.recovery_snapshot"
|
||||
BASE_IMG_PATH="${REPLICA_RECOVERY_SNAPS_DIR}/$DISK.base"
|
||||
RDIFF_PATH="${REPLICA_RECOVERY_SNAPS_DIR}/$DISK.rdiff"
|
||||
|
||||
SNAP_EXISTS_CMD=$(cat <<EOF
|
||||
if [ "$SNAP_PATH" -nt "$RDIFF_PATH" ] ; then
|
||||
true
|
||||
elif [ "$RDIFF_PATH" -nt "$SNAP_PATH" ] && [ -e "$BASE_IMG_PATH" ]; then
|
||||
rm -f "$SNAP_PATH"
|
||||
$RDIFF patch "$BASE_IMG_PATH" "$RDIFF_PATH" "$SNAP_PATH"
|
||||
true
|
||||
else
|
||||
false
|
||||
fi
|
||||
EOF
|
||||
)
|
||||
ssh "$REPLICA_HOST" "$SNAP_EXISTS_CMD"
|
||||
}
|
||||
@@ -226,6 +226,7 @@ class VmmAction
|
||||
end
|
||||
|
||||
stdin = step[:stdin] || @xml_data.to_s
|
||||
no_extra_params = step[:no_extra_params] || false
|
||||
|
||||
result, info = @vmm.do_action(get_parameters(step[:parameters]),
|
||||
@id,
|
||||
@@ -233,7 +234,8 @@ class VmmAction
|
||||
step[:action],
|
||||
:ssh_stream => ssh,
|
||||
:respond => false,
|
||||
:stdin => stdin)
|
||||
:stdin => stdin,
|
||||
:no_extra_params => no_extra_params)
|
||||
when :vnm
|
||||
if step[:destination]
|
||||
vnm = @vnm_dst
|
||||
@@ -408,12 +410,28 @@ class ExecDriver < VirtualMachineDriver
|
||||
}
|
||||
end
|
||||
|
||||
vm_dir=File.dirname(action.data[:remote_dfile])
|
||||
|
||||
steps.concat([
|
||||
# Execute pre-boot networking setup
|
||||
{
|
||||
:driver => :vnm,
|
||||
:action => :pre
|
||||
},
|
||||
# Store vm.xml and ds.xml
|
||||
{
|
||||
:driver => :vmm,
|
||||
:action => "/bin/cat - >#{vm_dir}/vm.xml",
|
||||
:stdin => xml_data.elements['VM'].to_s,
|
||||
:no_extra_params => true
|
||||
},
|
||||
{
|
||||
:driver => :vmm,
|
||||
:action => "/bin/cat - >#{vm_dir}/ds.xml",
|
||||
:stdin => xml_data.elements['DATASTORE'].to_s,
|
||||
:no_extra_params => true
|
||||
},
|
||||
|
||||
# Boot the Virtual Machine
|
||||
{
|
||||
:driver => :vmm,
|
||||
|
||||
@@ -23,10 +23,12 @@ DOMAIN="$1"
|
||||
SOURCE="$2"
|
||||
TARGET="$3"
|
||||
TARGET_INDEX="$4"
|
||||
VMID="$5"
|
||||
|
||||
DETACH_PARAMS="--domain $DOMAIN --target $TARGET"
|
||||
|
||||
exec_and_log "virsh --connect $LIBVIRT_URI detach-disk $DETACH_PARAMS" \
|
||||
\
|
||||
exec_and_log "virsh --connect $LIBVIRT_URI detach-disk $DETACH_PARAMS" \
|
||||
"Could not detach $TARGET from $DOMAIN"
|
||||
|
||||
virsh --connect $LIBVIRT_URI dumpxml $DOMAIN | \
|
||||
|
||||
@@ -93,7 +93,6 @@ done < <(echo $STDIN| $XPATH \
|
||||
SHARED="${XPATH_ELEMENTS[j++]}"
|
||||
VM_DIR="${XPATH_ELEMENTS[j++]}"
|
||||
|
||||
|
||||
# use "force-share" param for qemu >= 2.10
|
||||
[ "$(get_qemu_img_version)" -ge 2010000 ] && QEMU_IMG_PARAM="-U"
|
||||
|
||||
@@ -109,7 +108,7 @@ for SNAP in $SNAPS; do
|
||||
done
|
||||
|
||||
if [ "$SHARED" = "YES" ]; then
|
||||
virsh --connect $LIBVIRT_URI migrate \
|
||||
retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \
|
||||
--live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system
|
||||
|
||||
RC=$?
|
||||
@@ -163,6 +162,7 @@ else
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
# freeze/suspend domain and rsync raw disks again
|
||||
if [ -n "$RAW_DISKS" ]; then
|
||||
if virsh --connect $LIBVIRT_URI domfsfreeze $DEPLOY_ID; then
|
||||
@@ -191,7 +191,8 @@ else
|
||||
DISK_OPTS="--copy-storage-all --migrate-disks ${MIGRATE_DISKS}"
|
||||
fi
|
||||
|
||||
virsh --connect $LIBVIRT_URI migrate \
|
||||
retry_if "active block job" 3 5 \
|
||||
virsh --connect $LIBVIRT_URI migrate \
|
||||
--live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system \
|
||||
$DISK_OPTS
|
||||
RC=$?
|
||||
|
||||
@@ -20,6 +20,7 @@ source $(dirname $0)/../../etc/vmm/kvm/kvmrc
|
||||
source $(dirname $0)/../../scripts_common.sh
|
||||
|
||||
deploy_id=$1
|
||||
VMID=$3
|
||||
|
||||
exec_and_log "virsh --connect $LIBVIRT_URI reset $deploy_id" \
|
||||
"Could not reset domain $deploy_id"
|
||||
|
||||
@@ -20,11 +20,13 @@ DOMAIN=$1
|
||||
DISK_ID=$2
|
||||
DISK_SIZE=$3
|
||||
DRV_ACTION=$4
|
||||
VMID=$6
|
||||
|
||||
source $(dirname $0)/../../etc/vmm/kvm/kvmrc
|
||||
source $(dirname $0)/../../scripts_common.sh
|
||||
|
||||
get_disk_information "DISK_ID=$DISK_ID"
|
||||
|
||||
exec_and_log "virsh --connect ${LIBVIRT_URI} blockresize ${DOMAIN} ${DISK_TARGET} ${DISK_SIZE}M" \
|
||||
retry_if "is in use" 3 5 \
|
||||
exec_and_log "virsh --connect ${LIBVIRT_URI} blockresize ${DOMAIN} ${DISK_TARGET} ${DISK_SIZE}M" \
|
||||
"Failed to resize disk ${DISK_ID} on ${DOMAIN}"
|
||||
@@ -69,7 +69,7 @@ virsh --connect $LIBVIRT_URI save-image-dumpxml $FILE > $FILE_XML
|
||||
# system DS migration
|
||||
|
||||
sed -i "s%$DS_LOCATION/[0-9]\+/$VM_ID/%$DS_LOCATION/$DS_ID/$VM_ID/%g" $FILE_XML
|
||||
sed -i "s%$DS_LOCATION_NON_DOUBLE_SLASH/[0-9]\+/$VMID/%$DS_LOCATION/$DS_ID/$VMID/%g" $FILE_XML
|
||||
sed -i "s%$DS_LOCATION_NON_DOUBLE_SLASH/[0-9]\+/$VM_ID/%$DS_LOCATION/$DS_ID/$VM_ID/%g" $FILE_XML
|
||||
EOF
|
||||
)
|
||||
|
||||
|
||||
@@ -38,7 +38,8 @@ fi
|
||||
touch "$FILE"
|
||||
chmod 666 "$FILE"
|
||||
|
||||
exec_and_log "virsh --connect $LIBVIRT_URI save $DEPLOY_ID $FILE" \
|
||||
retry_if "active block job" 3 5 \
|
||||
exec_and_log "virsh --connect $LIBVIRT_URI save $DEPLOY_ID $FILE" \
|
||||
"Could not save $DEPLOY_ID to $FILE"
|
||||
|
||||
#-------------------------------------------------------------------------------
|
||||
|
||||
@@ -22,6 +22,7 @@ source $(dirname $0)/../../scripts_common.sh
|
||||
count=0
|
||||
|
||||
deploy_id=$1
|
||||
VMID=$3
|
||||
|
||||
shutdown_command="virsh --connect $LIBVIRT_URI shutdown $deploy_id"
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ DATASTORE_PATH="${XPATH_ELEMENTS[i++]}"
|
||||
# -------- Create the snapshot and dump its metadata to xml file ------------
|
||||
|
||||
SNAP_NAME="snap-${SNAP_ID}"
|
||||
data=`virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
|
||||
data=`retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
|
||||
|
||||
if [ "$?" = "0" ]; then
|
||||
echo "$data" | awk '{print $3}'
|
||||
@@ -52,7 +52,7 @@ if [ "$?" = "0" ]; then
|
||||
SNAP_XML_PATH="${DATASTORE_PATH}/${SNAP_NAME}.xml"
|
||||
|
||||
# dump snapshot metadata xml to the VM location
|
||||
virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
|
||||
retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
|
||||
fi
|
||||
else
|
||||
error_message "Could not create snapshot $NAME for domain $DOMAIN."
|
||||
|
||||
@@ -21,8 +21,9 @@ source $(dirname $0)/../../scripts_common.sh
|
||||
|
||||
DOMAIN="$1"
|
||||
NAME="$2"
|
||||
VMID="$3"
|
||||
|
||||
exec_and_log \
|
||||
retry_if "active block job" 3 5 exec_and_log \
|
||||
"virsh --connect $LIBVIRT_URI snapshot-revert --force $DOMAIN $NAME" \
|
||||
"Could not revert to snapshot $NAME for domain $DOMAIN."
|
||||
|
||||
|
||||
Reference in new issue
Block a user