diff --git a/install.sh b/install.sh
index 23c7b031fd..8959b9e0d0 100755
--- a/install.sh
+++ b/install.sh
@@ -1722,7 +1722,8 @@ TM_SSH_FILES="src/tm_mad/ssh/clone \
src/tm_mad/ssh/monitor_ds \
src/tm_mad/ssh/cpds \
src/tm_mad/ssh/resize \
- src/tm_mad/ssh/ssh_utils.sh"
+ src/tm_mad/ssh/ssh_utils.sh \
+ src/tm_mad/ssh/recovery_snap_create_live"
TM_SSH_ETC_FILES="src/tm_mad/ssh/sshrc"
diff --git a/src/im_mad/remotes/node-probes.d/monitor_ds.rb b/src/im_mad/remotes/node-probes.d/monitor_ds.rb
index 31b12e8fea..256ce0e652 100755
--- a/src/im_mad/remotes/node-probes.d/monitor_ds.rb
+++ b/src/im_mad/remotes/node-probes.d/monitor_ds.rb
@@ -48,7 +48,8 @@ class DSMonitor
# Skip if datastore is not marked for local monitoring
next unless File.exist? "#{@ds_location}/#{ds_id}/.monitor"
- puts ds_usage(ds_id)
+ tm = File.read("#{@ds_location}/#{ds_id}/.monitor").strip
+ puts ds_usage(ds_id, tm)
end
end
@@ -64,16 +65,33 @@ class DSMonitor
EOS
end
- def ds_usage(ds_id)
+ def replica_usage(path)
+ rs = 0
+ o, _e, s = Open3.capture3("du -sh #{path}")
+
+ if s.exitstatus == 0 && !o.empty?
+ du_a = o.split
+ rs = du_a.first if du_a
+ end
+
+ 'REPLICA_CACHE = "YES",' <<
+ "REPLICA_CACHE_SIZE = #{rs}," <<
+ "REPLICA_IMAGES = #{num_images(path)},"
+ end
+
+ def ds_usage(ds_id, tm)
sizes = path_usage("#{@ds_location}/#{ds_id}")
- unindent(<<-EOS)
- DS = [ ID = #{ds_id},
+ usage = "DS = [ ID = #{ds_id},"
+ usage += replica_usage("#{@ds_location}/#{ds_id}") if tm == "replica"
+ usage << <<-EOS
USED_MB = #{sizes[0]},
TOTAL_MB = #{sizes[1]},
FREE_MB = #{sizes[2]}
]
EOS
+
+ unindent(usage)
end
def path_usage(path)
@@ -87,6 +105,15 @@ class DSMonitor
[metrics[2], metrics[1], metrics[3]]
end
+ def num_images(path)
+ o, _e, s = Open3.capture3(
+ "find #{path} -maxdepth 1 -mindepth 1 -type f " <<
+ "-regex '.*/[-a-f0-9]*' | wc -l 2>/dev/null")
+
+ return "unknown" if s.exitstatus != 0 || o.empty?
+ o.strip
+ end
+
def unindent(str)
mrc = str.match(/^(\s*)/)
spaces = mrc[1].size
diff --git a/src/mad/ruby/DriverExecHelper.rb b/src/mad/ruby/DriverExecHelper.rb
index be55e5f389..485f4631c1 100644
--- a/src/mad/ruby/DriverExecHelper.rb
+++ b/src/mad/ruby/DriverExecHelper.rb
@@ -57,6 +57,8 @@ module DriverExecHelper
# actions is remote or local. If the local actions has defined an special
# script name this is used, otherwise the action name in downcase is
# used as the script name.
+ # When action is a String starting with '/' it's considered alreay full
+ # path command and no modification is performed apart from adding params.
#
# @param [String, Symbol] action name of the action
# @param [String] parameters arguments for the script
@@ -64,7 +66,10 @@ module DriverExecHelper
# @param [String, ''] directory to append to the scripts path for actions
# @return [String] command line needed to execute the action
def action_command_line(action, parameters, default_name=nil, directory='')
- if action_is_local? action
+ if action.is_a?(String) && action[0] == '/'
+ return action + " " + parameters if parameters
+ return action
+ elsif action_is_local? action
script_path=File.join(@local_scripts_path, directory)
else
script_path=File.join(@remote_scripts_path, directory)
diff --git a/src/mad/ruby/OpenNebulaDriver.rb b/src/mad/ruby/OpenNebulaDriver.rb
index 83860e5392..bf6aeb8ff5 100644
--- a/src/mad/ruby/OpenNebulaDriver.rb
+++ b/src/mad/ruby/OpenNebulaDriver.rb
@@ -90,10 +90,12 @@ class OpenNebulaDriver < ActionManager
:respond => true,
:ssh_stream => nil,
:base64 => false,
- :zip => false
+ :zip => false,
+ :no_extra_params => false
}.merge(ops)
- params = parameters + " #{id} #{host}"
+ params = parameters
+ params = "#{params} #{id} #{host}" unless options[:no_extra_params]
command = action_command_line(aname, params, options[:script_name])
if action_is_local?(aname)
diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh
index 91e19acbb0..8f5b2b43a7 100644
--- a/src/mad/sh/scripts_common.sh
+++ b/src/mad/sh/scripts_common.sh
@@ -1,4 +1,4 @@
-# -------------------------------------------------------------------------- #
+e -------------------------------------------------------------------------- #
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
# #
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
@@ -42,6 +42,7 @@ MKSWAP=${MKSWAP:-mkswap}
QEMU_IMG=${QEMU_IMG:-qemu-img}
RADOS=${RADOS:-rados}
RBD=${RBD:-rbd}
+RDIFF=${RDIFF:-rdiff}
READLINK=${READLINK:-readlink}
RM=${RM:-rm}
CP=${CP:-cp}
@@ -162,6 +163,38 @@ function exclusive
return $EXEC_RC
}
+# Retries if command fails and STDERR matches
+function retry_if
+{
+ MATCH=$1
+ TRIES=$2
+ SLEEP=$3
+ shift 3
+
+ unset TSTD TERR RC
+
+ while [ $TRIES -gt 0 ]; do
+ TRIES=$(( TRIES - 1 ))
+
+ eval "$( ("$@" ) \
+ 2> >(TERR=$(cat); typeset -p TERR) \
+ > >(TSTD=$(cat); typeset -p TSTD); RC=$?; typeset -p RC )"
+
+ [ $RC -eq 0 ] && break
+
+ if echo "$TERR" | grep -q "$MATCH"; then
+ sleep $SLEEP;
+ continue
+ fi
+
+ break
+ done
+
+ echo $TERR >&2
+ echo $TSTD
+ return $RC
+}
+
# Executes a command, if it fails returns error message and exits
# If a second parameter is present it is used as the error message when
# the command fails
@@ -1154,4 +1187,4 @@ function send_to_monitor {
# Send message
echo "$msg_type $msg_result $msg_oid $payload_b64" |
nc -u -w1 $mon_address $mon_port
-}
\ No newline at end of file
+}
diff --git a/src/tm_mad/ssh/clone b/src/tm_mad/ssh/clone
index b9ad8cca4f..7b9e8527de 100755
--- a/src/tm_mad/ssh/clone
+++ b/src/tm_mad/ssh/clone
@@ -39,6 +39,7 @@ else
fi
. $TMCOMMON
+. $SSH_UTILS
DRIVER_PATH=$(dirname $0)
XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin"
diff --git a/src/tm_mad/ssh/clone.replica b/src/tm_mad/ssh/clone.replica
index 037b5bc7d3..b2b53cb6b0 100755
--- a/src/tm_mad/ssh/clone.replica
+++ b/src/tm_mad/ssh/clone.replica
@@ -76,10 +76,12 @@ while IFS= read -r -d '' element; do
done < <(onevm show -x $VMID| $XPATH \
/VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \
/VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \
+ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/RECOVERY_SNAPSHOT_FREQ \
/VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID)
SIZE="${XPATH_ELEMENTS[j++]}"
ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}"
+RECOVERY_SNAPSHOT_FREQ="${XPATH_ELEMENTS[j++]}"
SYS_DS_ID="${XPATH_ELEMENTS[j++]}"
# ------------------------------------------------------------------------------
@@ -109,12 +111,11 @@ REPLICA_STORAGE_IP=$(awk 'gsub(/[\0]/, x)' \
# Synchronize Image Datastore in the Replica Host. Use a recovery snapshot
# if present in the RECOVERY_SNAPS_DIR
# ------------------------------------------------------------------------------
-if ssh $REPLICA_HOST \
- "test -e ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID/${DST_FILE}.replica_snap"; then
-
- # point to [disk].replica_snap files
+if recovery_snap_exists $REPLICA_HOST $VMID/$DST_FILE; then
+ # point to [disk].recovery_snaphost files
SRC_DIR=${REPLICA_RECOVERY_SNAPS_DIR}/$VMID
- SRC_FILE="${DST_FILE}.replica_snap"
+ SRC_FILE="${DST_FILE}.recovery_snapshot"
+ SRC_PATH="$SRC_DIR/$SRC_FILE"
else
rsync_img_to_replica "$SRC_PATH" "$REPLICA_HOST"
@@ -156,9 +157,38 @@ fi
ssh_forward ssh_exec_and_log $REPLICA_HOST "$CLONE_CMD" \
"Error copying $SRC to $DST"
+# Create uncopressed base image copy on REPLICA for recovery snapshots
+# and upload it's signature to the DST host
+
+if [ -n "$RECOVERY_SNAPSHOT_FREQ" ]; then
+ SNAP_DIR="$REPLICA_RECOVERY_SNAPS_DIR/$VMID"
+
+ BASE_SIGN_CMD=$(cat < vmid dsid
+# - host is the target host to deploy the VM
+# - remote_system_ds is the path for the system datastore in the host
+# - vmid is the id of the VM
+# - dsid is the target datastore (0 is the system datastore)
+
+DST=$1
+
+VMID=$2
+DSID=$3
+
+if [ -z "${ONE_LOCATION}" ]; then
+ TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh
+ SSH_RC=/var/lib/one/remotes/etc/tm/ssh/sshrc
+ SSH_UTILS=/var/lib/one/remotes/tm/ssh/ssh_utils.sh
+else
+ TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh
+ SSH_RC=$ONE_LOCATION/var/remotes/etc/tm/ssh/sshrc
+ SSH_UTILS=$ONE_LOCATION/var/remotes/tm/ssh/ssh_utils.sh
+fi
+
+. $TMCOMMON
+. $SSH_RC
+. $SSH_UTILS
+
+#-------------------------------------------------------------------------------
+# Delete recovery snapshots from replica_host, skip it for ACTION=14 (RECREATE)
+#-------------------------------------------------------------------------------
+REPLICA_HOST=$(get_replica_host $DSID)
+if [ -n "$REPLICA_HOST" ]; then
+ XPATH="$(dirname $0)/../../datastore/xpath.rb"
+ HIST='/VM/HISTORY_RECORDS/HISTORY[last()]/ACTION'
+ ACTION="$(awk 'gsub(/[\0]/, x)' <(onevm show $VMID -x|$XPATH $HIST))"
+
+ if [ $ACTION != "14" ]; then
+ ssh_exec_and_log "$REPLICA_HOST" \
+ "rm -rf ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID" \
+ "Error deleting ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID"
+ fi
+fi
+
+#-------------------------------------------------------------------------------
+# Return if deleting a disk, we will delete them when removing the
+# remote_system_ds directory for the VM (remotely)
+#-------------------------------------------------------------------------------
+DST_PATH=`arg_path $DST`
+DST_HOST=`arg_host $DST`
+
+log "Deleting $DST_PATH"
+
+delete_file=$(cat < "${IMG_PATH}.md5sum"
- fi
+ SRC_MD5SUM=$(md5sum_with_snaps "$IMG_PATH")
- SRC_MD5SUM=$(cat "${IMG_PATH}.md5sum")
-
- # replica can not reuse md5sum_with_snaps, duplicate it
- SCRIPT=$(cat </dev/null | cut -d " " -f 1 > ${IMG_PATH}.md5sum
- else
- CMD="md5sum $IMG_PATH"
-
- for S in \$SNAPS; do
- CMD+=" ${IMG_PATH}.snap/${S}"
- done
-
- \$CMD | md5sum | cut -d " " -f 1 > ${IMG_PATH}.md5sum
- fi
-
- cat ${IMG_PATH}.md5sum
- fi
-EOF
-)
-
- DST_MD5SUM=$(ssh "$REPLICA_HOST" "$SCRIPT")
+ DST_MD5SUM=`ssh $REPLICA_HOST bash -s < ssh,
:respond => false,
- :stdin => stdin)
+ :stdin => stdin,
+ :no_extra_params => no_extra_params)
when :vnm
if step[:destination]
vnm = @vnm_dst
@@ -408,12 +410,28 @@ class ExecDriver < VirtualMachineDriver
}
end
+ vm_dir=File.dirname(action.data[:remote_dfile])
+
steps.concat([
# Execute pre-boot networking setup
{
:driver => :vnm,
:action => :pre
},
+ # Store vm.xml and ds.xml
+ {
+ :driver => :vmm,
+ :action => "/bin/cat - >#{vm_dir}/vm.xml",
+ :stdin => xml_data.elements['VM'].to_s,
+ :no_extra_params => true
+ },
+ {
+ :driver => :vmm,
+ :action => "/bin/cat - >#{vm_dir}/ds.xml",
+ :stdin => xml_data.elements['DATASTORE'].to_s,
+ :no_extra_params => true
+ },
+
# Boot the Virtual Machine
{
:driver => :vmm,
diff --git a/src/vmm_mad/remotes/kvm/detach_disk b/src/vmm_mad/remotes/kvm/detach_disk
index 8bb172ccba..2b54271fa5 100755
--- a/src/vmm_mad/remotes/kvm/detach_disk
+++ b/src/vmm_mad/remotes/kvm/detach_disk
@@ -23,10 +23,12 @@ DOMAIN="$1"
SOURCE="$2"
TARGET="$3"
TARGET_INDEX="$4"
+VMID="$5"
DETACH_PARAMS="--domain $DOMAIN --target $TARGET"
-exec_and_log "virsh --connect $LIBVIRT_URI detach-disk $DETACH_PARAMS" \
+\
+ exec_and_log "virsh --connect $LIBVIRT_URI detach-disk $DETACH_PARAMS" \
"Could not detach $TARGET from $DOMAIN"
virsh --connect $LIBVIRT_URI dumpxml $DOMAIN | \
diff --git a/src/vmm_mad/remotes/kvm/migrate b/src/vmm_mad/remotes/kvm/migrate
index 50a3f8abfd..ce81af4e63 100755
--- a/src/vmm_mad/remotes/kvm/migrate
+++ b/src/vmm_mad/remotes/kvm/migrate
@@ -93,7 +93,6 @@ done < <(echo $STDIN| $XPATH \
SHARED="${XPATH_ELEMENTS[j++]}"
VM_DIR="${XPATH_ELEMENTS[j++]}"
-
# use "force-share" param for qemu >= 2.10
[ "$(get_qemu_img_version)" -ge 2010000 ] && QEMU_IMG_PARAM="-U"
@@ -109,7 +108,7 @@ for SNAP in $SNAPS; do
done
if [ "$SHARED" = "YES" ]; then
- virsh --connect $LIBVIRT_URI migrate \
+ retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \
--live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system
RC=$?
@@ -163,6 +162,7 @@ else
fi
done
+
# freeze/suspend domain and rsync raw disks again
if [ -n "$RAW_DISKS" ]; then
if virsh --connect $LIBVIRT_URI domfsfreeze $DEPLOY_ID; then
@@ -191,7 +191,8 @@ else
DISK_OPTS="--copy-storage-all --migrate-disks ${MIGRATE_DISKS}"
fi
- virsh --connect $LIBVIRT_URI migrate \
+ retry_if "active block job" 3 5 \
+ virsh --connect $LIBVIRT_URI migrate \
--live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system \
$DISK_OPTS
RC=$?
diff --git a/src/vmm_mad/remotes/kvm/reset b/src/vmm_mad/remotes/kvm/reset
index 6a4b8ccf6d..2505e313b1 100755
--- a/src/vmm_mad/remotes/kvm/reset
+++ b/src/vmm_mad/remotes/kvm/reset
@@ -20,6 +20,7 @@ source $(dirname $0)/../../etc/vmm/kvm/kvmrc
source $(dirname $0)/../../scripts_common.sh
deploy_id=$1
+VMID=$3
exec_and_log "virsh --connect $LIBVIRT_URI reset $deploy_id" \
"Could not reset domain $deploy_id"
diff --git a/src/vmm_mad/remotes/kvm/resize_disk b/src/vmm_mad/remotes/kvm/resize_disk
index 2ab89be94b..34f2558849 100755
--- a/src/vmm_mad/remotes/kvm/resize_disk
+++ b/src/vmm_mad/remotes/kvm/resize_disk
@@ -20,11 +20,13 @@ DOMAIN=$1
DISK_ID=$2
DISK_SIZE=$3
DRV_ACTION=$4
+VMID=$6
source $(dirname $0)/../../etc/vmm/kvm/kvmrc
source $(dirname $0)/../../scripts_common.sh
get_disk_information "DISK_ID=$DISK_ID"
-exec_and_log "virsh --connect ${LIBVIRT_URI} blockresize ${DOMAIN} ${DISK_TARGET} ${DISK_SIZE}M" \
+retry_if "is in use" 3 5 \
+ exec_and_log "virsh --connect ${LIBVIRT_URI} blockresize ${DOMAIN} ${DISK_TARGET} ${DISK_SIZE}M" \
"Failed to resize disk ${DISK_ID} on ${DOMAIN}"
diff --git a/src/vmm_mad/remotes/kvm/restore b/src/vmm_mad/remotes/kvm/restore
index ec5760400a..ffc63c1e47 100755
--- a/src/vmm_mad/remotes/kvm/restore
+++ b/src/vmm_mad/remotes/kvm/restore
@@ -69,7 +69,7 @@ virsh --connect $LIBVIRT_URI save-image-dumpxml $FILE > $FILE_XML
# system DS migration
sed -i "s%$DS_LOCATION/[0-9]\+/$VM_ID/%$DS_LOCATION/$DS_ID/$VM_ID/%g" $FILE_XML
-sed -i "s%$DS_LOCATION_NON_DOUBLE_SLASH/[0-9]\+/$VMID/%$DS_LOCATION/$DS_ID/$VMID/%g" $FILE_XML
+sed -i "s%$DS_LOCATION_NON_DOUBLE_SLASH/[0-9]\+/$VM_ID/%$DS_LOCATION/$DS_ID/$VM_ID/%g" $FILE_XML
EOF
)
diff --git a/src/vmm_mad/remotes/kvm/save b/src/vmm_mad/remotes/kvm/save
index 55271b3338..dc9de6fcf8 100755
--- a/src/vmm_mad/remotes/kvm/save
+++ b/src/vmm_mad/remotes/kvm/save
@@ -38,7 +38,8 @@ fi
touch "$FILE"
chmod 666 "$FILE"
-exec_and_log "virsh --connect $LIBVIRT_URI save $DEPLOY_ID $FILE" \
+retry_if "active block job" 3 5 \
+ exec_and_log "virsh --connect $LIBVIRT_URI save $DEPLOY_ID $FILE" \
"Could not save $DEPLOY_ID to $FILE"
#-------------------------------------------------------------------------------
diff --git a/src/vmm_mad/remotes/kvm/shutdown b/src/vmm_mad/remotes/kvm/shutdown
index 1d6c6d8c20..4d2b09223d 100755
--- a/src/vmm_mad/remotes/kvm/shutdown
+++ b/src/vmm_mad/remotes/kvm/shutdown
@@ -22,6 +22,7 @@ source $(dirname $0)/../../scripts_common.sh
count=0
deploy_id=$1
+VMID=$3
shutdown_command="virsh --connect $LIBVIRT_URI shutdown $deploy_id"
diff --git a/src/vmm_mad/remotes/kvm/snapshot_create b/src/vmm_mad/remotes/kvm/snapshot_create
index da0ef2b6a4..c3342c0d86 100755
--- a/src/vmm_mad/remotes/kvm/snapshot_create
+++ b/src/vmm_mad/remotes/kvm/snapshot_create
@@ -43,7 +43,7 @@ DATASTORE_PATH="${XPATH_ELEMENTS[i++]}"
# -------- Create the snapshot and dump its metadata to xml file ------------
SNAP_NAME="snap-${SNAP_ID}"
-data=`virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
+data=`retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"`
if [ "$?" = "0" ]; then
echo "$data" | awk '{print $3}'
@@ -52,7 +52,7 @@ if [ "$?" = "0" ]; then
SNAP_XML_PATH="${DATASTORE_PATH}/${SNAP_NAME}.xml"
# dump snapshot metadata xml to the VM location
- virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
+ retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true
fi
else
error_message "Could not create snapshot $NAME for domain $DOMAIN."
diff --git a/src/vmm_mad/remotes/kvm/snapshot_revert b/src/vmm_mad/remotes/kvm/snapshot_revert
index 21c577eac4..44d5a51183 100755
--- a/src/vmm_mad/remotes/kvm/snapshot_revert
+++ b/src/vmm_mad/remotes/kvm/snapshot_revert
@@ -21,8 +21,9 @@ source $(dirname $0)/../../scripts_common.sh
DOMAIN="$1"
NAME="$2"
+VMID="$3"
-exec_and_log \
+retry_if "active block job" 3 5 exec_and_log \
"virsh --connect $LIBVIRT_URI snapshot-revert --force $DOMAIN $NAME" \
"Could not revert to snapshot $NAME for domain $DOMAIN."