diff --git a/install.sh b/install.sh index 23c7b031fd..8959b9e0d0 100755 --- a/install.sh +++ b/install.sh @@ -1722,7 +1722,8 @@ TM_SSH_FILES="src/tm_mad/ssh/clone \ src/tm_mad/ssh/monitor_ds \ src/tm_mad/ssh/cpds \ src/tm_mad/ssh/resize \ - src/tm_mad/ssh/ssh_utils.sh" + src/tm_mad/ssh/ssh_utils.sh \ + src/tm_mad/ssh/recovery_snap_create_live" TM_SSH_ETC_FILES="src/tm_mad/ssh/sshrc" diff --git a/src/im_mad/remotes/node-probes.d/monitor_ds.rb b/src/im_mad/remotes/node-probes.d/monitor_ds.rb index 31b12e8fea..256ce0e652 100755 --- a/src/im_mad/remotes/node-probes.d/monitor_ds.rb +++ b/src/im_mad/remotes/node-probes.d/monitor_ds.rb @@ -48,7 +48,8 @@ class DSMonitor # Skip if datastore is not marked for local monitoring next unless File.exist? "#{@ds_location}/#{ds_id}/.monitor" - puts ds_usage(ds_id) + tm = File.read("#{@ds_location}/#{ds_id}/.monitor").strip + puts ds_usage(ds_id, tm) end end @@ -64,16 +65,33 @@ class DSMonitor EOS end - def ds_usage(ds_id) + def replica_usage(path) + rs = 0 + o, _e, s = Open3.capture3("du -sh #{path}") + + if s.exitstatus == 0 && !o.empty? + du_a = o.split + rs = du_a.first if du_a + end + + 'REPLICA_CACHE = "YES",' << + "REPLICA_CACHE_SIZE = #{rs}," << + "REPLICA_IMAGES = #{num_images(path)}," + end + + def ds_usage(ds_id, tm) sizes = path_usage("#{@ds_location}/#{ds_id}") - unindent(<<-EOS) - DS = [ ID = #{ds_id}, + usage = "DS = [ ID = #{ds_id}," + usage += replica_usage("#{@ds_location}/#{ds_id}") if tm == "replica" + usage << <<-EOS USED_MB = #{sizes[0]}, TOTAL_MB = #{sizes[1]}, FREE_MB = #{sizes[2]} ] EOS + + unindent(usage) end def path_usage(path) @@ -87,6 +105,15 @@ class DSMonitor [metrics[2], metrics[1], metrics[3]] end + def num_images(path) + o, _e, s = Open3.capture3( + "find #{path} -maxdepth 1 -mindepth 1 -type f " << + "-regex '.*/[-a-f0-9]*' | wc -l 2>/dev/null") + + return "unknown" if s.exitstatus != 0 || o.empty? + o.strip + end + def unindent(str) mrc = str.match(/^(\s*)/) spaces = mrc[1].size diff --git a/src/mad/ruby/DriverExecHelper.rb b/src/mad/ruby/DriverExecHelper.rb index be55e5f389..485f4631c1 100644 --- a/src/mad/ruby/DriverExecHelper.rb +++ b/src/mad/ruby/DriverExecHelper.rb @@ -57,6 +57,8 @@ module DriverExecHelper # actions is remote or local. If the local actions has defined an special # script name this is used, otherwise the action name in downcase is # used as the script name. + # When action is a String starting with '/' it's considered alreay full + # path command and no modification is performed apart from adding params. # # @param [String, Symbol] action name of the action # @param [String] parameters arguments for the script @@ -64,7 +66,10 @@ module DriverExecHelper # @param [String, ''] directory to append to the scripts path for actions # @return [String] command line needed to execute the action def action_command_line(action, parameters, default_name=nil, directory='') - if action_is_local? action + if action.is_a?(String) && action[0] == '/' + return action + " " + parameters if parameters + return action + elsif action_is_local? action script_path=File.join(@local_scripts_path, directory) else script_path=File.join(@remote_scripts_path, directory) diff --git a/src/mad/ruby/OpenNebulaDriver.rb b/src/mad/ruby/OpenNebulaDriver.rb index 83860e5392..bf6aeb8ff5 100644 --- a/src/mad/ruby/OpenNebulaDriver.rb +++ b/src/mad/ruby/OpenNebulaDriver.rb @@ -90,10 +90,12 @@ class OpenNebulaDriver < ActionManager :respond => true, :ssh_stream => nil, :base64 => false, - :zip => false + :zip => false, + :no_extra_params => false }.merge(ops) - params = parameters + " #{id} #{host}" + params = parameters + params = "#{params} #{id} #{host}" unless options[:no_extra_params] command = action_command_line(aname, params, options[:script_name]) if action_is_local?(aname) diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh index 91e19acbb0..8f5b2b43a7 100644 --- a/src/mad/sh/scripts_common.sh +++ b/src/mad/sh/scripts_common.sh @@ -1,4 +1,4 @@ -# -------------------------------------------------------------------------- # +e -------------------------------------------------------------------------- # # Copyright 2002-2020, OpenNebula Project, OpenNebula Systems # # # # Licensed under the Apache License, Version 2.0 (the "License"); you may # @@ -42,6 +42,7 @@ MKSWAP=${MKSWAP:-mkswap} QEMU_IMG=${QEMU_IMG:-qemu-img} RADOS=${RADOS:-rados} RBD=${RBD:-rbd} +RDIFF=${RDIFF:-rdiff} READLINK=${READLINK:-readlink} RM=${RM:-rm} CP=${CP:-cp} @@ -162,6 +163,38 @@ function exclusive return $EXEC_RC } +# Retries if command fails and STDERR matches +function retry_if +{ + MATCH=$1 + TRIES=$2 + SLEEP=$3 + shift 3 + + unset TSTD TERR RC + + while [ $TRIES -gt 0 ]; do + TRIES=$(( TRIES - 1 )) + + eval "$( ("$@" ) \ + 2> >(TERR=$(cat); typeset -p TERR) \ + > >(TSTD=$(cat); typeset -p TSTD); RC=$?; typeset -p RC )" + + [ $RC -eq 0 ] && break + + if echo "$TERR" | grep -q "$MATCH"; then + sleep $SLEEP; + continue + fi + + break + done + + echo $TERR >&2 + echo $TSTD + return $RC +} + # Executes a command, if it fails returns error message and exits # If a second parameter is present it is used as the error message when # the command fails @@ -1154,4 +1187,4 @@ function send_to_monitor { # Send message echo "$msg_type $msg_result $msg_oid $payload_b64" | nc -u -w1 $mon_address $mon_port -} \ No newline at end of file +} diff --git a/src/tm_mad/ssh/clone b/src/tm_mad/ssh/clone index b9ad8cca4f..7b9e8527de 100755 --- a/src/tm_mad/ssh/clone +++ b/src/tm_mad/ssh/clone @@ -39,6 +39,7 @@ else fi . $TMCOMMON +. $SSH_UTILS DRIVER_PATH=$(dirname $0) XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" diff --git a/src/tm_mad/ssh/clone.replica b/src/tm_mad/ssh/clone.replica index 037b5bc7d3..b2b53cb6b0 100755 --- a/src/tm_mad/ssh/clone.replica +++ b/src/tm_mad/ssh/clone.replica @@ -76,10 +76,12 @@ while IFS= read -r -d '' element; do done < <(onevm show -x $VMID| $XPATH \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SIZE \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/ORIGINAL_SIZE \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/RECOVERY_SNAPSHOT_FREQ \ /VM/HISTORY_RECORDS/HISTORY[last\(\)]/DS_ID) SIZE="${XPATH_ELEMENTS[j++]}" ORIGINAL_SIZE="${XPATH_ELEMENTS[j++]}" +RECOVERY_SNAPSHOT_FREQ="${XPATH_ELEMENTS[j++]}" SYS_DS_ID="${XPATH_ELEMENTS[j++]}" # ------------------------------------------------------------------------------ @@ -109,12 +111,11 @@ REPLICA_STORAGE_IP=$(awk 'gsub(/[\0]/, x)' \ # Synchronize Image Datastore in the Replica Host. Use a recovery snapshot # if present in the RECOVERY_SNAPS_DIR # ------------------------------------------------------------------------------ -if ssh $REPLICA_HOST \ - "test -e ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID/${DST_FILE}.replica_snap"; then - - # point to [disk].replica_snap files +if recovery_snap_exists $REPLICA_HOST $VMID/$DST_FILE; then + # point to [disk].recovery_snaphost files SRC_DIR=${REPLICA_RECOVERY_SNAPS_DIR}/$VMID - SRC_FILE="${DST_FILE}.replica_snap" + SRC_FILE="${DST_FILE}.recovery_snapshot" + SRC_PATH="$SRC_DIR/$SRC_FILE" else rsync_img_to_replica "$SRC_PATH" "$REPLICA_HOST" @@ -156,9 +157,38 @@ fi ssh_forward ssh_exec_and_log $REPLICA_HOST "$CLONE_CMD" \ "Error copying $SRC to $DST" +# Create uncopressed base image copy on REPLICA for recovery snapshots +# and upload it's signature to the DST host + +if [ -n "$RECOVERY_SNAPSHOT_FREQ" ]; then + SNAP_DIR="$REPLICA_RECOVERY_SNAPS_DIR/$VMID" + + BASE_SIGN_CMD=$(cat < vmid dsid +# - host is the target host to deploy the VM +# - remote_system_ds is the path for the system datastore in the host +# - vmid is the id of the VM +# - dsid is the target datastore (0 is the system datastore) + +DST=$1 + +VMID=$2 +DSID=$3 + +if [ -z "${ONE_LOCATION}" ]; then + TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh + SSH_RC=/var/lib/one/remotes/etc/tm/ssh/sshrc + SSH_UTILS=/var/lib/one/remotes/tm/ssh/ssh_utils.sh +else + TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh + SSH_RC=$ONE_LOCATION/var/remotes/etc/tm/ssh/sshrc + SSH_UTILS=$ONE_LOCATION/var/remotes/tm/ssh/ssh_utils.sh +fi + +. $TMCOMMON +. $SSH_RC +. $SSH_UTILS + +#------------------------------------------------------------------------------- +# Delete recovery snapshots from replica_host, skip it for ACTION=14 (RECREATE) +#------------------------------------------------------------------------------- +REPLICA_HOST=$(get_replica_host $DSID) +if [ -n "$REPLICA_HOST" ]; then + XPATH="$(dirname $0)/../../datastore/xpath.rb" + HIST='/VM/HISTORY_RECORDS/HISTORY[last()]/ACTION' + ACTION="$(awk 'gsub(/[\0]/, x)' <(onevm show $VMID -x|$XPATH $HIST))" + + if [ $ACTION != "14" ]; then + ssh_exec_and_log "$REPLICA_HOST" \ + "rm -rf ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID" \ + "Error deleting ${REPLICA_RECOVERY_SNAPS_DIR}/$VMID" + fi +fi + +#------------------------------------------------------------------------------- +# Return if deleting a disk, we will delete them when removing the +# remote_system_ds directory for the VM (remotely) +#------------------------------------------------------------------------------- +DST_PATH=`arg_path $DST` +DST_HOST=`arg_host $DST` + +log "Deleting $DST_PATH" + +delete_file=$(cat < "${IMG_PATH}.md5sum" - fi + SRC_MD5SUM=$(md5sum_with_snaps "$IMG_PATH") - SRC_MD5SUM=$(cat "${IMG_PATH}.md5sum") - - # replica can not reuse md5sum_with_snaps, duplicate it - SCRIPT=$(cat </dev/null | cut -d " " -f 1 > ${IMG_PATH}.md5sum - else - CMD="md5sum $IMG_PATH" - - for S in \$SNAPS; do - CMD+=" ${IMG_PATH}.snap/${S}" - done - - \$CMD | md5sum | cut -d " " -f 1 > ${IMG_PATH}.md5sum - fi - - cat ${IMG_PATH}.md5sum - fi -EOF -) - - DST_MD5SUM=$(ssh "$REPLICA_HOST" "$SCRIPT") + DST_MD5SUM=`ssh $REPLICA_HOST bash -s < ssh, :respond => false, - :stdin => stdin) + :stdin => stdin, + :no_extra_params => no_extra_params) when :vnm if step[:destination] vnm = @vnm_dst @@ -408,12 +410,28 @@ class ExecDriver < VirtualMachineDriver } end + vm_dir=File.dirname(action.data[:remote_dfile]) + steps.concat([ # Execute pre-boot networking setup { :driver => :vnm, :action => :pre }, + # Store vm.xml and ds.xml + { + :driver => :vmm, + :action => "/bin/cat - >#{vm_dir}/vm.xml", + :stdin => xml_data.elements['VM'].to_s, + :no_extra_params => true + }, + { + :driver => :vmm, + :action => "/bin/cat - >#{vm_dir}/ds.xml", + :stdin => xml_data.elements['DATASTORE'].to_s, + :no_extra_params => true + }, + # Boot the Virtual Machine { :driver => :vmm, diff --git a/src/vmm_mad/remotes/kvm/detach_disk b/src/vmm_mad/remotes/kvm/detach_disk index 8bb172ccba..2b54271fa5 100755 --- a/src/vmm_mad/remotes/kvm/detach_disk +++ b/src/vmm_mad/remotes/kvm/detach_disk @@ -23,10 +23,12 @@ DOMAIN="$1" SOURCE="$2" TARGET="$3" TARGET_INDEX="$4" +VMID="$5" DETACH_PARAMS="--domain $DOMAIN --target $TARGET" -exec_and_log "virsh --connect $LIBVIRT_URI detach-disk $DETACH_PARAMS" \ +\ + exec_and_log "virsh --connect $LIBVIRT_URI detach-disk $DETACH_PARAMS" \ "Could not detach $TARGET from $DOMAIN" virsh --connect $LIBVIRT_URI dumpxml $DOMAIN | \ diff --git a/src/vmm_mad/remotes/kvm/migrate b/src/vmm_mad/remotes/kvm/migrate index 50a3f8abfd..ce81af4e63 100755 --- a/src/vmm_mad/remotes/kvm/migrate +++ b/src/vmm_mad/remotes/kvm/migrate @@ -93,7 +93,6 @@ done < <(echo $STDIN| $XPATH \ SHARED="${XPATH_ELEMENTS[j++]}" VM_DIR="${XPATH_ELEMENTS[j++]}" - # use "force-share" param for qemu >= 2.10 [ "$(get_qemu_img_version)" -ge 2010000 ] && QEMU_IMG_PARAM="-U" @@ -109,7 +108,7 @@ for SNAP in $SNAPS; do done if [ "$SHARED" = "YES" ]; then - virsh --connect $LIBVIRT_URI migrate \ + retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI migrate \ --live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system RC=$? @@ -163,6 +162,7 @@ else fi done + # freeze/suspend domain and rsync raw disks again if [ -n "$RAW_DISKS" ]; then if virsh --connect $LIBVIRT_URI domfsfreeze $DEPLOY_ID; then @@ -191,7 +191,8 @@ else DISK_OPTS="--copy-storage-all --migrate-disks ${MIGRATE_DISKS}" fi - virsh --connect $LIBVIRT_URI migrate \ + retry_if "active block job" 3 5 \ + virsh --connect $LIBVIRT_URI migrate \ --live $MIGRATE_OPTIONS $DEPLOY_ID $QEMU_PROTOCOL://$DEST_HOST/system \ $DISK_OPTS RC=$? diff --git a/src/vmm_mad/remotes/kvm/reset b/src/vmm_mad/remotes/kvm/reset index 6a4b8ccf6d..2505e313b1 100755 --- a/src/vmm_mad/remotes/kvm/reset +++ b/src/vmm_mad/remotes/kvm/reset @@ -20,6 +20,7 @@ source $(dirname $0)/../../etc/vmm/kvm/kvmrc source $(dirname $0)/../../scripts_common.sh deploy_id=$1 +VMID=$3 exec_and_log "virsh --connect $LIBVIRT_URI reset $deploy_id" \ "Could not reset domain $deploy_id" diff --git a/src/vmm_mad/remotes/kvm/resize_disk b/src/vmm_mad/remotes/kvm/resize_disk index 2ab89be94b..34f2558849 100755 --- a/src/vmm_mad/remotes/kvm/resize_disk +++ b/src/vmm_mad/remotes/kvm/resize_disk @@ -20,11 +20,13 @@ DOMAIN=$1 DISK_ID=$2 DISK_SIZE=$3 DRV_ACTION=$4 +VMID=$6 source $(dirname $0)/../../etc/vmm/kvm/kvmrc source $(dirname $0)/../../scripts_common.sh get_disk_information "DISK_ID=$DISK_ID" -exec_and_log "virsh --connect ${LIBVIRT_URI} blockresize ${DOMAIN} ${DISK_TARGET} ${DISK_SIZE}M" \ +retry_if "is in use" 3 5 \ + exec_and_log "virsh --connect ${LIBVIRT_URI} blockresize ${DOMAIN} ${DISK_TARGET} ${DISK_SIZE}M" \ "Failed to resize disk ${DISK_ID} on ${DOMAIN}" diff --git a/src/vmm_mad/remotes/kvm/restore b/src/vmm_mad/remotes/kvm/restore index ec5760400a..ffc63c1e47 100755 --- a/src/vmm_mad/remotes/kvm/restore +++ b/src/vmm_mad/remotes/kvm/restore @@ -69,7 +69,7 @@ virsh --connect $LIBVIRT_URI save-image-dumpxml $FILE > $FILE_XML # system DS migration sed -i "s%$DS_LOCATION/[0-9]\+/$VM_ID/%$DS_LOCATION/$DS_ID/$VM_ID/%g" $FILE_XML -sed -i "s%$DS_LOCATION_NON_DOUBLE_SLASH/[0-9]\+/$VMID/%$DS_LOCATION/$DS_ID/$VMID/%g" $FILE_XML +sed -i "s%$DS_LOCATION_NON_DOUBLE_SLASH/[0-9]\+/$VM_ID/%$DS_LOCATION/$DS_ID/$VM_ID/%g" $FILE_XML EOF ) diff --git a/src/vmm_mad/remotes/kvm/save b/src/vmm_mad/remotes/kvm/save index 55271b3338..dc9de6fcf8 100755 --- a/src/vmm_mad/remotes/kvm/save +++ b/src/vmm_mad/remotes/kvm/save @@ -38,7 +38,8 @@ fi touch "$FILE" chmod 666 "$FILE" -exec_and_log "virsh --connect $LIBVIRT_URI save $DEPLOY_ID $FILE" \ +retry_if "active block job" 3 5 \ + exec_and_log "virsh --connect $LIBVIRT_URI save $DEPLOY_ID $FILE" \ "Could not save $DEPLOY_ID to $FILE" #------------------------------------------------------------------------------- diff --git a/src/vmm_mad/remotes/kvm/shutdown b/src/vmm_mad/remotes/kvm/shutdown index 1d6c6d8c20..4d2b09223d 100755 --- a/src/vmm_mad/remotes/kvm/shutdown +++ b/src/vmm_mad/remotes/kvm/shutdown @@ -22,6 +22,7 @@ source $(dirname $0)/../../scripts_common.sh count=0 deploy_id=$1 +VMID=$3 shutdown_command="virsh --connect $LIBVIRT_URI shutdown $deploy_id" diff --git a/src/vmm_mad/remotes/kvm/snapshot_create b/src/vmm_mad/remotes/kvm/snapshot_create index da0ef2b6a4..c3342c0d86 100755 --- a/src/vmm_mad/remotes/kvm/snapshot_create +++ b/src/vmm_mad/remotes/kvm/snapshot_create @@ -43,7 +43,7 @@ DATASTORE_PATH="${XPATH_ELEMENTS[i++]}" # -------- Create the snapshot and dump its metadata to xml file ------------ SNAP_NAME="snap-${SNAP_ID}" -data=`virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"` +data=`retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-create-as $DOMAIN --name "$SNAP_NAME"` if [ "$?" = "0" ]; then echo "$data" | awk '{print $3}' @@ -52,7 +52,7 @@ if [ "$?" = "0" ]; then SNAP_XML_PATH="${DATASTORE_PATH}/${SNAP_NAME}.xml" # dump snapshot metadata xml to the VM location - virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true + retry_if "active block job" 3 5 virsh --connect $LIBVIRT_URI snapshot-dumpxml $DOMAIN $SNAP_NAME > $SNAP_XML_PATH || true fi else error_message "Could not create snapshot $NAME for domain $DOMAIN." diff --git a/src/vmm_mad/remotes/kvm/snapshot_revert b/src/vmm_mad/remotes/kvm/snapshot_revert index 21c577eac4..44d5a51183 100755 --- a/src/vmm_mad/remotes/kvm/snapshot_revert +++ b/src/vmm_mad/remotes/kvm/snapshot_revert @@ -21,8 +21,9 @@ source $(dirname $0)/../../scripts_common.sh DOMAIN="$1" NAME="$2" +VMID="$3" -exec_and_log \ +retry_if "active block job" 3 5 exec_and_log \ "virsh --connect $LIBVIRT_URI snapshot-revert --force $DOMAIN $NAME" \ "Could not revert to snapshot $NAME for domain $DOMAIN."