diff --git a/install.sh b/install.sh index 75184fc389..a14c09a7d2 100755 --- a/install.sh +++ b/install.sh @@ -1181,7 +1181,8 @@ DATASTORE_DRIVER_CEPH_SCRIPTS="src/datastore_mad/remotes/ceph/cp \ src/datastore_mad/remotes/ceph/snap_delete \ src/datastore_mad/remotes/ceph/snap_revert \ src/datastore_mad/remotes/ceph/snap_flatten \ - src/datastore_mad/remotes/ceph/ceph.conf" + src/datastore_mad/remotes/ceph/ceph.conf \ + src/datastore_mad/remotes/ceph/ceph_utils.sh" DATASTORE_DRIVER_DEV_SCRIPTS="src/datastore_mad/remotes/dev/cp \ src/datastore_mad/remotes/dev/mkfs \ diff --git a/share/etc/oned.conf b/share/etc/oned.conf index 93333acbce..1c8a82cc2e 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -432,7 +432,7 @@ IM_MAD = [ VM_MAD = [ name = "kvm", executable = "one_vmm_exec", - arguments = "-t 15 -r 0 kvm", + arguments = "-t 15 -r 0 -i kvm", default = "vmm_exec/vmm_exec_kvm.conf", type = "kvm" ] @@ -840,6 +840,7 @@ VNET_RESTRICTED_ATTR = "AR/BRIDGE" INHERIT_DATASTORE_ATTR = "CEPH_HOST" INHERIT_DATASTORE_ATTR = "CEPH_SECRET" INHERIT_DATASTORE_ATTR = "CEPH_USER" +INHERIT_DATASTORE_ATTR = "CEPH_CONF" INHERIT_DATASTORE_ATTR = "GLUSTER_HOST" INHERIT_DATASTORE_ATTR = "GLUSTER_VOLUME" diff --git a/src/datastore_mad/remotes/ceph/ceph_utils.sh b/src/datastore_mad/remotes/ceph/ceph_utils.sh new file mode 100644 index 0000000000..b79d6bb467 --- /dev/null +++ b/src/datastore_mad/remotes/ceph/ceph_utils.sh @@ -0,0 +1,171 @@ +# ----------------------------------------------------------------------------- # +# Copyright 2002-2015, OpenNebula Project (OpenNebula.org), C12G Labs # +# # +# Licensed under the Apache License, Version 2.0 (the "License"); you may # +# not use this file except in compliance with the License. You may obtain # +# a copy of the License at # +# # +# http://www.apache.org/licenses/LICENSE-2.0 # +# # +# Unless required by applicable law or agreed to in writing, software # +# distributed under the License is distributed on an "AS IS" BASIS, # +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # +# See the License for the specific language governing permissions and # +# limitations under the License. # +#------------------------------------------------------------------------------ # + +#-------------------------------------------------------------------------------- +# Make a base @snap for image clones +# @param $1 the volume +#-------------------------------------------------------------------------------- +rbd_make_snap() { + if [ "$(rbd_format $1)" = "2" ]; then + $RBD info "$1@snap" >/dev/null 2>&1 + + if [ "$?" != "0" ]; then + $RBD snap create "$1@snap" + $RBD snap protect "$1@snap" + fi + fi +} + +#-------------------------------------------------------------------------------- +# Remove the base @snap for image clones +# @param $1 the volume +# @param $2 (Optional) the snapshot name. If empty it defaults to 'snap' +#-------------------------------------------------------------------------------- +rbd_rm_snap() { + local snap + snap=${2:-snap} + + if [ "$(rbd_format $1)" = "2" ]; then + $RBD info "$1@$snap" >/dev/null 2>&1 + + if [ "$?" = "0" ]; then + $RBD snap unprotect "$1@$snap" + $RBD snap rm "$1@$snap" + fi + fi +} + +#-------------------------------------------------------------------------------- +# Find the snapshot in current volume or any of the snapshot volumes +# @param $1 volume base, i.e. /one-[--] +# @param $2 snapshot id +# @return volume name, exits if not found +#-------------------------------------------------------------------------------- +rbd_find_snap() { + local rbd_tgt pool vol + + $RBD --format json snap ls $1 2>/dev/null | grep -q "\"name\":\"$2\"" + + if [ "$?" = "0" ]; then + rbd_tgt=$1 + else + pool=$(echo $1 | cut -f1 -d'/') + vol=$(echo $1 | cut -f2 -d'/') + + rbd_tgt=$($RBD ls $pool | grep -E "$vol-(.+:)?$2(:|$)") + + if [ -z "${rbd_tgt}" ]; then + echo "Could not find a volume with snapshot $2" >&2 + exit 1 + fi + + rbd_tgt=$pool/$rbd_tgt + fi + + echo $rbd_tgt +} + +#-------------------------------------------------------------------------------- +# Rename the target volume to include the snapshot list or remove it if it has +# no snapshots +# @param $1 volume base, i.e. /one-[--] +# @param $2 volume to rename or remove +#-------------------------------------------------------------------------------- +rbd_rename_rm() { + local snapids + + snapids=$($RBD --format json snap ls $2 | \ + grep -Po '(?<="name":")\d+' | \ + paste -d: -s) + + if [ -z "$snapids" ]; then + $RBD rm $2 + else + $RBD rename $2 $1-$snapids + fi +} + +#-------------------------------------------------------------------------------- +# Return format of rbd volume +# @param $1 the volume +# @return the format of the volume +#-------------------------------------------------------------------------------- +rbd_format() { + $RBD info $1 | sed -n 's/.*format: // p' +} + +#-------------------------------------------------------------------------------- +# Check if given volume id formta 2, exits otherwise +# @param $1 the volume +#-------------------------------------------------------------------------------- +rbd_check_2() { + if [ "$(rbd_format $1)" != "2" ]; then + echo "Only RBD Format 2 is supported for this operation" >&2 + exit 1 + fi +} + +#-------------------------------------------------------------------------------- +# Get top parent of a snapshot hierarchy if the volume has snapshots. +# @param $1 the volume +# @return the top parent or volume in no snapshots found +#-------------------------------------------------------------------------------- +rbd_top_parent() { + local pool snap0 volume + + pool=$(echo $1 | cut -f1 -d'/') + volume=$(echo $1 | cut -f2 -d'/') + + snap0=$($RBD ls -l $pool | awk '{print $1}' | grep -E "$volume(-.+)?@0") + + if [ -n "$snap0" ]; then + volume=$pool/${snap0%%@*} + else + volume=$1 + fi + + echo $volume +} + +#-------------------------------------------------------------------------------- +# Remove all the images and snapshots from a given volume +# @param $1 the volume (or snapshot to delete) +#-------------------------------------------------------------------------------- +rbd_rm_r() { + local rbd rbd_base children snaps + + rbd=$1 + rbd_base=${rbd%%@*} + + if [ "$rbd" != "$rbd_base" ]; then + children=$($RBD children $rbd 2>/dev/null) + + for child in $children; do + rbd_rm_r $child + done + + $RBD snap unprotect $rbd + $RBD snap rm $rbd + else + snaps=$($RBD snap ls $rbd 2>/dev/null| awk 'NR > 1 {print $2}') + + for snap in $snaps; do + rbd_rm_r $rbd@$snap + done + + $RBD rm $rbd + fi +} diff --git a/src/datastore_mad/remotes/ceph/clone b/src/datastore_mad/remotes/ceph/clone index 4253d3f5ca..0e176d2be5 100755 --- a/src/datastore_mad/remotes/ceph/clone +++ b/src/datastore_mad/remotes/ceph/clone @@ -47,18 +47,22 @@ while IFS= read -r -d '' element; do done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RBD_FORMAT \ /DS_DRIVER_ACTION_DATA/IMAGE/PATH \ /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) unset i BASE_PATH="${XPATH_ELEMENTS[i++]}" BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" POOL_NAME="${XPATH_ELEMENTS[i++]:-$POOL_NAME}" +RBD_FORMAT="${XPATH_ELEMENTS[i++]:-$RBD_FORMAT}" SRC="${XPATH_ELEMENTS[i++]}" SIZE="${XPATH_ELEMENTS[i++]}" CEPH_USER="${XPATH_ELEMENTS[i++]}" +CEPH_CONF="${XPATH_ELEMENTS[i++]}" DST_HOST=`get_destination_host $ID` @@ -67,6 +71,10 @@ if [ -z "$DST_HOST" ]; then exit -1 fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi @@ -79,5 +87,4 @@ RBD_DST="${POOL_NAME}/${IMAGE_NAME}" ssh_exec_and_log "$DST_HOST" "$RBD copy $SRC $RBD_DST" \ "Error cloning $SRC to $RBD_DST in $DST_HOST" - echo "$RBD_DST" diff --git a/src/datastore_mad/remotes/ceph/cp b/src/datastore_mad/remotes/ceph/cp index 5510fc5aa8..4918e88dc1 100755 --- a/src/datastore_mad/remotes/ceph/cp +++ b/src/datastore_mad/remotes/ceph/cp @@ -61,7 +61,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) unset i @@ -79,6 +80,7 @@ SHA1="${XPATH_ELEMENTS[i++]}" NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}" LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}" CEPH_USER="${XPATH_ELEMENTS[i++]}" +CEPH_CONF="${XPATH_ELEMENTS[i++]}" DST_HOST=`get_destination_host $ID` @@ -91,6 +93,10 @@ if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" IMAGE_HASH=`generate_image_hash` diff --git a/src/datastore_mad/remotes/ceph/mkfs b/src/datastore_mad/remotes/ceph/mkfs index 6887672489..0fbb31b6e1 100755 --- a/src/datastore_mad/remotes/ceph/mkfs +++ b/src/datastore_mad/remotes/ceph/mkfs @@ -55,7 +55,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RBD_FORMAT \ /DS_DRIVER_ACTION_DATA/IMAGE/FSTYPE \ /DS_DRIVER_ACTION_DATA/IMAGE/SIZE \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) unset i @@ -69,6 +70,7 @@ RBD_FORMAT="${XPATH_ELEMENTS[i++]:-$RBD_FORMAT}" FSTYPE="${XPATH_ELEMENTS[i++]}" SIZE="${XPATH_ELEMENTS[i++]}" CEPH_USER="${XPATH_ELEMENTS[i++]}" +CEPH_CONF="${XPATH_ELEMENTS[i++]}" DST_HOST=`get_destination_host $ID` @@ -81,6 +83,10 @@ if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" IMAGE_HASH=`generate_image_hash` diff --git a/src/datastore_mad/remotes/ceph/monitor b/src/datastore_mad/remotes/ceph/monitor index 5a0ad57a24..23430fcc7e 100755 --- a/src/datastore_mad/remotes/ceph/monitor +++ b/src/datastore_mad/remotes/ceph/monitor @@ -47,11 +47,13 @@ while IFS= read -r -d '' element; do XPATH_ELEMENTS[i++]="$element" done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) BRIDGE_LIST="${XPATH_ELEMENTS[j++]}" POOL_NAME="${XPATH_ELEMENTS[j++]:-$POOL_NAME}" CEPH_USER="${XPATH_ELEMENTS[j++]}" +CEPH_CONF="${XPATH_ELEMENTS[j++]}" HOST=`get_destination_host` @@ -64,6 +66,11 @@ if [ -n "$CEPH_USER" ]; then CEPH="$CEPH --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + CEPH="$CEPH --conf ${CEPH_CONF}" + RADOS="$RADOS --conf ${CEPH_CONF}" +fi + # ------------ Compute datastore usage ------------- MONITOR_SCRIPT=$(cat </dev/null) - - for child in \$CHILDREN; do - snap_id=\${child##*-} - child=\$child@\$snap_id - rm_children \$child - done - - $RBD snap unprotect \$rbd_snap - $RBD snap rm \$rbd_snap - fi - - $RBD rm \$rbd - } - - RBD_FORMAT=\$($RBD info $RBD_SRC | sed -n 's/.*format: // p') - - if [ "\$RBD_FORMAT" = "2" ]; then - has_snap_shots=\$($RBD info $RBD_SRC-0@0 2>/dev/null) - - if [ -n "\$has_snap_shots" ]; then - rm_children $RBD_SRC-0@0 - else - $RBD rm $RBD_SRC - fi + if [ "\$(rbd_format $RBD_SRC)" = "2" ]; then + rbd_rm_r \$(rbd_top_parent $RBD_SRC) else $RBD rm $RBD_SRC fi EOF ) -ssh_exec_and_log "$DST_HOST" "$DELETE_CMD" \ +ssh_exec_and_log_stdin "$DST_HOST" "$DELETE_CMD" "$CEPH_UTILS" \ "Error deleting $RBD_SRC in $DST_HOST" - -exit 0 diff --git a/src/datastore_mad/remotes/ceph/snap_delete b/src/datastore_mad/remotes/ceph/snap_delete index a5bc821540..f3e62eb304 100755 --- a/src/datastore_mad/remotes/ceph/snap_delete +++ b/src/datastore_mad/remotes/ceph/snap_delete @@ -31,6 +31,8 @@ fi . $LIB_LOCATION/sh/scripts_common.sh DRIVER_PATH=$(dirname $0) +CEPH_UTILS=${DRIVER_PATH}/ceph_utils.sh + source ${DRIVER_PATH}/../libfs.sh source ${DRIVER_PATH}/ceph.conf @@ -49,7 +51,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \ /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \ /DS_DRIVER_ACTION_DATA/IMAGE/TARGET_SNAPSHOT \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) unset i @@ -58,6 +61,7 @@ POOL_NAME="${XPATH_ELEMENTS[i++]:-$POOL_NAME}" RBD_SRC="${XPATH_ELEMENTS[i++]}" SNAP_ID="${XPATH_ELEMENTS[i++]}" CEPH_USER="${XPATH_ELEMENTS[i++]}" +CEPH_CONF="${XPATH_ELEMENTS[i++]}" DST_HOST=`get_destination_host $ID` @@ -70,21 +74,30 @@ if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + SNAP_DELETE_CMD=$(cat <&2 + RBD_TGT=\$(rbd_find_snap $RBD_SRC $SNAP_ID) + + if [ -z "\$RBD_TGT" ]; then + echo "Target RBD not found." >&2 exit 1 fi - $RBD snap unprotect $RBD_SRC-$SNAP_ID@$SNAP_ID - $RBD snap rm $RBD_SRC-$SNAP_ID@$SNAP_ID - $RBD rm $RBD_SRC-$SNAP_ID + $RBD snap unprotect \${RBD_TGT}@$SNAP_ID + $RBD snap rm \${RBD_TGT}@$SNAP_ID + + if [ "\${RBD_TGT}" != "$RBD_SRC" ]; then + rbd_rename_rm $RBD_SRC \${RBD_TGT} + fi EOF ) -ssh_exec_and_log "$DST_HOST" "$SNAP_DELETE_CMD" \ +ssh_exec_and_log_stdin "$DST_HOST" "$SNAP_DELETE_CMD" "$CEPH_UTILS" \ "Error deleting snapshot $RBD_SRC-$SNAP_ID@$SNAP_ID" diff --git a/src/datastore_mad/remotes/ceph/snap_flatten b/src/datastore_mad/remotes/ceph/snap_flatten index f5ace4fd10..a35b25911e 100755 --- a/src/datastore_mad/remotes/ceph/snap_flatten +++ b/src/datastore_mad/remotes/ceph/snap_flatten @@ -31,6 +31,8 @@ fi . $LIB_LOCATION/sh/scripts_common.sh DRIVER_PATH=$(dirname $0) +CEPH_UTILS=${DRIVER_PATH}/ceph_utils.sh + source ${DRIVER_PATH}/../libfs.sh source ${DRIVER_PATH}/ceph.conf @@ -49,7 +51,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \ /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \ /DS_DRIVER_ACTION_DATA/IMAGE/TARGET_SNAPSHOT \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) unset i @@ -58,6 +61,7 @@ POOL_NAME="${XPATH_ELEMENTS[i++]:-$POOL_NAME}" RBD_SRC="${XPATH_ELEMENTS[i++]}" SNAP_ID="${XPATH_ELEMENTS[i++]}" CEPH_USER="${XPATH_ELEMENTS[i++]}" +CEPH_CONF="${XPATH_ELEMENTS[i++]}" DST_HOST=`get_destination_host $ID` @@ -70,46 +74,31 @@ if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + SNAP_FLATTEN_CMD=$(cat </dev/null) - - for child in \$CHILDREN; do - snap_id=\${child##*-} - child=\$child@\$snap_id - rm_children \$child - done - - $RBD snap unprotect \$rbd_snap - $RBD snap rm \$rbd_snap - fi - - $RBD rm \$rbd - } - - RBD_FORMAT=\$($RBD info $RBD_SRC | sed -n 's/.*format: // p') - - if [ "\${RBD_FORMAT}" != "2" ]; then - echo "Only RBD Format 2 is supported for this operation" >&2 + if [ -z "\$RBD_TGT" ]; then + echo "Target RBD not found." >&2 exit 1 fi - $RBD clone $RBD_SRC-$SNAP_ID@$SNAP_ID $RBD_SRC.flatten + $RBD clone \${RBD_TGT}@$SNAP_ID $RBD_SRC.flatten + $RBD flatten $RBD_SRC.flatten - rm_children $RBD_SRC-0@0 + rbd_rm_r \$(rbd_top_parent $RBD_SRC) $RBD rename $RBD_SRC.flatten $RBD_SRC EOF ) -ssh_exec_and_log "$DST_HOST" "$SNAP_FLATTEN_CMD" \ - "Error flattening snapshot $RBD_SRC-$SNAP_ID@$SNAP_ID" +ssh_exec_and_log_stdin "$DST_HOST" "$SNAP_FLATTEN_CMD" "$CEPH_UTILS" \ + "Error flattening snapshot $SNAP_ID for $RBD_SRC" diff --git a/src/datastore_mad/remotes/ceph/snap_revert b/src/datastore_mad/remotes/ceph/snap_revert index 0412c89724..d81f8e80d8 100755 --- a/src/datastore_mad/remotes/ceph/snap_revert +++ b/src/datastore_mad/remotes/ceph/snap_revert @@ -31,6 +31,8 @@ fi . $LIB_LOCATION/sh/scripts_common.sh DRIVER_PATH=$(dirname $0) +CEPH_UTILS=${DRIVER_PATH}/ceph_utils.sh + source ${DRIVER_PATH}/../libfs.sh source ${DRIVER_PATH}/ceph.conf @@ -49,7 +51,8 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \ /DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \ /DS_DRIVER_ACTION_DATA/IMAGE/TARGET_SNAPSHOT \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_USER \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CEPH_CONF) unset i @@ -58,6 +61,7 @@ POOL_NAME="${XPATH_ELEMENTS[i++]:-$POOL_NAME}" RBD_SRC="${XPATH_ELEMENTS[i++]}" SNAP_ID="${XPATH_ELEMENTS[i++]}" CEPH_USER="${XPATH_ELEMENTS[i++]}" +CEPH_CONF="${XPATH_ELEMENTS[i++]}" DST_HOST=`get_destination_host $ID` @@ -70,20 +74,27 @@ if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + SNAP_REVERT_CMD=$(cat <&2 + rbd_rename_rm $RBD_SRC $RBD_SRC + + RBD_TGT=\$(rbd_find_snap $RBD_SRC $SNAP_ID) + + if [ -z "\$RBD_TGT" ]; then + echo "Target RBD not found." >&2 exit 1 fi - $RBD rm $RBD_SRC - $RBD clone $RBD_SRC-$SNAP_ID@$SNAP_ID $RBD_SRC + $RBD clone \${RBD_TGT}@$SNAP_ID $RBD_SRC EOF ) -ssh_exec_and_log "$DST_HOST" "$SNAP_REVERT_CMD" \ - "Error reverting snapshot $RBD_SRC-$SNAP_ID@$SNAP_ID" +ssh_exec_and_log_stdin "$DST_HOST" "$SNAP_REVERT_CMD" "$CEPH_UTILS" \ + "Error reverting snapshot $SNAP_ID for $RBD_DST" diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh index 95ba79bcd6..5c0d1a85b7 100644 --- a/src/mad/sh/scripts_common.sh +++ b/src/mad/sh/scripts_common.sh @@ -387,6 +387,36 @@ EOF` fi } +# Remote command execution over SSH preloading a local file +# $1: HOSTNAME +# $2: COMMAND +# $3: file to be loaded into the script +# $4: ERROR_REPORT +function ssh_exec_and_log_stdin +{ + SSH_EXEC_ERR=`$SSH $1 sh -s 2>&1 1>/dev/null </dev/null 2>&1 && exit 0 + + rbd_make_snap $SRC_PATH + set -e - $RBD info $RBD_DST > /dev/null 2>&1 && exit 0 - - RBD_FORMAT=\$($RBD info $SRC_PATH | sed -n 's/.*format: // p') - - if [ "\$RBD_FORMAT" = "2" ]; then - $RBD snap create "$SRC_PATH@$RBD_SNAP" - $RBD snap protect "$SRC_PATH@$RBD_SNAP" - $RBD clone "$SRC_PATH@$RBD_SNAP" $RBD_DST + if [ "\$(rbd_format $SRC_PATH)" = "2" ]; then + $RBD clone "$SRC_PATH@snap" $RBD_DST else $RBD copy $SRC_PATH $RBD_DST fi @@ -97,6 +110,6 @@ CLONE_CMD=$(cat <&2 + exit 1 + fi + + RBD_DST=\$RBD_DST@$SNAP_ID + fi + + $RBD copy $RBD_DST $DST +EOF +) + +ssh_exec_and_log_stdin "$SRC_HOST" "$COPY_CMD" "$CEPH_UTILS" \ + "Error cloning $RBD_DST to $DST in $SRC_HOST" diff --git a/src/tm_mad/ceph/delete b/src/tm_mad/ceph/delete index e4101fb365..720b4ad934 100755 --- a/src/tm_mad/ceph/delete +++ b/src/tm_mad/ceph/delete @@ -24,17 +24,24 @@ DST=$1 VM_ID=$2 DS_ID=$3 +#-------------------------------------------------------------------------------- + if [ -z "${ONE_LOCATION}" ]; then TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh + LIB_LOCATION=/usr/lib/one else TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh + LIB_LOCATION=$ONE_LOCATION/lib fi DRIVER_PATH=$(dirname $0) +source $LIB_LOCATION/sh/scripts_common.sh source $TMCOMMON source ${DRIVER_PATH}/../../datastore/ceph/ceph.conf +CEPH_UTILS=${DRIVER_PATH}/../../datastore/ceph/ceph_utils.sh + #------------------------------------------------------------------------------- # Process destination #------------------------------------------------------------------------------- @@ -68,11 +75,13 @@ while IFS= read -r -d '' element; do done < <(onevm show -x $VM_ID| $XPATH \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE \ - /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_USER) + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_USER \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_CONF) SRC="${XPATH_ELEMENTS[j++]}" CLONE="${XPATH_ELEMENTS[j++]}" CEPH_USER="${XPATH_ELEMENTS[j++]}" +CEPH_CONF="${XPATH_ELEMENTS[j++]}" # No need to delete no cloned images if [ "$CLONE" = "NO" ]; then @@ -83,6 +92,10 @@ if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + # cloned, so the name will be "/one---" RBD_SRC="${SRC}-${VM_ID}-${DISK_ID}" RBD_SNAP="${VM_ID}-${DISK_ID}" @@ -93,57 +106,17 @@ RBD_SNAP="${VM_ID}-${DISK_ID}" log "Deleting $DST_PATH" -# Note that this command, as opposed to the rest of $RBD commands in this set of -# drivers, is executed in the worker node and not in the CEPH frontend. - DELETE_CMD=$(cat </dev/null) - - for child in \$CHILDREN; do - snap_id=\${child##*-} - child=\$child@\$snap_id - rm_children \$child - done - - $RBD snap unprotect \$rbd_snap - $RBD snap rm \$rbd_snap - fi - - $RBD rm \$rbd - } - - RBD_FORMAT=\$($RBD info $RBD_SRC | sed -n 's/.*format: // p') - - if [ "\$RBD_FORMAT" = "2" ]; then - has_snap_shots=\$($RBD info $RBD_SRC-0@0 2>/dev/null) - - if [ -n "\$has_snap_shots" ]; then - rm_children $RBD_SRC-0@0 - else - $RBD rm $RBD_SRC - fi + if [ "\$(rbd_format $RBD_SRC)" = "2" ]; then + rbd_rm_r \$(rbd_top_parent $RBD_SRC) + rbd_rm_snap $SRC $RBD_SNAP else $RBD rm $RBD_SRC fi - - # Remove the snapshot of the original image used to create a CLONE - if [ "\$RBD_FORMAT" = "2" ]; then - $RBD snap unprotect $SRC@$RBD_SNAP - $RBD snap rm $SRC@$RBD_SNAP - fi EOF ) -ssh_exec_and_log "$DST_HOST" "$DELETE_CMD" \ +ssh_exec_and_log_stdin "$DST_HOST" "$DELETE_CMD" "$CEPH_UTILS" \ "Error deleting $RBD_SRC in $DST_HOST" - -exit 0 diff --git a/src/tm_mad/ceph/ln b/src/tm_mad/ceph/ln index bf7bd35deb..5d08b8216d 100755 --- a/src/tm_mad/ceph/ln +++ b/src/tm_mad/ceph/ln @@ -22,4 +22,73 @@ # - host is the target host to deploy the VM # - remote_system_ds is the path for the system datastore in the host +SRC=$1 +DST=$2 +VM_ID=$3 +DS_ID=$4 + +#-------------------------------------------------------------------------------- + +if [ -z "${ONE_LOCATION}" ]; then + TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh + LIB_LOCATION=/usr/lib/one +else + TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh + LIB_LOCATION=$ONE_LOCATION/lib +fi + +DRIVER_PATH=$(dirname $0) + +source $LIB_LOCATION/sh/scripts_common.sh +source $TMCOMMON +source ${DRIVER_PATH}/../../datastore/ceph/ceph.conf + +CEPH_UTILS=${DRIVER_PATH}/../../datastore/ceph/ceph_utils.sh + +#------------------------------------------------------------------------------- +# Process destination +#------------------------------------------------------------------------------- + +DST_HOST=`arg_host $DST` + +#------------------------------------------------------------------------------- +# Get Image information & destination image +#------------------------------------------------------------------------------- + +SRC_PATH=`arg_path $SRC` + +XPATH="${DRIVER_PATH}/../../datastore/xpath.rb --stdin" + +unset i j XPATH_ELEMENTS + +while IFS= read -r -d '' element; do + XPATH_ELEMENTS[i++]="$element" + done < <(onevm show -x $VM_ID| $XPATH \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_USER \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_CONF) + +CEPH_USER="${XPATH_ELEMENTS[j++]}" +CEPH_CONF="${XPATH_ELEMENTS[j++]}" + +#------------------------------------------------------------------------------- +# Remove any @snap in image (non persistent -> persistnet) +#------------------------------------------------------------------------------- + +if [ -n "$CEPH_USER" ]; then + RBD="$RBD --id ${CEPH_USER}" +fi + +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + +CLONE_CMD=$(cat <&2 - exit 1 - fi + rbd_check_2 $RBD_DST $RBD snap create $RBD_DST@$SNAP_ID $RBD snap protect $RBD_DST@$SNAP_ID @@ -93,7 +103,7 @@ SNAP_CREATE_CMD=$(cat < +# 2.- any of the snapshot volumes: one-[--]- #------------------------------------------------------------------------------- if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + SNAP_DELETE_CMD=$(cat <&2 + RBD_TGT=\$(rbd_find_snap $RBD_DST $SNAP_ID) + + if [ -z "\$RBD_TGT" ]; then + echo "Target RBD not found." >&2 exit 1 fi - $RBD snap unprotect $RBD_DST-$SNAP_ID@$SNAP_ID - $RBD snap rm $RBD_DST-$SNAP_ID@$SNAP_ID - $RBD rm $RBD_DST-$SNAP_ID + $RBD snap unprotect \${RBD_TGT}@$SNAP_ID + $RBD snap rm \${RBD_TGT}@$SNAP_ID + + if [ "\${RBD_TGT}" != "$RBD_DST" ]; then + rbd_rename_rm $RBD_DST \${RBD_TGT} + fi EOF ) -ssh_exec_and_log "$SRC_HOST" "$SNAP_DELETE_CMD" \ - "Error deleting snapshot $RBD_DST-$SNAP_ID@$SNAP_ID" - -exit 0 +ssh_exec_and_log_stdin "$SRC_HOST" "$SNAP_DELETE_CMD" "$CEPH_UTILS" \ + "Error deleting snapshot $SNAP_ID" diff --git a/src/tm_mad/ceph/snap_revert b/src/tm_mad/ceph/snap_revert index 971962e797..909aa9cd0f 100755 --- a/src/tm_mad/ceph/snap_revert +++ b/src/tm_mad/ceph/snap_revert @@ -23,17 +23,24 @@ SNAP_ID=$2 VM_ID=$3 DS_ID=$4 +#-------------------------------------------------------------------------------- + if [ -z "${ONE_LOCATION}" ]; then TMCOMMON=/var/lib/one/remotes/tm/tm_common.sh + LIB_LOCATION=/usr/lib/one else TMCOMMON=$ONE_LOCATION/var/remotes/tm/tm_common.sh + LIB_LOCATION=$ONE_LOCATION/lib fi DRIVER_PATH=$(dirname $0) +source $LIB_LOCATION/sh/scripts_common.sh source $TMCOMMON source ${DRIVER_PATH}/../../datastore/ceph/ceph.conf +CEPH_UTILS=${DRIVER_PATH}/../../datastore/ceph/ceph_utils.sh + #------------------------------------------------------------------------------- # Set dst path and dir #------------------------------------------------------------------------------- @@ -56,11 +63,13 @@ while IFS= read -r -d '' element; do done < <(onevm show -x $VM_ID| $XPATH \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/SOURCE \ /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CLONE \ - /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_USER) + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_USER \ + /VM/TEMPLATE/DISK[DISK_ID=$DISK_ID]/CEPH_CONF) RBD_SRC="${XPATH_ELEMENTS[j++]}" CLONE="${XPATH_ELEMENTS[j++]}" CEPH_USER="${XPATH_ELEMENTS[j++]}" +CEPH_CONF="${XPATH_ELEMENTS[j++]}" if [ "$CLONE" = "NO" ]; then RBD_DST="${RBD_SRC}" @@ -69,29 +78,54 @@ else fi #------------------------------------------------------------------------------- -# Create snapshots +# Revert to Snapshot. Using the following tree structure +# +# one-3-13-0-1:2:3 +# ---------------- +# | @1 (snap 1) | revert to 2 +# | @2 (snap 2) |------------ +# | @3 (snap 3) | clone | +# ---------------- | +# V +# one-3-13-0-4:5 (stores snaps @4,@5 children of @2) +# -------------- +# revert to 4 | @4 (snap 4)| +# --------------| @5 (snap 5)| +# | clone -------------- +# | +# V +# one-3-13-0 (current volume) +# -------------- +# | @6 (snap 6)| +# -------------- #------------------------------------------------------------------------------- if [ -n "$CEPH_USER" ]; then RBD="$RBD --id ${CEPH_USER}" fi +if [ -n "$CEPH_CONF" ]; then + RBD="$RBD --conf ${CEPH_CONF}" +fi + SNAP_REVERT_CMD=$(cat <&2 + rbd_rename_rm $RBD_DST $RBD_DST + + RBD_TGT=\$(rbd_find_snap $RBD_DST $SNAP_ID) + + if [ -z "\$RBD_TGT" ]; then + echo "Target RBD not found." >&2 exit 1 fi - $RBD rm $RBD_DST - $RBD clone $RBD_DST-$SNAP_ID@$SNAP_ID $RBD_DST + $RBD clone \${RBD_TGT}@$SNAP_ID $RBD_DST EOF ) -ssh_exec_and_log "$SRC_HOST" "$SNAP_REVERT_CMD" \ - "Error reverting snapshot $RBD_DST-$SNAP_ID@$SNAP_ID" +ssh_exec_and_log_stdin "$SRC_HOST" "$SNAP_REVERT_CMD" "$CEPH_UTILS" \ + "Error reverting snapshot $SNAP_ID for $RBD_DST" -exit 0 diff --git a/src/vmm_mad/exec/vmm_execrc b/src/vmm_mad/exec/vmm_execrc index e3aba75579..8c899f9934 100644 --- a/src/vmm_mad/exec/vmm_execrc +++ b/src/vmm_mad/exec/vmm_execrc @@ -16,4 +16,4 @@ # Space separated list of VMM-TM pairs that support live disk snapshots. VMM # and TM must be separated by '-' -LIVE_DISK_SNAPSHOTS="kvm-qcow2" +LIVE_DISK_SNAPSHOTS="kvm-qcow2 kvm-ceph"