From 5d530dd70ce7dbb293712a258d37082338486d50 Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Fri, 20 Feb 2015 16:18:07 +0100 Subject: [PATCH 1/9] Feature #3574: ds_mad/fs/cp now support BRIDGE_LIST --- src/datastore_mad/remotes/fs/cp | 59 ++++++++++++++++++------------ src/datastore_mad/remotes/libfs.sh | 2 +- 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index a84e4c8f8b..3da6ea64f1 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -50,51 +50,64 @@ while IFS= read -r -d '' element; do done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/STAGING_DIR \ /DS_DRIVER_ACTION_DATA/IMAGE/PATH \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW) -BASE_PATH="${XPATH_ELEMENTS[0]}" -RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}" -SAFE_DIRS="${XPATH_ELEMENTS[2]}" -SRC="${XPATH_ELEMENTS[3]}" -MD5="${XPATH_ELEMENTS[4]}" -SHA1="${XPATH_ELEMENTS[5]}" -NO_DECOMPRESS="${XPATH_ELEMENTS[6]}" -LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[7]}" +unset i + +BASE_PATH="${XPATH_ELEMENTS[i++]}" +RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}" +SAFE_DIRS="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +STAGING_DIR="${XPATH_ELEMENTS[i++]:-/var/tmp}" +SRC="${XPATH_ELEMENTS[i++]}" +MD5="${XPATH_ELEMENTS[i++]}" +SHA1="${XPATH_ELEMENTS[i++]}" +NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}" +LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}" mkdir -p "$BASE_PATH" - set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" DST=`generate_image_path` +IMAGE_HASH=`basename $DST` -DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$DST"` +if [ -n "$BRIDGE_LIST" ]; then + DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -` +else + DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" "$DST"` +fi COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS" -# ------------ Copy the image to the repository ------------- - -case $SRC in -http://*|https://*) - log "Downloading $SRC to the image repository" - - exec_and_log "$COPY_COMMAND" "Error downloading $SRC" - ;; - -*) - if [ `check_restricted $SRC` -eq 1 ]; then +if echo "$SRC" | grep -vq '^https\?://'; then + if [ `check_restricted 1` -eq 1 ]; then log_error "Not allowed to copy images from $RESTRICTED_DIRS" error_message "Not allowed to copy image file $SRC" exit -1 fi log "Copying local image $SRC to the image repository" +else + log "Downloading image $SRC to the image repository" +fi +if [ -n "$BRIDGE_LIST" ]; then + DST_HOST=`get_destination_host $ID` + TMP_DST="$STAGING_DIR/$IMAGE_HASH" + + exec_and_log "eval $COPY_COMMAND | $SSH $DST_HOST $DD of=$TMP_DST bs=64k" \ + "Error dumping $SRC to $DST_HOST:$TMP_DST" + + ssh_exec_and_log "$DST_HOST" "mkdir -p $BASE_PATH; mv -f $TMP_DST $DST" \ + "Error moving $TMP_DST to $DST in $DST_HOST" +else exec_and_log "$COPY_COMMAND" "Error copying $SRC to $DST" - ;; -esac +fi echo "$DST" diff --git a/src/datastore_mad/remotes/libfs.sh b/src/datastore_mad/remotes/libfs.sh index 2f4f28bd3c..9ed6103ab9 100644 --- a/src/datastore_mad/remotes/libfs.sh +++ b/src/datastore_mad/remotes/libfs.sh @@ -154,7 +154,7 @@ function gzip_file_size { function fs_size { case $1 in - http://*) + http://*|https://*) HEADERS=`curl -LIk --max-time 60 $1 2>&1` if echo "$HEADERS" | grep -q "OpenNebula-AppMarket-Size"; then From 697fd4e3f8f2acff6216e953a878b7618cbeb791 Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Fri, 20 Feb 2015 19:20:58 +0100 Subject: [PATCH 2/9] Feature #3574: adds BRIDGE_LIST support for the fs DS_MAD actions: clone, cp, mkfs and rm --- src/datastore_mad/remotes/common/clone | 21 +++++++++---- src/datastore_mad/remotes/fs/cp | 6 ++-- src/datastore_mad/remotes/fs/mkfs | 42 ++++++++++++++++++++------ src/datastore_mad/remotes/fs/rm | 30 ++++++++++++------ 4 files changed, 70 insertions(+), 29 deletions(-) diff --git a/src/datastore_mad/remotes/common/clone b/src/datastore_mad/remotes/common/clone index e6fa699ceb..078985bdc5 100755 --- a/src/datastore_mad/remotes/common/clone +++ b/src/datastore_mad/remotes/common/clone @@ -46,20 +46,29 @@ unset i XPATH_ELEMENTS while IFS= read -r -d '' element; do XPATH_ELEMENTS[i++]="$element" done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/IMAGE/PATH) -BASE_PATH="${XPATH_ELEMENTS[0]}" -SRC="${XPATH_ELEMENTS[1]}" +unset i + +BASE_PATH="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +SRC="${XPATH_ELEMENTS[i++]}" -mkdir -p "$BASE_PATH" set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" DST=`generate_image_path` # ------------ Copy the image to the repository ------------- -log "Copying local image $SRC to the image repository" - -exec_and_log "cp -f $SRC $DST" "Error copying $SRC to $DST" +if [ -n "$BRIDGE_LIST" ]; then + log "Copying remotely local image $SRC to the image repository" + DST_HOST=`get_destination_host $ID` + ssh_exec_and_log "$DST_HOST" "mkdir -p $BASE_PATH; cp -f $SRC $DST" "Error copying $SRC to $DST in $DST_HOST" +else + log "Copying local image $SRC to the image repository" + mkdir -p "$BASE_PATH" + exec_and_log "cp -f $SRC $DST" "Error copying $SRC to $DST" +fi echo "$DST" diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index 3da6ea64f1..4f60d6ec8a 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -71,12 +71,11 @@ SHA1="${XPATH_ELEMENTS[i++]}" NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}" LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}" -mkdir -p "$BASE_PATH" -set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" - DST=`generate_image_path` IMAGE_HASH=`basename $DST` +set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" + if [ -n "$BRIDGE_LIST" ]; then DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -` else @@ -107,6 +106,7 @@ if [ -n "$BRIDGE_LIST" ]; then ssh_exec_and_log "$DST_HOST" "mkdir -p $BASE_PATH; mv -f $TMP_DST $DST" \ "Error moving $TMP_DST to $DST in $DST_HOST" else + mkdir -p "$BASE_PATH" exec_and_log "$COPY_COMMAND" "Error copying $SRC to $DST" fi diff --git a/src/datastore_mad/remotes/fs/mkfs b/src/datastore_mad/remotes/fs/mkfs index a3e46daba1..2d181763e1 100755 --- a/src/datastore_mad/remotes/fs/mkfs +++ b/src/datastore_mad/remotes/fs/mkfs @@ -48,17 +48,19 @@ while IFS= read -r -d '' element; do done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \ /DS_DRIVER_ACTION_DATA/IMAGE/FSTYPE \ /DS_DRIVER_ACTION_DATA/IMAGE/SIZE) +unset i -BASE_PATH="${XPATH_ELEMENTS[0]}" -RESTRICTED_DIRS="${XPATH_ELEMENTS[1]}" -SAFE_DIRS="${XPATH_ELEMENTS[2]}" -FSTYPE="${XPATH_ELEMENTS[3]}" -SIZE="${XPATH_ELEMENTS[4]}" +BASE_PATH="${XPATH_ELEMENTS[i++]}" +RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}" +SAFE_DIRS="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" +FSTYPE="${XPATH_ELEMENTS[i++]}" +SIZE="${XPATH_ELEMENTS[i++]}" -mkdir -p "$BASE_PATH" set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS" DST=`generate_image_path` @@ -74,9 +76,29 @@ fi MKFS_CMD=`mkfs_command $DST $FSTYPE $SIZE` -exec_and_log "$DD if=/dev/zero of=$DST bs=1 count=1 seek=${SIZE}M" \ - "Could not create image $DST" -exec_and_log "$MKFS_CMD" \ - "Unable to create filesystem $FSTYPE in $DST" +REMOTE_REGISTER_CMD=$(cat < Date: Fri, 20 Feb 2015 20:28:07 +0100 Subject: [PATCH 3/9] Feature #3574: Improve error message reporting in ssh_monitor_and_log --- src/mad/sh/scripts_common.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh index 7288e0e9e0..edda6d63c5 100644 --- a/src/mad/sh/scripts_common.sh +++ b/src/mad/sh/scripts_common.sh @@ -358,7 +358,9 @@ EOF` fi } -#This function executes $2 at $1 host and returns stdout +# This function executes $2 at $1 host and returns stdout +# If $3 is present, it is used as the error message when +# the command fails function ssh_monitor_and_log { SSH_EXEC_OUT=`$SSH $1 sh -s 2>/dev/null < Date: Fri, 20 Feb 2015 20:28:39 +0100 Subject: [PATCH 4/9] Feature #3574: Create a local version of the ssh_monitor_and_log function --- src/mad/sh/scripts_common.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/mad/sh/scripts_common.sh b/src/mad/sh/scripts_common.sh index edda6d63c5..2fa24db3c0 100644 --- a/src/mad/sh/scripts_common.sh +++ b/src/mad/sh/scripts_common.sh @@ -149,6 +149,33 @@ function exec_and_log fi } +# This function executes $1 and returns stdout +# If a second parameter is present it is used as the error message when +# the command fails +function monitor_and_log +{ + EXEC_OUT=`bash -s 2>/dev/null < Date: Fri, 20 Feb 2015 20:29:03 +0100 Subject: [PATCH 5/9] Feature #3574: Handle BRIDGE_LIST in the fs/monitor script --- src/datastore_mad/remotes/fs/monitor | 49 +++++++++++++++++++--------- 1 file changed, 34 insertions(+), 15 deletions(-) diff --git a/src/datastore_mad/remotes/fs/monitor b/src/datastore_mad/remotes/fs/monitor index 8b7f55306f..c6827f9f44 100755 --- a/src/datastore_mad/remotes/fs/monitor +++ b/src/datastore_mad/remotes/fs/monitor @@ -44,31 +44,50 @@ unset i XPATH_ELEMENTS while IFS= read -r -d '' element; do XPATH_ELEMENTS[i++]="$element" -done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH) +done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST) -BASE_PATH="${XPATH_ELEMENTS[0]}" +unset i + +BASE_PATH="${XPATH_ELEMENTS[i++]}" +BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" # ------------ Compute datastore usage ------------- -if [ ! -d $BASE_PATH ]; then - mkdir -p $BASE_PATH +MONITOR_SCRIPT=$(cat </dev/null | cut -f1` -TOTAL_MB=`df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print $2}'` -FREE_MB=`df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print $4}'` +# USED_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$3}') +TOTAL_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$2}') +FREE_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$4}') -if [ -z "$USED_MB" -o -z "$TOTAL_MB" -o -z "$FREE_MB"]; then - msg="Empty monitoring data" - error_message "$msg: USED_MB=$USED_MB, TOTAL_MB=$TOTAL_MB, FREE_MB=$FREE_MB" +if [ -z "\$USED_MB" -o -z "\$TOTAL_MB" -o -z "\$FREE_MB" ]; then + msg="Empty value found" + echo "\$msg: USED_MB=\$USED_MB, TOTAL_MB=\$TOTAL_MB, FREE_MB=\$FREE_MB" exit 1 fi -MONITOR_DATA=$(cat <&1) +else + MONITOR_DATA=$(monitor_and_log "$MONITOR_SCRIPT" "Monitor script" 2>&1) +fi + +MONITOR_STATUS=$? + +if [ "$MONITOR_STATUS" = "0" ]; then + echo "$MONITOR_DATA" | tr ' ' '\n' + exit 0 +else + echo "$MONITOR_DATA" + exit $MONITOR_STATUS +fi From fa1eeeadcd766d9d4b3e860f7e46618e953b0ad2 Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Fri, 20 Feb 2015 20:37:27 +0100 Subject: [PATCH 6/9] typo --- src/datastore_mad/remotes/fs/monitor | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datastore_mad/remotes/fs/monitor b/src/datastore_mad/remotes/fs/monitor index c6827f9f44..63ab1d272f 100755 --- a/src/datastore_mad/remotes/fs/monitor +++ b/src/datastore_mad/remotes/fs/monitor @@ -59,7 +59,7 @@ if [ ! -d \$BASE_PATH ]; then mkdir -p \$BASE_PATH fi -# USED_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$3}') +USED_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$3}') TOTAL_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$2}') FREE_MB=\$(df -B1M -P $BASE_PATH 2>/dev/null | tail -n 1 | awk '{print \$4}') From 41f50998142c515a901f3d02b5ca17b6af8bf7cb Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Mon, 23 Feb 2015 12:53:38 +0100 Subject: [PATCH 7/9] Feature #3574: Bug in the remote directory creation --- src/datastore_mad/remotes/fs/monitor | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/datastore_mad/remotes/fs/monitor b/src/datastore_mad/remotes/fs/monitor index 63ab1d272f..2901b9486c 100755 --- a/src/datastore_mad/remotes/fs/monitor +++ b/src/datastore_mad/remotes/fs/monitor @@ -55,8 +55,8 @@ BRIDGE_LIST="${XPATH_ELEMENTS[i++]}" # ------------ Compute datastore usage ------------- MONITOR_SCRIPT=$(cat </dev/null | tail -n 1 | awk '{print \$3}') From ce2c08dd5b07c1a74355020dfe26201b1f6a6e5c Mon Sep 17 00:00:00 2001 From: Jaime Melis Date: Mon, 23 Feb 2015 18:48:33 +0100 Subject: [PATCH 8/9] Feature #3574: Make sure we are deleting a file --- src/datastore_mad/remotes/fs/rm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datastore_mad/remotes/fs/rm b/src/datastore_mad/remotes/fs/rm index 98c53e823e..46e86b7e6c 100755 --- a/src/datastore_mad/remotes/fs/rm +++ b/src/datastore_mad/remotes/fs/rm @@ -59,7 +59,7 @@ BASE_PATH="${XPATH_ELEMENTS[i++]}" if [ -n "$BRIDGE_LIST" ]; then DST_HOST=`get_destination_host $ID` - ssh_exec_and_log "$DST_HOST" "rm -f $SRC" \ + ssh_exec_and_log "$DST_HOST" "[ -f $SRC ] && rm -f $SRC" \ "Error deleting $SRC in $DST_HOST" else BASENAME_SRC=`basename "${SRC##$REMOTE_RM_CMD}"` From 4c9171ae580423c1dc9b20df9329787b8125ff4b Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Mon, 23 Feb 2015 18:50:02 +0100 Subject: [PATCH 9/9] feature #3574: Fix checkresticted arg --- src/datastore_mad/remotes/fs/cp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index 4f60d6ec8a..9e1a87f4b1 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -85,7 +85,7 @@ fi COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS" if echo "$SRC" | grep -vq '^https\?://'; then - if [ `check_restricted 1` -eq 1 ]; then + if [ `check_restricted $SRC` -eq 1 ]; then log_error "Not allowed to copy images from $RESTRICTED_DIRS" error_message "Not allowed to copy image file $SRC" exit -1