From a2a1e830567567a38e41798189c4816500dd19ff Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 18 May 2017 12:54:47 +0000 Subject: [PATCH 1/7] B #3705: Override image DRIVER attribute by the datastore DRIVER --- src/datastore/Datastore.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/datastore/Datastore.cc b/src/datastore/Datastore.cc index cb1d9bf890..de59b18ef4 100644 --- a/src/datastore/Datastore.cc +++ b/src/datastore/Datastore.cc @@ -147,6 +147,13 @@ void Datastore::disk_attribute( { disk->replace("DISK_TYPE", Image::disk_type_to_str(get_disk_type())); } + + get_template_attribute("DRIVER", st); + + if(!st.empty()) + { + disk->replace("DRIVER", st); + } } /* ************************************************************************ */ From 1393bff1c5d3dfea3f0fde91e57fd0b29bf38cc1 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Thu, 18 May 2017 19:32:40 +0200 Subject: [PATCH 2/7] B #3705: Update DS_MAD fs/cp to support CONVERT/DRIVER --- src/datastore_mad/remotes/fs/cp | 35 ++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index 91665aebef..9c61227cf9 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -59,7 +59,10 @@ done < <($XPATH /DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \ /DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \ /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \ - /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW) + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/CONVERT \ + /DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/DRIVER \ + /DS_DRIVER_ACTION_DATA/IMAGE/TYPE) unset i @@ -74,6 +77,9 @@ MD5="${XPATH_ELEMENTS[i++]}" SHA1="${XPATH_ELEMENTS[i++]}" NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}" LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}" +CONVERT="${XPATH_ELEMENTS[i++]}" +DRIVER="${XPATH_ELEMENTS[i++]}" +IMAGE_TYPE="${XPATH_ELEMENTS[i++]}" DST=`generate_image_path` IMAGE_HASH=`basename $DST` @@ -85,6 +91,11 @@ if [ "$TYPE" = "2" ]; then NO_DECOMPRESS="${NO_DECOMPRESS:-yes}" fi +# Disable conversion for CD-ROM and 'files' datastores +if [ "$IMAGE_TYPE" = "1" ] || [ "$TYPE" = "2" ]; then + CONVERT=no +fi + if [ -n "$BRIDGE_LIST" ]; then DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -` else @@ -105,6 +116,19 @@ else log "Downloading image $SRC to the image repository" fi +CONVERT_CMD=$(cat < Date: Thu, 18 May 2017 20:31:51 +0200 Subject: [PATCH 3/7] B #3705: DRIVER configuration for the TM_MAD_CONF --- include/NebulaTemplate.h | 3 ++- share/etc/oned.conf | 5 +++-- src/datastore/Datastore.cc | 7 +++++++ src/nebula/NebulaTemplate.cc | 22 ++++++++++++---------- 4 files changed, 24 insertions(+), 13 deletions(-) diff --git a/include/NebulaTemplate.h b/include/NebulaTemplate.h index 4710a71026..224cefec08 100644 --- a/include/NebulaTemplate.h +++ b/include/NebulaTemplate.h @@ -156,7 +156,8 @@ private: const std::string& ln_target, const std::string& clone_target, const std::string& shared, - const std::string& ds_migrate); + const std::string& ds_migrate, + const std::string& driver); /** * Sets a the defaults for a Market diff --git a/share/etc/oned.conf b/share/etc/oned.conf index d941454beb..3b0408f20a 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -990,7 +990,8 @@ TM_MAD_CONF = [ ] TM_MAD_CONF = [ - NAME = "fs_lvm", LN_TARGET = "SYSTEM", CLONE_TARGET = "SYSTEM", SHARED="YES" + NAME = "fs_lvm", LN_TARGET = "SYSTEM", CLONE_TARGET = "SYSTEM", SHARED="YES", + DRIVER = "raw" ] TM_MAD_CONF = [ @@ -1004,7 +1005,7 @@ TM_MAD_CONF = [ TM_MAD_CONF = [ NAME = "ceph", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES", - DS_MIGRATE = "NO" + DS_MIGRATE = "NO", DRIVER = "raw" ] TM_MAD_CONF = [ diff --git a/src/datastore/Datastore.cc b/src/datastore/Datastore.cc index de59b18ef4..aa7830a117 100644 --- a/src/datastore/Datastore.cc +++ b/src/datastore/Datastore.cc @@ -342,6 +342,13 @@ int Datastore::set_tm_mad(string &tm_mad, string &error_str) replace_template_attribute("CLONE_TARGET", st); + st = vatt->vector_value("DRIVER"); + + if (!st.empty()) + { + replace_template_attribute("DRIVER", st); + } + remove_template_attribute("SHARED"); } diff --git a/src/nebula/NebulaTemplate.cc b/src/nebula/NebulaTemplate.cc index 363970757f..536a8b28d9 100644 --- a/src/nebula/NebulaTemplate.cc +++ b/src/nebula/NebulaTemplate.cc @@ -123,15 +123,15 @@ void OpenNebulaTemplate::set_multiple_conf_default() # dev #******************************************************************************* */ - set_conf_tm("dummy", "NONE", "SYSTEM", "YES", "YES"); - set_conf_tm("lvm", "NONE", "SELF", "YES", "NO"); - set_conf_tm("shared", "NONE", "SYSTEM", "YES", "YES"); - set_conf_tm("fs_lvm", "SYSTEM", "SYSTEM", "YES", "NO"); - set_conf_tm("qcow2", "NONE", "SYSTEM", "YES", "NO"); - set_conf_tm("ssh", "SYSTEM", "SYSTEM", "NO", "YES"); - set_conf_tm("vmfs", "NONE", "SYSTEM", "YES", "NO"); - set_conf_tm("ceph", "NONE", "SELF", "YES", "NO"); - set_conf_tm("dev", "NONE", "NONE", "YES", "NO"); + set_conf_tm("dummy", "NONE", "SYSTEM", "YES", "YES", ""); + set_conf_tm("lvm", "NONE", "SELF", "YES", "NO", ""); + set_conf_tm("shared", "NONE", "SYSTEM", "YES", "YES", ""); + set_conf_tm("fs_lvm", "SYSTEM", "SYSTEM", "YES", "NO", "raw"); + set_conf_tm("qcow2", "NONE", "SYSTEM", "YES", "NO", ""); + set_conf_tm("ssh", "SYSTEM", "SYSTEM", "NO", "YES", ""); + set_conf_tm("vmfs", "NONE", "SYSTEM", "YES", "NO", ""); + set_conf_tm("ceph", "NONE", "SELF", "YES", "NO", "raw"); + set_conf_tm("dev", "NONE", "NONE", "YES", "NO", ""); register_multiple_conf_default("TM_MAD_CONF"); /* @@ -301,7 +301,8 @@ void OpenNebulaTemplate::set_conf_tm(const std::string& name, const std::string& ln_target, const std::string& clone_target, const std::string& shared, - const std::string& ds_migrate) + const std::string& ds_migrate, + const std::string& driver) { VectorAttribute * vattribute; std::map vvalue; @@ -311,6 +312,7 @@ void OpenNebulaTemplate::set_conf_tm(const std::string& name, vvalue.insert(make_pair("CLONE_TARGET", clone_target)); vvalue.insert(make_pair("SHARED", shared)); vvalue.insert(make_pair("DS_MIGRATE", ds_migrate)); + vvalue.insert(make_pair("DRIVER", driver)); vattribute = new VectorAttribute("TM_MAD_CONF", vvalue); conf_default.insert(make_pair(vattribute->name(), vattribute)); From 38e7822328d1dec51a4fab4523a29424bdfa9276 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Fri, 19 May 2017 10:37:19 +0200 Subject: [PATCH 4/7] B #3705: Default CONVERT=yes in fs/cp --- src/datastore_mad/remotes/fs/cp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/datastore_mad/remotes/fs/cp b/src/datastore_mad/remotes/fs/cp index 9c61227cf9..15ff9dee72 100755 --- a/src/datastore_mad/remotes/fs/cp +++ b/src/datastore_mad/remotes/fs/cp @@ -77,7 +77,7 @@ MD5="${XPATH_ELEMENTS[i++]}" SHA1="${XPATH_ELEMENTS[i++]}" NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}" LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}" -CONVERT="${XPATH_ELEMENTS[i++]}" +CONVERT="${XPATH_ELEMENTS[i++]:-yes}" DRIVER="${XPATH_ELEMENTS[i++]}" IMAGE_TYPE="${XPATH_ELEMENTS[i++]}" @@ -92,6 +92,7 @@ if [ "$TYPE" = "2" ]; then fi # Disable conversion for CD-ROM and 'files' datastores +CONVERT=$(echo "${CONVERT}" | $TR '[:upper:]' '[:lower:]') if [ "$IMAGE_TYPE" = "1" ] || [ "$TYPE" = "2" ]; then CONVERT=no fi From 3c9065fec40825587c8dbf6c26c6204af9f872cd Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Fri, 19 May 2017 10:51:13 +0200 Subject: [PATCH 5/7] B #3705: Set DRIVER=qcow2 for TM_MAD qcow2 --- share/etc/oned.conf | 3 ++- src/nebula/NebulaTemplate.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/share/etc/oned.conf b/share/etc/oned.conf index 3b0408f20a..d59884b4f5 100644 --- a/share/etc/oned.conf +++ b/share/etc/oned.conf @@ -995,7 +995,8 @@ TM_MAD_CONF = [ ] TM_MAD_CONF = [ - NAME = "qcow2", LN_TARGET = "NONE", CLONE_TARGET = "SYSTEM", SHARED = "YES" + NAME = "qcow2", LN_TARGET = "NONE", CLONE_TARGET = "SYSTEM", SHARED = "YES", + DRIVER = "qcow2" ] TM_MAD_CONF = [ diff --git a/src/nebula/NebulaTemplate.cc b/src/nebula/NebulaTemplate.cc index 536a8b28d9..1b19234efc 100644 --- a/src/nebula/NebulaTemplate.cc +++ b/src/nebula/NebulaTemplate.cc @@ -127,7 +127,7 @@ void OpenNebulaTemplate::set_multiple_conf_default() set_conf_tm("lvm", "NONE", "SELF", "YES", "NO", ""); set_conf_tm("shared", "NONE", "SYSTEM", "YES", "YES", ""); set_conf_tm("fs_lvm", "SYSTEM", "SYSTEM", "YES", "NO", "raw"); - set_conf_tm("qcow2", "NONE", "SYSTEM", "YES", "NO", ""); + set_conf_tm("qcow2", "NONE", "SYSTEM", "YES", "NO", "qcow2"); set_conf_tm("ssh", "SYSTEM", "SYSTEM", "NO", "YES", ""); set_conf_tm("vmfs", "NONE", "SYSTEM", "YES", "NO", ""); set_conf_tm("ceph", "NONE", "SELF", "YES", "NO", "raw"); From ae96f4eaf0ee3c50ce9308d0ea1727e9f7a230c7 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Fri, 19 May 2017 16:54:03 +0200 Subject: [PATCH 6/7] B #3705: Don't override image DRIVER if CDROM --- src/datastore/Datastore.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/datastore/Datastore.cc b/src/datastore/Datastore.cc index aa7830a117..899c23c3cd 100644 --- a/src/datastore/Datastore.cc +++ b/src/datastore/Datastore.cc @@ -107,6 +107,7 @@ void Datastore::disk_attribute( string st; string inherit_val; string current_val; + string type; vector::const_iterator it; @@ -148,11 +149,16 @@ void Datastore::disk_attribute( disk->replace("DISK_TYPE", Image::disk_type_to_str(get_disk_type())); } - get_template_attribute("DRIVER", st); + type = disk->vector_value("TYPE"); - if(!st.empty()) + if (type != "CDROM") { - disk->replace("DRIVER", st); + get_template_attribute("DRIVER", st); + + if(!st.empty()) + { + disk->replace("DRIVER", st); + } } } From 6383b7529c3376e60bd7fbed4f547787e8fd44f6 Mon Sep 17 00:00:00 2001 From: Vlastimil Holer Date: Fri, 19 May 2017 23:04:52 +0200 Subject: [PATCH 7/7] B #3705: Add TEMPLATE/DRIVER=raw into exist. CEPH/LVM DS on migration --- src/onedb/local/4.90.0_to_5.3.80.rb | 37 +++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/src/onedb/local/4.90.0_to_5.3.80.rb b/src/onedb/local/4.90.0_to_5.3.80.rb index 8b34908001..6dda3c6b54 100644 --- a/src/onedb/local/4.90.0_to_5.3.80.rb +++ b/src/onedb/local/4.90.0_to_5.3.80.rb @@ -44,6 +44,8 @@ module Migrator feature_2347() + bug_3705() + log_time() return true @@ -137,4 +139,39 @@ module Migrator def feature_2347 create_table(:vmgroup_pool) end + + ############################################################################ + # Bug 3705 + # Adds DRIVER to CEPH and LVM image datastores + ############################################################################ + def bug_3705 + @db.run "ALTER TABLE datastore_pool RENAME TO old_datastore_pool;" + create_table(:datastore_pool) + + @db.transaction do + @db.fetch("SELECT * FROM old_datastore_pool") do |row| + doc = Nokogiri::XML(row[:body], nil, NOKOGIRI_ENCODING) { |c| + c.default_xml.noblanks + } + + type = xpath(doc, 'TYPE').to_i + tm_mad = xpath(doc, 'TM_MAD') + + if (type == 0) && (["ceph", "fs_lvm"].include?(tm_mad)) + doc.root.xpath("TEMPLATE/DRIVER").each do |d| + d.remove + end + + driver = doc.create_element "DRIVER", "raw" + doc.root.at_xpath("TEMPLATE").add_child(driver) + + row[:body] = doc.root.to_s + end + + @db[:datastore_pool].insert(row) + end + end + + @db.run "DROP TABLE old_datastore_pool;" + end end