diff --git a/install.sh b/install.sh index 512fefa211..63030b2047 100755 --- a/install.sh +++ b/install.sh @@ -1157,7 +1157,8 @@ ONEDB_FILES="src/onedb/fsck.rb \ src/onedb/import_slave.rb \ src/onedb/onedb.rb \ src/onedb/onedb_backend.rb \ - src/onedb/sqlite2mysql.rb" + src/onedb/sqlite2mysql.rb \ + src/onedb/fsck" ONEDB_SHARED_MIGRATOR_FILES="src/onedb/shared/2.0_to_2.9.80.rb \ src/onedb/shared/2.9.80_to_2.9.85.rb \ diff --git a/src/onedb/fsck.rb b/src/onedb/fsck.rb index eb17ccdb41..d79d34c452 100644 --- a/src/onedb/fsck.rb +++ b/src/onedb/fsck.rb @@ -29,12 +29,42 @@ require 'set' require 'nokogiri' +require 'opennebula' + +require 'fsck/pool_control' +require 'fsck/user' +require 'fsck/group' + +require 'fsck/cluster' +require 'fsck/host' +require 'fsck/datastore' +require 'fsck/network' + +require 'fsck/image' +require 'fsck/marketplaceapp' +require 'fsck/marketplace' +require 'fsck/vm' +require 'fsck/cluster_vnc_bitmap' +require 'fsck/history' +require 'fsck/vrouter' +require 'fsck/template' + +require 'fsck/quotas' + module OneDBFsck VERSION = "5.2.0" LOCAL_VERSION = "5.3.80" + def db_version + if defined?(@db_version) && @db_version + @db_version + else + @db_version = read_db_version + end + end + def check_db_version() - db_version = read_db_version() + # db_version = read_db_version() if ( db_version[:version] != VERSION || db_version[:local_version] != LOCAL_VERSION ) @@ -53,87 +83,178 @@ EOT "OpenNebula #{VERSION}" end - def db_version - one_version() - end - - IMAGE_STATES=%w{INIT READY USED DISABLED LOCKED ERROR CLONE DELETE USED_PERS} + # def db_version + # one_version() + # end VM_BIN = 0x0000001000000000 NET_BIN = 0x0000004000000000 VROUTER_BIN = 0x0004000000000000 HOLD = 0xFFFFFFFF + TABLES = ["group_pool", "user_pool", "acl", "image_pool", "host_pool", + "network_pool", "template_pool", "vm_pool", "cluster_pool", + "datastore_pool", "document_pool", "zone_pool", "secgroup_pool", + "vdc_pool", "vrouter_pool", "marketplace_pool", + "marketplaceapp_pool"].freeze + + FEDERATED_TABLES = ["group_pool", "user_pool", "acl", "zone_pool", + "vdc_pool", "marketplace_pool", "marketplaceapp_pool"].freeze + + SCHEMA = { + cluster_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " << + "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " << + "group_u INTEGER, other_u INTEGER, UNIQUE(name)", + cluster_datastore_relation: "cid INTEGER, oid INTEGER, " << + "PRIMARY KEY(cid, oid)", + cluster_network_relation: "cid INTEGER, oid INTEGER, " << + "PRIMARY KEY(cid, oid)", + datastore_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " << + "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " << + "group_u INTEGER, other_u INTEGER", + cluster_vnc_bitmap: "id INTEGER, map LONGTEXT, PRIMARY KEY(id)", + host_pool: "oid INTEGER PRIMARY KEY, " << + "name VARCHAR(128), body MEDIUMTEXT, state INTEGER, " << + "last_mon_time INTEGER, uid INTEGER, gid INTEGER, " << + "owner_u INTEGER, group_u INTEGER, other_u INTEGER, " << + "cid INTEGER", + image_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " << + "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " << + "group_u INTEGER, other_u INTEGER, UNIQUE(name,uid)", + network_pool: "oid INTEGER PRIMARY KEY, name VARCHAR(128), " << + "body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, " << + "group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid)", + user_quotas: "user_oid INTEGER PRIMARY KEY, body MEDIUMTEXT", + group_quotas: "group_oid INTEGER PRIMARY KEY, body MEDIUMTEXT" + } + + def tables + TABLES + end + + def federated_tables + FEDERATED_TABLES + end + + def create_table(type, name = nil) + if name + n = name.to_s + else + n = type.to_s + end + + schema = SCHEMA[type] + + if !schema + STDERR.puts "Schema not found (#{type})" + exit(-1) + end + + sql = "CREATE TABLE #{n} (#{schema});" + + STDERR.puts sql + + @db.run sql + end + + def nokogiri_doc(body) + Nokogiri::XML(body, nil, NOKOGIRI_ENCODING) do |c| + c.default_xml.noblanks + end + end + + + ######################################################################## + # Acl + ######################################################################## + + ######################################################################## + # Users + # + # USER/GNAME + ######################################################################## + + ######################################################################## + # Datastore + # + # DATASTORE/UID + # DATASTORE/UNAME + # DATASTORE/GID + # DATASTORE/GNAME + # DATASTORE/SYSTEM ?? + ######################################################################## + + ######################################################################## + # VM Template + # + # VMTEMPLATE/UID + # VMTEMPLATE/UNAME + # VMTEMPLATE/GID + # VMTEMPLATE/GNAME + ######################################################################## + + ######################################################################## + # Documents + # + # DOCUMENT/UID + # DOCUMENT/UNAME + # DOCUMENT/GID + # DOCUMENT/GNAME + ######################################################################## + + ######################################################################## + # VM + # + # VM/UID + # VM/UNAME + # VM/GID + # VM/GNAME + # + # VM/STATE <--- Check transitioning states? + # VM/LCM_STATE <---- Check consistency state/lcm_state ? + ######################################################################## + + ######################################################################## + # Image + # + # IMAGE/UID + # IMAGE/UNAME + # IMAGE/GID + # IMAGE/GNAME + ######################################################################## + + ######################################################################## + # VNet + # + # VNET/UID + # VNET/UNAME + # VNET/GID + # VNET/GNAME + ######################################################################## + + + def counters + if !defined?(@counters) + @counters = {} + @counters[:host] = {} + @counters[:image] = {} + @counters[:vnet] = {} + @counters[:vrouter] = {} + end + + @counters + end + + # Initialize all the vrouter counters to 0 + def init_vrouter_counters + @db.fetch("SELECT oid FROM vrouter_pool") do |row| + counters[:vrouter][row[:oid]] = { + :vms => Set.new + } + end + end + def fsck - - ######################################################################## - # Acl - ######################################################################## - - ######################################################################## - # Users - # - # USER/GNAME - ######################################################################## - - ######################################################################## - # Datastore - # - # DATASTORE/UID - # DATASTORE/UNAME - # DATASTORE/GID - # DATASTORE/GNAME - # DATASTORE/SYSTEM ?? - ######################################################################## - - ######################################################################## - # VM Template - # - # VMTEMPLATE/UID - # VMTEMPLATE/UNAME - # VMTEMPLATE/GID - # VMTEMPLATE/GNAME - ######################################################################## - - ######################################################################## - # Documents - # - # DOCUMENT/UID - # DOCUMENT/UNAME - # DOCUMENT/GID - # DOCUMENT/GNAME - ######################################################################## - - ######################################################################## - # VM - # - # VM/UID - # VM/UNAME - # VM/GID - # VM/GNAME - # - # VM/STATE <--- Check transitioning states? - # VM/LCM_STATE <---- Check consistency state/lcm_state ? - ######################################################################## - - ######################################################################## - # Image - # - # IMAGE/UID - # IMAGE/UNAME - # IMAGE/GID - # IMAGE/GNAME - ######################################################################## - - ######################################################################## - # VNet - # - # VNET/UID - # VNET/UNAME - # VNET/GID - # VNET/GNAME - ######################################################################## - init_log_time() @errors = 0 @@ -148,55 +269,9 @@ EOT # pool_control ######################################################################## - tables = ["group_pool", "user_pool", "acl", "image_pool", "host_pool", - "network_pool", "template_pool", "vm_pool", "cluster_pool", - "datastore_pool", "document_pool", "zone_pool", "secgroup_pool", - "vdc_pool", "vrouter_pool", "marketplace_pool", "marketplaceapp_pool"] + check_pool_control - federated_tables = ["group_pool", "user_pool", "acl", "zone_pool", - "vdc_pool", "marketplace_pool", "marketplaceapp_pool"] - - tables.each do |table| - max_oid = -1 - - @db.fetch("SELECT MAX(oid) FROM #{table}") do |row| - max_oid = row[:"MAX(oid)"].to_i - end - - # max(oid) will return 0 if there is none, - # or if the max is actually 0. Check this: - if ( max_oid == 0 ) - max_oid = -1 - - @db.fetch("SELECT oid FROM #{table} WHERE oid=0") do |row| - max_oid = 0 - end - end - - control_oid = -1 - - @db.fetch("SELECT last_oid FROM pool_control WHERE tablename='#{table}'") do |row| - control_oid = row[:last_oid].to_i - end - - if ( max_oid > control_oid ) - msg = "pool_control for table #{table} has last_oid #{control_oid}, but it is #{max_oid}" - - if control_oid != -1 - if db_version[:is_slave] && federated_tables.include?(table) - log_error(msg, false) - log_msg("^ Needs to be fixed in the master OpenNebula") - else - log_error(msg) - @db.run("UPDATE pool_control SET last_oid=#{max_oid} WHERE tablename='#{table}'") - end - else - @db[:pool_control].insert( - :tablename => table, - :last_oid => max_oid) - end - end - end + fix_pool_control log_time() @@ -212,146 +287,15 @@ EOT # USER/GID ######################################################################## - group = {} + check_user + fix_user - @db.fetch("SELECT oid FROM group_pool") do |row| - group[row[:oid]] = [] - end + log_time - users_fix = {} + check_group + fix_group - @db.fetch("SELECT oid,body,gid FROM user_pool") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - gid = doc.root.at_xpath('GID').text.to_i - user_gid = gid - user_gids = Set.new - - if group[gid].nil? - log_error("User #{row[:oid]} has primary group #{gid}, but it does not exist", !db_version[:is_slave]) - - user_gid = 1 - - doc.root.xpath('GID').each do |e| - e.content = "1" - end - - doc.root.xpath('GNAME').each do |e| - e.content = "users" - end - - doc.root.xpath("GROUPS").each { |e| - e.xpath("ID[.=#{gid}]").each{|x| x.remove} - - e.add_child(doc.create_element("ID")).content = user_gid.to_s - } - - users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} - end - - doc.root.xpath("GROUPS/ID").each { |e| - user_gids.add e.text.to_i - } - - if !user_gids.include?(user_gid) - log_error("User #{row[:oid]} does not have his primary group #{user_gid} in the list of secondary groups", !db_version[:is_slave]) - - doc.root.xpath("GROUPS").each { |e| - e.add_child(doc.create_element("ID")).content = user_gid.to_s - } - - user_gids.add user_gid.to_i - - users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} - end - - user_gids.each do |secondary_gid| - if group[secondary_gid].nil? - log_error("User #{row[:oid]} has secondary group #{secondary_gid}, but it does not exist", !db_version[:is_slave]) - - doc.root.xpath("GROUPS").each { |e| - e.xpath("ID[.=#{secondary_gid}]").each{|x| x.remove} - } - - users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} - else - group[secondary_gid] << row[:oid] - end - end - - if gid != row[:gid] - log_error( - "User #{row[:oid]} is in group #{gid}, but the DB "<< - "table has GID column #{row[:gid]}", !db_version[:is_slave]) - - users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} - end - end - - if !db_version[:is_slave] - @db.transaction do - users_fix.each do |id, user| - @db[:user_pool].where(:oid => id).update( - :body => user[:body], - :gid => user[:gid]) - end - end - elsif !users_fix.empty? - log_msg("^ User errors need to be fixed in the master OpenNebula") - end - - log_time() - - groups_fix = {} - - @db.fetch("SELECT oid,body from group_pool") do |row| - gid = row[:oid] - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - users_elem = doc.root.at_xpath("USERS") - users_elem.remove if !users_elem.nil? - - users_new_elem = doc.create_element("USERS") - doc.root.add_child(users_new_elem) - - error_found = false - - group[gid].each do |id| - id_elem = users_elem.at_xpath("ID[.=#{id}]") - - if id_elem.nil? - log_error("User #{id} is missing from Group #{gid} users id list", !db_version[:is_slave]) - error_found = true - else - id_elem.remove - end - - users_new_elem.add_child(doc.create_element("ID")).content = id.to_s - end - - users_elem.xpath("ID").each do |id_elem| - log_error("User #{id_elem.text} is in Group #{gid} users id list, but it should not", !db_version[:is_slave]) - error_found = true - end - - - if error_found - groups_fix[row[:oid]] = doc.root.to_s - end - end - - if !db_version[:is_slave] - @db.transaction do - groups_fix.each do |id, body| - @db[:group_pool].where(:oid => id).update( - :body => body) - end - end - elsif !groups_fix.empty? - log_msg("^ Group errors need to be fixed in the master OpenNebula") - end - - log_time() + log_time ######################################################################## # Clusters @@ -377,247 +321,30 @@ EOT # HOST/CLUSTER ######################################################################## - cluster = {} + init_cluster - @db.fetch("SELECT oid, name FROM cluster_pool") do |row| - cluster[row[:oid]] = {} + check_host_cluster + fix_host_cluster - cluster[row[:oid]][:name] = row[:name] + log_time - cluster[row[:oid]][:hosts] = [] - cluster[row[:oid]][:datastores] = Set.new - cluster[row[:oid]][:vnets] = Set.new - end + check_datastore_cluster + fix_datastore_cluster - hosts_fix = {} - datastores_fix = {} - vnets_fix = {} + log_time - @db.transaction do - @db.fetch("SELECT oid,body,cid FROM host_pool") do |row| - doc = Document.new(row[:body]) + check_network_cluster + fix_network_cluster - cluster_id = doc.root.get_text('CLUSTER_ID').to_s.to_i - cluster_name = doc.root.get_text('CLUSTER') + log_time - if cluster_id != row[:cid] - log_error("Host #{row[:oid]} is in cluster #{cluster_id}, but cid column has cluster #{row[:cid]}") - hosts_fix[row[:oid]] = {:body => row[:body], :cid => cluster_id} - end + check_fix_cluster - if cluster_id != -1 - cluster_entry = cluster[cluster_id] + log_time - if cluster_entry.nil? - log_error("Host #{row[:oid]} is in cluster #{cluster_id}, but it does not exist") + check_fix_cluster_relations - doc.root.each_element('CLUSTER_ID') do |e| - e.text = "-1" - end - - doc.root.each_element('CLUSTER') do |e| - e.text = "" - end - - hosts_fix[row[:oid]] = {:body => doc.root.to_s, :cid => -1} - else - if cluster_name != cluster_entry[:name] - log_error("Host #{row[:oid]} has a wrong name for cluster #{cluster_id}, #{cluster_name}. It will be changed to #{cluster_entry[:name]}") - - doc.root.each_element('CLUSTER') do |e| - e.text = cluster_entry[:name] - end - - hosts_fix[row[:oid]] = {:body => doc.root.to_s, :cid => cluster_id} - end - - cluster_entry[:hosts] << row[:oid] - end - end - end - - hosts_fix.each do |id, entry| - @db[:host_pool].where(:oid => id).update(:body => entry[:body], :cid => entry[:cid]) - end - - log_time() - - @db.fetch("SELECT oid,body FROM datastore_pool") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - doc.root.xpath("CLUSTERS/ID").each do |e| - cluster_id = e.text.to_i - - cluster_entry = cluster[cluster_id] - - if cluster_entry.nil? - log_error("Datastore #{row[:oid]} is in cluster #{cluster_id}, but it does not exist") - - e.remove - - datastores_fix[row[:oid]] = {:body => doc.root.to_s} - else - cluster_entry[:datastores] << row[:oid] - end - end - end - - datastores_fix.each do |id, entry| - @db[:datastore_pool].where(:oid => id).update(:body => entry[:body]) - end - - log_time() - - @db.fetch("SELECT oid,body FROM network_pool") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - doc.root.xpath("CLUSTERS/ID").each do |e| - cluster_id = e.text.to_i - - cluster_entry = cluster[cluster_id] - - if cluster_entry.nil? - log_error("VNet #{row[:oid]} is in cluster #{cluster_id}, but it does not exist") - - e.remove - - vnets_fix[row[:oid]] = {:body => doc.root.to_s} - else - cluster_entry[:vnets] << row[:oid] - end - end - end - - vnets_fix.each do |id, entry| - @db[:network_pool].where(:oid => id).update(:body => entry[:body]) - end - end - - log_time() - - @db.run "CREATE TABLE cluster_pool_new (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name));" - - @db.transaction do - @db.fetch("SELECT * from cluster_pool") do |row| - cluster_id = row[:oid] - doc = Document.new(row[:body]) - - # Hosts - hosts_elem = doc.root.elements.delete("HOSTS") - - hosts_new_elem = doc.root.add_element("HOSTS") - - cluster[cluster_id][:hosts].each do |id| - id_elem = hosts_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error("Host #{id} is missing from Cluster #{cluster_id} host id list") - end - - hosts_new_elem.add_element("ID").text = id.to_s - end - - hosts_elem.each_element("ID") do |id_elem| - log_error("Host #{id_elem.text} is in Cluster #{cluster_id} host id list, but it should not") - end - - - # Datastores - ds_elem = doc.root.elements.delete("DATASTORES") - - ds_new_elem = doc.root.add_element("DATASTORES") - - cluster[cluster_id][:datastores].each do |id| - id_elem = ds_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error("Datastore #{id} is missing from Cluster #{cluster_id} datastore id list") - end - - ds_new_elem.add_element("ID").text = id.to_s - - if @db.fetch("SELECT * FROM cluster_datastore_relation WHERE cid=#{cluster_id} AND oid=#{id}").empty? - log_error("Table cluster_datastore_relation is missing relation cluster #{cluster_id}, datastore #{id}") - @db[:cluster_datastore_relation].insert(:cid => cluster_id, :oid => id) - end - end - - ds_elem.each_element("ID") do |id_elem| - log_error("Datastore #{id_elem.text} is in Cluster #{cluster_id} datastore id list, but it should not") - end - - - # VNets - vnets_elem = doc.root.elements.delete("VNETS") - - vnets_new_elem = doc.root.add_element("VNETS") - - cluster[cluster_id][:vnets].each do |id| - id_elem = vnets_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error("VNet #{id} is missing from Cluster #{cluster_id} vnet id list") - end - - vnets_new_elem.add_element("ID").text = id.to_s - - if @db.fetch("SELECT * FROM cluster_network_relation WHERE cid=#{cluster_id} AND oid=#{id}").empty? - log_error("Table cluster_network_relation is missing relation cluster #{cluster_id}, vnet #{id}") - @db[:cluster_network_relation].insert(:cid => cluster_id, :oid => id) - end - end - - vnets_elem.each_element("ID") do |id_elem| - log_error("VNet #{id_elem.text} is in Cluster #{cluster_id} vnet id list, but it should not") - end - - - row[:body] = doc.root.to_s - - # commit - @db[:cluster_pool_new].insert(row) - end - end - - # Rename table - @db.run("DROP TABLE cluster_pool") - @db.run("ALTER TABLE cluster_pool_new RENAME TO cluster_pool") - - log_time() - - @db.transaction do - @db.run "CREATE TABLE cluster_datastore_relation_new (cid INTEGER, oid INTEGER, PRIMARY KEY(cid, oid));" - - @db.fetch("SELECT * from cluster_datastore_relation") do |row| - if (cluster[row[:cid]][:datastores].count(row[:oid]) != 1) - log_error("Table cluster_datastore_relation contains relation cluster #{row[:cid]}, datastore #{row[:oid]}, but it should not") - else - @db[:cluster_datastore_relation_new].insert(row) - end - end - - @db.run("DROP TABLE cluster_datastore_relation") - @db.run("ALTER TABLE cluster_datastore_relation_new RENAME TO cluster_datastore_relation") - end - - log_time() - - @db.transaction do - @db.run "CREATE TABLE cluster_network_relation_new (cid INTEGER, oid INTEGER, PRIMARY KEY(cid, oid));" - - @db.fetch("SELECT * from cluster_network_relation") do |row| - if (cluster[row[:cid]][:vnets].count(row[:oid]) != 1) - log_error("Table cluster_network_relation contains relation cluster #{row[:cid]}, vnet #{row[:oid]}, but it should not") - else - @db[:cluster_network_relation_new].insert(row) - end - end - - @db.run("DROP TABLE cluster_network_relation") - @db.run("ALTER TABLE cluster_network_relation_new RENAME TO cluster_network_relation") - end - - log_time() + log_time ######################################################################## # Datastore @@ -630,448 +357,59 @@ EOT # IMAGE/DATASTORE ######################################################################## - datastore = {} + init_datastore_counters - @db.fetch("SELECT oid, name FROM datastore_pool") do |row| - datastore[row[:oid]] = {:name => row[:name], :images => []} - end + log_time - images_fix = {} + check_datastore_image + fix_datastore_image - @db.fetch("SELECT oid,body FROM image_pool") do |row| - doc = Document.new(row[:body]) + log_time - ds_id = doc.root.get_text('DATASTORE_ID').to_s.to_i - ds_name = doc.root.get_text('DATASTORE') + check_fix_datastore - if ds_id != -1 - ds_entry = datastore[ds_id] - - if ds_entry.nil? - log_error("Image #{row[:oid]} has datastore #{ds_id}, but it does not exist. The image is probably unusable, and needs to be deleted manually:\n"<< - " * The image contents should be deleted manually:\n"<< - " #{doc.root.get_text('SOURCE')}\n"<< - " * The DB entry can be then deleted with the command:\n"<< - " DELETE FROM image_pool WHERE oid=#{row[:oid]};\n"<< - " * Run fsck again.\n", false) - else - if ds_name != ds_entry[:name] - log_error("Image #{row[:oid]} has a wrong name for datastore #{ds_id}, #{ds_name}. It will be changed to #{ds_entry[:name]}") - - doc.root.each_element('DATASTORE') do |e| - e.text = ds_entry[:name] - end - - images_fix[row[:oid]] = doc.root.to_s - end - - ds_entry[:images] << row[:oid] - end - end - end - - @db.transaction do - images_fix.each do |id, body| - @db[:image_pool].where(:oid => id).update(:body => body) - end - end - - log_time() - - @db.run "CREATE TABLE datastore_pool_new (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER);" - - @db.transaction do - @db.fetch("SELECT * from datastore_pool") do |row| - ds_id = row[:oid] - doc = Document.new(row[:body]) - - images_elem = doc.root.elements.delete("IMAGES") - - images_new_elem = doc.root.add_element("IMAGES") - - datastore[ds_id][:images].each do |id| - id_elem = images_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error( - "Image #{id} is missing from Datastore #{ds_id} "<< - "image id list") - end - - images_new_elem.add_element("ID").text = id.to_s - end - - images_elem.each_element("ID") do |id_elem| - log_error( - "Image #{id_elem.text} is in Datastore #{ds_id} "<< - "image id list, but it should not") - end - - - row[:body] = doc.root.to_s - - # commit - @db[:datastore_pool_new].insert(row) - end - end - - # Rename table - @db.run("DROP TABLE datastore_pool") - @db.run("ALTER TABLE datastore_pool_new RENAME TO datastore_pool") - - log_time() + log_time ######################################################################## # VM Counters for host, image and vnet usage ######################################################################## - counters = {} - counters[:host] = {} - counters[:image] = {} - counters[:vnet] = {} - counters[:vrouter] = {} + init_host_counters - # Initialize all the host counters to 0 - @db.fetch("SELECT oid, name FROM host_pool") do |row| - counters[:host][row[:oid]] = { - :name => row[:name], - :memory => 0, - :cpu => 0, - :rvms => Set.new - } - end + log_time - log_time() + init_image_counters - # Init image counters - @db.fetch("SELECT oid,body FROM image_pool") do |row| - if counters[:image][row[:oid]].nil? - counters[:image][row[:oid]] = { - :vms => Set.new, - :clones => Set.new, - :app_clones => Set.new - } - end + log_time - doc = Document.new(row[:body]) + init_network_counters - doc.root.each_element("CLONING_ID") do |e| - img_id = e.text.to_i + log_time - if counters[:image][img_id].nil? - counters[:image][img_id] = { - :vms => Set.new, - :clones => Set.new, - :app_clones => Set.new - } - end + init_vrouter_counters - counters[:image][img_id][:clones].add(row[:oid]) - end - end + log_time - log_time() + check_vm + fix_vm - # Init vnet counters - @db.fetch("SELECT oid,body FROM network_pool") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + log_time - ar_leases = {} + # VNC - doc.root.xpath("AR_POOL/AR/AR_ID").each do |ar_id| - ar_leases[ar_id.text.to_i] = {} - end + # DATA: VNC Bitmap - counters[:vnet][row[:oid]] = { - :ar_leases => ar_leases, - :no_ar_leases => {} - } - end + check_cluster_vnc_bitmap + fix_cluster_vnc_bitmap - log_time() + log_time - # Initialize all the vrouter counters to 0 - @db.fetch("SELECT oid FROM vrouter_pool") do |row| - counters[:vrouter][row[:oid]] = { - :vms => Set.new - } - end + # history - log_time() + check_history + fix_history - vms_fix = {} - cluster_vnc = {} - - # Aggregate information of the RUNNING vms - @db.transaction do - @db.fetch("SELECT oid,body FROM vm_pool WHERE state<>6") do |row| - vm_doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - state = vm_doc.root.at_xpath('STATE').text.to_i - lcm_state = vm_doc.root.at_xpath('LCM_STATE').text.to_i - - # VNC ports per cluster - cid = vm_doc.root.at_xpath("HISTORY_RECORDS/HISTORY[last()]/CID").text.to_i rescue nil - port = vm_doc.root.at_xpath('TEMPLATE/GRAPHICS[translate(TYPE,"vnc","VNC")="VNC"]/PORT').text.to_i rescue nil - - if cid && port - cluster_vnc[cid] ||= Set.new - cluster_vnc[cid] << port - end - - # Images used by this VM - vm_doc.root.xpath("TEMPLATE/DISK/IMAGE_ID").each do |e| - img_id = e.text.to_i - - if counters[:image][img_id].nil? - log_error("VM #{row[:oid]} is using Image #{img_id}, but "<< - "it does not exist", false) - else - counters[:image][img_id][:vms].add(row[:oid]) - end - end - - # VNets used by this VM - vm_doc.root.xpath("TEMPLATE/NIC").each do |nic| - net_id = nil - nic.xpath("NETWORK_ID").each do |nid| - net_id = nid.text.to_i - end - - if !net_id.nil? - if counters[:vnet][net_id].nil? - log_error("VM #{row[:oid]} is using VNet #{net_id}, "<< - "but it does not exist", false) - else - mac = nic.at_xpath("MAC").nil? ? nil : nic.at_xpath("MAC").text - - ar_id_e = nic.at_xpath('AR_ID') - - if ar_id_e.nil? - if !counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)].nil? - log_error("VNet #{net_id} has more than one lease with the same MAC address (#{mac}). "<< - "FSCK can't handle this, and consistency is not guaranteed", false) - end - - counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)] = { - :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, - :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, - :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, - :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, - :mac => mac, - :vm => row[:oid], - :vnet => nil, - :vrouter => nil - } - else - ar_id = ar_id_e.text.to_i - - if counters[:vnet][net_id][:ar_leases][ar_id].nil? - log_error("VM #{row[:oid]} is using VNet #{net_id}, AR #{ar_id}, "<< - "but the AR does not exist", false) - else - counters[:vnet][net_id][:ar_leases][ar_id][mac_s_to_i(mac)] = { - :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, - :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, - :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, - :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, - :mac => mac, - :vm => row[:oid], - :vnet => nil, - :vrouter => nil - } - end - end - end - end - end - - # See if it's part of a Virtual Router - vrouter_e = vm_doc.root.at_xpath("TEMPLATE/VROUTER_ID") - - if !vrouter_e.nil? - vr_id = vrouter_e.text.to_i - counters_vrouter = counters[:vrouter][vr_id] - - if counters_vrouter.nil? - log_error("VM #{row[:oid]} is part of VRouter #{vr_id}, but "<< - "it does not exist", false) - else - counters_vrouter[:vms].add(row[:oid]) - end - end - - # Host resources - - # Only states that add to Host resources consumption are - # ACTIVE, SUSPENDED, POWEROFF - next if !([3,5,8].include? state) - - # Get memory (integer) - memory = vm_doc.root.at_xpath("TEMPLATE/MEMORY").text.to_i - - # Get CPU (float) - cpu = vm_doc.root.at_xpath("TEMPLATE/CPU").text.to_f - - # Get hostid, hostname - hid = -1 - vm_doc.root.xpath("HISTORY_RECORDS/HISTORY[last()]/HID").each { |e| - hid = e.text.to_i - } - - hostname = "" - vm_doc.root.xpath("HISTORY_RECORDS/HISTORY[last()]/HOSTNAME").each { |e| - hostname = e.text - } - - counters_host = counters[:host][hid] - - if counters_host.nil? - log_error("VM #{row[:oid]} is using Host #{hid}, "<< - "but it does not exist", false) - else - if counters_host[:name] != hostname - log_error("VM #{row[:oid]} has a wrong hostname for "<< - "Host #{hid}, #{hostname}. It will be changed to "<< - "#{counters_host[:name]}") - - vm_doc.root.xpath( - "HISTORY_RECORDS/HISTORY[last()]/HOSTNAME").each { |e| - e.content = counters_host[:name] - } - - vms_fix[row[:oid]] = vm_doc.root.to_s - end - - counters_host[:memory] += memory - counters_host[:cpu] += cpu - counters_host[:rvms].add(row[:oid]) - end - - @db.fetch("SELECT * FROM history WHERE vid=#{row[:oid]}") do |hrow| - hdoc = Nokogiri::XML(hrow[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - found = false - - # Rename VMMMAD -> VM_MAD and TMMAD -> TM_MAD - hdoc.root.xpath("VMMMAD").each {|e| - e.name = "VM_MAD" - found = true - } - - hdoc.root.xpath("TMMAD").each {|e| - e.name = "TM_MAD" - found = true - } - - if found - @db[:history].where(:vid => hrow[:vid], :seq => hrow[:seq]).update( - :body => hdoc.root.to_s) - end - end - end - end - - @db.transaction do - vms_fix.each do |id, body| - @db[:vm_pool].where(:oid => id).update(:body => body) - end - end - - log_time() - - # VNC Bitmap - - # Re-create cluster_vnc_bitmap table - @db.run("ALTER TABLE cluster_vnc_bitmap RENAME TO old_cluster_vnc_bitmap") - @db.run("CREATE TABLE cluster_vnc_bitmap (id INTEGER, map LONGTEXT, PRIMARY KEY(id));") - - vnc_pool_size = 65536 - - @db.transaction do - @db.fetch("SELECT * FROM cluster_pool") do |row| - cluster_id = row[:oid] - - if cluster_vnc[cluster_id] - map = "" - vnc_pool_size.times.each do |i| - map << (cluster_vnc[cluster_id].include?(vnc_pool_size - 1 - i) ? "1" : "0") - end - - map_encoded = Base64::strict_encode64(Zlib::Deflate.deflate(map)) - else - map_encoded = "eJztwYEAAAAAgCCl/ekWqQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqFo8C0Q==" - end - - old_map_encoded = @db[:old_cluster_vnc_bitmap].first(:id => cluster_id)[:map] rescue nil - - if old_map_encoded != map_encoded - log_error("Cluster #{cluster_id} has not the proper reserved VNC ports") - end - - @db[:cluster_vnc_bitmap].insert( - :id => cluster_id, - :map => map_encoded - ) - end - end - - @db.run("DROP TABLE old_cluster_vnc_bitmap") - - log_time() - - # Bug #4000 may cause history records with etime=0 when they should - # be closed. The last history can be fixed with the VM etime, but - # previous history entries need to be fixed manually - - # Query to select history elements that: - # - have etime = 0 - # - are not the last seq - @db.fetch("SELECT vid,seq FROM history WHERE (etime = 0 AND seq <> (SELECT MAX(seq) FROM history AS subhistory WHERE history.vid = subhistory.vid) )") do |row| - log_error("History record for VM #{row[:vid]} seq # #{row[:seq]} is not closed (etime = 0)", false) - end - - log_time() - - history_fix = [] - - # Query to select history elements that have: - # - etime = 0 - # - is last seq - # - VM is DONE - @db.fetch("SELECT * FROM history WHERE (etime = 0 AND vid IN (SELECT oid FROM vm_pool WHERE state=6) AND seq = (SELECT MAX(seq) FROM history AS subhistory WHERE history.vid=subhistory.vid))") do |row| - log_error("History record for VM #{row[:vid]} seq # #{row[:seq]} is not closed (etime = 0), but the VM is in state DONE") - - etime = 0 - - @db.fetch("SELECT body FROM vm_pool WHERE oid=#{row[:vid]}") do |vm_row| - vm_doc = Nokogiri::XML(vm_row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - etime = vm_doc.root.at_xpath("ETIME").text.to_i - end - - history_doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - ["RETIME", "ESTIME", "EETIME", "ETIME"].each do |att| - elem = history_doc.root.at_xpath(att) - if (elem.text == "0") - elem.content = etime - end - end - - row[:body] = history_doc.root.to_s - row[:etime] = etime - - history_fix.push(row) - end - - @db.transaction do - history_fix.each do |row| - @db[:history].where({:vid => row[:vid], :seq => row[:seq]}).update(row) - end - end - - log_time() + log_time ######################################################################## # Virtual Routers @@ -1079,121 +417,13 @@ EOT # VROUTER/VMS/ID ######################################################################## - vrouters_fix = {} + check_vrouter + fix_vrouter - # Aggregate information of the RUNNING vms - @db.fetch("SELECT oid,body FROM vrouter_pool") do |row| - vrouter_doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - # VNets used by this Virtual Router - vrouter_doc.root.xpath("TEMPLATE/NIC").each do |nic| - net_id = nil - nic.xpath("NETWORK_ID").each do |nid| - net_id = nid.text.to_i - end - - floating = false - - nic.xpath("FLOATING_IP").each do |floating_e| - floating = (floating_e.text.upcase == "YES") - end - - if !net_id.nil? && floating - if counters[:vnet][net_id].nil? - log_error("VRouter #{row[:oid]} is using VNet #{net_id}, "<< - "but it does not exist", false) - else - mac = nic.at_xpath("MAC").nil? ? nil : nic.at_xpath("MAC").text - - ar_id_e = nic.at_xpath('AR_ID') - - if ar_id_e.nil? - if !counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)].nil? - log_error("VNet #{net_id} has more than one lease with the same MAC address (#{mac}). "<< - "FSCK can't handle this, and consistency is not guaranteed", false) - end - - counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)] = { - :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, - :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, - :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, - :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, - :mac => mac, - :vm => nil, - :vnet => nil, - :vrouter => row[:oid], - } - else - ar_id = ar_id_e.text.to_i - - if counters[:vnet][net_id][:ar_leases][ar_id].nil? - log_error("VRouter #{row[:oid]} is using VNet #{net_id}, AR #{ar_id}, "<< - "but the AR does not exist", false) - else - counters[:vnet][net_id][:ar_leases][ar_id][mac_s_to_i(mac)] = { - :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, - :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, - :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, - :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, - :mac => mac, - :vm => nil, - :vnet => nil, - :vrouter => row[:oid], - } - end - end - end - end - end - - # re-do list of VM IDs - error = false - - counters_vrouter = counters[:vrouter][row[:oid]] - - vms_elem = vrouter_doc.root.at_xpath("VMS").remove - - vms_new_elem = vrouter_doc.create_element("VMS") - vrouter_doc.root.add_child(vms_new_elem) - - counters_vrouter[:vms].each do |id| - id_elem = vms_elem.at_xpath("ID[.=#{id}]") - - if id_elem.nil? - log_error( - "VM #{id} is missing from VRouter #{row[:oid]} VM id list") - - error = true - else - id_elem.remove - end - - vms_new_elem.add_child(vrouter_doc.create_element("ID")).content = id.to_s - end - - vms_elem.xpath("ID").each do |id_elem| - log_error( - "VM #{id_elem.text} is in VRouter #{row[:oid]} VM id list, "<< - "but it should not") - - error = true - end - - if (error) - vrouters_fix[row[:oid]] = vrouter_doc.root.to_s - end - end - - @db.transaction do - vrouters_fix.each do |id, body| - @db[:vrouter_pool].where(:oid => id).update(:body => body) - end - end - - log_time() + log_time ######################################################################## - # Hosts + # DATA: Hosts # # HOST/HOST_SHARE/MEM_USAGE # HOST/HOST_SHARE/CPU_USAGE @@ -1201,251 +431,37 @@ EOT # HOST/VMS/ID ######################################################################## - # Create a new empty table where we will store the new calculated values - @db.run "CREATE TABLE host_pool_new (oid INTEGER PRIMARY KEY, " << - "name VARCHAR(128), body MEDIUMTEXT, state INTEGER, " << - "last_mon_time INTEGER, uid INTEGER, gid INTEGER, " << - "owner_u INTEGER, group_u INTEGER, other_u INTEGER, " << - "cid INTEGER);" + check_host + fix_host - # Calculate the host's xml and write them to host_pool_new - @db.transaction do - @db[:host_pool].each do |row| - host_doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - hid = row[:oid] - - counters_host = counters[:host][hid] - - rvms = counters_host[:rvms].size - cpu_usage = (counters_host[:cpu]*100).to_i - mem_usage = counters_host[:memory]*1024 - - # rewrite running_vms - host_doc.root.xpath("HOST_SHARE/RUNNING_VMS").each {|e| - if e.text != rvms.to_s - log_error( - "Host #{hid} RUNNING_VMS has #{e.text} \tis\t#{rvms}") - e.content = rvms - end - } - - - # re-do list of VM IDs - vms_elem = host_doc.root.at_xpath("VMS").remove - - vms_new_elem = host_doc.create_element("VMS") - host_doc.root.add_child(vms_new_elem) - - counters_host[:rvms].each do |id| - id_elem = vms_elem.at_xpath("ID[.=#{id}]") - - if id_elem.nil? - log_error( - "VM #{id} is missing from Host #{hid} VM id list") - else - id_elem.remove - end - - vms_new_elem.add_child(host_doc.create_element("ID")).content = id.to_s - end - - vms_elem.xpath("ID").each do |id_elem| - log_error( - "VM #{id_elem.text} is in Host #{hid} VM id list, "<< - "but it should not") - end - - host_doc.root.xpath("HOST_SHARE/PCI_DEVICES/PCI").each do |pci| - if !pci.at_xpath("VMID").nil? - vmid = pci.at_xpath("VMID").text.to_i - - if vmid != -1 && !counters_host[:rvms].include?(vmid) - log_error("VM #{vmid} has a PCI device assigned in host #{hid}, but it should not. Device: #{pci.at_xpath('DEVICE_NAME').text}") - pci.at_xpath("VMID").content = "-1" - end - end - end - - # rewrite cpu - host_doc.root.xpath("HOST_SHARE/CPU_USAGE").each {|e| - if e.text != cpu_usage.to_s - log_error( - "Host #{hid} CPU_USAGE has #{e.text} "<< - "\tis\t#{cpu_usage}") - e.content = cpu_usage - end - } - - # rewrite memory - host_doc.root.xpath("HOST_SHARE/MEM_USAGE").each {|e| - if e.text != mem_usage.to_s - log_error("Host #{hid} MEM_USAGE has #{e.text} "<< - "\tis\t#{mem_usage}") - e.content = mem_usage - end - } - - row[:body] = host_doc.root.to_s - - # commit - @db[:host_pool_new].insert(row) - end - end - - # Rename table - @db.run("DROP TABLE host_pool") - @db.run("ALTER TABLE host_pool_new RENAME TO host_pool") - - log_time() + log_time ######################################################################## - # Marketplace + # DATA: Marketplace # # MARKETPLACE/MARKETPLACEAPPS/ID ######################################################################## - # App + # DATA: App # # MARKETPLACEAPP/MARKETPLACE_ID # MARKETPLACEAPP/MARKETPLACE # MARKETPLACEAPP/ORIGIN_ID ######################################################################## - marketplace = {} + check_marketplaceapp - @db.fetch("SELECT oid, name FROM marketplace_pool") do |row| - marketplace[row[:oid]] = {:name => row[:name], :apps => []} - end - - apps_fix = {} - - @db.fetch("SELECT oid,body FROM marketplaceapp_pool") do |row| - doc = Document.new(row[:body]) - - market_id = doc.root.get_text('MARKETPLACE_ID').to_s.to_i - market_name = doc.root.get_text('MARKETPLACE') - - #################################################################### - # TODO, BUG: this code will only work for a standalone oned. - # In a federation, the image ID will refer to a different image - # in each zone - #################################################################### - - origin_id = doc.root.get_text('ORIGIN_ID').to_s.to_i - if origin_id >= 0 && doc.root.get_text('STATE').to_s.to_i == 2 # LOCKED - counters[:image][origin_id][:app_clones].add(row[:oid]) - end - - #################################################################### - ##################################################################### - - if market_id != -1 - market_entry = marketplace[market_id] - - if market_entry.nil? - log_error("Marketplace App #{row[:oid]} has marketplace #{market_id}, but it does not exist. The app is probably unusable, and needs to be deleted manually:\n"<< - " * The DB entry can be deleted with the command:\n"<< - " DELETE FROM marketplaceapp_pool WHERE oid=#{row[:oid]};\n"<< - " * Run fsck again.\n", false) - else - if market_name != market_entry[:name] - log_error("Marketplace App #{row[:oid]} has a wrong name for marketplace #{market_id}, #{market_name}. It will be changed to #{market_entry[:name]}") - - doc.root.each_element('MARKETPLACE') do |e| - e.text = market_entry[:name] - end - - apps_fix[row[:oid]] = doc.root.to_s - end - - market_entry[:apps] << row[:oid] - end - end - end - - if !db_version[:is_slave] - @db.transaction do - apps_fix.each do |id, body| - @db[:marketplaceapp_pool].where(:oid => id).update(:body => body) - end - end - elsif !apps_fix.empty? - log_msg("^ Marketplace App errors need to be fixed in the master OpenNebula") - end + fix_marketplaceapp log_time() - markets_fix = {} + check_marketplace - @db.fetch("SELECT oid,body FROM marketplace_pool") do |row| - market_id = row[:oid] - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - apps_elem = doc.root.at_xpath("MARKETPLACEAPPS") - apps_elem.remove if !apps_elem.nil? - - apps_new_elem = doc.create_element("MARKETPLACEAPPS") - doc.root.add_child(apps_new_elem) - - error = false - - marketplace[market_id][:apps].each do |id| - id_elem = apps_elem.at_xpath("ID[.=#{id}]") - - if id_elem.nil? - error = true - - log_error( - "Marketplace App #{id} is missing from Marketplace #{market_id} "<< - "app id list") - else - id_elem.remove - end - - apps_new_elem.add_child(doc.create_element("ID")).content = id.to_s - end - - apps_elem.xpath("ID").each do |id_elem| - error = true - - log_error( - "Marketplace App #{id_elem.text} is in Marketplace #{market_id} "<< - "app id list, but it should not") - end - - zone_elem = doc.root.at_xpath("ZONE_ID") - - if (zone_elem.nil? || zone_elem.text == "-1") - error = true - - log_error("Marketplace #{market_id} has an invalid ZONE_ID. Will be set to 0") - - if (zone_elem.nil?) - zone_elem = doc.root.add_child(doc.create_element("ZONE_ID")) - end - - zone_elem.content = "0" - end - - if (error) - markets_fix[row[:oid]] = doc.root.to_s - end - end - - if !db_version[:is_slave] - @db.transaction do - markets_fix.each do |id, body| - @db[:marketplace_pool].where(:oid => id).update(:body => body) - end - end - elsif !markets_fix.empty? - log_msg("^ Marketplace errors need to be fixed in the master OpenNebula") - end + fix_marketplace log_time() ######################################################################## - # Image + # DATA: Image # # IMAGE/RUNNING_VMS # IMAGE/VMS/ID @@ -1459,149 +475,11 @@ EOT # IMAGE/STATE ######################################################################## - # Create a new empty table where we will store the new calculated values - @db.run "CREATE TABLE image_pool_new (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, UNIQUE(name,uid) );" + check_image - @db.transaction do - @db[:image_pool].each do |row| - doc = Document.new(row[:body]) + fix_image - oid = row[:oid] - - persistent = ( doc.root.get_text('PERSISTENT').to_s == "1" ) - current_state = doc.root.get_text('STATE').to_s.to_i - - counters_img = counters[:image][oid] - - rvms = counters_img[:vms].size - n_cloning_ops = counters_img[:clones].size + counters_img[:app_clones].size - - # rewrite running_vms - doc.root.each_element("RUNNING_VMS") {|e| - if e.text != rvms.to_s - log_error("Image #{oid} RUNNING_VMS has #{e.text} \tis\t#{rvms}") - e.text = rvms - end - } - - # re-do list of VM IDs - vms_elem = doc.root.elements.delete("VMS") - - vms_new_elem = doc.root.add_element("VMS") - - counters_img[:vms].each do |id| - id_elem = vms_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error("VM #{id} is missing from Image #{oid} VM id list") - end - - vms_new_elem.add_element("ID").text = id.to_s - end - - vms_elem.each_element("ID") do |id_elem| - log_error("VM #{id_elem.text} is in Image #{oid} VM id list, but it should not") - end - - - if ( persistent && rvms > 0 ) - n_cloning_ops = 0 - counters_img[:clones] = Set.new - counters_img[:app_clones] = Set.new - end - - # Check number of clones - doc.root.each_element("CLONING_OPS") { |e| - if e.text != n_cloning_ops.to_s - log_error("Image #{oid} CLONING_OPS has #{e.text} \tis\t#{n_cloning_ops}") - e.text = n_cloning_ops - end - } - - # re-do list of Images cloning this one - clones_elem = doc.root.elements.delete("CLONES") - - clones_new_elem = doc.root.add_element("CLONES") - - counters_img[:clones].each do |id| - id_elem = clones_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error("Image #{id} is missing from Image #{oid} CLONES id list") - end - - clones_new_elem.add_element("ID").text = id.to_s - end - - clones_elem.each_element("ID") do |id_elem| - log_error("Image #{id_elem.text} is in Image #{oid} CLONES id list, but it should not") - end - - # re-do list of Apps cloning this one - clones_elem = doc.root.elements.delete("APP_CLONES") - - clones_new_elem = doc.root.add_element("APP_CLONES") - - counters_img[:app_clones].each do |id| - id_elem = clones_elem.elements.delete("ID[.=#{id}]") - - if id_elem.nil? - log_error("Marketplace App #{id} is missing from Image #{oid} APP_CLONES id list") - end - - clones_new_elem.add_element("ID").text = id.to_s - end - - clones_elem.each_element("ID") do |id_elem| - log_error("Marketplace App #{id_elem.text} is in Image #{oid} APP_CLONES id list, but it should not") - end - - - # Check state - - state = current_state - - if persistent - if ( rvms > 0 ) - state = 8 # USED_PERS - elsif ( n_cloning_ops > 0 ) - state = 6 # CLONE - elsif ( current_state == 8 || current_state == 6 ) - # rvms == 0 && n_cloning_ops == 0, but image is in state - # USED_PERS or CLONE - - state = 1 # READY - end - else - if ( rvms > 0 || n_cloning_ops > 0 ) - state = 2 # USED - elsif ( current_state == 2 ) - # rvms == 0 && n_cloning_ops == 0, but image is in state - # USED - - state = 1 # READY - end - end - - doc.root.each_element("STATE") { |e| - if e.text != state.to_s - log_error("Image #{oid} has STATE #{IMAGE_STATES[e.text.to_i]} \tis\t#{IMAGE_STATES[state]}") - e.text = state - end - } - - row[:body] = doc.root.to_s - - # commit - @db[:image_pool_new].insert(row) - end - end - - # Rename table - @db.run("DROP TABLE image_pool") - @db.run("ALTER TABLE image_pool_new RENAME TO image_pool") - - log_time() + log_time ######################################################################## # VNet @@ -1609,409 +487,12 @@ EOT # LEASES ######################################################################## - @db.fetch("SELECT oid,body,pid FROM network_pool WHERE pid<>-1") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + init_network_lease_counters - parent_vnet = doc.root.at_xpath("PARENT_NETWORK_ID").text.to_i - - if (row[:pid] != parent_vnet) - # TODO - end - - doc.root.xpath("AR_POOL/AR").each do |ar| - parent_ar_e = ar.at_xpath("PARENT_NETWORK_AR_ID") - if !(parent_ar_e.nil? || parent_ar_e.text == "") - - parent_ar = parent_ar_e.text.to_i - - if counters[:vnet][parent_vnet][:ar_leases][parent_ar].nil? - log_error( - "VNet #{row[:oid]} is using parent "<< - "VNet #{parent_vnet}, AR #{parent_ar}, "<< - "but the AR does not exist", false) - end - - # MAC - first_mac = mac_s_to_i(ar.at_xpath("MAC").text) - - # IP - first_ip = nil - if (!ar.at_xpath("IP").nil?) - first_ip = IPAddr.new(ar.at_xpath("IP").text.strip, Socket::AF_INET) - end - - # IP6 - global_prefix = nil - if !ar.at_xpath("GLOBAL_PREFIX").nil? - global_prefix = ip6_prefix_s_to_i( - ar.at_xpath("GLOBAL_PREFIX").text) - end - - ula_prefix = nil - if !ar.at_xpath("ULA_PREFIX").nil? - ula_prefix = ip6_prefix_s_to_i( - ar.at_xpath("ULA_PREFIX").text) - end - - link_prefix = nil - type = ar.at_xpath("TYPE").text - if ( type == "IP6" || type == "IP4_6" ) - link_prefix = 0xfe80000000000000 - end - - # Parent vnet has a lease for each address of this reservation - ar.at_xpath("SIZE").text.to_i.times do |index| - - lease = { - :ip => nil, - :ip6_global => nil, - :ip6_link => nil, - :ip6_ula => nil, - :mac => nil, - :vm => nil, - :vnet => row[:oid], - :vrouter => nil - } - - #MAC - mac = (first_mac & 0xFFFF00000000) + - (((first_mac & 0xFFFFFFFF) + index) % 0x100000000) - lease[:mac] = mac_i_to_s(mac) - - # IP - if (!first_ip.nil?) - lease[:ip] = IPAddr.new(first_ip.to_i + index, - Socket::AF_INET).to_s - end - - # IP6 - ip6_suffix = mac_to_ip6_suffix(mac) - - if (!global_prefix.nil?) - lease[:ip6_global] = IPAddr.new( - (global_prefix << 64) | ip6_suffix, - Socket::AF_INET6 ).to_s - end - - if (!ula_prefix.nil?) - lease[:ip6_ula] = IPAddr.new( - (ula_prefix << 64) | ip6_suffix, - Socket::AF_INET6 ).to_s - end - - if (!link_prefix.nil?) - lease[:ip6_link] = IPAddr.new( - (link_prefix << 64) | ip6_suffix, - Socket::AF_INET6 ).to_s - end - - counters[:vnet][parent_vnet][ - :ar_leases][parent_ar][mac] = lease - end - end - end - end - - - # Create a new empty table where we will store the new calculated values - @db.run "CREATE TABLE network_pool_new (oid INTEGER PRIMARY KEY, name VARCHAR(128), body MEDIUMTEXT, uid INTEGER, gid INTEGER, owner_u INTEGER, group_u INTEGER, other_u INTEGER, pid INTEGER, UNIQUE(name,uid));" - - @db.transaction do - @db[:network_pool].each do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - oid = row[:oid] - - used_leases = doc.root.at_xpath("USED_LEASES").text.to_i - new_used_leases = 0 - - counter_no_ar = counters[:vnet][row[:oid]][:no_ar_leases] - - counters[:vnet][row[:oid]][:ar_leases].each do |ar_id, counter_ar| - net_ar = doc.root.at_xpath("AR_POOL/AR[AR_ID=#{ar_id}]") - - if (net_ar.nil?) - # TODO shouldn't happen? - end - - # MAC - first_mac = mac_s_to_i(net_ar.at_xpath("MAC").text) - - # IP - first_ip = nil - if !net_ar.at_xpath("IP").nil? - first_ip_st = net_ar.at_xpath("IP").text - - if (first_ip_st != first_ip_st.strip) - log_error("VNet #{oid} AR #{ar_id} "<< - "IP \"#{first_ip_st}\" contains whitespaces") - - first_ip_st.strip! - - net_ar.at_xpath("IP").content = first_ip_st - end - - first_ip = IPAddr.new(first_ip_st, Socket::AF_INET) - end - - # IP6 - global_prefix = nil - if !net_ar.at_xpath("GLOBAL_PREFIX").nil? - global_prefix = ip6_prefix_s_to_i( - net_ar.at_xpath("GLOBAL_PREFIX").text) - end - - ula_prefix = nil - if !net_ar.at_xpath("ULA_PREFIX").nil? - ula_prefix = ip6_prefix_s_to_i( - net_ar.at_xpath("ULA_PREFIX").text) - end - - link_prefix = nil - type = net_ar.at_xpath("TYPE").text - if ( type == "IP6" || type == "IP4_6" ) - link_prefix = 0xfe80000000000000 - end - - # Allocated leases - allocated_e = net_ar.at_xpath("ALLOCATED") - - allocated = allocated_e.nil? ? "" : allocated_e.text - - leases = allocated.scan(/(\d+) (\d+)/) - - new_leases = [] - - leases.each do |lease_str| - index = lease_str[0].to_i - binary_magic = lease_str[1].to_i - - lease = { - :ip => nil, - :ip6_global => nil, - :ip6_link => nil, - :ip6_ula => nil, - :mac => nil, - :vm => nil, - :vnet => nil, - :vrouter => nil - } - - # MAC - mac = (first_mac & 0xFFFF00000000) + - (((first_mac & 0xFFFFFFFF) + index) % 0x100000000) - - lease[:mac] = mac_i_to_s(mac) - - # IP - if (!first_ip.nil?) - lease[:ip] = IPAddr.new(first_ip.to_i + index, - Socket::AF_INET).to_s - end - - # IP6 - ip6_suffix = mac_to_ip6_suffix(mac) - - if (!global_prefix.nil?) - lease[:ip6_global] = IPAddr.new( - (global_prefix << 64) | ip6_suffix, - Socket::AF_INET6 ).to_s - end - - if (!ula_prefix.nil?) - lease[:ip6_ula] = IPAddr.new( - (ula_prefix << 64) | ip6_suffix, - Socket::AF_INET6 ).to_s - end - - if (!link_prefix.nil?) - lease[:ip6_link] = IPAddr.new( - (link_prefix << 64) | ip6_suffix, - Socket::AF_INET6 ).to_s - end - - # OID - lease_oid = binary_magic & 0x00000000FFFFFFFF - lease_obj = "" - - if (binary_magic & VM_BIN != 0) - lease[:vm] = lease_oid - lease_obj = "VM" - elsif (binary_magic & NET_BIN != 0) - lease[:vnet] = lease_oid - lease_obj = "VNet" - else #(binary_magic & VROUTER_BIN != 0) - lease[:vrouter] = lease_oid - lease_obj = "VRouter" - end - - counter_lease = counter_ar[mac] - counter_ar.delete(mac) - - if counter_lease.nil? - counter_lease = counter_no_ar[mac] - counter_no_ar.delete(mac) - end - - if counter_lease.nil? - if(lease[:vm] != HOLD) - log_error( - "VNet #{oid} AR #{ar_id} has leased #{lease_to_s(lease)} "<< - "to #{lease_obj} #{lease_oid}, but it is actually free") - else - new_leases << lease_str - end - else - if counter_lease != lease - - # Things that can be fixed - if (counter_lease[:vm] != lease[:vm] || - counter_lease[:vnet] != lease[:vnet] || - counter_lease[:vrouter] != lease[:vrouter]) - - new_lease_obj = "" - new_lease_oid = 0 - new_binary_magic = 0 - - if !counter_lease[:vm].nil? - new_lease_obj = "VM" - new_lease_oid = counter_lease[:vm].to_i - - new_binary_magic = (VM_BIN | - (new_lease_oid & 0xFFFFFFFF)) - elsif !counter_lease[:vnet].nil? - new_lease_obj = "VNet" - new_lease_oid = counter_lease[:vnet].to_i - - new_binary_magic = (NET_BIN | - (new_lease_oid & 0xFFFFFFFF)) - else #if !counter_lease[:vrouter].nil? - new_lease_obj = "VRouter" - new_lease_oid = counter_lease[:vrouter].to_i - - new_binary_magic = (VROUTER_BIN | - (new_lease_oid & 0xFFFFFFFF)) - end - - if (lease[:vm] == HOLD) - log_error( - "VNet #{oid} AR #{ar_id} has lease "<< - "#{lease_to_s(lease)} on hold, but it is "<< - "actually used by "<< - "#{new_lease_obj} #{new_lease_oid}") - else - log_error( - "VNet #{oid} AR #{ar_id} has leased #{lease_to_s(lease)} "<< - "to #{lease_obj} #{lease_oid}, but it is "<< - "actually used by "<< - "#{new_lease_obj} #{new_lease_oid}") - end - - lease_str[1] = new_binary_magic.to_s - end - - # Things that can't be fixed - - [:ip, :ip6_global, :ip6_link, :ip6_ula].each do |key| - if (counter_lease[key] != lease[key]) - log_error( - "VNet #{oid} AR #{ar_id} has a wrong "<< - "lease for "<< - "#{lease_obj} #{lease_oid}. #{key.to_s.upcase} "<< - "does not match: "<< - "#{counter_lease[key]} != #{lease[key]}. "<< - "This can't be fixed", false) - end - end - end - - new_leases << lease_str - end - end - - counter_ar.each do |mac, counter_lease| - index = ((mac & 0xFFFFFFFF) - (first_mac & 0xFFFFFFFF) ) % 0x100000000 - - new_lease_obj = "" - new_lease_oid = 0 - new_binary_magic = 0 - - if !counter_lease[:vm].nil? - new_lease_obj = "VM" - new_lease_oid = counter_lease[:vm].to_i - - new_binary_magic = (VM_BIN | - (new_lease_oid & 0xFFFFFFFF)) - elsif !counter_lease[:vnet].nil? - new_lease_obj = "VNet" - new_lease_oid = counter_lease[:vnet].to_i - - new_binary_magic = (NET_BIN | - (new_lease_oid & 0xFFFFFFFF)) - else #if !counter_lease[:vrouter].nil? - new_lease_obj = "VRouter" - new_lease_oid = counter_lease[:vrouter].to_i - - new_binary_magic = (VROUTER_BIN | - (new_lease_oid & 0xFFFFFFFF)) - end - - log_error("VNet #{oid} AR #{ar_id} does not have a lease "<< - "for #{mac_i_to_s(mac)}, but it is in use by "<< - "#{new_lease_obj} #{new_lease_oid}") - - new_leases << [index.to_s, new_binary_magic.to_s] - end - - new_used_leases += new_leases.size - - if new_leases.size > 0 - allocated_e.content = " #{new_leases.join(" ")}" - else - allocated_e.remove if !allocated_e.nil? - end - end - - if (new_used_leases != used_leases) - log_error("VNet #{oid} has #{used_leases} used leases, "<< - "but it is actually #{new_used_leases}") - - doc.root.at_xpath("USED_LEASES").content = - new_used_leases.to_s - end - - counter_no_ar.each do |mac, counter_lease| - log_error("VM #{counter_lease[:vm]} has a lease from "<< - "VNet #{oid}, but it could not be matched to any AR", false) - end - - vn_mad_e = doc.root.at_xpath("VN_MAD") - if vn_mad_e.nil? - log_error("VNet #{oid} VN_MAD element is missing", false) - else - vn_mad = vn_mad_e.text - vn_mad_tmpl_e = doc.root.at_xpath("TEMPLATE/VN_MAD") - - if (vn_mad_tmpl_e.nil? || vn_mad_tmpl_e.text != vn_mad) - log_error("VNet #{oid} VN_MAD element is missing from the TEMPLATE") - - doc.root.at_xpath("TEMPLATE").add_child( - doc.create_element("VN_MAD")).content = vn_mad - end - end - - row[:body] = doc.root.to_s - - # commit - @db[:network_pool_new].insert(row) - end - end - - # Rename table - @db.run("DROP TABLE network_pool") - @db.run("ALTER TABLE network_pool_new RENAME TO network_pool") - - log_time() + check_network + fix_network + log_time ######################################################################## # Users @@ -2019,47 +500,9 @@ EOT # USER QUOTAS ######################################################################## - # This block is not needed for now -=begin - @db.transaction do - @db.fetch("SELECT oid FROM user_pool") do |row| - found = false + check_fix_user_quotas - @db.fetch("SELECT user_oid FROM user_quotas WHERE user_oid=#{row[:oid]}") do |q_row| - found = true - end - - if !found - log_error("User #{row[:oid]} does not have a quotas entry") - - @db.run "INSERT INTO user_quotas VALUES(#{row[:oid]},'#{row[:oid]}');" - end - end - end -=end - @db.run "ALTER TABLE user_quotas RENAME TO old_user_quotas;" - @db.run "CREATE TABLE user_quotas (user_oid INTEGER PRIMARY KEY, body MEDIUMTEXT);" - - @db.transaction do - # oneadmin does not have quotas - @db.fetch("SELECT * FROM old_user_quotas WHERE user_oid=0") do |row| - @db[:user_quotas].insert(row) - end - - @db.fetch("SELECT * FROM old_user_quotas WHERE user_oid>0") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - calculate_quotas(doc, "uid=#{row[:user_oid]}", "User") - - @db[:user_quotas].insert( - :user_oid => row[:user_oid], - :body => doc.root.to_s) - end - end - - @db.run "DROP TABLE old_user_quotas;" - - log_time() + log_time ######################################################################## # Groups @@ -2067,47 +510,9 @@ EOT # GROUP QUOTAS ######################################################################## - # This block is not needed for now -=begin - @db.transaction do - @db.fetch("SELECT oid FROM group_pool") do |row| - found = false + check_fix_group_quotas - @db.fetch("SELECT group_oid FROM group_quotas WHERE group_oid=#{row[:oid]}") do |q_row| - found = true - end - - if !found - log_error("Group #{row[:oid]} does not have a quotas entry") - - @db.run "INSERT INTO group_quotas VALUES(#{row[:oid]},'#{row[:oid]}');" - end - end - end -=end - @db.run "ALTER TABLE group_quotas RENAME TO old_group_quotas;" - @db.run "CREATE TABLE group_quotas (group_oid INTEGER PRIMARY KEY, body MEDIUMTEXT);" - - @db.transaction do - # oneadmin does not have quotas - @db.fetch("SELECT * FROM old_group_quotas WHERE group_oid=0") do |row| - @db[:group_quotas].insert(row) - end - - @db.fetch("SELECT * FROM old_group_quotas WHERE group_oid>0") do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - calculate_quotas(doc, "gid=#{row[:group_oid]}", "Group") - - @db[:group_quotas].insert( - :group_oid => row[:group_oid], - :body => doc.root.to_s) - end - end - - @db.run "DROP TABLE old_group_quotas;" - - log_time() + log_time ######################################################################## # VM Templates @@ -2115,92 +520,12 @@ EOT # TEMPLATE/OS/BOOT ######################################################################## - templates_fix = {} + check_template + fix_template - @db.transaction do - @db[:template_pool].each do |row| - doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + log_time - boot = doc.root.at_xpath("TEMPLATE/OS/BOOT") - - if boot.nil? || boot.text.downcase.match(/fd|hd|cdrom|network/).nil? - next - end - - # Note: this code assumes that disks are ordered in the same order as - # their target, and may break boot order if the target is not left - # completely to oned. - # If, for example, the third disk ends with target="vda", - # boot="hd" should be updated to boot="disk2", but it is not - - devs = [] - - hd_i = 0 - cdrom_i = 0 - network_i = 0 - - error = false - - boot.text.split(",").each do |dev| - dev.downcase! - - case dev - when "hd", "cdrom" - index = nil - if dev == "hd" - index = hd_i - hd_i += 1 - else - index = cdrom_i - cdrom_i += 1 - end - - id = get_disk_id(dev, index, doc) - if id.nil? - log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", but DISK ##{index} of type #{dev} could not be found to fix it automatically", false) - error = true - end - devs.push("disk#{id}") - - when "network" - devs.push("nic#{network_i}") - network_i += 1 - - when "fd" - log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", but \"fd\" is not supported anymore and can't be fixed automatically", false) - error = true - - else - log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", but it can't be parsed to be fixed automatically", false) - error = true - - end - end - - if error - next - end - - new_boot = devs.join(",") - - log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", is was updated to #{new_boot}") - - boot.content = new_boot - - templates_fix[row[:oid]] = doc.root.to_s - end - end - - - @db.transaction do - templates_fix.each do |id, body| - @db[:template_pool].where(:oid => id).update(:body => body) - end - end - - log_time() - - log_total_errors() + log_total_errors return true end @@ -2239,321 +564,6 @@ EOT puts "A copy of this output was stored in #{LOG}" end - - - def calculate_quotas(doc, where_filter, resource) - - oid = doc.root.at_xpath("ID").text.to_i - - # VM quotas - cpu_used = 0 - mem_used = 0 - vms_used = 0 - sys_used = 0 - - # VNet quotas - vnet_usage = {} - - # Image quotas - img_usage = {} - - @db.fetch("SELECT body FROM vm_pool WHERE #{where_filter} AND state<>6") do |vm_row| - vmdoc = Nokogiri::XML(vm_row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - # VM quotas - vmdoc.root.xpath("TEMPLATE/CPU").each { |e| - # truncate to 2 decimals - cpu = (e.text.to_f * 100).to_i - cpu_used += cpu - } - - vmdoc.root.xpath("TEMPLATE/MEMORY").each { |e| - mem_used += e.text.to_i - } - - vmdoc.root.xpath("TEMPLATE/DISK").each { |e| - type = "" - - e.xpath("TYPE").each { |t_elem| - type = t_elem.text.upcase - } - - size = 0 - - if !e.at_xpath("SIZE").nil? - size = e.at_xpath("SIZE").text.to_i - end - - if ( type == "SWAP" || type == "FS") - sys_used += size - else - if !e.at_xpath("CLONE").nil? - clone = (e.at_xpath("CLONE").text.upcase == "YES") - - target = nil - - if clone - target = e.at_xpath("CLONE_TARGET").text if !e.at_xpath("CLONE_TARGET").nil? - else - target = e.at_xpath("LN_TARGET").text if !e.at_xpath("LN_TARGET").nil? - end - - if !target.nil? && target == "SYSTEM" - sys_used += size - - if !e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").nil? - sys_used += e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").text.to_i - end - end - end - end - } - - vms_used += 1 - - # VNet quotas - vmdoc.root.xpath("TEMPLATE/NIC/NETWORK_ID").each { |e| - vnet_usage[e.text] = 0 if vnet_usage[e.text].nil? - vnet_usage[e.text] += 1 - } - - # Image quotas - vmdoc.root.xpath("TEMPLATE/DISK/IMAGE_ID").each { |e| - img_usage[e.text] = 0 if img_usage[e.text].nil? - img_usage[e.text] += 1 - } - end - - - @db.fetch("SELECT body FROM vrouter_pool WHERE #{where_filter}") do |vrouter_row| - vrouter_doc = Nokogiri::XML(vrouter_row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - # VNet quotas - vrouter_doc.root.xpath("TEMPLATE/NIC").each { |nic| - net_id = nil - nic.xpath("NETWORK_ID").each do |nid| - net_id = nid.text - end - - floating = false - - nic.xpath("FLOATING_IP").each do |floating_e| - floating = (floating_e.text.upcase == "YES") - end - - if !net_id.nil? && floating - vnet_usage[net_id] = 0 if vnet_usage[net_id].nil? - - vnet_usage[net_id] += 1 - end - } - - end - - # VM quotas - - vm_elem = nil - doc.root.xpath("VM_QUOTA/VM").each { |e| vm_elem = e } - - if vm_elem.nil? - doc.root.xpath("VM_QUOTA").each { |e| e.remove } - - vm_quota = doc.root.add_child(doc.create_element("VM_QUOTA")) - vm_elem = vm_quota.add_child(doc.create_element("VM")) - - vm_elem.add_child(doc.create_element("CPU")).content = "-1" - vm_elem.add_child(doc.create_element("CPU_USED")).content = "0" - - vm_elem.add_child(doc.create_element("MEMORY")).content = "-1" - vm_elem.add_child(doc.create_element("MEMORY_USED")).content = "0" - - vm_elem.add_child(doc.create_element("VMS")).content = "-1" - vm_elem.add_child(doc.create_element("VMS_USED")).content = "0" - - vm_elem.add_child(doc.create_element("SYSTEM_DISK_SIZE")).content = "-1" - vm_elem.add_child(doc.create_element("SYSTEM_DISK_SIZE_USED")).content = "0" - end - - - vm_elem.xpath("CPU_USED").each { |e| - - # Because of bug http://dev.opennebula.org/issues/1567 the element - # may contain a float number in scientific notation. - - # Check if the float value or the string representation mismatch, - # but ignoring the precision - - cpu_used = (cpu_used / 100.0) - - different = ( e.text.to_f != cpu_used || - ![sprintf('%.2f', cpu_used), sprintf('%.1f', cpu_used), sprintf('%.0f', cpu_used)].include?(e.text) ) - - cpu_used_str = sprintf('%.2f', cpu_used) - - if different - log_error("#{resource} #{oid} quotas: CPU_USED has #{e.text} \tis\t#{cpu_used_str}") - e.content = cpu_used_str - end - } - - vm_elem.xpath("MEMORY_USED").each { |e| - if e.text != mem_used.to_s - log_error("#{resource} #{oid} quotas: MEMORY_USED has #{e.text} \tis\t#{mem_used}") - e.content = mem_used.to_s - end - } - - vm_elem.xpath("VMS_USED").each { |e| - if e.text != vms_used.to_s - log_error("#{resource} #{oid} quotas: VMS_USED has #{e.text} \tis\t#{vms_used}") - e.content = vms_used.to_s - end - } - - vm_elem.xpath("SYSTEM_DISK_SIZE_USED").each { |e| - if e.text != sys_used.to_s - log_error("#{resource} #{oid} quotas: SYSTEM_DISK_SIZE_USED has #{e.text} \tis\t#{sys_used}") - e.content = sys_used.to_s - end - } - - # VNet quotas - - net_quota = nil - doc.root.xpath("NETWORK_QUOTA").each { |e| net_quota = e } - - if net_quota.nil? - net_quota = doc.root.add_child(doc.create_element("NETWORK_QUOTA")) - end - - net_quota.xpath("NETWORK").each { |net_elem| - vnet_id = net_elem.at_xpath("ID").text - - leases_used = vnet_usage.delete(vnet_id) - - leases_used = 0 if leases_used.nil? - - net_elem.xpath("LEASES_USED").each { |e| - if e.text != leases_used.to_s - log_error("#{resource} #{oid} quotas: VNet #{vnet_id}\tLEASES_USED has #{e.text} \tis\t#{leases_used}") - e.content = leases_used.to_s - end - } - } - - vnet_usage.each { |vnet_id, leases_used| - log_error("#{resource} #{oid} quotas: VNet #{vnet_id}\tLEASES_USED has 0 \tis\t#{leases_used}") - - new_elem = net_quota.add_child(doc.create_element("NETWORK")) - - new_elem.add_child(doc.create_element("ID")).content = vnet_id - new_elem.add_child(doc.create_element("LEASES")).content = "-1" - new_elem.add_child(doc.create_element("LEASES_USED")).content = leases_used.to_s - } - - - # Image quotas - - img_quota = nil - doc.root.xpath("IMAGE_QUOTA").each { |e| img_quota = e } - - if img_quota.nil? - img_quota = doc.root.add_child(doc.create_element("IMAGE_QUOTA")) - end - - img_quota.xpath("IMAGE").each { |img_elem| - img_id = img_elem.at_xpath("ID").text - - rvms = img_usage.delete(img_id) - - rvms = 0 if rvms.nil? - - img_elem.xpath("RVMS_USED").each { |e| - if e.text != rvms.to_s - log_error("#{resource} #{oid} quotas: Image #{img_id}\tRVMS has #{e.text} \tis\t#{rvms}") - e.content = rvms.to_s - end - } - } - - img_usage.each { |img_id, rvms| - log_error("#{resource} #{oid} quotas: Image #{img_id}\tRVMS has 0 \tis\t#{rvms}") - - new_elem = img_quota.add_child(doc.create_element("IMAGE")) - - new_elem.add_child(doc.create_element("ID")).content = img_id - new_elem.add_child(doc.create_element("RVMS")).content = "-1" - new_elem.add_child(doc.create_element("RVMS_USED")).content = rvms.to_s - } - - # Datastore quotas - ds_usage = {} - - @db.fetch("SELECT body FROM image_pool WHERE #{where_filter}") do |img_row| - img_doc = Nokogiri::XML(img_row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} - - img_doc.root.xpath("DATASTORE_ID").each { |e| - ds_usage[e.text] = [0,0] if ds_usage[e.text].nil? - ds_usage[e.text][0] += 1 - - img_doc.root.xpath("SIZE").each { |size| - ds_usage[e.text][1] += size.text.to_i - } - - img_doc.root.xpath("SNAPSHOTS/SNAPSHOT/SIZE").each { |size| - ds_usage[e.text][1] += size.text.to_i - } - } - end - - ds_quota = nil - doc.root.xpath("DATASTORE_QUOTA").each { |e| ds_quota = e } - - if ds_quota.nil? - ds_quota = doc.root.add_child(doc.create_element("DATASTORE_QUOTA")) - end - - ds_quota.xpath("DATASTORE").each { |ds_elem| - ds_id = ds_elem.at_xpath("ID").text - - images_used,size_used = ds_usage.delete(ds_id) - - images_used = 0 if images_used.nil? - size_used = 0 if size_used.nil? - - ds_elem.xpath("IMAGES_USED").each { |e| - if e.text != images_used.to_s - log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tIMAGES_USED has #{e.text} \tis\t#{images_used}") - e.content = images_used.to_s - end - } - - ds_elem.xpath("SIZE_USED").each { |e| - if e.text != size_used.to_s - log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tSIZE_USED has #{e.text} \tis\t#{size_used}") - e.content = size_used.to_s - end - } - } - - ds_usage.each { |ds_id, array| - images_used,size_used = array - - log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tIMAGES_USED has 0 \tis\t#{images_used}") - log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tSIZE_USED has 0 \tis\t#{size_used}") - - new_elem = ds_quota.add_child(doc.create_element("DATASTORE")) - - new_elem.add_child(doc.create_element("ID")).content = ds_id - - new_elem.add_child(doc.create_element("IMAGES")).content = "-1" - new_elem.add_child(doc.create_element("IMAGES_USED")).content = images_used.to_s - - new_elem.add_child(doc.create_element("SIZE")).content = "-1" - new_elem.add_child(doc.create_element("SIZE_USED")).content = size_used.to_s - } - end - def mac_s_to_i(mac) return nil if mac.empty? return mac.split(":").map {|e| diff --git a/src/onedb/fsck/cluster.rb b/src/onedb/fsck/cluster.rb new file mode 100644 index 0000000000..4de9ddb767 --- /dev/null +++ b/src/onedb/fsck/cluster.rb @@ -0,0 +1,175 @@ + +module OneDBFsck + def init_cluster + cluster = @data_cluster = {} + + @db.fetch("SELECT oid, name FROM cluster_pool") do |row| + cluster[row[:oid]] = {} + + cluster[row[:oid]][:name] = row[:name] + + cluster[row[:oid]][:hosts] = [] + cluster[row[:oid]][:datastores] = Set.new + cluster[row[:oid]][:vnets] = Set.new + end + end + + def check_fix_cluster + cluster = @data_cluster + create_table(:cluster_pool, :cluster_pool_new) + + @db.transaction do + @db.fetch("SELECT * from cluster_pool") do |row| + cluster_id = row[:oid] + doc = Document.new(row[:body]) + + # Hosts + hosts_elem = doc.root.elements.delete("HOSTS") + + hosts_new_elem = doc.root.add_element("HOSTS") + + cluster[cluster_id][:hosts].each do |id| + id_elem = hosts_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error("Host #{id} is missing from Cluster " << + "#{cluster_id} host id list") + end + + hosts_new_elem.add_element("ID").text = id.to_s + end + + hosts_elem.each_element("ID") do |id_elem| + log_error("Host #{id_elem.text} is in Cluster " << + "#{cluster_id} host id list, but it should not") + end + + + # Datastores + ds_elem = doc.root.elements.delete("DATASTORES") + + ds_new_elem = doc.root.add_element("DATASTORES") + + cluster[cluster_id][:datastores].each do |id| + id_elem = ds_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error("Datastore #{id} is missing from Cluster " << + "#{cluster_id} datastore id list") + end + + ds_new_elem.add_element("ID").text = id.to_s + + if @db.fetch("SELECT * FROM cluster_datastore_relation " << + "WHERE cid=#{cluster_id} AND oid=#{id}").empty? + + log_error("Table cluster_datastore_relation is " << + "missing relation cluster #{cluster_id}, " << + "datastore #{id}") + + @db[:cluster_datastore_relation].insert( + cid: cluster_id, + oid: id + ) + end + end + + ds_elem.each_element("ID") do |id_elem| + log_error("Datastore #{id_elem.text} is in Cluster " << + "#{cluster_id} datastore id list, but " << + "it should not") + end + + + # VNets + vnets_elem = doc.root.elements.delete("VNETS") + + vnets_new_elem = doc.root.add_element("VNETS") + + cluster[cluster_id][:vnets].each do |id| + id_elem = vnets_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error("VNet #{id} is missing from Cluster " << + "#{cluster_id} vnet id list") + end + + vnets_new_elem.add_element("ID").text = id.to_s + + if @db.fetch("SELECT * FROM cluster_network_relation " << + "WHERE cid=#{cluster_id} AND oid=#{id}").empty? + + log_error("Table cluster_network_relation is " << + "missing relation cluster #{cluster_id}, " << + "vnet #{id}") + + @db[:cluster_network_relation].insert( + cid: cluster_id, + oid: id + ) + end + end + + vnets_elem.each_element("ID") do |id_elem| + log_error("VNet #{id_elem.text} is in Cluster " << + "#{cluster_id} vnet id list, but it should not") + end + + + row[:body] = doc.root.to_s + + # commit + @db[:cluster_pool_new].insert(row) + end + end + + # Rename table + @db.run("DROP TABLE cluster_pool") + @db.run("ALTER TABLE cluster_pool_new RENAME TO cluster_pool") + end + + def check_fix_cluster_relations + cluster = @data_cluster + + @db.transaction do + create_table(:cluster_datastore_relation, + :cluster_datastore_relation_new) + + @db.fetch("SELECT * from cluster_datastore_relation") do |row| + if (cluster[row[:cid]][:datastores].count(row[:oid]) != 1) + log_error("Table cluster_datastore_relation contains " << + "relation cluster #{row[:cid]}, datastore " << + "#{row[:oid]}, but it should not") + else + @db[:cluster_datastore_relation_new].insert(row) + end + end + + @db.run("DROP TABLE cluster_datastore_relation") + @db.run("ALTER TABLE cluster_datastore_relation_new " << + "RENAME TO cluster_datastore_relation") + end + + log_time() + + @db.transaction do + create_table(:cluster_network_relation, + :cluster_network_relation_new) + + @db.fetch("SELECT * from cluster_network_relation") do |row| + if (cluster[row[:cid]][:vnets].count(row[:oid]) != 1) + log_error("Table cluster_network_relation contains " << + "relation cluster #{row[:cid]}, " << + "vnet #{row[:oid]}, but it should not") + else + @db[:cluster_network_relation_new].insert(row) + end + end + + @db.run("DROP TABLE cluster_network_relation") + @db.run("ALTER TABLE cluster_network_relation_new " << + "RENAME TO cluster_network_relation") + end + end +end + diff --git a/src/onedb/fsck/cluster_vnc_bitmap.rb b/src/onedb/fsck/cluster_vnc_bitmap.rb new file mode 100644 index 0000000000..79bda0c083 --- /dev/null +++ b/src/onedb/fsck/cluster_vnc_bitmap.rb @@ -0,0 +1,40 @@ + +module OneDBFsck + def check_cluster_vnc_bitmap + fixes = @fixes_cluster_vnc_bitmap = {} + cluster_vnc = @data_vm[:vnc] + + vnc_pool_size = 65536 + + @db.fetch("SELECT * FROM cluster_pool") do |row| + cluster_id = row[:oid] + + if cluster_vnc[cluster_id] + map = "" + vnc_pool_size.times.each do |i| + map << (cluster_vnc[cluster_id].include?(vnc_pool_size - 1 - i) ? "1" : "0") + end + + map_encoded = Base64::strict_encode64(Zlib::Deflate.deflate(map)) + else + map_encoded = "eJztwYEAAAAAgCCl/ekWqQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABqFo8C0Q==" + end + + old_map_encoded = @db[:cluster_vnc_bitmap].first(:id => cluster_id)[:map] rescue nil + + if old_map_encoded != map_encoded + log_error("Cluster #{cluster_id} has not the proper reserved VNC ports") + fixes[cluster_id] = map_encoded + end + end + end + + def fix_cluster_vnc_bitmap + @db.transaction do + @fixes_cluster_vnc_bitmap.each do |id, map| + @db[:cluster_vnc_bitmap].where(oid: id).update(map: map) + end + end + end +end + diff --git a/src/onedb/fsck/datastore.rb b/src/onedb/fsck/datastore.rb new file mode 100644 index 0000000000..53ad4ab3bf --- /dev/null +++ b/src/onedb/fsck/datastore.rb @@ -0,0 +1,138 @@ + +module OneDBFsck + def init_datastore_counters + @data_datastore = {} + + @db.fetch("SELECT oid, name FROM datastore_pool") do |row| + @data_datastore[row[:oid]] = { + name: row[:name], + images: [] + } + end + end + + def check_datastore_cluster + cluster = @data_cluster + @fixes_datastore_cluster = {} + + @db.fetch("SELECT oid,body FROM datastore_pool") do |row| + doc = nokogiri_doc(row[:body]) + + doc.root.xpath("CLUSTERS/ID").each do |e| + cluster_id = e.text.to_i + + cluster_entry = cluster[cluster_id] + + if cluster_entry.nil? + log_error("Datastore #{row[:oid]} is in cluster " << + "#{cluster_id}, but it does not exist") + + e.remove + + @fixes_datastore_cluster[row[:oid]] = { body: doc.root.to_s } + else + cluster_entry[:datastores] << row[:oid] + end + end + end + end + + def fix_datastore_cluster + @db.transaction do + @fixes_datastore_cluster.each do |id, entry| + @db[:datastore_pool].where(oid: id).update(body: entry[:body]) + end + end + end + + def check_datastore_image + datastore = @data_datastore + @fixes_datastore_image = {} + + @db.fetch("SELECT oid,body FROM image_pool") do |row| + doc = Document.new(row[:body]) + + ds_id = doc.root.get_text('DATASTORE_ID').to_s.to_i + ds_name = doc.root.get_text('DATASTORE') + + if ds_id != -1 + ds_entry = datastore[ds_id] + + if ds_entry.nil? + log_error("Image #{row[:oid]} has datastore #{ds_id}, but it does not exist. The image is probably unusable, and needs to be deleted manually:\n"<< + " * The image contents should be deleted manually:\n"<< + " #{doc.root.get_text('SOURCE')}\n"<< + " * The DB entry can be then deleted with the command:\n"<< + " DELETE FROM image_pool WHERE oid=#{row[:oid]};\n"<< + " * Run fsck again.\n", false) + else + if ds_name != ds_entry[:name] + log_error("Image #{row[:oid]} has a wrong name for datastore #{ds_id}, #{ds_name}. It will be changed to #{ds_entry[:name]}") + + doc.root.each_element('DATASTORE') do |e| + e.text = ds_entry[:name] + end + + @fixes_datastore_image[row[:oid]] = doc.root.to_s + end + + ds_entry[:images] << row[:oid] + end + end + end + end + + def fix_datastore_image + @db.transaction do + @fixes_datastore_image.each do |id, body| + @db[:image_pool].where(:oid => id).update(:body => body) + end + end + end + + def check_fix_datastore + datastore = @data_datastore + + create_table(:datastore_pool, :datastore_pool_new) + + @db.transaction do + @db.fetch("SELECT * from datastore_pool") do |row| + ds_id = row[:oid] + doc = Document.new(row[:body]) + + images_elem = doc.root.elements.delete("IMAGES") + + images_new_elem = doc.root.add_element("IMAGES") + + datastore[ds_id][:images].each do |id| + id_elem = images_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error( + "Image #{id} is missing from Datastore #{ds_id} "<< + "image id list") + end + + images_new_elem.add_element("ID").text = id.to_s + end + + images_elem.each_element("ID") do |id_elem| + log_error( + "Image #{id_elem.text} is in Datastore #{ds_id} "<< + "image id list, but it should not") + end + + + row[:body] = doc.root.to_s + + # commit + @db[:datastore_pool_new].insert(row) + end + end + + # Rename table + @db.run("DROP TABLE datastore_pool") + @db.run("ALTER TABLE datastore_pool_new RENAME TO datastore_pool") + end +end + diff --git a/src/onedb/fsck/group.rb b/src/onedb/fsck/group.rb new file mode 100644 index 0000000000..3ffe6178c7 --- /dev/null +++ b/src/onedb/fsck/group.rb @@ -0,0 +1,58 @@ + +module OneDBFsck + def check_group + @fixes_group = groups_fix = {} + + group = @data_user[:group] + + @db.fetch("SELECT oid,body from group_pool") do |row| + gid = row[:oid] + doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + users_elem = doc.root.at_xpath("USERS") + users_elem.remove if !users_elem.nil? + + users_new_elem = doc.create_element("USERS") + doc.root.add_child(users_new_elem) + + error_found = false + + group[gid].each do |id| + id_elem = users_elem.at_xpath("ID[.=#{id}]") + + if id_elem.nil? + log_error("User #{id} is missing from Group #{gid} users id list", !db_version[:is_slave]) + error_found = true + else + id_elem.remove + end + + users_new_elem.add_child(doc.create_element("ID")).content = id.to_s + end + + users_elem.xpath("ID").each do |id_elem| + log_error("User #{id_elem.text} is in Group #{gid} users id list, but it should not", !db_version[:is_slave]) + error_found = true + end + + + if error_found + groups_fix[row[:oid]] = doc.root.to_s + end + end + end + + def fix_group + groups_fix = @fixes_group + + if !db_version[:is_slave] + @db.transaction do + groups_fix.each do |id, body| + @db[:group_pool].where(:oid => id).update(:body => body) + end + end + elsif !groups_fix.empty? + log_msg("^ Group errors need to be fixed in the master OpenNebula") + end + end +end diff --git a/src/onedb/fsck/history.rb b/src/onedb/fsck/history.rb new file mode 100644 index 0000000000..1229988627 --- /dev/null +++ b/src/onedb/fsck/history.rb @@ -0,0 +1,73 @@ + +module OneDBFsck + def check_history + check_history_etime + + log_time + + check_history_opened + end + + def check_history_etime + # DATA: check history etime + + # Bug #4000 may cause history records with etime=0 when they should + # be closed. The last history can be fixed with the VM etime, but + # previous history entries need to be fixed manually + + # Query to select history elements that: + # - have etime = 0 + # - are not the last seq + @db.fetch("SELECT vid,seq FROM history WHERE (etime = 0 AND seq <> (SELECT MAX(seq) FROM history AS subhistory WHERE history.vid = subhistory.vid) )") do |row| + log_error("History record for VM #{row[:vid]} seq # #{row[:seq]} is not closed (etime = 0)", false) + end + end + + def check_history_opened + history_fix = @fixes_history = [] + + # DATA: go through all bad history records (etime=0) and ask + # DATA: new time values to fix them + + # Query to select history elements that have: + # - etime = 0 + # - is last seq + # - VM is DONE + @db.fetch("SELECT * FROM history WHERE (etime = 0 AND vid IN (SELECT oid FROM vm_pool WHERE state=6) AND seq = (SELECT MAX(seq) FROM history AS subhistory WHERE history.vid=subhistory.vid))") do |row| + log_error("History record for VM #{row[:vid]} seq # #{row[:seq]} is not closed (etime = 0), but the VM is in state DONE") + + etime = 0 + + @db.fetch("SELECT body FROM vm_pool WHERE oid=#{row[:vid]}") do |vm_row| + vm_doc = nokogiri_doc(vm_row[:body]) + + etime = vm_doc.root.at_xpath("ETIME").text.to_i + end + + history_doc = nokogiri_doc(row[:body]) + + ["RETIME", "ESTIME", "EETIME", "ETIME"].each do |att| + elem = history_doc.root.at_xpath(att) + if (elem.text == "0") + elem.content = etime + end + end + + row[:body] = history_doc.root.to_s + row[:etime] = etime + + history_fix.push(row) + end + end + + def fix_history + # DATA: FIX: update history records with fixed data + # DATA: TODO: check all fixes to always do the same (update vs rewrite) + @db.transaction do + @fixes_history.each do |row| + @db[:history].where(vid: row[:vid], seq: row[:seq]).update(row) + end + end + end +end + diff --git a/src/onedb/fsck/host.rb b/src/onedb/fsck/host.rb new file mode 100644 index 0000000000..0ecdc27a28 --- /dev/null +++ b/src/onedb/fsck/host.rb @@ -0,0 +1,179 @@ + +module OneDBFsck + # Initialize all the host counters to 0 + def init_host_counters + @db.fetch("SELECT oid, name FROM host_pool") do |row| + counters[:host][row[:oid]] = { + :name => row[:name], + :memory => 0, + :cpu => 0, + :rvms => Set.new + } + end + end + + def check_host_cluster + cluster = @data_cluster + hosts_fix = @fixes_host_cluster = {} + + @db.fetch("SELECT oid,body,cid FROM host_pool") do |row| + doc = Document.new(row[:body]) + + cluster_id = doc.root.get_text('CLUSTER_ID').to_s.to_i + cluster_name = doc.root.get_text('CLUSTER') + + if cluster_id != row[:cid] + log_error("Host #{row[:oid]} is in cluster #{cluster_id}, " << + "but cid column has cluster #{row[:cid]}") + hosts_fix[row[:oid]] = {:body => row[:body], :cid => cluster_id} + end + + if cluster_id != -1 + cluster_entry = cluster[cluster_id] + + if cluster_entry.nil? + log_error("Host #{row[:oid]} is in cluster " << + "#{cluster_id}, but it does not exist") + + doc.root.each_element('CLUSTER_ID') do |e| + e.text = "-1" + end + + doc.root.each_element('CLUSTER') do |e| + e.text = "" + end + + hosts_fix[row[:oid]] = {:body => doc.root.to_s, :cid => -1} + else + if cluster_name != cluster_entry[:name] + log_error("Host #{row[:oid]} has a wrong name for " << + "cluster #{cluster_id}, #{cluster_name}. " << + "It will be changed to #{cluster_entry[:name]}") + + doc.root.each_element('CLUSTER') do |e| + e.text = cluster_entry[:name] + end + + hosts_fix[row[:oid]] = { + body: doc.root.to_s, + cid: cluster_id + } + end + + cluster_entry[:hosts] << row[:oid] + end + end + end + end + + def fix_host_cluster + @db.transaction do + @fixes_host_cluster.each do |id, entry| + @db[:host_pool].where(oid: id).update( + body: entry[:body], + cid: entry[:cid] + ) + end + end + end + + def check_host + @fixes_host = {} + + # DATA: FIX: Calculate the host's xml and write them to host_pool_new + @db[:host_pool].each do |row| + host_doc = nokogiri_doc(row[:body]) + + error = false + + hid = row[:oid] + + counters_host = counters[:host][hid] + + rvms = counters_host[:rvms].size + cpu_usage = (counters_host[:cpu]*100).to_i + mem_usage = counters_host[:memory]*1024 + + # rewrite running_vms + host_doc.root.xpath("HOST_SHARE/RUNNING_VMS").each {|e| + if e.text != rvms.to_s + log_error( + "Host #{hid} RUNNING_VMS has #{e.text} \tis\t#{rvms}") + e.content = rvms + error = true + end + } + + # re-do list of VM IDs + vms_elem = host_doc.root.at_xpath("VMS").remove + + vms_new_elem = host_doc.create_element("VMS") + host_doc.root.add_child(vms_new_elem) + + counters_host[:rvms].each do |id| + id_elem = vms_elem.at_xpath("ID[.=#{id}]") + + if id_elem.nil? + log_error( + "VM #{id} is missing from Host #{hid} VM id list") + error = true + else + id_elem.remove + end + + vms_new_elem.add_child(host_doc.create_element("ID")).content = id.to_s + end + + vms_elem.xpath("ID").each do |id_elem| + log_error( + "VM #{id_elem.text} is in Host #{hid} VM id list, "<< + "but it should not") + error = true + end + + host_doc.root.xpath("HOST_SHARE/PCI_DEVICES/PCI").each do |pci| + if !pci.at_xpath("VMID").nil? + vmid = pci.at_xpath("VMID").text.to_i + + if vmid != -1 && !counters_host[:rvms].include?(vmid) + log_error("VM #{vmid} has a PCI device assigned in host #{hid}, but it should not. Device: #{pci.at_xpath('DEVICE_NAME').text}") + pci.at_xpath("VMID").content = "-1" + error = true + end + end + end + + # rewrite cpu + host_doc.root.xpath("HOST_SHARE/CPU_USAGE").each {|e| + if e.text != cpu_usage.to_s + log_error( + "Host #{hid} CPU_USAGE has #{e.text} "<< + "\tis\t#{cpu_usage}") + e.content = cpu_usage + error = true + end + } + + # rewrite memory + host_doc.root.xpath("HOST_SHARE/MEM_USAGE").each {|e| + if e.text != mem_usage.to_s + log_error("Host #{hid} MEM_USAGE has #{e.text} "<< + "\tis\t#{mem_usage}") + e.content = mem_usage + error = true + end + } + + @fixes_host[hid] = host_doc.root.to_s if error + end + end + + def fix_host + @db.transaction do + @fixes_host.each do |id, body| + @db[:host_pool].where(oid: id).update(body: body) + end + end + end +end + diff --git a/src/onedb/fsck/image.rb b/src/onedb/fsck/image.rb new file mode 100644 index 0000000000..65c12bdddf --- /dev/null +++ b/src/onedb/fsck/image.rb @@ -0,0 +1,191 @@ + +module OneDBFsck + # Init image counters + def init_image_counters + @db.fetch("SELECT oid,body FROM image_pool") do |row| + if counters[:image][row[:oid]].nil? + counters[:image][row[:oid]] = { + :vms => Set.new, + :clones => Set.new, + :app_clones => Set.new + } + end + + doc = Document.new(row[:body]) + + doc.root.each_element("CLONING_ID") do |e| + img_id = e.text.to_i + + if counters[:image][img_id].nil? + counters[:image][img_id] = { + :vms => Set.new, + :clones => Set.new, + :app_clones => Set.new + } + end + + counters[:image][img_id][:clones].add(row[:oid]) + end + end + end + + def check_image + @fixes_image = {} + + @db.transaction do + @db[:image_pool].each do |row| + doc = Document.new(row[:body]) + + oid = row[:oid] + + persistent = ( doc.root.get_text('PERSISTENT').to_s == "1" ) + current_state = doc.root.get_text('STATE').to_s.to_i + + counters_img = counters[:image][oid] + + rvms = counters_img[:vms].size + n_cloning_ops = counters_img[:clones].size + counters_img[:app_clones].size + + # DATA: CHECK: running vm counter with this image + # rewrite running_vms + doc.root.each_element("RUNNING_VMS") {|e| + if e.text != rvms.to_s + log_error("Image #{oid} RUNNING_VMS has #{e.text} \tis\t#{rvms}") + e.text = rvms + end + } + + # re-do list of VM IDs + vms_elem = doc.root.elements.delete("VMS") + + vms_new_elem = doc.root.add_element("VMS") + + # DATA: CHECK: running vm list with this image + counters_img[:vms].each do |id| + id_elem = vms_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error("VM #{id} is missing from Image #{oid} VM id list") + end + + vms_new_elem.add_element("ID").text = id.to_s + end + + vms_elem.each_element("ID") do |id_elem| + log_error("VM #{id_elem.text} is in Image #{oid} VM id list, but it should not") + end + + + if ( persistent && rvms > 0 ) + n_cloning_ops = 0 + counters_img[:clones] = Set.new + counters_img[:app_clones] = Set.new + end + + # DATA: CHECK: Check number of clones + doc.root.each_element("CLONING_OPS") { |e| + if e.text != n_cloning_ops.to_s + log_error("Image #{oid} CLONING_OPS has #{e.text} \tis\t#{n_cloning_ops}") + e.text = n_cloning_ops + end + } + + # re-do list of Images cloning this one + clones_elem = doc.root.elements.delete("CLONES") + + clones_new_elem = doc.root.add_element("CLONES") + + # DATA: CHECK: image clones (is it used?) + counters_img[:clones].each do |id| + id_elem = clones_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error("Image #{id} is missing from Image #{oid} CLONES id list") + end + + clones_new_elem.add_element("ID").text = id.to_s + end + + clones_elem.each_element("ID") do |id_elem| + log_error("Image #{id_elem.text} is in Image #{oid} CLONES id list, but it should not") + end + + # re-do list of Apps cloning this one + clones_elem = doc.root.elements.delete("APP_CLONES") + + clones_new_elem = doc.root.add_element("APP_CLONES") + + # DATA: CHECK: check app clones + # DATA: TODO: understand app clones and image clones + counters_img[:app_clones].each do |id| + id_elem = clones_elem.elements.delete("ID[.=#{id}]") + + if id_elem.nil? + log_error("Marketplace App #{id} is missing from Image #{oid} APP_CLONES id list") + end + + clones_new_elem.add_element("ID").text = id.to_s + end + + clones_elem.each_element("ID") do |id_elem| + log_error("Marketplace App #{id_elem.text} is in Image #{oid} APP_CLONES id list, but it should not") + end + + + # DATA: Check state + # DATA: TODO: Error state is taken into account? + + state = current_state + + if persistent + if ( rvms > 0 ) + state = 8 # USED_PERS + elsif ( n_cloning_ops > 0 ) + state = 6 # CLONE + elsif ( current_state == 8 || current_state == 6 ) + # rvms == 0 && n_cloning_ops == 0, but image is in state + # USED_PERS or CLONE + + state = 1 # READY + end + else + if ( rvms > 0 || n_cloning_ops > 0 ) + state = 2 # USED + elsif ( current_state == 2 ) + # rvms == 0 && n_cloning_ops == 0, but image is in state + # USED + + state = 1 # READY + end + end + + doc.root.each_element("STATE") { |e| + if e.text != state.to_s + log_error("Image #{oid} has STATE " << + OpenNebula::Image::IMAGE_STATES[e.text.to_i] << + " \tis\t#{OpenNebula::Image::IMAGE_STATES[state]}") + e.text = state + end + } + + # row[:body] = doc.root.to_s + + # # commit + # @db[:image_pool_new].insert(row) + + @fixes_image[oid] = doc.root.to_s + end + end + end + + def fix_image + # # Rename table + # @db.run("DROP TABLE image_pool") + # @db.run("ALTER TABLE image_pool_new RENAME TO image_pool") + + @fixes_image.each do |oid, body| + @db[:image_pool].where(oid: oid).update(body: body) + end + end + +end diff --git a/src/onedb/fsck/marketplace.rb b/src/onedb/fsck/marketplace.rb new file mode 100644 index 0000000000..78bc54d74e --- /dev/null +++ b/src/onedb/fsck/marketplace.rb @@ -0,0 +1,90 @@ + +module OneDBFsck + # Needs: + # + # @data_marketplaceapp: set by do_check_marketplaceapp + # + # Sets: + # + # @fixes_marketplace: used by do_check_marketplace + + def check_marketplace + marketplace = @data_marketplaceapp[:marketplace] + + @fixes_marketplace = {} + markets_fix = {} + + # DATA: check marketplace pool + @db.fetch("SELECT oid,body FROM marketplace_pool") do |row| + market_id = row[:oid] + doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + apps_elem = doc.root.at_xpath("MARKETPLACEAPPS") + apps_elem.remove if !apps_elem.nil? + + apps_new_elem = doc.create_element("MARKETPLACEAPPS") + doc.root.add_child(apps_new_elem) + + error = false + + # DATA: CHECK: are all apps in the marketplace? + marketplace[market_id][:apps].each do |id| + id_elem = apps_elem.at_xpath("ID[.=#{id}]") + + if id_elem.nil? + error = true + + log_error( + "Marketplace App #{id} is missing from Marketplace #{market_id} "<< + "app id list") + else + id_elem.remove + end + + apps_new_elem.add_child(doc.create_element("ID")).content = id.to_s + end + + # DATA: CHECK: listed apps that don't belong to the marketplace + apps_elem.xpath("ID").each do |id_elem| + error = true + + log_error( + "Marketplace App #{id_elem.text} is in Marketplace #{market_id} "<< + "app id list, but it should not") + end + + zone_elem = doc.root.at_xpath("ZONE_ID") + + # DATA: CHECK: zone id + if (zone_elem.nil? || zone_elem.text == "-1") + error = true + + log_error("Marketplace #{market_id} has an invalid ZONE_ID. Will be set to 0") + + if (zone_elem.nil?) + zone_elem = doc.root.add_child(doc.create_element("ZONE_ID")) + end + + zone_elem.content = "0" + end + + if (error) + @fixes_marketplace[row[:oid]] = doc.root.to_s + end + end + end + + def fix_marketplace + # DATA: FIX: update each marketplace that needs fixing + if !db_version[:is_slave] + @db.transaction do + @fixes_marketplace.each do |id, body| + @db[:marketplace_pool].where(:oid => id).update(:body => body) + end + end + elsif !markets_fix.empty? + log_msg("^ Marketplace errors need to be fixed in the master OpenNebula") + end + end +end + diff --git a/src/onedb/fsck/marketplaceapp.rb b/src/onedb/fsck/marketplaceapp.rb new file mode 100644 index 0000000000..fd7a3d6ad2 --- /dev/null +++ b/src/onedb/fsck/marketplaceapp.rb @@ -0,0 +1,86 @@ + +module OneDBFsck + # Sets: + # + # @data_marketplaceapp: used also by marketplace check + # @fixes_marketplaceapp: used by do_check_marketplaceapp + + def check_marketplaceapp + @data_marketplaceapp = { + marketplace: {} + } + + marketplace = @data_marketplaceapp[:marketplace] + + # DATA: create marketplace hash with its name and empty apps array + @db.fetch("SELECT oid, name FROM marketplace_pool") do |row| + marketplace[row[:oid]] = {:name => row[:name], :apps => []} + end + + @fixes_marketplaceapp = {} + apps_fix = @fixes_marketplaceapp + + # DATA: go through all apps + @db.fetch("SELECT oid,body FROM marketplaceapp_pool") do |row| + doc = Document.new(row[:body]) + + market_id = doc.root.get_text('MARKETPLACE_ID').to_s.to_i + market_name = doc.root.get_text('MARKETPLACE') + + #################################################################### + # DATA: TODO, BUG: this code will only work for a standalone oned. + # In a federation, the image ID will refer to a different image + # in each zone + #################################################################### + + # DATA: get image origin id. Does it work? + origin_id = doc.root.get_text('ORIGIN_ID').to_s.to_i + if origin_id >= 0 && doc.root.get_text('STATE').to_s.to_i == 2 # LOCKED + counters[:image][origin_id][:app_clones].add(row[:oid]) + end + + #################################################################### + ##################################################################### + + if market_id != -1 + market_entry = marketplace[market_id] + + # DATA: CHECK: does marketplace for this app exist? + if market_entry.nil? + log_error("Marketplace App #{row[:oid]} has marketplace #{market_id}, but it does not exist. The app is probably unusable, and needs to be deleted manually:\n"<< + " * The DB entry can be deleted with the command:\n"<< + " DELETE FROM marketplaceapp_pool WHERE oid=#{row[:oid]};\n"<< + " * Run fsck again.\n", false) + else + # DATA: CHECK: marketplace name is correct + if market_name != market_entry[:name] + log_error("Marketplace App #{row[:oid]} has a wrong name for marketplace #{market_id}, #{market_name}. It will be changed to #{market_entry[:name]}") + + doc.root.each_element('MARKETPLACE') do |e| + e.text = market_entry[:name] + end + + apps_fix[row[:oid]] = doc.root.to_s + end + + # DATA: Add app to marketplace list. Used in marketplace check + market_entry[:apps] << row[:oid] + end + end + end + end + + def fix_marketplaceapp + # DATA: FIX: fix marketplace app data + if !db_version[:is_slave] + @db.transaction do + @fixes_marketplaceapp.each do |id, body| + @db[:marketplaceapp_pool].where(:oid => id).update(:body => body) + end + end + elsif !apps_fix.empty? + log_msg("^ Marketplace App errors need to be fixed in the master OpenNebula") + end + end +end + diff --git a/src/onedb/fsck/network.rb b/src/onedb/fsck/network.rb new file mode 100644 index 0000000000..ad757bd506 --- /dev/null +++ b/src/onedb/fsck/network.rb @@ -0,0 +1,471 @@ + +module OneDBFsck + + # Init vnet counters + def init_network_counters + @db.fetch("SELECT oid,body FROM network_pool") do |row| + doc = nokogiri_doc(row[:body]) + + ar_leases = {} + + doc.root.xpath("AR_POOL/AR/AR_ID").each do |ar_id| + ar_leases[ar_id.text.to_i] = {} + end + + counters[:vnet][row[:oid]] = { + :ar_leases => ar_leases, + :no_ar_leases => {} + } + end + end + + def check_network_cluster + cluster = @data_cluster + @fixes_host_cluster = {} + + @db.fetch("SELECT oid,body FROM network_pool") do |row| + doc = nokogiri_doc(row[:body]) + + doc.root.xpath("CLUSTERS/ID").each do |e| + cluster_id = e.text.to_i + + cluster_entry = cluster[cluster_id] + + if cluster_entry.nil? + log_error("VNet #{row[:oid]} is in cluster " << + "#{cluster_id}, but it does not exist") + + e.remove + + @fixes_host_cluster[row[:oid]] = { body: doc.root.to_s } + else + cluster_entry[:vnets] << row[:oid] + end + end + end + end + + def fix_network_cluster + @db.transaction do + @fixes_host_cluster.each do |id, entry| + @db[:host_pool].where(oid: id).update(body: entry[:body]) + end + end + end + + def init_network_lease_counters + @db.fetch("SELECT oid,body,pid FROM network_pool WHERE pid<>-1") do |row| + doc = nokogiri_doc(row[:body]) + + parent_vnet = doc.root.at_xpath("PARENT_NETWORK_ID").text.to_i + + if (row[:pid] != parent_vnet) + # TODO + end + + doc.root.xpath("AR_POOL/AR").each do |ar| + parent_ar_e = ar.at_xpath("PARENT_NETWORK_AR_ID") + if !(parent_ar_e.nil? || parent_ar_e.text == "") + + parent_ar = parent_ar_e.text.to_i + + if counters[:vnet][parent_vnet][:ar_leases][parent_ar].nil? + log_error( + "VNet #{row[:oid]} is using parent "<< + "VNet #{parent_vnet}, AR #{parent_ar}, "<< + "but the AR does not exist", false) + end + + # MAC + first_mac = mac_s_to_i(ar.at_xpath("MAC").text) + + # IP + first_ip = nil + if (!ar.at_xpath("IP").nil?) + first_ip = IPAddr.new(ar.at_xpath("IP").text.strip, Socket::AF_INET) + end + + # IP6 + global_prefix = nil + if !ar.at_xpath("GLOBAL_PREFIX").nil? + global_prefix = ip6_prefix_s_to_i( + ar.at_xpath("GLOBAL_PREFIX").text) + end + + ula_prefix = nil + if !ar.at_xpath("ULA_PREFIX").nil? + ula_prefix = ip6_prefix_s_to_i( + ar.at_xpath("ULA_PREFIX").text) + end + + link_prefix = nil + type = ar.at_xpath("TYPE").text + if ( type == "IP6" || type == "IP4_6" ) + link_prefix = 0xfe80000000000000 + end + + # Parent vnet has a lease for each address of this reservation + ar.at_xpath("SIZE").text.to_i.times do |index| + + lease = { + :ip => nil, + :ip6_global => nil, + :ip6_link => nil, + :ip6_ula => nil, + :mac => nil, + :vm => nil, + :vnet => row[:oid], + :vrouter => nil + } + + #MAC + mac = (first_mac & 0xFFFF00000000) + + (((first_mac & 0xFFFFFFFF) + index) % 0x100000000) + lease[:mac] = mac_i_to_s(mac) + + # IP + if (!first_ip.nil?) + lease[:ip] = IPAddr.new(first_ip.to_i + index, + Socket::AF_INET).to_s + end + + # IP6 + ip6_suffix = mac_to_ip6_suffix(mac) + + if (!global_prefix.nil?) + lease[:ip6_global] = IPAddr.new( + (global_prefix << 64) | ip6_suffix, + Socket::AF_INET6 ).to_s + end + + if (!ula_prefix.nil?) + lease[:ip6_ula] = IPAddr.new( + (ula_prefix << 64) | ip6_suffix, + Socket::AF_INET6 ).to_s + end + + if (!link_prefix.nil?) + lease[:ip6_link] = IPAddr.new( + (link_prefix << 64) | ip6_suffix, + Socket::AF_INET6 ).to_s + end + + counters[:vnet][parent_vnet][ + :ar_leases][parent_ar][mac] = lease + end + end + end + end + end + + def check_network + @fixes_network = {} + + @db[:network_pool].each do |row| + doc = nokogiri_doc(row[:body]) + oid = row[:oid] + + used_leases = doc.root.at_xpath("USED_LEASES").text.to_i + new_used_leases = 0 + + counter_no_ar = counters[:vnet][row[:oid]][:no_ar_leases] + + counters[:vnet][row[:oid]][:ar_leases].each do |ar_id, counter_ar| + error = false + + net_ar = doc.root.at_xpath("AR_POOL/AR[AR_ID=#{ar_id}]") + + if (net_ar.nil?) + # TODO shouldn't happen? + end + + # MAC + first_mac = mac_s_to_i(net_ar.at_xpath("MAC").text) + + # IP + first_ip = nil + if !net_ar.at_xpath("IP").nil? + first_ip_st = net_ar.at_xpath("IP").text + + if (first_ip_st != first_ip_st.strip) + log_error("VNet #{oid} AR #{ar_id} "<< + "IP \"#{first_ip_st}\" contains whitespaces") + error = true + + first_ip_st.strip! + + net_ar.at_xpath("IP").content = first_ip_st + end + + first_ip = IPAddr.new(first_ip_st, Socket::AF_INET) + end + + # IP6 + global_prefix = nil + if !net_ar.at_xpath("GLOBAL_PREFIX").nil? + global_prefix = ip6_prefix_s_to_i( + net_ar.at_xpath("GLOBAL_PREFIX").text) + end + + ula_prefix = nil + if !net_ar.at_xpath("ULA_PREFIX").nil? + ula_prefix = ip6_prefix_s_to_i( + net_ar.at_xpath("ULA_PREFIX").text) + end + + link_prefix = nil + type = net_ar.at_xpath("TYPE").text + if ( type == "IP6" || type == "IP4_6" ) + link_prefix = 0xfe80000000000000 + end + + # Allocated leases + allocated_e = net_ar.at_xpath("ALLOCATED") + + allocated = allocated_e.nil? ? "" : allocated_e.text + + leases = allocated.scan(/(\d+) (\d+)/) + + new_leases = [] + + leases.each do |lease_str| + index = lease_str[0].to_i + binary_magic = lease_str[1].to_i + + lease = { + :ip => nil, + :ip6_global => nil, + :ip6_link => nil, + :ip6_ula => nil, + :mac => nil, + :vm => nil, + :vnet => nil, + :vrouter => nil + } + + # MAC + mac = (first_mac & 0xFFFF00000000) + + (((first_mac & 0xFFFFFFFF) + index) % 0x100000000) + + lease[:mac] = mac_i_to_s(mac) + + # IP + if (!first_ip.nil?) + lease[:ip] = IPAddr.new(first_ip.to_i + index, + Socket::AF_INET).to_s + end + + # IP6 + ip6_suffix = mac_to_ip6_suffix(mac) + + if (!global_prefix.nil?) + lease[:ip6_global] = IPAddr.new( + (global_prefix << 64) | ip6_suffix, + Socket::AF_INET6 ).to_s + end + + if (!ula_prefix.nil?) + lease[:ip6_ula] = IPAddr.new( + (ula_prefix << 64) | ip6_suffix, + Socket::AF_INET6 ).to_s + end + + if (!link_prefix.nil?) + lease[:ip6_link] = IPAddr.new( + (link_prefix << 64) | ip6_suffix, + Socket::AF_INET6 ).to_s + end + + # OID + lease_oid = binary_magic & 0x00000000FFFFFFFF + lease_obj = "" + + if (binary_magic & VM_BIN != 0) + lease[:vm] = lease_oid + lease_obj = "VM" + elsif (binary_magic & NET_BIN != 0) + lease[:vnet] = lease_oid + lease_obj = "VNet" + else #(binary_magic & VROUTER_BIN != 0) + lease[:vrouter] = lease_oid + lease_obj = "VRouter" + end + + counter_lease = counter_ar[mac] + counter_ar.delete(mac) + + if counter_lease.nil? + counter_lease = counter_no_ar[mac] + counter_no_ar.delete(mac) + end + + if counter_lease.nil? + if(lease[:vm] != HOLD) + log_error( + "VNet #{oid} AR #{ar_id} has leased #{lease_to_s(lease)} "<< + "to #{lease_obj} #{lease_oid}, but it is actually free") + + error = true + else + new_leases << lease_str + end + else + if counter_lease != lease + + # Things that can be fixed + if (counter_lease[:vm] != lease[:vm] || + counter_lease[:vnet] != lease[:vnet] || + counter_lease[:vrouter] != lease[:vrouter]) + + new_lease_obj = "" + new_lease_oid = 0 + new_binary_magic = 0 + + if !counter_lease[:vm].nil? + new_lease_obj = "VM" + new_lease_oid = counter_lease[:vm].to_i + + new_binary_magic = (VM_BIN | + (new_lease_oid & 0xFFFFFFFF)) + elsif !counter_lease[:vnet].nil? + new_lease_obj = "VNet" + new_lease_oid = counter_lease[:vnet].to_i + + new_binary_magic = (NET_BIN | + (new_lease_oid & 0xFFFFFFFF)) + else #if !counter_lease[:vrouter].nil? + new_lease_obj = "VRouter" + new_lease_oid = counter_lease[:vrouter].to_i + + new_binary_magic = (VROUTER_BIN | + (new_lease_oid & 0xFFFFFFFF)) + end + + if (lease[:vm] == HOLD) + log_error( + "VNet #{oid} AR #{ar_id} has lease "<< + "#{lease_to_s(lease)} on hold, but it is "<< + "actually used by "<< + "#{new_lease_obj} #{new_lease_oid}") + error = true + else + log_error( + "VNet #{oid} AR #{ar_id} has leased #{lease_to_s(lease)} "<< + "to #{lease_obj} #{lease_oid}, but it is "<< + "actually used by "<< + "#{new_lease_obj} #{new_lease_oid}") + error = true + end + + lease_str[1] = new_binary_magic.to_s + end + + # Things that can't be fixed + + [:ip, :ip6_global, :ip6_link, :ip6_ula].each do |key| + if (counter_lease[key] != lease[key]) + log_error( + "VNet #{oid} AR #{ar_id} has a wrong "<< + "lease for "<< + "#{lease_obj} #{lease_oid}. #{key.to_s.upcase} "<< + "does not match: "<< + "#{counter_lease[key]} != #{lease[key]}. "<< + "This can't be fixed", false) + end + end + end + + new_leases << lease_str + end + end + + counter_ar.each do |mac, counter_lease| + index = ((mac & 0xFFFFFFFF) - (first_mac & 0xFFFFFFFF) ) % 0x100000000 + + new_lease_obj = "" + new_lease_oid = 0 + new_binary_magic = 0 + + if !counter_lease[:vm].nil? + new_lease_obj = "VM" + new_lease_oid = counter_lease[:vm].to_i + + new_binary_magic = (VM_BIN | + (new_lease_oid & 0xFFFFFFFF)) + elsif !counter_lease[:vnet].nil? + new_lease_obj = "VNet" + new_lease_oid = counter_lease[:vnet].to_i + + new_binary_magic = (NET_BIN | + (new_lease_oid & 0xFFFFFFFF)) + else #if !counter_lease[:vrouter].nil? + new_lease_obj = "VRouter" + new_lease_oid = counter_lease[:vrouter].to_i + + new_binary_magic = (VROUTER_BIN | + (new_lease_oid & 0xFFFFFFFF)) + end + + log_error("VNet #{oid} AR #{ar_id} does not have a lease "<< + "for #{mac_i_to_s(mac)}, but it is in use by "<< + "#{new_lease_obj} #{new_lease_oid}") + + error = true + + new_leases << [index.to_s, new_binary_magic.to_s] + end + + new_used_leases += new_leases.size + + if new_leases.size > 0 + allocated_e.content = " #{new_leases.join(" ")}" + else + allocated_e.remove if !allocated_e.nil? + end + end + + if (new_used_leases != used_leases) + log_error("VNet #{oid} has #{used_leases} used leases, "<< + "but it is actually #{new_used_leases}") + + error = true + + doc.root.at_xpath("USED_LEASES").content = + new_used_leases.to_s + end + + counter_no_ar.each do |mac, counter_lease| + log_error("VM #{counter_lease[:vm]} has a lease from "<< + "VNet #{oid}, but it could not be matched to any AR", false) + end + + vn_mad_e = doc.root.at_xpath("VN_MAD") + if vn_mad_e.nil? + log_error("VNet #{oid} VN_MAD element is missing", false) + else + vn_mad = vn_mad_e.text + vn_mad_tmpl_e = doc.root.at_xpath("TEMPLATE/VN_MAD") + + if (vn_mad_tmpl_e.nil? || vn_mad_tmpl_e.text != vn_mad) + log_error("VNet #{oid} VN_MAD element is missing from the TEMPLATE") + + error = true + + doc.root.at_xpath("TEMPLATE").add_child( + doc.create_element("VN_MAD")).content = vn_mad + end + end + + @fixes_network[oid] = doc.root.to_s if error + end + end + + def fix_network + @db.transaction do + @fixes_network.each do |id, body| + @db[:network_pool].where(oid: id).update(body: body) + end + end + end +end + diff --git a/src/onedb/fsck/pool_control.rb b/src/onedb/fsck/pool_control.rb new file mode 100644 index 0000000000..b5abd9e8a9 --- /dev/null +++ b/src/onedb/fsck/pool_control.rb @@ -0,0 +1,63 @@ + +module OneDBFsck + def check_pool_control + @fixes_pool_control = {} + + tables.each do |table| + max_oid = -1 + + @db.fetch("SELECT MAX(oid) FROM #{table}") do |row| + max_oid = row[:"MAX(oid)"].to_i + end + + # max(oid) will return 0 if there is none, + # or if the max is actually 0. Check this: + if ( max_oid == 0 ) + max_oid = -1 + + @db.fetch("SELECT oid FROM #{table} WHERE oid=0") do |row| + max_oid = 0 + end + end + + control_oid = -1 + + @db.fetch("SELECT last_oid FROM pool_control WHERE tablename='#{table}'") do |row| + control_oid = row[:last_oid].to_i + end + + if ( max_oid > control_oid ) + msg = "pool_control for table #{table} has last_oid #{control_oid}, but it is #{max_oid}" + + if control_oid != -1 + if db_version[:is_slave] && federated_tables.include?(table) + log_error(msg, false) + log_msg("^ Needs to be fixed in the master OpenNebula") + else + log_error(msg) + # @db.run("UPDATE pool_control SET last_oid=#{max_oid} WHERE tablename='#{table}'") + @fixes_pool_control[table] = max_oid + end + else + # @db[:pool_control].insert( + # :tablename => table, + # :last_oid => max_oid) + @fixes_pool_control[table] = max_oid + end + end + end + end + + def fix_pool_control + db = @db[:image_pool] + + @db.transaction do + @fixes_pool_control.each do |name, last_oid| + # If it can not update use insert + if 1 != db.where(tablename: name).update(last_oid: last_oid) + db.insert(tablename: name, last_oid: last_oid) + end + end + end + end +end diff --git a/src/onedb/fsck/quotas.rb b/src/onedb/fsck/quotas.rb new file mode 100644 index 0000000000..bbc490be79 --- /dev/null +++ b/src/onedb/fsck/quotas.rb @@ -0,0 +1,406 @@ + +module OneDBFsck + def check_fix_user_quotas + # This block is not needed for now +=begin + @db.transaction do + @db.fetch("SELECT oid FROM user_pool") do |row| + found = false + + @db.fetch("SELECT user_oid FROM user_quotas WHERE user_oid=#{row[:oid]}") do |q_row| + found = true + end + + if !found + log_error("User #{row[:oid]} does not have a quotas entry") + + @db.run "INSERT INTO user_quotas VALUES(#{row[:oid]},'#{row[:oid]}');" + end + end + end +=end + @db.run "ALTER TABLE user_quotas RENAME TO old_user_quotas;" + create_table(:user_quotas) + + @db.transaction do + # oneadmin does not have quotas + @db.fetch("SELECT * FROM old_user_quotas WHERE user_oid=0") do |row| + @db[:user_quotas].insert(row) + end + + @db.fetch("SELECT * FROM old_user_quotas WHERE user_oid>0") do |row| + doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + calculate_quotas(doc, "uid=#{row[:user_oid]}", "User") + + @db[:user_quotas].insert( + :user_oid => row[:user_oid], + :body => doc.root.to_s) + end + end + + @db.run "DROP TABLE old_user_quotas;" + end + + def check_fix_group_quotas + # This block is not needed for now +=begin + @db.transaction do + @db.fetch("SELECT oid FROM group_pool") do |row| + found = false + + @db.fetch("SELECT group_oid FROM group_quotas WHERE group_oid=#{row[:oid]}") do |q_row| + found = true + end + + if !found + log_error("Group #{row[:oid]} does not have a quotas entry") + + @db.run "INSERT INTO group_quotas VALUES(#{row[:oid]},'#{row[:oid]}');" + end + end + end +=end + @db.run "ALTER TABLE group_quotas RENAME TO old_group_quotas;" + create_table(:group_quotas) + + @db.transaction do + # oneadmin does not have quotas + @db.fetch("SELECT * FROM old_group_quotas WHERE group_oid=0") do |row| + @db[:group_quotas].insert(row) + end + + @db.fetch("SELECT * FROM old_group_quotas WHERE group_oid>0") do |row| + doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + calculate_quotas(doc, "gid=#{row[:group_oid]}", "Group") + + @db[:group_quotas].insert( + :group_oid => row[:group_oid], + :body => doc.root.to_s) + end + end + + @db.run "DROP TABLE old_group_quotas;" + end + + def calculate_quotas(doc, where_filter, resource) + oid = doc.root.at_xpath("ID").text.to_i + + # VM quotas + cpu_used = 0 + mem_used = 0 + vms_used = 0 + sys_used = 0 + + # VNet quotas + vnet_usage = {} + + # Image quotas + img_usage = {} + datastore_usage = {} + + @db.fetch("SELECT body FROM vm_pool WHERE #{where_filter} AND state<>6") do |vm_row| + vmdoc = nokogiri_doc(vm_row[:body]) + + # VM quotas + vmdoc.root.xpath("TEMPLATE/CPU").each { |e| + # truncate to 2 decimals + cpu = (e.text.to_f * 100).to_i + cpu_used += cpu + } + + vmdoc.root.xpath("TEMPLATE/MEMORY").each { |e| + mem_used += e.text.to_i + } + + vmdoc.root.xpath("TEMPLATE/DISK").each { |e| + type = "" + + e.xpath("TYPE").each { |t_elem| + type = t_elem.text.upcase + } + + size = 0 + + if !e.at_xpath("SIZE").nil? + size = e.at_xpath("SIZE").text.to_i + end + + if ( type == "SWAP" || type == "FS") + sys_used += size + else + if !e.at_xpath("CLONE").nil? + clone = (e.at_xpath("CLONE").text.upcase == "YES") + + target = nil + + if clone + target = e.at_xpath("CLONE_TARGET").text if !e.at_xpath("CLONE_TARGET").nil? + else + target = e.at_xpath("LN_TARGET").text if !e.at_xpath("LN_TARGET").nil? + end + + if !target.nil? && target == "SYSTEM" + sys_used += size + + if !e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").nil? + sys_used += e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").text.to_i + end + elsif !target.nil? && target == "SELF" + datastore_id = e.at_xpath("DATASTORE_ID").text + datastore_usage[datastore_id] ||= 0 + datastore_usage[datastore_id] += size + + if !e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").nil? + datastore_usage[datastore_id] += e.at_xpath("DISK_SNAPSHOT_TOTAL_SIZE").text.to_i + end + end + end + end + } + + vms_used += 1 + + # VNet quotas + vmdoc.root.xpath("TEMPLATE/NIC/NETWORK_ID").each { |e| + vnet_usage[e.text] = 0 if vnet_usage[e.text].nil? + vnet_usage[e.text] += 1 + } + + # Image quotas + vmdoc.root.xpath("TEMPLATE/DISK/IMAGE_ID").each { |e| + img_usage[e.text] = 0 if img_usage[e.text].nil? + img_usage[e.text] += 1 + } + end + + @db.fetch("SELECT body FROM vrouter_pool WHERE #{where_filter}") do |vrouter_row| + vrouter_doc = Nokogiri::XML(vrouter_row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + # VNet quotas + vrouter_doc.root.xpath("TEMPLATE/NIC").each { |nic| + net_id = nil + nic.xpath("NETWORK_ID").each do |nid| + net_id = nid.text + end + + floating = false + + nic.xpath("FLOATING_IP").each do |floating_e| + floating = (floating_e.text.upcase == "YES") + end + + if !net_id.nil? && floating + vnet_usage[net_id] = 0 if vnet_usage[net_id].nil? + + vnet_usage[net_id] += 1 + end + } + end + + # VM quotas + + vm_elem = nil + doc.root.xpath("VM_QUOTA/VM").each { |e| vm_elem = e } + + if vm_elem.nil? + doc.root.xpath("VM_QUOTA").each { |e| e.remove } + + vm_quota = doc.root.add_child(doc.create_element("VM_QUOTA")) + vm_elem = vm_quota.add_child(doc.create_element("VM")) + + vm_elem.add_child(doc.create_element("CPU")).content = "-1" + vm_elem.add_child(doc.create_element("CPU_USED")).content = "0" + + vm_elem.add_child(doc.create_element("MEMORY")).content = "-1" + vm_elem.add_child(doc.create_element("MEMORY_USED")).content = "0" + + vm_elem.add_child(doc.create_element("VMS")).content = "-1" + vm_elem.add_child(doc.create_element("VMS_USED")).content = "0" + + vm_elem.add_child(doc.create_element("SYSTEM_DISK_SIZE")).content = "-1" + vm_elem.add_child(doc.create_element("SYSTEM_DISK_SIZE_USED")).content = "0" + end + + vm_elem.xpath("CPU_USED").each { |e| + # Because of bug http://dev.opennebula.org/issues/1567 the element + # may contain a float number in scientific notation. + + # Check if the float value or the string representation mismatch, + # but ignoring the precision + + cpu_used = (cpu_used / 100.0) + + different = ( e.text.to_f != cpu_used || + ![sprintf('%.2f', cpu_used), sprintf('%.1f', cpu_used), sprintf('%.0f', cpu_used)].include?(e.text) ) + + cpu_used_str = sprintf('%.2f', cpu_used) + + if different + log_error("#{resource} #{oid} quotas: CPU_USED has #{e.text} \tis\t#{cpu_used_str}") + e.content = cpu_used_str + end + } + + vm_elem.xpath("MEMORY_USED").each { |e| + if e.text != mem_used.to_s + log_error("#{resource} #{oid} quotas: MEMORY_USED has #{e.text} \tis\t#{mem_used}") + e.content = mem_used.to_s + end + } + + vm_elem.xpath("VMS_USED").each { |e| + if e.text != vms_used.to_s + log_error("#{resource} #{oid} quotas: VMS_USED has #{e.text} \tis\t#{vms_used}") + e.content = vms_used.to_s + end + } + + vm_elem.xpath("SYSTEM_DISK_SIZE_USED").each { |e| + if e.text != sys_used.to_s + log_error("#{resource} #{oid} quotas: SYSTEM_DISK_SIZE_USED has #{e.text} \tis\t#{sys_used}") + e.content = sys_used.to_s + end + } + + # VNet quotas + + net_quota = nil + doc.root.xpath("NETWORK_QUOTA").each { |e| net_quota = e } + + if net_quota.nil? + net_quota = doc.root.add_child(doc.create_element("NETWORK_QUOTA")) + end + + net_quota.xpath("NETWORK").each { |net_elem| + vnet_id = net_elem.at_xpath("ID").text + + leases_used = vnet_usage.delete(vnet_id) + + leases_used = 0 if leases_used.nil? + + net_elem.xpath("LEASES_USED").each { |e| + if e.text != leases_used.to_s + log_error("#{resource} #{oid} quotas: VNet #{vnet_id}\tLEASES_USED has #{e.text} \tis\t#{leases_used}") + e.content = leases_used.to_s + end + } + } + + vnet_usage.each { |vnet_id, leases_used| + log_error("#{resource} #{oid} quotas: VNet #{vnet_id}\tLEASES_USED has 0 \tis\t#{leases_used}") + + new_elem = net_quota.add_child(doc.create_element("NETWORK")) + + new_elem.add_child(doc.create_element("ID")).content = vnet_id + new_elem.add_child(doc.create_element("LEASES")).content = "-1" + new_elem.add_child(doc.create_element("LEASES_USED")).content = leases_used.to_s + } + + # Image quotas + + img_quota = nil + doc.root.xpath("IMAGE_QUOTA").each { |e| img_quota = e } + + if img_quota.nil? + img_quota = doc.root.add_child(doc.create_element("IMAGE_QUOTA")) + end + + img_quota.xpath("IMAGE").each { |img_elem| + img_id = img_elem.at_xpath("ID").text + + rvms = img_usage.delete(img_id) + + rvms = 0 if rvms.nil? + + img_elem.xpath("RVMS_USED").each { |e| + if e.text != rvms.to_s + log_error("#{resource} #{oid} quotas: Image #{img_id}\tRVMS has #{e.text} \tis\t#{rvms}") + e.content = rvms.to_s + end + } + } + + img_usage.each { |img_id, rvms| + log_error("#{resource} #{oid} quotas: Image #{img_id}\tRVMS has 0 \tis\t#{rvms}") + + new_elem = img_quota.add_child(doc.create_element("IMAGE")) + + new_elem.add_child(doc.create_element("ID")).content = img_id + new_elem.add_child(doc.create_element("RVMS")).content = "-1" + new_elem.add_child(doc.create_element("RVMS_USED")).content = rvms.to_s + } + + # Datastore quotas + ds_usage = {} + + @db.fetch("SELECT body FROM image_pool WHERE #{where_filter}") do |img_row| + img_doc = Nokogiri::XML(img_row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + img_doc.root.xpath("DATASTORE_ID").each { |e| + ds_usage[e.text] = [0,0] if ds_usage[e.text].nil? + ds_usage[e.text][0] += 1 + + img_doc.root.xpath("SIZE").each { |size| + ds_usage[e.text][1] += size.text.to_i + } + + img_doc.root.xpath("SNAPSHOTS/SNAPSHOT/SIZE").each { |size| + ds_usage[e.text][1] += size.text.to_i + } + } + end + + ds_quota = nil + doc.root.xpath("DATASTORE_QUOTA").each { |e| ds_quota = e } + + if ds_quota.nil? + ds_quota = doc.root.add_child(doc.create_element("DATASTORE_QUOTA")) + end + + ds_quota.xpath("DATASTORE").each { |ds_elem| + ds_id = ds_elem.at_xpath("ID").text + + images_used,size_used = ds_usage.delete(ds_id) + + images_used = 0 if images_used.nil? + size_used = 0 if size_used.nil? + + cloned_usage = datastore_usage[ds_id] || 0 + size_used += cloned_usage + + ds_elem.xpath("IMAGES_USED").each { |e| + if e.text != images_used.to_s + log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tIMAGES_USED has #{e.text} \tis\t#{images_used}") + e.content = images_used.to_s + end + } + + ds_elem.xpath("SIZE_USED").each { |e| + if e.text != size_used.to_s + log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tSIZE_USED has #{e.text} \tis\t#{size_used}") + e.content = size_used.to_s + end + } + } + + ds_usage.each { |ds_id, array| + images_used,size_used = array + + log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tIMAGES_USED has 0 \tis\t#{images_used}") + log_error("#{resource} #{oid} quotas: Datastore #{ds_id}\tSIZE_USED has 0 \tis\t#{size_used}") + + new_elem = ds_quota.add_child(doc.create_element("DATASTORE")) + + new_elem.add_child(doc.create_element("ID")).content = ds_id + + new_elem.add_child(doc.create_element("IMAGES")).content = "-1" + new_elem.add_child(doc.create_element("IMAGES_USED")).content = images_used.to_s + + new_elem.add_child(doc.create_element("SIZE")).content = "-1" + new_elem.add_child(doc.create_element("SIZE_USED")).content = size_used.to_s + } + end +end + diff --git a/src/onedb/fsck/template.rb b/src/onedb/fsck/template.rb new file mode 100644 index 0000000000..b8e873854b --- /dev/null +++ b/src/onedb/fsck/template.rb @@ -0,0 +1,87 @@ + +module OneDBFsck + def check_template + templates_fix = @fixes_template = {} + + @db[:template_pool].each do |row| + doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + boot = doc.root.at_xpath("TEMPLATE/OS/BOOT") + + if boot.nil? || boot.text.downcase.match(/fd|hd|cdrom|network/).nil? + next + end + + # Note: this code assumes that disks are ordered in the same order as + # their target, and may break boot order if the target is not left + # completely to oned. + # If, for example, the third disk ends with target="vda", + # boot="hd" should be updated to boot="disk2", but it is not + + devs = [] + + hd_i = 0 + cdrom_i = 0 + network_i = 0 + + error = false + + boot.text.split(",").each do |dev| + dev.downcase! + + case dev + when "hd", "cdrom" + index = nil + if dev == "hd" + index = hd_i + hd_i += 1 + else + index = cdrom_i + cdrom_i += 1 + end + + id = get_disk_id(dev, index, doc) + if id.nil? + log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", but DISK ##{index} of type #{dev} could not be found to fix it automatically", false) + error = true + end + devs.push("disk#{id}") + + when "network" + devs.push("nic#{network_i}") + network_i += 1 + + when "fd" + log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", but \"fd\" is not supported anymore and can't be fixed automatically", false) + error = true + + else + log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", but it can't be parsed to be fixed automatically", false) + error = true + + end + end + + if error + next + end + + new_boot = devs.join(",") + + log_error("VM Template #{row[:oid]} OS/BOOT contains deprecated format \"#{boot.content}\", is was updated to #{new_boot}") + + boot.content = new_boot + + templates_fix[row[:oid]] = doc.root.to_s + end + end + + def fix_template + @db.transaction do + @fixes_template.each do |id, body| + @db[:template_pool].where(oid: id).update(body: body) + end + end + end +end + diff --git a/src/onedb/fsck/user.rb b/src/onedb/fsck/user.rb new file mode 100644 index 0000000000..5ef585ba5e --- /dev/null +++ b/src/onedb/fsck/user.rb @@ -0,0 +1,97 @@ + +module OneDBFsck + def check_user + @data_user = { group: {} } + group = @data_user[:group] + + @db.fetch("SELECT oid FROM group_pool") do |row| + group[row[:oid]] = [] + end + + @fixes_user = users_fix = {} + + @db.fetch("SELECT oid,body,gid FROM user_pool") do |row| + doc = Nokogiri::XML(row[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + + gid = doc.root.at_xpath('GID').text.to_i + user_gid = gid + user_gids = Set.new + + if group[gid].nil? + log_error("User #{row[:oid]} has primary group #{gid}, but it does not exist", !db_version[:is_slave]) + + user_gid = 1 + + doc.root.xpath('GID').each do |e| + e.content = "1" + end + + doc.root.xpath('GNAME').each do |e| + e.content = "users" + end + + doc.root.xpath("GROUPS").each { |e| + e.xpath("ID[.=#{gid}]").each{|x| x.remove} + + e.add_child(doc.create_element("ID")).content = user_gid.to_s + } + + users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} + end + + doc.root.xpath("GROUPS/ID").each { |e| + user_gids.add e.text.to_i + } + + if !user_gids.include?(user_gid) + log_error("User #{row[:oid]} does not have his primary group #{user_gid} in the list of secondary groups", !db_version[:is_slave]) + + doc.root.xpath("GROUPS").each { |e| + e.add_child(doc.create_element("ID")).content = user_gid.to_s + } + + user_gids.add user_gid.to_i + + users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} + end + + user_gids.each do |secondary_gid| + if group[secondary_gid].nil? + log_error("User #{row[:oid]} has secondary group #{secondary_gid}, but it does not exist", !db_version[:is_slave]) + + doc.root.xpath("GROUPS").each { |e| + e.xpath("ID[.=#{secondary_gid}]").each{|x| x.remove} + } + + users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} + else + group[secondary_gid] << row[:oid] + end + end + + if gid != row[:gid] + log_error( + "User #{row[:oid]} is in group #{gid}, but the DB "<< + "table has GID column #{row[:gid]}", !db_version[:is_slave]) + + users_fix[row[:oid]] = {:body => doc.root.to_s, :gid => user_gid} + end + end + end + + def fix_user + users_fix = @fixes_user + + if !db_version[:is_slave] + @db.transaction do + users_fix.each do |id, user| + @db[:user_pool].where(:oid => id).update( + :body => user[:body], + :gid => user[:gid]) + end + end + elsif !users_fix.empty? + log_msg("^ User errors need to be fixed in the master OpenNebula") + end + end +end diff --git a/src/onedb/fsck/vm.rb b/src/onedb/fsck/vm.rb new file mode 100644 index 0000000000..604b5b2a9f --- /dev/null +++ b/src/onedb/fsck/vm.rb @@ -0,0 +1,202 @@ + +module OneDBFsck + def check_vm + vms_fix = @fixes_vm = {} + @fixes_vm_history = {} + + @data_vm = {} + cluster_vnc = @data_vm[:vnc] = {} + + # DATA: Aggregate information of the RUNNING vms + @db.fetch("SELECT oid,body FROM vm_pool WHERE state<>6") do |row| + vm_doc = nokogiri_doc(row[:body]) + + state = vm_doc.root.at_xpath('STATE').text.to_i + lcm_state = vm_doc.root.at_xpath('LCM_STATE').text.to_i + + # DATA: VNC ports per cluster + cid = vm_doc.root.at_xpath("HISTORY_RECORDS/HISTORY[last()]/CID").text.to_i rescue nil + port = vm_doc.root.at_xpath('TEMPLATE/GRAPHICS[translate(TYPE,"vnc","VNC")="VNC"]/PORT').text.to_i rescue nil + # DATA: TODO: get also spice port + + if cid && port + cluster_vnc[cid] ||= Set.new + cluster_vnc[cid] << port + end + + # DATA: Images used by this VM + vm_doc.root.xpath("TEMPLATE/DISK/IMAGE_ID").each do |e| + img_id = e.text.to_i + + if counters[:image][img_id].nil? + log_error("VM #{row[:oid]} is using Image #{img_id}, but "<< + "it does not exist", false) + else + counters[:image][img_id][:vms].add(row[:oid]) + end + end + + # DATA: VNets used by this VM + vm_doc.root.xpath("TEMPLATE/NIC").each do |nic| + net_id = nil + nic.xpath("NETWORK_ID").each do |nid| + net_id = nid.text.to_i + end + + if !net_id.nil? + if counters[:vnet][net_id].nil? + log_error("VM #{row[:oid]} is using VNet #{net_id}, "<< + "but it does not exist", false) + else + mac = nic.at_xpath("MAC").nil? ? nil : nic.at_xpath("MAC").text + + ar_id_e = nic.at_xpath('AR_ID') + + if ar_id_e.nil? + if !counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)].nil? + log_error("VNet #{net_id} has more than one lease with the same MAC address (#{mac}). "<< + "FSCK can't handle this, and consistency is not guaranteed", false) + end + + # DATA: IPs per network no ar + counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)] = { + :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, + :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, + :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, + :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, + :mac => mac, + :vm => row[:oid], + :vnet => nil, + :vrouter => nil + } + else + ar_id = ar_id_e.text.to_i + + if counters[:vnet][net_id][:ar_leases][ar_id].nil? + log_error("VM #{row[:oid]} is using VNet #{net_id}, AR #{ar_id}, "<< + "but the AR does not exist", false) + # DATA: why these are not added to counters? + else + # DATA: IPs per network with ar + counters[:vnet][net_id][:ar_leases][ar_id][mac_s_to_i(mac)] = { + :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, + :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, + :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, + :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, + :mac => mac, + :vm => row[:oid], + :vnet => nil, + :vrouter => nil + } + end + end + end + end + end + + # See if it's part of a Virtual Router + vrouter_e = vm_doc.root.at_xpath("TEMPLATE/VROUTER_ID") + + # DATA: add vrouter counters + if !vrouter_e.nil? + vr_id = vrouter_e.text.to_i + counters_vrouter = counters[:vrouter][vr_id] + + if counters_vrouter.nil? + log_error("VM #{row[:oid]} is part of VRouter #{vr_id}, but "<< + "it does not exist", false) + else + counters_vrouter[:vms].add(row[:oid]) + end + end + + # DATA: Host resources + + # Only states that add to Host resources consumption are + # ACTIVE, SUSPENDED, POWEROFF + next if !([3,5,8].include? state) + + # DATA: Get memory (integer) + memory = vm_doc.root.at_xpath("TEMPLATE/MEMORY").text.to_i + + # DATA: Get CPU (float) + cpu = vm_doc.root.at_xpath("TEMPLATE/CPU").text.to_f + + # DATA: Get hostid, hostname + hid = -1 + vm_doc.root.xpath("HISTORY_RECORDS/HISTORY[last()]/HID").each { |e| + hid = e.text.to_i + } + + hostname = "" + vm_doc.root.xpath("HISTORY_RECORDS/HISTORY[last()]/HOSTNAME").each { |e| + hostname = e.text + } + + counters_host = counters[:host][hid] + + if counters_host.nil? + log_error("VM #{row[:oid]} is using Host #{hid}, "<< + "but it does not exist", false) + else + # DATA: FIX: hostname is wrong, fix inline + if counters_host[:name] != hostname + log_error("VM #{row[:oid]} has a wrong hostname for "<< + "Host #{hid}, #{hostname}. It will be changed to "<< + "#{counters_host[:name]}") + + vm_doc.root.xpath( + "HISTORY_RECORDS/HISTORY[last()]/HOSTNAME").each { |e| + e.content = counters_host[:name] + } + + vms_fix[row[:oid]] = vm_doc.root.to_s + end + + # DATA: add resources to host counters + counters_host[:memory] += memory + counters_host[:cpu] += cpu + counters_host[:rvms].add(row[:oid]) + end + + # DATA: search history for VMMMAD and TMMAD to translate + @db.fetch("SELECT * FROM history WHERE vid=#{row[:oid]}") do |hrow| + # hdoc = Nokogiri::XML(hrow[:body],nil,NOKOGIRI_ENCODING){|c| c.default_xml.noblanks} + hdoc = nokogiri_doc(hrow[:body]) + + found = false + + # Rename VMMMAD -> VM_MAD and TMMAD -> TM_MAD + hdoc.root.xpath("VMMMAD").each {|e| + e.name = "VM_MAD" + found = true + } + + hdoc.root.xpath("TMMAD").each {|e| + e.name = "TM_MAD" + found = true + } + + # DATA: translate VMMMAD and TMMAD to VM_MAD and TM_MAD + if found + index = [hrow[:vid], hrow[:seq]] + @fixes_vm_history[index] = hdoc.root.to_s + end + end + end + end + + def fix_vm + # DATA: FIX: do vm_pool fixes + @db.transaction do + @fixes_vm.each do |id, body| + @db[:vm_pool].where(:oid => id).update(:body => body) + end + + @fixes_vm_history.each do |index, body| + vid, seq = index + @db[:history].where(vid: vid, seq: seq).update(body: body) + end + end + end +end diff --git a/src/onedb/fsck/vrouter.rb b/src/onedb/fsck/vrouter.rb new file mode 100644 index 0000000000..afde41422b --- /dev/null +++ b/src/onedb/fsck/vrouter.rb @@ -0,0 +1,120 @@ + +module OneDBFsck + def check_vrouter + vrouters_fix = @fixes_vrouter = {} + + # DATA: Aggregate information of the RUNNING vms + @db.fetch("SELECT oid,body FROM vrouter_pool") do |row| + vrouter_doc = nokogiri_doc(row[:body]) + + # DATA: VNets used by this Virtual Router + vrouter_doc.root.xpath("TEMPLATE/NIC").each do |nic| + net_id = nil + nic.xpath("NETWORK_ID").each do |nid| + net_id = nid.text.to_i + end + + floating = false + + nic.xpath("FLOATING_IP").each do |floating_e| + floating = (floating_e.text.upcase == "YES") + end + + if !net_id.nil? && floating + if counters[:vnet][net_id].nil? + log_error("VRouter #{row[:oid]} is using VNet #{net_id}, "<< + "but it does not exist", false) + else + # DATA: this part is copied from "VNets used by this VM" + mac = nic.at_xpath("MAC").nil? ? nil : nic.at_xpath("MAC").text + + ar_id_e = nic.at_xpath('AR_ID') + + if ar_id_e.nil? + if !counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)].nil? + log_error("VNet #{net_id} has more than one lease with the same MAC address (#{mac}). "<< + "FSCK can't handle this, and consistency is not guaranteed", false) + end + + counters[:vnet][net_id][:no_ar_leases][mac_s_to_i(mac)] = { + :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, + :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, + :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, + :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, + :mac => mac, + :vm => nil, + :vnet => nil, + :vrouter => row[:oid], + } + else + ar_id = ar_id_e.text.to_i + + if counters[:vnet][net_id][:ar_leases][ar_id].nil? + log_error("VRouter #{row[:oid]} is using VNet #{net_id}, AR #{ar_id}, "<< + "but the AR does not exist", false) + else + counters[:vnet][net_id][:ar_leases][ar_id][mac_s_to_i(mac)] = { + :ip => nic.at_xpath("IP").nil? ? nil : nic.at_xpath("IP").text, + :ip6_global => nic.at_xpath("IP6_GLOBAL").nil? ? nil : nic.at_xpath("IP6_GLOBAL").text, + :ip6_link => nic.at_xpath("IP6_LINK").nil? ? nil : nic.at_xpath("IP6_LINK").text, + :ip6_ula => nic.at_xpath("IP6_ULA").nil? ? nil : nic.at_xpath("IP6_ULA").text, + :mac => mac, + :vm => nil, + :vnet => nil, + :vrouter => row[:oid], + } + end + end + end + end + end + + # DATA: re-do list of VM IDs per vrouter + error = false + + counters_vrouter = counters[:vrouter][row[:oid]] + + vms_elem = vrouter_doc.root.at_xpath("VMS").remove + + vms_new_elem = vrouter_doc.create_element("VMS") + vrouter_doc.root.add_child(vms_new_elem) + + counters_vrouter[:vms].each do |id| + id_elem = vms_elem.at_xpath("ID[.=#{id}]") + + if id_elem.nil? + log_error( + "VM #{id} is missing from VRouter #{row[:oid]} VM id list") + + error = true + else + id_elem.remove + end + + vms_new_elem.add_child(vrouter_doc.create_element("ID")).content = id.to_s + end + + vms_elem.xpath("ID").each do |id_elem| + log_error( + "VM #{id_elem.text} is in VRouter #{row[:oid]} VM id list, "<< + "but it should not") + + error = true + end + + if (error) + vrouters_fix[row[:oid]] = vrouter_doc.root.to_s + end + end + end + + def fix_vrouter + # DATA: FIX: update vrouter_pool with regenerated documents (ids) + @db.transaction do + @fixes_vrouter.each do |id, body| + @db[:vrouter_pool].where(oid: id).update(body: body) + end + end + end +end +