diff --git a/flow/client/bin/appflow b/flow/client/bin/appflow index 886a5ff40b..e2a0dfb022 100755 --- a/flow/client/bin/appflow +++ b/flow/client/bin/appflow @@ -459,7 +459,7 @@ cmd=CommandParser::CmdParser.new(ARGV) do shutdown_desc = <<-EOT.unindent Shutdown a service. - From RUNNING or UNKNOWN shuts down the Service + From RUNNING or WARNING shuts down the Service EOT command :shutdown, shutdown_desc, [:range, :service_id_list] do diff --git a/flow/client/lib/appflow_client.rb b/flow/client/lib/appflow_client.rb index 6e5a0b2ddc..f707012605 100644 --- a/flow/client/lib/appflow_client.rb +++ b/flow/client/lib/appflow_client.rb @@ -43,14 +43,13 @@ module Role 'DEPLOYING' => 1, 'RUNNING' => 2, 'UNDEPLOYING' => 3, - 'FAILED' => 4, - 'UNKNOWN' => 5, - 'DONE' => 6, - 'FAILED_UNDEPLOYING' => 7, - 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9, - 'FAILED_SCALING' => 10, - 'COOLDOWN' => 11 + 'WARNING' => 4, + 'DONE' => 5, + 'FAILED_UNDEPLOYING' => 6, + 'FAILED_DEPLOYING' => 7, + 'SCALING' => 8, + 'FAILED_SCALING' => 9, + 'COOLDOWN' => 10 } STATE_STR = [ @@ -58,8 +57,7 @@ module Role 'DEPLOYING', 'RUNNING', 'UNDEPLOYING', - 'FAILED', - 'UNKNOWN', + 'WARNING', 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', @@ -83,14 +81,13 @@ module Service 'DEPLOYING' => 1, 'RUNNING' => 2, 'UNDEPLOYING' => 3, - 'FAILED' => 4, - 'UNKNOWN' => 5, - 'DONE' => 6, - 'FAILED_UNDEPLOYING' => 7, - 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9, - 'FAILED_SCALING' => 10, - 'COOLDOWN' => 11 + 'WARNING' => 4, + 'DONE' => 5, + 'FAILED_UNDEPLOYING' => 6, + 'FAILED_DEPLOYING' => 7, + 'SCALING' => 8, + 'FAILED_SCALING' => 9, + 'COOLDOWN' => 10 } STATE_STR = [ @@ -98,8 +95,7 @@ module Service 'DEPLOYING', 'RUNNING', 'UNDEPLOYING', - 'FAILED', - 'UNKNOWN', + 'WARNING', 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', diff --git a/flow/doc/states.dot b/flow/doc/states.dot index 4de9f7d932..5069203a17 100644 --- a/flow/doc/states.dot +++ b/flow/doc/states.dot @@ -19,7 +19,7 @@ digraph AppFlow { end [ width=0.15, height=0.15, shape=point ]; - running [ label="running /\nunknown"]; + running [ label="running /\nwarning"]; subgraph { rank = min; user; pending} subgraph { rank = same; deploying; failed_deploying; color="white"} @@ -37,8 +37,8 @@ digraph AppFlow { done -> end [label="delete"]; -// running -> unknown [style="dotted"]; -// unknown -> running [style="dotted"]; +// running -> warning [style="dotted"]; +// warning -> running [style="dotted"]; running -> scaling [style="dashed"]; scaling -> running [style="dashed"]; diff --git a/flow/lcm/LifeCycleManager.rb b/flow/lcm/LifeCycleManager.rb index dc14361949..555f322fd8 100644 --- a/flow/lcm/LifeCycleManager.rb +++ b/flow/lcm/LifeCycleManager.rb @@ -69,16 +69,16 @@ class ServiceLCM service.set_state(Service::STATE['FAILED_DEPLOYING']) end end - when Service::STATE['RUNNING'], Service::STATE['UNKNOWN'] + when Service::STATE['RUNNING'], Service::STATE['WARNING'] strategy.monitor_step(service) if service.all_roles_running? - if service.state() == Service::STATE['UNKNOWN'] + if service.state() == Service::STATE['WARNING'] service.set_state(Service::STATE['RUNNING']) end else if service.state() == Service::STATE['RUNNING'] - service.set_state(Service::STATE['UNKNOWN']) + service.set_state(Service::STATE['WARNING']) end end @@ -117,12 +117,6 @@ class ServiceLCM if !service.any_role_failed_scaling? service.set_state(Service::STATE['SCALING']) end - when Service::STATE['FAILED'] - strategy.monitor_step(service) - - if service.all_roles_running? - service.set_state(Service::STATE['RUNNING']) - end when Service::STATE['UNDEPLOYING'] strategy.monitor_step(service) @@ -133,21 +127,21 @@ class ServiceLCM else rc = strategy.shutdown_step(service) if !rc[0] - service.set_state(Service::STATE['FAILED']) + service.set_state(Service::STATE['FAILED_UNDEPLOYING']) end end - #when Service::STATE['FAILED_DEPLOYING'] - # strategy.monitor_step(service) -# - # if !service.any_role_failed? - # service.set_state(Service::STATE['DEPLOYING']) - # end - #when Service::STATE['FAILED_UNDEPLOYING'] - # strategy.monitor_step(service) -# - # if !service.any_role_failed? - # service.set_state(Service::STATE['UNDEPLOYING']) - # end + when Service::STATE['FAILED_DEPLOYING'] + strategy.monitor_step(service) + + if !service.any_role_failed? + service.set_state(Service::STATE['DEPLOYING']) + end + when Service::STATE['FAILED_UNDEPLOYING'] + strategy.monitor_step(service) + + if !service.any_role_failed? + service.set_state(Service::STATE['UNDEPLOYING']) + end end rc = service.update() diff --git a/flow/lcm/strategy.rb b/flow/lcm/strategy.rb index 36e346e46a..24a7c6b211 100644 --- a/flow/lcm/strategy.rb +++ b/flow/lcm/strategy.rb @@ -138,9 +138,9 @@ class Strategy case role.state() when Role::STATE['RUNNING'] if OpenNebula.is_error?(rc) - role.set_state(Role::STATE['UNKNOWN']) + role.set_state(Role::STATE['WARNING']) elsif !role_nodes_running?(role) - role.set_state(Role::STATE['UNKNOWN']) + role.set_state(Role::STATE['WARNING']) end when Role::STATE['DEPLOYING'] if OpenNebula.is_error?(rc) @@ -166,16 +166,10 @@ class Strategy if role.cooldown_over? role.set_state(Role::STATE['RUNNING']) end - when Role::STATE['UNKNOWN'] + when Role::STATE['WARNING'] if role_nodes_running?(role) role.set_state(Role::STATE['RUNNING']) end - when Role::STATE['FAILED'] - if OpenNebula.is_error?(rc) - role.set_state(Role::STATE['UNKNOWN']) - elsif role_nodes_running?(role) - role.set_state(Role::STATE['RUNNING']) - end when Role::STATE['UNDEPLOYING'] if OpenNebula.is_error?(rc) role.set_state(Role::STATE['FAILED_UNDEPLOYING']) @@ -186,7 +180,7 @@ class Strategy end when Role::STATE['FAILED_DEPLOYING'] if !OpenNebula.is_error?(rc) && role_nodes_running?(role) - role.set_state(Role::STATE['PENDING']) + role.set_state(Role::STATE['RUNNING']) end when Role::STATE['FAILED_UNDEPLOYING'] if !OpenNebula.is_error?(rc) && role_nodes_done?(role) @@ -241,7 +235,6 @@ protected def get_roles_shutdown(service) result = service.get_roles.select {|name, role| ![Role::STATE['UNDEPLOYING'], - Role::STATE['FAILED'], Role::STATE['DONE'], Role::STATE['FAILED_UNDEPLOYING'], Role::STATE['FAILED_DEPLOYING']].include?(role.state) @@ -259,6 +252,10 @@ protected # @param [Role] role # @return [true|false] def role_nodes_running?(role) + if role.get_nodes.size() != role.cardinality() + return false + end + role.get_nodes.each { |node| if node && node['vm_info'] vm_state = node['vm_info']['VM']['STATE'] diff --git a/flow/lcm/strategy/straight.rb b/flow/lcm/strategy/straight.rb index 06dba65108..c0ab39bb6c 100644 --- a/flow/lcm/strategy/straight.rb +++ b/flow/lcm/strategy/straight.rb @@ -121,7 +121,6 @@ module Straight roles.each { |name, role| # All roles can be shutdown, except the ones in these states if (![Role::STATE['UNDEPLOYING'], - Role::STATE['FAILED'], Role::STATE['DONE'], Role::STATE['FAILED_UNDEPLOYING'], Role::STATE['FAILED_DEPLOYING']].include?(role.state) ) diff --git a/flow/models/Role.rb b/flow/models/Role.rb index 5bd721309d..0bfc9ac020 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -44,14 +44,13 @@ module OpenNebula 'DEPLOYING' => 1, 'RUNNING' => 2, 'UNDEPLOYING' => 3, - 'FAILED' => 4, - 'UNKNOWN' => 5, - 'DONE' => 6, - 'FAILED_UNDEPLOYING' => 7, - 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9, - 'FAILED_SCALING' => 10, - 'COOLDOWN' => 11 + 'WARNING' => 4, + 'DONE' => 5, + 'FAILED_UNDEPLOYING' => 6, + 'FAILED_DEPLOYING' => 7, + 'SCALING' => 8, + 'FAILED_SCALING' => 9, + 'COOLDOWN' => 10 } STATE_STR = [ @@ -59,8 +58,7 @@ module OpenNebula 'DEPLOYING', 'RUNNING', 'UNDEPLOYING', - 'FAILED', - 'UNKNOWN', + 'WARNING', 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', @@ -766,5 +764,80 @@ module OpenNebula def update(template) set_cardinality(template["cardinality"]) end + + ######################################################################## + # Recover + ######################################################################## + + + def recover_deployment() + delete_failed_done() + end + + def recover_warning() + @body['nodes'].each do |node| + vm_state = nil + lcm_state = nil + vm_id = node['deploy_id'] + + if node['vm_info'] && node['vm_info']['VM'] + vm_state = node['vm_info']['VM']['STATE'] + lcm_state = node['vm_info']['VM']['LCM_STATE'] + end + + if vm_state == '3' && lcm_state == '16' # UNKNOWN + vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client) + vm.boot + end + end + + delete_failed_done() + + deploy() + end + + def delete_failed_done() + + nodes = @body['nodes'] + new_nodes = [] + disposed_nodes = @body['disposed_nodes'] + + nodes.each do |node| + vm_state = nil + vm_id = node['deploy_id'] + + if node['vm_info'] && node['vm_info']['VM'] && node['vm_info']['VM']['STATE'] + vm_state = node['vm_info']['VM']['STATE'] + end + + if vm_state == '6' # DONE + # Store the VM id in the array of disposed nodes + disposed_nodes << vm_id + + elsif vm_state == '7' # FAILED + vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client) + rc = vm.finalize + + if !OpenNebula.is_error?(rc) + # Store the VM id in the array of disposed nodes + disposed_nodes << vm_id + + Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id() + else + msg = "Role #{name} : Delete failed for VM #{vm_id}; #{rc.message}" + Log.error LOG_COMP, msg, @service.id() + @service.log_error(msg) + + success = false + + new_nodes << node + end + else + new_nodes << node + end + end + + @body['nodes'] = new_nodes + end end end diff --git a/flow/models/Service.rb b/flow/models/Service.rb index a267a5e0a0..0ece3d74f4 100644 --- a/flow/models/Service.rb +++ b/flow/models/Service.rb @@ -24,14 +24,13 @@ module OpenNebula 'DEPLOYING' => 1, 'RUNNING' => 2, 'UNDEPLOYING' => 3, - 'FAILED' => 4, - 'UNKNOWN' => 5, - 'DONE' => 6, - 'FAILED_UNDEPLOYING' => 7, - 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9, - 'FAILED_SCALING' => 10, - 'COOLDOWN' => 11 + 'WARNING' => 4, + 'DONE' => 5, + 'FAILED_UNDEPLOYING' => 6, + 'FAILED_DEPLOYING' => 7, + 'SCALING' => 8, + 'FAILED_SCALING' => 9, + 'COOLDOWN' => 10 } STATE_STR = [ @@ -39,8 +38,7 @@ module OpenNebula 'DEPLOYING', 'RUNNING', 'UNDEPLOYING', - 'FAILED', - 'UNKNOWN', + 'WARNING', 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', @@ -132,8 +130,7 @@ module OpenNebula # @return [true, false] true if any of the roles is in failed state def any_role_failed?() failed_states = [ - Role::STATE['FAILED'], - Role::STATE['FAILED_DEPLOYING'], + Role::STATE['FAILED_DEPLOYING'], Role::STATE['FAILED_UNDEPLOYING']] @roles.each { |name, role| @@ -194,7 +191,7 @@ module OpenNebula def shutdown if [Service::STATE['RUNNING'], Service::STATE['FAILED_UNDEPLOYING'], - Service::STATE['UNKNOWN']].include?(self.state) + Service::STATE['WARNING']].include?(self.state) self.set_state(Service::STATE['UNDEPLOYING']) return self.update else @@ -210,6 +207,13 @@ module OpenNebula # otherwise def recover if [Service::STATE['FAILED_DEPLOYING']].include?(self.state) + @roles.each do |name, role| + if role.state == Role::STATE['FAILED_DEPLOYING'] + role.set_state(Role::STATE['PENDING']) + role.recover_deployment() + end + end + self.set_state(Service::STATE['DEPLOYING']) elsif self.state == Service::STATE['FAILED_SCALING'] @@ -223,6 +227,12 @@ module OpenNebula self.set_state(Service::STATE['SCALING']) elsif self.state == Service::STATE['FAILED_UNDEPLOYING'] + @roles.each do |name, role| + if role.state == Role::STATE['FAILED_UNDEPLOYING'] + role.set_state(Role::STATE['RUNNING']) + end + end + self.set_state(Service::STATE['UNDEPLOYING']) elsif self.state == Service::STATE['COOLDOWN'] @@ -234,6 +244,13 @@ module OpenNebula self.set_state(Service::STATE['RUNNING']) + elsif self.state == Service::STATE['WARNING'] + @roles.each do |name, role| + if role.state == Role::STATE['WARNING'] + role.recover_warning() + end + end + else return OpenNebula::Error.new("Action recover: Wrong state" \ " #{self.state_str()}") @@ -336,7 +353,7 @@ module OpenNebula def update_role(role_name, template_json) - if ![Service::STATE['RUNNING'], Service::STATE['UNKNOWN']].include?(self.state) + if ![Service::STATE['RUNNING'], Service::STATE['WARNING']].include?(self.state) return OpenNebula::Error.new("Update role: Wrong state" \ " #{self.state_str()}") end diff --git a/flow/sunstone/public/js/apptools-appflow-services.js b/flow/sunstone/public/js/apptools-appflow-services.js index c2e96815ed..2e7acbdb78 100644 --- a/flow/sunstone/public/js/apptools-appflow-services.js +++ b/flow/sunstone/public/js/apptools-appflow-services.js @@ -60,8 +60,7 @@ var Service = { tr("DEPLOYING"), tr("RUNNING"), tr("UNDEPLOYING"), - tr("FAILED"), - tr("UNKNOWN"), + tr("WARNING"), tr("DONE"), tr("FAILED_UNDEPLOYING"), tr("FAILED_DEPLOYING"), @@ -83,8 +82,7 @@ var Role = { tr("DEPLOYING"), tr("RUNNING"), tr("UNDEPLOYING"), - tr("FAILED"), - tr("UNKNOWN"), + tr("WARNING"), tr("DONE"), tr("FAILED_UNDEPLOYING"), tr("FAILED_DEPLOYING"),