From efcc6c0b701ddcb451e40a12dddbebba6c59cba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Thu, 30 May 2013 17:09:39 +0200 Subject: [PATCH] Feature #1852: New state FAILED_SCALING. Scaling action can be recovered with appflow deploy --- flow/client/lib/appflow_client.rb | 12 ++++-- flow/lcm/LifeCycleManager.rb | 22 ++++++----- flow/lcm/strategy.rb | 48 ++++++++++++++++------ flow/lib/grammar.rb | 3 -- flow/lib/grammar.treetop | 3 -- flow/models/Role.rb | 66 ++++++++++++++++++++++++++----- flow/models/Service.rb | 28 ++++++++++++- 7 files changed, 138 insertions(+), 44 deletions(-) diff --git a/flow/client/lib/appflow_client.rb b/flow/client/lib/appflow_client.rb index acddeb56d4..c9ff61dab7 100644 --- a/flow/client/lib/appflow_client.rb +++ b/flow/client/lib/appflow_client.rb @@ -30,7 +30,8 @@ module Role 'DONE' => 6, 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9 + 'SCALING' => 9, + 'FAILED_SCALING' => 10 } STATE_STR = [ @@ -43,7 +44,8 @@ module Role 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', - 'SCALING' + 'SCALING', + 'FAILED_SCALING' ] # Returns the string representation of the role state @@ -66,7 +68,8 @@ module Service 'DONE' => 6, 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9 + 'SCALING' => 9, + 'FAILED_SCALING' => 10 } STATE_STR = [ @@ -79,7 +82,8 @@ module Service 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', - 'SCALING' + 'SCALING', + 'FAILED_SCALING' ] # Returns the string representation of the service state diff --git a/flow/lcm/LifeCycleManager.rb b/flow/lcm/LifeCycleManager.rb index 630cf61847..ab7dc8f6d2 100644 --- a/flow/lcm/LifeCycleManager.rb +++ b/flow/lcm/LifeCycleManager.rb @@ -77,11 +77,17 @@ class ServiceLCM else rc = strategy.scale_step(service) - if rc[0] - service.set_state(Service::STATE['SCALING']) - end + scale = rc[0] + succ = rc[1] + error_msg = rc[2] - # TODO: check error + if scale + if succ + service.set_state(Service::STATE['SCALING']) + else + service.set_state(Service::STATE['FAILED_SCALING']) + end + end end when Service::STATE['SCALING'] @@ -90,11 +96,9 @@ class ServiceLCM # TODO: Check scalability rules for roles that are # not scaling now? - # The service is moved to RUNNING even if the - # roles scaling ended in failure. Any error is - # handled the next cycle in the RUNNING case - - if !service.any_role_scaling? + if service.any_role_failed_scaling? + service.set_state(Service::STATE['FAILED_SCALING']) + elsif !service.any_role_scaling? service.set_state(Service::STATE['RUNNING']) end when Service::STATE['UNKNOWN'] diff --git a/flow/lcm/strategy.rb b/flow/lcm/strategy.rb index bac2434ac6..06ac7ad336 100644 --- a/flow/lcm/strategy.rb +++ b/flow/lcm/strategy.rb @@ -74,7 +74,11 @@ class Strategy return [true, nil] end - + # + # @return [Array, Array, Array] + # 1st elem: true if the service needed to scale + # 2nd elem: true if the operation succeeded + # 3rd elem: error reason, if any def scale_step(service) Log.debug LOG_COMP, "Scale step", service.id() @@ -93,13 +97,14 @@ class Strategy rc = role.scale_down(role.cardinality() + diff) end - # TODO: Handle error -# if !rc[0] -# role.set_state(Role::STATE['FAILED_DEPLOYING']) -# return rc -# end - if diff != 0 + + if !rc[0] + role.set_state(Role::STATE['FAILED_SCALING']) + + return [true] + rc + end + role.set_state(Role::STATE['SCALING']) scale = true @@ -107,14 +112,12 @@ class Strategy end end - return [scale, nil] + return [scale, true, nil] end # Performs a monitor step, check if the roles already deployed are running - # @param [Service] service service to boot - # @return [Array, Array] true if all the nodes - # were running, false and the error reason if there was a problem - # monitoring the nodes + # @param [Service] service service to monitor + # @return [nil] def monitor_step(service) Log.debug LOG_COMP, "Monitor step", service.id() @@ -132,7 +135,7 @@ class Strategy elsif !role_nodes_running?(role) role.set_state(Role::STATE['UNKNOWN']) end - when Role::STATE['DEPLOYING'], Role::STATE['SCALING'] + when Role::STATE['DEPLOYING'] if OpenNebula.is_error?(rc) role.set_state(Role::STATE['FAILED_DEPLOYING']) elsif role_nodes_running?(role) @@ -140,6 +143,14 @@ class Strategy elsif any_node_failed?(role) role.set_state(Role::STATE['FAILED_DEPLOYING']) end + when Role::STATE['SCALING'] + if OpenNebula.is_error?(rc) + role.set_state(Role::STATE['FAILED_SCALING']) + elsif role_nodes_running?(role) + role.set_state(Role::STATE['RUNNING']) + elsif any_node_failed_scaling?(role) + role.set_state(Role::STATE['FAILED_SCALING']) + end when Role::STATE['UNKNOWN'] if role_nodes_running?(role) role.set_state(Role::STATE['RUNNING']) @@ -281,4 +292,15 @@ protected return true end + + # Determine if any of the role nodes failed to scale + # @param [Role] role + # @return [true|false] + def any_node_failed_scaling?(role) + + # TODO: This method returns true if any VM is in FAILED, but + # it should only consider VMs being scaled (instatiated/shutdown). + + return any_node_failed?(role) + end end diff --git a/flow/lib/grammar.rb b/flow/lib/grammar.rb index bdbf588cfd..21f99b818f 100644 --- a/flow/lib/grammar.rb +++ b/flow/lib/grammar.rb @@ -862,9 +862,6 @@ module ElasticityGrammar n_nodes = 0 att = text_value.upcase - # TODO: search in 'TEMPLATE' also? - # TODO: what to do if value is not found - nodes.each { |node| if node && node['vm_info'] if node['vm_info']['VM']['USER_TEMPLATE'][att] diff --git a/flow/lib/grammar.treetop b/flow/lib/grammar.treetop index e290dc6c5f..4abe3e1292 100644 --- a/flow/lib/grammar.treetop +++ b/flow/lib/grammar.treetop @@ -158,9 +158,6 @@ grammar ElasticityGrammar n_nodes = 0 att = text_value.upcase - # TODO: search in 'TEMPLATE' also? - # TODO: what to do if value is not found - nodes.each { |node| if node && node['vm_info'] if node['vm_info']['VM']['USER_TEMPLATE'][att] diff --git a/flow/models/Role.rb b/flow/models/Role.rb index 612802061c..0ae734ecd0 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -31,7 +31,8 @@ module OpenNebula 'DONE' => 6, 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9 + 'SCALING' => 9, + 'FAILED_SCALING' => 10 } STATE_STR = [ @@ -44,7 +45,8 @@ module OpenNebula 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', - 'SCALING' + 'SCALING', + 'FAILED_SCALING' ] LOG_COMP = "ROL" @@ -249,6 +251,9 @@ module OpenNebula vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client) rc = vm.shutdown + if scale_down + node['disposed'] = '1' + end if OpenNebula.is_error?(rc) msg = "Role #{name} : Shutdown failed for VM #{vm_id}, will perform a Delete; #{rc.message}" @@ -266,17 +271,9 @@ module OpenNebula #return [false, rc.message] else Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id() - - if scale_down - node['disposed'] = '1' - end end else Log.debug LOG_COMP, "Role #{name} : Shutdown success for VM #{vm_id}", @service.id() - - if scale_down - node['disposed'] = '1' - end end } @@ -534,5 +531,54 @@ module OpenNebula return shutdown(true) end + + # For a failed scale up, the cardinality is updated to the actual value + # For a failed scale down, the shutdown actions are retried + def retry_scale() + n_dispose = 0 + + get_nodes.each do |node| + if node['disposed'] == "1" + + ############################################################ + # TODO: refactor code, copied from shutdown() + ############################################################ + + vm_id = node['deploy_id'] + + Log.debug LOG_COMP, "Role #{name} : Shutting down VM #{vm_id}", @service.id() + + vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client) + rc = vm.shutdown + + if OpenNebula.is_error?(rc) + msg = "Role #{name} : Shutdown failed for VM #{vm_id}, will perform a Delete; #{rc.message}" + Log.error LOG_COMP, msg, @service.id() + @service.log_error(msg) + + rc = vm.finalize + + if OpenNebula.is_error?(rc) + msg = "Role #{name} : Delete failed for VM #{vm_id}; #{rc.message}" + Log.error LOG_COMP, msg, @service.id() + @service.log_error(msg) + + success = false + #return [false, rc.message] + else + Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id() + end + else + Log.debug LOG_COMP, "Role #{name} : Shutdown success for VM #{vm_id}", @service.id() + end + + ############################################################ + + n_dispose += 1 + end + end + + set_cardinality( get_nodes.size() - n_dispose ) + end end end diff --git a/flow/models/Service.rb b/flow/models/Service.rb index 36ec571913..53c8ea89f8 100644 --- a/flow/models/Service.rb +++ b/flow/models/Service.rb @@ -29,7 +29,8 @@ module OpenNebula 'DONE' => 6, 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, - 'SCALING' => 9 + 'SCALING' => 9, + 'FAILED_SCALING' => 10 } STATE_STR = [ @@ -42,7 +43,8 @@ module OpenNebula 'DONE', 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', - 'SCALING' + 'SCALING', + 'FAILED_SCALING' ] LOG_COMP = "SER" @@ -151,6 +153,16 @@ module OpenNebula return false end + def any_role_failed_scaling?() + @roles.each do |name, role| + if role.state == Role::STATE['FAILED_SCALING'] + return true + end + end + + return false + end + # Create a new service based on the template provided # @param [String] template_json # @return [nil, OpenNebula::Error] nil in case of success, Error @@ -188,6 +200,18 @@ module OpenNebula if [Service::STATE['FAILED_DEPLOYING']].include?(self.state) self.set_state(Service::STATE['DEPLOYING']) return self.update + + # TODO: hijacked the deploy action to continue a failed scaling operation + elsif self.state == Service::STATE['FAILED_SCALING'] + @roles.each do |name, role| + if role.state == Role::STATE['FAILED_SCALING'] + role.retry_scale() + role.set_state(Role::STATE['SCALING']) + end + end + + self.set_state(Service::STATE['SCALING']) + return self.update else return OpenNebula::Error.new("Action deploy: Wrong state" \ " #{self.state_str()}")