Feature #1852: New state FAILED_SCALING.

Scaling action can be recovered with appflow deploy
This commit is contained in:
Carlos Martín committed 2013-05-30 17:09:39 +02:00
1 parent 2fd735934e
commit efcc6c0b70
7 files changed
+138 -44

No files matched your search

+8 -4
View File
@@ -30,7 +30,8 @@ module Role
'DONE' => 6,
'FAILED_UNDEPLOYING' => 7,
'FAILED_DEPLOYING' => 8,
'SCALING' => 9
'SCALING' => 9,
'FAILED_SCALING' => 10
}
STATE_STR = [
@@ -43,7 +44,8 @@ module Role
'DONE',
'FAILED_UNDEPLOYING',
'FAILED_DEPLOYING',
'SCALING'
'SCALING',
'FAILED_SCALING'
]
# Returns the string representation of the role state
@@ -66,7 +68,8 @@ module Service
'DONE' => 6,
'FAILED_UNDEPLOYING' => 7,
'FAILED_DEPLOYING' => 8,
'SCALING' => 9
'SCALING' => 9,
'FAILED_SCALING' => 10
}
STATE_STR = [
@@ -79,7 +82,8 @@ module Service
'DONE',
'FAILED_UNDEPLOYING',
'FAILED_DEPLOYING',
'SCALING'
'SCALING',
'FAILED_SCALING'
]
# Returns the string representation of the service state
+13 -9
View File
@@ -77,11 +77,17 @@ class ServiceLCM
else
rc = strategy.scale_step(service)
if rc[0]
service.set_state(Service::STATE['SCALING'])
end
scale = rc[0]
succ = rc[1]
error_msg = rc[2]
# TODO: check error
if scale
if succ
service.set_state(Service::STATE['SCALING'])
else
service.set_state(Service::STATE['FAILED_SCALING'])
end
end
end
when Service::STATE['SCALING']
@@ -90,11 +96,9 @@ class ServiceLCM
# TODO: Check scalability rules for roles that are
# not scaling now?
# The service is moved to RUNNING even if the
# roles scaling ended in failure. Any error is
# handled the next cycle in the RUNNING case
if !service.any_role_scaling?
if service.any_role_failed_scaling?
service.set_state(Service::STATE['FAILED_SCALING'])
elsif !service.any_role_scaling?
service.set_state(Service::STATE['RUNNING'])
end
when Service::STATE['UNKNOWN']
+35 -13
View File
@@ -74,7 +74,11 @@ class Strategy
return [true, nil]
end
#
# @return [Array<false, nil, nil>, Array<true, true, nil>, Array<true, false, String>]
# 1st elem: true if the service needed to scale
# 2nd elem: true if the operation succeeded
# 3rd elem: error reason, if any
def scale_step(service)
Log.debug LOG_COMP, "Scale step", service.id()
@@ -93,13 +97,14 @@ class Strategy
rc = role.scale_down(role.cardinality() + diff)
end
# TODO: Handle error
# if !rc[0]
# role.set_state(Role::STATE['FAILED_DEPLOYING'])
# return rc
# end
if diff != 0
if !rc[0]
role.set_state(Role::STATE['FAILED_SCALING'])
return [true] + rc
end
role.set_state(Role::STATE['SCALING'])
scale = true
@@ -107,14 +112,12 @@ class Strategy
end
end
return [scale, nil]
return [scale, true, nil]
end
# Performs a monitor step, check if the roles already deployed are running
# @param [Service] service service to boot
# @return [Array<true, nil>, Array<false, String>] true if all the nodes
# were running, false and the error reason if there was a problem
# monitoring the nodes
# @param [Service] service service to monitor
# @return [nil]
def monitor_step(service)
Log.debug LOG_COMP, "Monitor step", service.id()
@@ -132,7 +135,7 @@ class Strategy
elsif !role_nodes_running?(role)
role.set_state(Role::STATE['UNKNOWN'])
end
when Role::STATE['DEPLOYING'], Role::STATE['SCALING']
when Role::STATE['DEPLOYING']
if OpenNebula.is_error?(rc)
role.set_state(Role::STATE['FAILED_DEPLOYING'])
elsif role_nodes_running?(role)
@@ -140,6 +143,14 @@ class Strategy
elsif any_node_failed?(role)
role.set_state(Role::STATE['FAILED_DEPLOYING'])
end
when Role::STATE['SCALING']
if OpenNebula.is_error?(rc)
role.set_state(Role::STATE['FAILED_SCALING'])
elsif role_nodes_running?(role)
role.set_state(Role::STATE['RUNNING'])
elsif any_node_failed_scaling?(role)
role.set_state(Role::STATE['FAILED_SCALING'])
end
when Role::STATE['UNKNOWN']
if role_nodes_running?(role)
role.set_state(Role::STATE['RUNNING'])
@@ -281,4 +292,15 @@ protected
return true
end
# Determine if any of the role nodes failed to scale
# @param [Role] role
# @return [true|false]
def any_node_failed_scaling?(role)
# TODO: This method returns true if any VM is in FAILED, but
# it should only consider VMs being scaled (instatiated/shutdown).
return any_node_failed?(role)
end
end
-3
View File
@@ -862,9 +862,6 @@ module ElasticityGrammar
n_nodes = 0
att = text_value.upcase
# TODO: search in 'TEMPLATE' also?
# TODO: what to do if value is not found
nodes.each { |node|
if node && node['vm_info']
if node['vm_info']['VM']['USER_TEMPLATE'][att]
-3
View File
@@ -158,9 +158,6 @@ grammar ElasticityGrammar
n_nodes = 0
att = text_value.upcase
# TODO: search in 'TEMPLATE' also?
# TODO: what to do if value is not found
nodes.each { |node|
if node && node['vm_info']
if node['vm_info']['VM']['USER_TEMPLATE'][att]
+56 -10
View File
@@ -31,7 +31,8 @@ module OpenNebula
'DONE' => 6,
'FAILED_UNDEPLOYING' => 7,
'FAILED_DEPLOYING' => 8,
'SCALING' => 9
'SCALING' => 9,
'FAILED_SCALING' => 10
}
STATE_STR = [
@@ -44,7 +45,8 @@ module OpenNebula
'DONE',
'FAILED_UNDEPLOYING',
'FAILED_DEPLOYING',
'SCALING'
'SCALING',
'FAILED_SCALING'
]
LOG_COMP = "ROL"
@@ -249,6 +251,9 @@ module OpenNebula
vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client)
rc = vm.shutdown
if scale_down
node['disposed'] = '1'
end
if OpenNebula.is_error?(rc)
msg = "Role #{name} : Shutdown failed for VM #{vm_id}, will perform a Delete; #{rc.message}"
@@ -266,17 +271,9 @@ module OpenNebula
#return [false, rc.message]
else
Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id()
if scale_down
node['disposed'] = '1'
end
end
else
Log.debug LOG_COMP, "Role #{name} : Shutdown success for VM #{vm_id}", @service.id()
if scale_down
node['disposed'] = '1'
end
end
}
@@ -534,5 +531,54 @@ module OpenNebula
return shutdown(true)
end
# For a failed scale up, the cardinality is updated to the actual value
# For a failed scale down, the shutdown actions are retried
def retry_scale()
n_dispose = 0
get_nodes.each do |node|
if node['disposed'] == "1"
############################################################
# TODO: refactor code, copied from shutdown()
############################################################
vm_id = node['deploy_id']
Log.debug LOG_COMP, "Role #{name} : Shutting down VM #{vm_id}", @service.id()
vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client)
rc = vm.shutdown
if OpenNebula.is_error?(rc)
msg = "Role #{name} : Shutdown failed for VM #{vm_id}, will perform a Delete; #{rc.message}"
Log.error LOG_COMP, msg, @service.id()
@service.log_error(msg)
rc = vm.finalize
if OpenNebula.is_error?(rc)
msg = "Role #{name} : Delete failed for VM #{vm_id}; #{rc.message}"
Log.error LOG_COMP, msg, @service.id()
@service.log_error(msg)
success = false
#return [false, rc.message]
else
Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id()
end
else
Log.debug LOG_COMP, "Role #{name} : Shutdown success for VM #{vm_id}", @service.id()
end
############################################################
n_dispose += 1
end
end
set_cardinality( get_nodes.size() - n_dispose )
end
end
end
+26 -2
View File
@@ -29,7 +29,8 @@ module OpenNebula
'DONE' => 6,
'FAILED_UNDEPLOYING' => 7,
'FAILED_DEPLOYING' => 8,
'SCALING' => 9
'SCALING' => 9,
'FAILED_SCALING' => 10
}
STATE_STR = [
@@ -42,7 +43,8 @@ module OpenNebula
'DONE',
'FAILED_UNDEPLOYING',
'FAILED_DEPLOYING',
'SCALING'
'SCALING',
'FAILED_SCALING'
]
LOG_COMP = "SER"
@@ -151,6 +153,16 @@ module OpenNebula
return false
end
def any_role_failed_scaling?()
@roles.each do |name, role|
if role.state == Role::STATE['FAILED_SCALING']
return true
end
end
return false
end
# Create a new service based on the template provided
# @param [String] template_json
# @return [nil, OpenNebula::Error] nil in case of success, Error
@@ -188,6 +200,18 @@ module OpenNebula
if [Service::STATE['FAILED_DEPLOYING']].include?(self.state)
self.set_state(Service::STATE['DEPLOYING'])
return self.update
# TODO: hijacked the deploy action to continue a failed scaling operation
elsif self.state == Service::STATE['FAILED_SCALING']
@roles.each do |name, role|
if role.state == Role::STATE['FAILED_SCALING']
role.retry_scale()
role.set_state(Role::STATE['SCALING'])
end
end
self.set_state(Service::STATE['SCALING'])
return self.update
else
return OpenNebula::Error.new("Action deploy: Wrong state" \
" #{self.state_str()}")