mirror of
https://github.com/OpenNebula/one.git
synced 2026-11-21 04:12:37 +09:00
Feature #1852: New state FAILED_SCALING.
Scaling action can be recovered with appflow deploy
This commit is contained in:
1 parent
2fd735934e
commit
efcc6c0b70
7 files changed
+138
-44
No files matched your search
@@ -30,7 +30,8 @@ module Role
|
||||
'DONE' => 6,
|
||||
'FAILED_UNDEPLOYING' => 7,
|
||||
'FAILED_DEPLOYING' => 8,
|
||||
'SCALING' => 9
|
||||
'SCALING' => 9,
|
||||
'FAILED_SCALING' => 10
|
||||
}
|
||||
|
||||
STATE_STR = [
|
||||
@@ -43,7 +44,8 @@ module Role
|
||||
'DONE',
|
||||
'FAILED_UNDEPLOYING',
|
||||
'FAILED_DEPLOYING',
|
||||
'SCALING'
|
||||
'SCALING',
|
||||
'FAILED_SCALING'
|
||||
]
|
||||
|
||||
# Returns the string representation of the role state
|
||||
@@ -66,7 +68,8 @@ module Service
|
||||
'DONE' => 6,
|
||||
'FAILED_UNDEPLOYING' => 7,
|
||||
'FAILED_DEPLOYING' => 8,
|
||||
'SCALING' => 9
|
||||
'SCALING' => 9,
|
||||
'FAILED_SCALING' => 10
|
||||
}
|
||||
|
||||
STATE_STR = [
|
||||
@@ -79,7 +82,8 @@ module Service
|
||||
'DONE',
|
||||
'FAILED_UNDEPLOYING',
|
||||
'FAILED_DEPLOYING',
|
||||
'SCALING'
|
||||
'SCALING',
|
||||
'FAILED_SCALING'
|
||||
]
|
||||
|
||||
# Returns the string representation of the service state
|
||||
|
||||
@@ -77,11 +77,17 @@ class ServiceLCM
|
||||
else
|
||||
rc = strategy.scale_step(service)
|
||||
|
||||
if rc[0]
|
||||
service.set_state(Service::STATE['SCALING'])
|
||||
end
|
||||
scale = rc[0]
|
||||
succ = rc[1]
|
||||
error_msg = rc[2]
|
||||
|
||||
# TODO: check error
|
||||
if scale
|
||||
if succ
|
||||
service.set_state(Service::STATE['SCALING'])
|
||||
else
|
||||
service.set_state(Service::STATE['FAILED_SCALING'])
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
when Service::STATE['SCALING']
|
||||
@@ -90,11 +96,9 @@ class ServiceLCM
|
||||
# TODO: Check scalability rules for roles that are
|
||||
# not scaling now?
|
||||
|
||||
# The service is moved to RUNNING even if the
|
||||
# roles scaling ended in failure. Any error is
|
||||
# handled the next cycle in the RUNNING case
|
||||
|
||||
if !service.any_role_scaling?
|
||||
if service.any_role_failed_scaling?
|
||||
service.set_state(Service::STATE['FAILED_SCALING'])
|
||||
elsif !service.any_role_scaling?
|
||||
service.set_state(Service::STATE['RUNNING'])
|
||||
end
|
||||
when Service::STATE['UNKNOWN']
|
||||
|
||||
+35
-13
@@ -74,7 +74,11 @@ class Strategy
|
||||
return [true, nil]
|
||||
end
|
||||
|
||||
|
||||
#
|
||||
# @return [Array<false, nil, nil>, Array<true, true, nil>, Array<true, false, String>]
|
||||
# 1st elem: true if the service needed to scale
|
||||
# 2nd elem: true if the operation succeeded
|
||||
# 3rd elem: error reason, if any
|
||||
def scale_step(service)
|
||||
Log.debug LOG_COMP, "Scale step", service.id()
|
||||
|
||||
@@ -93,13 +97,14 @@ class Strategy
|
||||
rc = role.scale_down(role.cardinality() + diff)
|
||||
end
|
||||
|
||||
# TODO: Handle error
|
||||
# if !rc[0]
|
||||
# role.set_state(Role::STATE['FAILED_DEPLOYING'])
|
||||
# return rc
|
||||
# end
|
||||
|
||||
if diff != 0
|
||||
|
||||
if !rc[0]
|
||||
role.set_state(Role::STATE['FAILED_SCALING'])
|
||||
|
||||
return [true] + rc
|
||||
end
|
||||
|
||||
role.set_state(Role::STATE['SCALING'])
|
||||
scale = true
|
||||
|
||||
@@ -107,14 +112,12 @@ class Strategy
|
||||
end
|
||||
end
|
||||
|
||||
return [scale, nil]
|
||||
return [scale, true, nil]
|
||||
end
|
||||
|
||||
# Performs a monitor step, check if the roles already deployed are running
|
||||
# @param [Service] service service to boot
|
||||
# @return [Array<true, nil>, Array<false, String>] true if all the nodes
|
||||
# were running, false and the error reason if there was a problem
|
||||
# monitoring the nodes
|
||||
# @param [Service] service service to monitor
|
||||
# @return [nil]
|
||||
def monitor_step(service)
|
||||
Log.debug LOG_COMP, "Monitor step", service.id()
|
||||
|
||||
@@ -132,7 +135,7 @@ class Strategy
|
||||
elsif !role_nodes_running?(role)
|
||||
role.set_state(Role::STATE['UNKNOWN'])
|
||||
end
|
||||
when Role::STATE['DEPLOYING'], Role::STATE['SCALING']
|
||||
when Role::STATE['DEPLOYING']
|
||||
if OpenNebula.is_error?(rc)
|
||||
role.set_state(Role::STATE['FAILED_DEPLOYING'])
|
||||
elsif role_nodes_running?(role)
|
||||
@@ -140,6 +143,14 @@ class Strategy
|
||||
elsif any_node_failed?(role)
|
||||
role.set_state(Role::STATE['FAILED_DEPLOYING'])
|
||||
end
|
||||
when Role::STATE['SCALING']
|
||||
if OpenNebula.is_error?(rc)
|
||||
role.set_state(Role::STATE['FAILED_SCALING'])
|
||||
elsif role_nodes_running?(role)
|
||||
role.set_state(Role::STATE['RUNNING'])
|
||||
elsif any_node_failed_scaling?(role)
|
||||
role.set_state(Role::STATE['FAILED_SCALING'])
|
||||
end
|
||||
when Role::STATE['UNKNOWN']
|
||||
if role_nodes_running?(role)
|
||||
role.set_state(Role::STATE['RUNNING'])
|
||||
@@ -281,4 +292,15 @@ protected
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
# Determine if any of the role nodes failed to scale
|
||||
# @param [Role] role
|
||||
# @return [true|false]
|
||||
def any_node_failed_scaling?(role)
|
||||
|
||||
# TODO: This method returns true if any VM is in FAILED, but
|
||||
# it should only consider VMs being scaled (instatiated/shutdown).
|
||||
|
||||
return any_node_failed?(role)
|
||||
end
|
||||
end
|
||||
@@ -862,9 +862,6 @@ module ElasticityGrammar
|
||||
n_nodes = 0
|
||||
att = text_value.upcase
|
||||
|
||||
# TODO: search in 'TEMPLATE' also?
|
||||
# TODO: what to do if value is not found
|
||||
|
||||
nodes.each { |node|
|
||||
if node && node['vm_info']
|
||||
if node['vm_info']['VM']['USER_TEMPLATE'][att]
|
||||
|
||||
@@ -158,9 +158,6 @@ grammar ElasticityGrammar
|
||||
n_nodes = 0
|
||||
att = text_value.upcase
|
||||
|
||||
# TODO: search in 'TEMPLATE' also?
|
||||
# TODO: what to do if value is not found
|
||||
|
||||
nodes.each { |node|
|
||||
if node && node['vm_info']
|
||||
if node['vm_info']['VM']['USER_TEMPLATE'][att]
|
||||
|
||||
+56
-10
@@ -31,7 +31,8 @@ module OpenNebula
|
||||
'DONE' => 6,
|
||||
'FAILED_UNDEPLOYING' => 7,
|
||||
'FAILED_DEPLOYING' => 8,
|
||||
'SCALING' => 9
|
||||
'SCALING' => 9,
|
||||
'FAILED_SCALING' => 10
|
||||
}
|
||||
|
||||
STATE_STR = [
|
||||
@@ -44,7 +45,8 @@ module OpenNebula
|
||||
'DONE',
|
||||
'FAILED_UNDEPLOYING',
|
||||
'FAILED_DEPLOYING',
|
||||
'SCALING'
|
||||
'SCALING',
|
||||
'FAILED_SCALING'
|
||||
]
|
||||
|
||||
LOG_COMP = "ROL"
|
||||
@@ -249,6 +251,9 @@ module OpenNebula
|
||||
vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client)
|
||||
rc = vm.shutdown
|
||||
|
||||
if scale_down
|
||||
node['disposed'] = '1'
|
||||
end
|
||||
|
||||
if OpenNebula.is_error?(rc)
|
||||
msg = "Role #{name} : Shutdown failed for VM #{vm_id}, will perform a Delete; #{rc.message}"
|
||||
@@ -266,17 +271,9 @@ module OpenNebula
|
||||
#return [false, rc.message]
|
||||
else
|
||||
Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id()
|
||||
|
||||
if scale_down
|
||||
node['disposed'] = '1'
|
||||
end
|
||||
end
|
||||
else
|
||||
Log.debug LOG_COMP, "Role #{name} : Shutdown success for VM #{vm_id}", @service.id()
|
||||
|
||||
if scale_down
|
||||
node['disposed'] = '1'
|
||||
end
|
||||
end
|
||||
}
|
||||
|
||||
@@ -534,5 +531,54 @@ module OpenNebula
|
||||
|
||||
return shutdown(true)
|
||||
end
|
||||
|
||||
# For a failed scale up, the cardinality is updated to the actual value
|
||||
# For a failed scale down, the shutdown actions are retried
|
||||
def retry_scale()
|
||||
n_dispose = 0
|
||||
|
||||
get_nodes.each do |node|
|
||||
if node['disposed'] == "1"
|
||||
|
||||
############################################################
|
||||
# TODO: refactor code, copied from shutdown()
|
||||
############################################################
|
||||
|
||||
vm_id = node['deploy_id']
|
||||
|
||||
Log.debug LOG_COMP, "Role #{name} : Shutting down VM #{vm_id}", @service.id()
|
||||
|
||||
vm = OpenNebula::VirtualMachine.new_with_id(vm_id, @service.client)
|
||||
rc = vm.shutdown
|
||||
|
||||
if OpenNebula.is_error?(rc)
|
||||
msg = "Role #{name} : Shutdown failed for VM #{vm_id}, will perform a Delete; #{rc.message}"
|
||||
Log.error LOG_COMP, msg, @service.id()
|
||||
@service.log_error(msg)
|
||||
|
||||
rc = vm.finalize
|
||||
|
||||
if OpenNebula.is_error?(rc)
|
||||
msg = "Role #{name} : Delete failed for VM #{vm_id}; #{rc.message}"
|
||||
Log.error LOG_COMP, msg, @service.id()
|
||||
@service.log_error(msg)
|
||||
|
||||
success = false
|
||||
#return [false, rc.message]
|
||||
else
|
||||
Log.debug LOG_COMP, "Role #{name} : Delete success for VM #{vm_id}", @service.id()
|
||||
end
|
||||
else
|
||||
Log.debug LOG_COMP, "Role #{name} : Shutdown success for VM #{vm_id}", @service.id()
|
||||
end
|
||||
|
||||
############################################################
|
||||
|
||||
n_dispose += 1
|
||||
end
|
||||
end
|
||||
|
||||
set_cardinality( get_nodes.size() - n_dispose )
|
||||
end
|
||||
end
|
||||
end
|
||||
+26
-2
@@ -29,7 +29,8 @@ module OpenNebula
|
||||
'DONE' => 6,
|
||||
'FAILED_UNDEPLOYING' => 7,
|
||||
'FAILED_DEPLOYING' => 8,
|
||||
'SCALING' => 9
|
||||
'SCALING' => 9,
|
||||
'FAILED_SCALING' => 10
|
||||
}
|
||||
|
||||
STATE_STR = [
|
||||
@@ -42,7 +43,8 @@ module OpenNebula
|
||||
'DONE',
|
||||
'FAILED_UNDEPLOYING',
|
||||
'FAILED_DEPLOYING',
|
||||
'SCALING'
|
||||
'SCALING',
|
||||
'FAILED_SCALING'
|
||||
]
|
||||
|
||||
LOG_COMP = "SER"
|
||||
@@ -151,6 +153,16 @@ module OpenNebula
|
||||
return false
|
||||
end
|
||||
|
||||
def any_role_failed_scaling?()
|
||||
@roles.each do |name, role|
|
||||
if role.state == Role::STATE['FAILED_SCALING']
|
||||
return true
|
||||
end
|
||||
end
|
||||
|
||||
return false
|
||||
end
|
||||
|
||||
# Create a new service based on the template provided
|
||||
# @param [String] template_json
|
||||
# @return [nil, OpenNebula::Error] nil in case of success, Error
|
||||
@@ -188,6 +200,18 @@ module OpenNebula
|
||||
if [Service::STATE['FAILED_DEPLOYING']].include?(self.state)
|
||||
self.set_state(Service::STATE['DEPLOYING'])
|
||||
return self.update
|
||||
|
||||
# TODO: hijacked the deploy action to continue a failed scaling operation
|
||||
elsif self.state == Service::STATE['FAILED_SCALING']
|
||||
@roles.each do |name, role|
|
||||
if role.state == Role::STATE['FAILED_SCALING']
|
||||
role.retry_scale()
|
||||
role.set_state(Role::STATE['SCALING'])
|
||||
end
|
||||
end
|
||||
|
||||
self.set_state(Service::STATE['SCALING'])
|
||||
return self.update
|
||||
else
|
||||
return OpenNebula::Error.new("Action deploy: Wrong state" \
|
||||
" #{self.state_str()}")
|
||||
|
||||
Reference in new issue
Block a user