From 587221bfadf41605cfabd86654b29ecfc91963cd Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Tue, 19 Apr 2022 11:29:36 +0200 Subject: [PATCH] F #1765: Allow instantiation of OneFlow service with all VMs on hold (#1910) This applies commit 1df045fd0ecaeb823c1487d105ea7fb39dae6234. --- share/onegate/onegate | 6 +- src/cli/oneflow | 23 +++ src/flow/lib/EventManager.rb | 50 ++++- src/flow/lib/LifeCycleManager.rb | 195 ++++++++++++++++-- src/flow/lib/models/role.rb | 75 ++++++- src/flow/lib/models/service.rb | 26 ++- src/flow/lib/strategy.rb | 31 +++ src/flow/lib/strategy/straight.rb | 74 +++++++ src/flow/oneflow-server.rb | 3 + .../ruby/opennebula/flow/service_template.rb | 8 + src/oca/ruby/opennebula/oneflow_client.rb | 12 +- 11 files changed, 481 insertions(+), 22 deletions(-) diff --git a/share/onegate/onegate b/share/onegate/onegate index 8c3d8fe9d0..05b24a7c84 100755 --- a/share/onegate/onegate +++ b/share/onegate/onegate @@ -359,7 +359,8 @@ module OneGate 'DEPLOYING_NETS' => 11, 'UNDEPLOYING_NETS' => 12, 'FAILED_DEPLOYING_NETS' => 13, - 'FAILED_UNDEPLOYING_NETS' => 14 + 'FAILED_UNDEPLOYING_NETS' => 14, + 'HOLD' => 15 } STATE_STR = [ @@ -377,7 +378,8 @@ module OneGate 'DEPLOYING_NETS', 'UNDEPLOYING_NETS', 'FAILED_DEPLOYING_NETS', - 'FAILED_UNDEPLOYING_NETS' + 'FAILED_UNDEPLOYING_NETS', + 'HOLD' ] # Returns the string representation of the service state diff --git a/src/cli/oneflow b/src/cli/oneflow index 8ede10a7f2..7819156c07 100755 --- a/src/cli/oneflow +++ b/src/cli/oneflow @@ -521,4 +521,27 @@ CommandParser::CmdParser.new(ARGV) do 0 end end + + ### + + release_desc = <<-EOT.unindent + Release roles of a service on hold + EOT + + command :release, release_desc, :service_id do + service_id = args[0] + client = helper.client(options) + + params = {} + json = Service.build_json_action('release', params) + + response = client.post("#{RESOURCE_PATH}/#{service_id}/action", + json) + + if CloudClient.is_error?(response) + [response.code.to_i, response.to_s] + else + 0 + end + end end diff --git a/src/flow/lib/EventManager.rb b/src/flow/lib/EventManager.rb index c98c89344c..320eca5f24 100644 --- a/src/flow/lib/EventManager.rb +++ b/src/flow/lib/EventManager.rb @@ -35,7 +35,9 @@ class EventManager :wait_remove_action, :wait_cooldown_action, :wait_deploy_nets_action, - :wait_undeploy_nets_action + :wait_undeploy_nets_action, + :wait_hold, + :wait_release ] FAILURE_STATES = %w[ @@ -82,6 +84,7 @@ class EventManager @subscriber_endpoint = @cloud_auth.conf[:subscriber_endpoint] # Register Action Manager actions + ACTIONS.each do |m| @am.register_action(m, method(m.to_s)) end @@ -304,6 +307,51 @@ class EventManager role_name) end + # Wait for nodes to be in HOLD + # @param [service_id] the service id + # @param [role_name] the role name of the role which contains the VMs + # @param [nodes] the list of nodes (VMs) to wait for + def wait_hold_action(client, service_id, role_name, nodes) + Log.info LOG_COMP, "Waiting #{nodes} to be (HOLD, LCM_INIT)" + wait(nodes, 'HOLD', 'LCM_INIT') + + @lcm.trigger_action(:hold_cb, + service_id, + client, + service_id, + role_name) + end + + # Wait for nodes to be in RUNNING if OneGate check required it will trigger + # another action after VMs are RUNNING + # @param [Service] service the service + # @param [Role] the role which contains the VMs + # @param [Node] nodes the list of nodes (VMs) to wait for + def wait_release_action(client, service_id, role_name, nodes, report) + if report + Log.info LOG_COMP, "Waiting #{nodes} to report ready" + rc = wait_report_ready(nodes) + else + Log.info LOG_COMP, "Waiting #{nodes} to be (ACTIVE, RUNNING)" + rc = wait(nodes, 'ACTIVE', 'RUNNING') + end + + if rc[0] + @lcm.trigger_action(:release_cb, + service_id, + client, + service_id, + role_name, + rc[1]) + else + @lcm.trigger_action(:deploy_failure_cb, + service_id, + client, + service_id, + role_name) + end + end + private ############################################################################ diff --git a/src/flow/lib/LifeCycleManager.rb b/src/flow/lib/LifeCycleManager.rb index 47f4822686..6cb2e48632 100644 --- a/src/flow/lib/LifeCycleManager.rb +++ b/src/flow/lib/LifeCycleManager.rb @@ -52,7 +52,11 @@ class ServiceLCM # WD callbacks :error_wd_cb, :done_wd_cb, - :running_wd_cb + :running_wd_cb, + + # Hold/Release callbacks + :hold_cb, + :release_cb ] def initialize(client, concurrency, cloud_auth) @@ -199,6 +203,43 @@ class ServiceLCM rc end + # Release a service on hold state + # + # @param client [OpenNebula::Client] Client executing action + # @param service_id [Integer] Service ID + # + # @return [OpenNebula::Error] Error if any + def release_action(client, service_id) + rc = @srv_pool.get(service_id, client) do |service| + # Get roles that can be release + set_deploy_strategy(service) + roles = service.roles_release + + if roles.empty? + break OpenNebula::Error.new('Service has no roles in HOLD') + end + + rc = release_roles(client, + roles, + 'DEPLOYING', + 'FAILED_DEPLOYING', + :wait_release, + service.report_ready?) + + if !OpenNebula.is_error?(rc) + service.set_state(Service::STATE['DEPLOYING']) + else + service.set_state(Service::STATE['FAILED_DEPLOYING']) + end + + service.update + rc + end + + Log.error LOG_COMP, rc.message if OpenNebula.is_error?(rc) + rc + end + ############################################################################ # Life cycle manager actions ############################################################################ @@ -292,6 +333,16 @@ class ServiceLCM break end + # Set all roles on hold if the on_hold option + # is set at service level + roles.each do |_, role| + if service.on_hold? + role.hold(true) + elsif role.any_parent_on_hold? + role.hold(true) + end + end + rc = deploy_roles(client, roles, 'DEPLOYING', @@ -299,7 +350,9 @@ class ServiceLCM :wait_deploy_action, service.report_ready?) - if !OpenNebula.is_error?(rc) + if !OpenNebula.is_error?(rc) & service.on_hold? + service.set_state(Service::STATE['HOLD']) + elsif !OpenNebula.is_error?(rc) & !service.on_hold? service.set_state(Service::STATE['DEPLOYING']) else service.set_state(Service::STATE['FAILED_DEPLOYING']) @@ -647,7 +700,7 @@ class ServiceLCM nodes[node] end - # If the role has 0 nodes, delete role + # If the role has 0 nodes, deleteƒ role undeploy = service.check_role(service.roles[role_name]) if service.all_roles_running? @@ -978,6 +1031,80 @@ class ServiceLCM Log.error LOG_COMP, rc.message if OpenNebula.is_error?(rc) end + def hold_cb(client, service_id, role_name) + rc = @srv_pool.get(service_id, client) do |service| + if service.roles[role_name].state != Role::STATE['HOLD'] + service.roles[role_name].set_state(Role::STATE['HOLD']) + end + + if service.all_roles_hold? && + service.state != Service::STATE['HOLD'] + service.set_state(Service::STATE['HOLD']) + elsif service.strategy == 'straight' + set_deploy_strategy(service) + + deploy_roles(client, + service.roles_hold, + 'DEPLOYING', + 'FAILED_DEPLOYING', + :wait_deploy, + service.report_ready?) + end + + rc = service.update + + return rc if OpenNebula.is_error?(rc) + end + + Log.error 'WD', rc.message if OpenNebula.is_error?(rc) + end + + def release_cb(client, service_id, role_name, nodes) + undeploy = false + + rc = @srv_pool.get(service_id, client) do |service| + service.roles[role_name].set_state(Role::STATE['RUNNING']) + + service.roles[role_name].nodes.delete_if do |node| + if nodes[node] && service.roles[role_name].cardinalitty > 0 + service.roles[role_name].cardinality -= 1 + end + + nodes[node] + end + + # If the role has 0 nodes, delete role + undeploy = service.check_role(service.roles[role_name]) + + if service.all_roles_running? + service.set_state(Service::STATE['RUNNING']) + elsif service.strategy == 'straight' + set_deploy_strategy(service) + + release_roles(client, + service.roles_release, + 'DEPLOYING', + 'FAILED_DEPLOYING', + :wait_deploy, + service.report_ready?) + end + + rc = service.update + + return rc if OpenNebula.is_error?(rc) + + @wd.add_service(service) if service.all_roles_running? + end + + Log.error LOG_COMP, rc.message if OpenNebula.is_error?(rc) + + return unless undeploy + + Log.info LOG_COMP, "Automatically deleting service #{service_id}" + + undeploy_action(client, service_id) + end + ############################################################################ # WatchDog Callbacks ############################################################################ @@ -1105,6 +1232,16 @@ class ServiceLCM def deploy_roles(client, roles, success_state, error_state, action, report) # rubocop:enable Metrics/ParameterLists roles.each do |name, role| + if role.state == Role::STATE['PENDING'] + # Set all roles on hold if the on_hold option + # is set at service level + if role.service_on_hold? + role.hold(true) + elsif role.any_parent_on_hold? + role.hold(true) + end + end + rc = role.deploy if !rc[0] @@ -1114,15 +1251,24 @@ class ServiceLCM ) end - role.set_state(Role::STATE[success_state]) - - @event_manager.trigger_action(action, - role.service.id, - client, - role.service.id, - role.name, - rc[0], - report) + if role.on_hold? && role.state == Role::STATE['PENDING'] + role.set_state(Role::STATE['HOLD']) + @event_manager.trigger_action(:wait_hold, + role.service.id, + client, + role.service.id, + role.name, + rc[0]) + else + role.set_state(Role::STATE[success_state]) + @event_manager.trigger_action(action, + role.service.id, + client, + role.service.id, + role.name, + rc[0], + report) + end end end @@ -1150,6 +1296,31 @@ class ServiceLCM end end + # rubocop:disable Metrics/ParameterLists + def release_roles(client, roles, success_state, error_state, action, report) + # rubocop:enable Metrics/ParameterLists + roles.each do |name, role| + rc = role.release + + if !rc[1] + role.set_state(Role::STATE[error_state]) + break OpenNebula::Error.new( + "Error releasing role #{name}: #{rc[1]}" + ) + end + + role.set_state(Role::STATE[success_state]) + + @event_manager.trigger_action(action, + role.service.id, + client, + role.service.id, + role.name, + rc[0], + report) + end + end + def set_cardinality(role, cardinality, force) tmpl_json = "{ \"cardinality\" : #{cardinality},\n" \ " \"force\" : #{force} }" diff --git a/src/flow/lib/models/role.rb b/src/flow/lib/models/role.rb index 86d0c8c0f4..85a8db0417 100644 --- a/src/flow/lib/models/role.rb +++ b/src/flow/lib/models/role.rb @@ -65,7 +65,8 @@ module OpenNebula 'FAILED_DEPLOYING' => 7, 'SCALING' => 8, 'FAILED_SCALING' => 9, - 'COOLDOWN' => 10 + 'COOLDOWN' => 10, + 'HOLD' => 11 } STATE_STR = %w[ @@ -80,6 +81,7 @@ module OpenNebula SCALING FAILED_SCALING COOLDOWN + HOLD ] RECOVER_DEPLOY_STATES = %w[ @@ -147,6 +149,7 @@ module OpenNebula @body['cooldown'] = @@default_cooldown if @body['cooldown'].nil? @body['nodes'] ||= [] + @body['on_hold'] = false if @body['on_hold'].nil? end def name @@ -195,12 +198,25 @@ module OpenNebula true end + def can_release? + state == STATE['HOLD'] + end + # Returns the role parents # @return [Array] the role parents def parents @body['parents'] || [] end + def any_parent_on_hold? + parents.each do |parent| + next unless @service.roles[parent] + + return true if @service.roles[parent].on_hold? + end + false + end + # Returns the role cardinality # @return [Integer] the role cardinality def cardinality @@ -357,6 +373,23 @@ module OpenNebula @body.delete('scale_way') end + # Returns the on_hold role option + # @return [true, false] true if the on_hold option is enabled + def on_hold? + @body['on_hold'] + end + + # Returns the on_hold service option + # @return [true, false] true if the on_hold option is enabled + def service_on_hold? + @service.on_hold? + end + + # Set the on_hold vm option to true + def hold(hold) + @body['on_hold'] = hold + end + # Retrieves the VM information for each Node in this Role. If a Node # is to be disposed and it is found in DONE, it will be cleaned # @@ -436,7 +469,7 @@ module OpenNebula "template #{template_id}, with name #{vm_name}", @service.id - vm_id = template.instantiate(vm_name, false, extra_template) + vm_id = template.instantiate(vm_name, on_hold?, extra_template) deployed_nodes << vm_id @@ -546,6 +579,44 @@ module OpenNebula [true, nil] end + # Release all the nodes in this role + # @return [Array, Bool] true if all the VMs + # were released, false otherwise and Array with VMs released + def release + release_nodes = [] + success = true + + # Release all vms in the role + nodes.each do |node| + vm_id = node['deploy_id'] + + Log.debug(LOG_COMP, + "Role #{name}: Releasing VM #{vm_id}", + @service.id) + + vm = OpenNebula::VirtualMachine.new_with_id(vm_id, + @service.client) + rc = vm.release + + if OpenNebula.is_error?(rc) + msg = "Role #{name}: Release failed for VM #{vm_id}, " \ + "#{rc.message}" + + Log.error(LOG_COMP, msg, @service.id) + @service.log_error(msg) + success = false + else + Log.debug(LOG_COMP, + "Role #{name}: Release success for VM #{vm_id}", + @service.id) + + release_nodes << vm_id + end + end + + [release_nodes, success] + end + # Schedule the given action on all the VMs that belong to the Role # @param [String] action one of the available SCHEDULE_ACTIONS # @param [Integer] period diff --git a/src/flow/lib/models/service.rb b/src/flow/lib/models/service.rb index 5c97d1577e..de9f7b67c4 100644 --- a/src/flow/lib/models/service.rb +++ b/src/flow/lib/models/service.rb @@ -38,7 +38,8 @@ module OpenNebula 'DEPLOYING_NETS' => 11, 'UNDEPLOYING_NETS' => 12, 'FAILED_DEPLOYING_NETS' => 13, - 'FAILED_UNDEPLOYING_NETS' => 14 + 'FAILED_UNDEPLOYING_NETS' => 14, + 'HOLD' => 15 } STATE_STR = %w[ @@ -57,6 +58,7 @@ module OpenNebula UNDEPLOYING_NETS FAILED_DEPLOYING_NETS FAILED_UNDEPLOYING_NETS + HOLD ] TRANSIENT_STATES = %w[ @@ -216,6 +218,16 @@ module OpenNebula self['GID'].to_i end + # Returns the on_hold service option + # @return [true, false] true if the on_hold option is enabled + def on_hold? + @body['on_hold'] + end + + def hold? + state_str == 'HOLD' + end + # Replaces this object's client with a new one # @param [OpenNebula::Client] owner_client the new client def replace_client(owner_client) @@ -265,6 +277,18 @@ module OpenNebula true end + # Returns true if all the nodes are in hold state + # @return [true, false] true if all the nodes are in hold state + def all_roles_hold? + @roles.each do |_name, role| + if role.state != Role::STATE['HOLD'] + return false + end + end + + true + end + # Returns virtual networks IDs # @return [Array] Array of integers containing the IDs def networks(deploy) diff --git a/src/flow/lib/strategy.rb b/src/flow/lib/strategy.rb index e38c844395..969bf8e5b9 100644 --- a/src/flow/lib/strategy.rb +++ b/src/flow/lib/strategy.rb @@ -58,4 +58,35 @@ module Strategy result end + # Returns all node Roles ready to be set on hold + # @return [Hash] Roles + def roles_hold + result = roles.select do |_name, role| + role.state == Role::STATE['PENDING'] + end + + # Ruby 1.8 compatibility + if result.instance_of?(Array) + result = result.to_h + end + + result + end + + # Returns all node Roles ready to be released + # @param [Service] service + # @return [Hash] Roles + def roles_release + result = roles.select do |_name, role| + role.state == Role::STATE['HOLD'] + end + + # Ruby 1.8 compatibility + if result.instance_of?(Array) + result = result.to_h + end + + result + end + end diff --git a/src/flow/lib/strategy/straight.rb b/src/flow/lib/strategy/straight.rb index b10556ea72..77b8e62e5b 100644 --- a/src/flow/lib/strategy/straight.rb +++ b/src/flow/lib/strategy/straight.rb @@ -140,4 +140,78 @@ module Straight result end + # Returns all node Roles ready to be set on hold + # @return [Hash] Roles + def roles_hold + hold_roles = roles.select do |_name, role| + role.state == Role::STATE['HOLD'] + end + + # Ruby 1.8 compatibility + if hold_roles.instance_of?(Array) + hold_roles = hold_roles.to_h + end + + result = roles.select do |_name, role| + check = true + + if role.state == Role::STATE['PENDING'] + role.parents.each do |parent| + if !hold_roles.include?(parent) + check = false + break + end + end + else + check = false + end + + check + end + + # Ruby 1.8 compatibility + if result.instance_of?(Array) + result = result.to_h + end + + result + end + + # Returns all node Roles ready to be released + # @return [Hash] Roles + def roles_release + running_roles = roles.select do |_name, role| + role.state == Role::STATE['RUNNING'] + end + + # Ruby 1.8 compatibility + if running_roles.instance_of?(Array) + running_roles = running_roles.to_h + end + + result = roles.select do |_name, role| + check = true + + if role.state == Role::STATE['HOLD'] + role.parents.each do |parent| + if !running_roles.include?(parent) + check = false + break + end + end + else + check = false + end + + check + end + + # Ruby 1.8 compatibility + if result.instance_of?(Array) + result = result.to_h + end + + result + end + end diff --git a/src/flow/oneflow-server.rb b/src/flow/oneflow-server.rb index 2e152705d2..f0e64602c5 100644 --- a/src/flow/oneflow-server.rb +++ b/src/flow/oneflow-server.rb @@ -294,6 +294,9 @@ post '/service/:id/action' do rc = OpenNebula::Error.new("Action #{action['perform']}: " \ 'You have to specify a name') end + when 'release' + rc = lcm.release_action(@client, params[:id]) + when *Role::SCHEDULE_ACTIONS # Use defaults only if one of the options is supplied opts['period'] ||= conf[:action_period] diff --git a/src/oca/ruby/opennebula/flow/service_template.rb b/src/oca/ruby/opennebula/flow/service_template.rb index 9adfa726e0..a116f52ff7 100644 --- a/src/oca/ruby/opennebula/flow/service_template.rb +++ b/src/oca/ruby/opennebula/flow/service_template.rb @@ -73,6 +73,10 @@ module OpenNebula :required => false, :minimum => 0 }, + 'on_hold' => { + :type => :boolean, + :required => false + }, 'elasticity_policies' => { :type => :array, :items => { @@ -220,6 +224,10 @@ module OpenNebula :properties => {} }, :required => false + }, + 'on_hold' => { + :type => :boolean, + :required => false } } } diff --git a/src/oca/ruby/opennebula/oneflow_client.rb b/src/oca/ruby/opennebula/oneflow_client.rb index c7feb369ef..7a359c400f 100644 --- a/src/oca/ruby/opennebula/oneflow_client.rb +++ b/src/oca/ruby/opennebula/oneflow_client.rb @@ -59,7 +59,8 @@ module Role 'DEPLOYING_NETS' => 11, 'UNDEPLOYING_NETS' => 12, 'FAILED_DEPLOYING_NETS' => 13, - 'FAILED_UNDEPLOYING_NETS' => 14 + 'FAILED_UNDEPLOYING_NETS' => 14, + 'HOLD' => 15 } STATE_STR = [ @@ -77,7 +78,8 @@ module Role 'DEPLOYING_NETS', 'UNDEPLOYING_NETS', 'FAILED_DEPLOYING_NETS', - 'FAILED_UNDEPLOYING_NETS' + 'FAILED_UNDEPLOYING_NETS', + 'HOLD' ] # Returns the string representation of the role state @@ -105,7 +107,8 @@ module Service 'DEPLOYING_NETS' => 11, 'UNDEPLOYING_NETS' => 12, 'FAILED_DEPLOYING_NETS' => 13, - 'FAILED_UNDEPLOYING_NETS' => 14 + 'FAILED_UNDEPLOYING_NETS' => 14, + 'HOLD' => 15 } STATE_STR = [ @@ -123,7 +126,8 @@ module Service 'DEPLOYING_NETS', 'UNDEPLOYING_NETS', 'FAILED_DEPLOYING_NETS', - 'FAILED_UNDEPLOYING_NETS' + 'FAILED_UNDEPLOYING_NETS', + 'HOLD' ] # Returns the string representation of the service state