From cb68721926d42f771ac48ad65a17ce476c033e58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Wed, 12 Jun 2013 15:41:19 +0200 Subject: [PATCH 1/7] Feature #2118: New state COOLDOWN after SCALING --- flow/client/lib/appflow_client.rb | 12 ++++++---- flow/lcm/LifeCycleManager.rb | 11 ++++++--- flow/lcm/strategy.rb | 13 +++++++++-- flow/models/Role.rb | 39 ++++++++++++++++++++++++++----- flow/models/Service.rb | 16 +++++++++++-- 5 files changed, 74 insertions(+), 17 deletions(-) diff --git a/flow/client/lib/appflow_client.rb b/flow/client/lib/appflow_client.rb index af32cdd268..6e5a0b2ddc 100644 --- a/flow/client/lib/appflow_client.rb +++ b/flow/client/lib/appflow_client.rb @@ -49,7 +49,8 @@ module Role 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -63,7 +64,8 @@ module Role 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] # Returns the string representation of the role state @@ -87,7 +89,8 @@ module Service 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -101,7 +104,8 @@ module Service 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] # Returns the string representation of the service state diff --git a/flow/lcm/LifeCycleManager.rb b/flow/lcm/LifeCycleManager.rb index 0ed01695a0..dc14361949 100644 --- a/flow/lcm/LifeCycleManager.rb +++ b/flow/lcm/LifeCycleManager.rb @@ -93,11 +93,10 @@ class ServiceLCM when Service::STATE['SCALING'] strategy.monitor_step(service) - # TODO: Check scalability rules for roles that are - # not scaling now? - if service.any_role_failed_scaling? service.set_state(Service::STATE['FAILED_SCALING']) + elsif service.any_role_cooldown? + service.set_state(Service::STATE['COOLDOWN']) elsif !service.any_role_scaling? service.set_state(Service::STATE['RUNNING']) else @@ -106,6 +105,12 @@ class ServiceLCM service.set_state(Service::STATE['FAILED_SCALING']) end end + when Service::STATE['COOLDOWN'] + strategy.monitor_step(service) + + if !service.any_role_cooldown? + service.set_state(Service::STATE['RUNNING']) + end when Service::STATE['FAILED_SCALING'] strategy.monitor_step(service) diff --git a/flow/lcm/strategy.rb b/flow/lcm/strategy.rb index 3576dad7e3..498c124dca 100644 --- a/flow/lcm/strategy.rb +++ b/flow/lcm/strategy.rb @@ -82,7 +82,7 @@ class Strategy Log.debug LOG_COMP, "Apply scaling policies", service.id() service.get_roles.each do |name, role| - diff = role.scale? + diff, cooldown_duration = role.scale? if diff != 0 Log.debug LOG_COMP, "Role #{name} needs to scale #{diff} nodes", service.id() @@ -90,6 +90,7 @@ class Strategy role.set_cardinality(role.cardinality() + diff) role.set_state(Role::STATE['SCALING']) + role.set_cooldown_duration(cooldown_duration) return true end @@ -153,10 +154,18 @@ class Strategy if OpenNebula.is_error?(rc) role.set_state(Role::STATE['FAILED_SCALING']) elsif role_finished_scaling?(role) - role.set_state(Role::STATE['RUNNING']) + if apply_cooldown = role.apply_cooldown_duration() + role.set_state(Role::STATE['COOLDOWN']) + else + role.set_state(Role::STATE['RUNNING']) + end elsif any_node_failed_scaling?(role) role.set_state(Role::STATE['FAILED_SCALING']) end + when Role::STATE['COOLDOWN'] + if role.cooldown_over? + role.set_state(Role::STATE['RUNNING']) + end when Role::STATE['UNKNOWN'] if role_nodes_running?(role) role.set_state(Role::STATE['RUNNING']) diff --git a/flow/models/Role.rb b/flow/models/Role.rb index 0a9407cc2b..c077027e18 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -50,7 +50,8 @@ module OpenNebula 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -64,7 +65,8 @@ module OpenNebula 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] LOG_COMP = "ROL" @@ -442,7 +444,8 @@ module OpenNebula # Returns a positive, 0, or negative number of nodes to adjust, # according to the elasticity and scheduled policies - # @return [Integer] positive, 0, or negative number of nodes to adjust + # @return [Array] positive, 0, or negative number of nodes to + # adjust, plus the cooldown period duration def scale?() elasticity_pol = @body['elasticity_policies'] scheduled_pol = @body['scheduled_policies'] @@ -450,17 +453,20 @@ module OpenNebula elasticity_pol ||= [] scheduled_pol ||= [] + # TODO: Fixed duration + cooldown_duration = 60 + scheduled_pol.each do |policy| diff = scale_time?(policy) - return diff if diff != 0 + return [diff, 0] if diff != 0 end elasticity_pol.each do |policy| diff = scale_attributes?(policy) - return diff if diff != 0 + return [diff, cooldown_duration] if diff != 0 end - return 0 + return [0, 0] end # Returns a positive, 0, or negative number of nodes to adjust, @@ -715,6 +721,27 @@ module OpenNebula set_cardinality( get_nodes.size() - n_dispose ) end + # Updates the duration for the next cooldown + # @param cooldown_duration [Integer] duration for the next cooldown + def set_cooldown_duration(cooldown_duration) + @body['cooldown_duration'] = cooldown_duration.to_i + end + + # Sets the cooldown end time from now + the duration set in set_cooldown_duration + # @return [true, false] true if the cooldown duration is bigger than 0 + def apply_cooldown_duration() + cooldown_duration = @body['cooldown_duration'].to_i + + @body['cooldown_end'] = Time.now + cooldown_duration + @body.delete('cooldown_duration') + end + + # Returns true if the cooldown period ended + # @return [true, false] true if the cooldown period ended + def cooldown_over?() + return Time.now.to_i >= @body['cooldown_end'].to_i + end + def update(template) set_cardinality(template["cardinality"]) end diff --git a/flow/models/Service.rb b/flow/models/Service.rb index 8de2047a60..081f6ce29f 100644 --- a/flow/models/Service.rb +++ b/flow/models/Service.rb @@ -30,7 +30,8 @@ module OpenNebula 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -44,7 +45,8 @@ module OpenNebula 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] LOG_COMP = "SER" @@ -163,6 +165,16 @@ module OpenNebula return false end + def any_role_cooldown?() + @roles.each do |name, role| + if role.state == Role::STATE['COOLDOWN'] + return true + end + end + + return false + end + # Create a new service based on the template provided # @param [String] template_json # @return [nil, OpenNebula::Error] nil in case of success, Error From 8b9bf82ad432f071bbee8ab307e2ae4e93b98246 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Wed, 12 Jun 2013 16:25:12 +0200 Subject: [PATCH 2/7] Feature #2118: Add cooldown attribute to elasticity_policies and Role. --- flow/lcm/strategy.rb | 2 +- flow/models/Role.rb | 29 +++++++++++++++++++---------- flow/models/ServiceTemplate.rb | 19 ++++++++----------- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/flow/lcm/strategy.rb b/flow/lcm/strategy.rb index 498c124dca..36e346e46a 100644 --- a/flow/lcm/strategy.rb +++ b/flow/lcm/strategy.rb @@ -154,7 +154,7 @@ class Strategy if OpenNebula.is_error?(rc) role.set_state(Role::STATE['FAILED_SCALING']) elsif role_finished_scaling?(role) - if apply_cooldown = role.apply_cooldown_duration() + if role.apply_cooldown_duration() role.set_state(Role::STATE['COOLDOWN']) else role.set_state(Role::STATE['RUNNING']) diff --git a/flow/models/Role.rb b/flow/models/Role.rb index c077027e18..91d8bd8daf 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -453,17 +453,19 @@ module OpenNebula elasticity_pol ||= [] scheduled_pol ||= [] - # TODO: Fixed duration - cooldown_duration = 60 - scheduled_pol.each do |policy| diff = scale_time?(policy) return [diff, 0] if diff != 0 end elasticity_pol.each do |policy| - diff = scale_attributes?(policy) - return [diff, cooldown_duration] if diff != 0 + diff, cooldown_duration = scale_attributes?(policy) + if diff != 0 + cooldown_duration = @body['cooldown'] if cooldown_duration.nil? + cooldown_duration = 0 if cooldown_duration.nil? + + return [diff, cooldown_duration] + end end return [0, 0] @@ -535,7 +537,8 @@ module OpenNebula # Returns a positive, 0, or negative number of nodes to adjust, # according to a policy based on attributes # @param [Hash] A policy based on attributes - # @return [Integer] positive, 0, or negative number of nodes to adjust + # @return [Array] positive, 0, or negative number of nodes to + # adjust, plus the cooldown period duration def scale_attributes?(elasticity_pol) now = Time.now.to_i @@ -552,7 +555,7 @@ module OpenNebula if now < (last_eval + period_duration) Log.debug "ELAS", "Role #{name} expression '#{expression}' evaluation ignored, time < period" - return 0 + return [0, 0] end end @@ -576,7 +579,7 @@ module OpenNebula elasticity_pol['true_evals'] = new_evals - return new_cardinality - cardinality() + return [new_cardinality - cardinality(), elasticity_pol['cooldown']] end # Returns true if the scalability rule is triggered @@ -732,8 +735,14 @@ module OpenNebula def apply_cooldown_duration() cooldown_duration = @body['cooldown_duration'].to_i - @body['cooldown_end'] = Time.now + cooldown_duration - @body.delete('cooldown_duration') + if cooldown_duration != 0 + @body['cooldown_end'] = Time.now.to_i + cooldown_duration + @body.delete('cooldown_duration') + + return true + end + + return false end # Returns true if the cooldown period ended diff --git a/flow/models/ServiceTemplate.rb b/flow/models/ServiceTemplate.rb index 285aa63276..f7d5eb3e81 100644 --- a/flow/models/ServiceTemplate.rb +++ b/flow/models/ServiceTemplate.rb @@ -42,13 +42,14 @@ module OpenNebula :type => :integer, :default => 1 }, - #'cooldown' => { - # :type => :integer - #} 'max_vms' => { :type => :integer, :required => false }, + 'cooldown' => { + :type => :integer, + :required => false + }, 'elasticity_policies' => { :type => :array, :items => { @@ -79,11 +80,11 @@ module OpenNebula 'expression' => { :type => :string, :required => false + }, + 'cooldown' => { + :type => :integer, + :required => false } - - #'cooldown' => { - # :type => :integer - #} #'statistic' => { # # SampleCount | Average | Sum | Minimum | Maximum # :type => :string @@ -117,10 +118,6 @@ module OpenNebula :type => :string, :required => false } - - #'cooldown' => { - # :type => :integer - #} } } } From dafe3cd58af9e9eaa4e18d8404bfc39658e39fec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Wed, 12 Jun 2013 16:34:15 +0200 Subject: [PATCH 3/7] Feature #2118: Show cooldown attributes in cli output --- flow/client/bin/appflow | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/flow/client/bin/appflow b/flow/client/bin/appflow index c626f608ca..820ecb5c23 100755 --- a/flow/client/bin/appflow +++ b/flow/client/bin/appflow @@ -223,6 +223,7 @@ def show_service(client, args, options) puts "ELASTICITY RULES" puts str % ["MIN VMS", role['min_vms']] puts str % ["MAX VMS", role['max_vms']] + puts str % ["COOLDOWN", "#{role['cooldown']}s"] if role['cooldown'] if role['elasticity_policies'] puts @@ -232,7 +233,7 @@ def show_service(client, args, options) adjust_str(d) end - column :EXPRESSION, "", :left, :size=>49 do |d| + column :EXPRESSION, "", :left, :size=>44 do |d| d['expression'] end @@ -244,7 +245,11 @@ def show_service(client, args, options) "#{d['period']}s" end - default :ADJUST, :EXPRESSION, :EVALUATION, :PERIOD + column :COOLDOWN, "", :size=>4 do |d| + d['cooldown'] ? "#{d['cooldown']}s" : '-' + end + + default :ADJUST, :EXPRESSION, :EVALUATION, :PERIOD, :COOLDOWN end.show([role['elasticity_policies']].flatten, {}) end From bf0245d17c2150dab4ef7dd30c8a7b77204cd8e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Wed, 12 Jun 2013 17:54:18 +0200 Subject: [PATCH 4/7] Feature #2118: Add cooldown time after a set-cardinality action --- flow/models/Role.rb | 5 +++++ flow/models/Service.rb | 2 ++ 2 files changed, 7 insertions(+) diff --git a/flow/models/Role.rb b/flow/models/Role.rb index 91d8bd8daf..a665c71f20 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -730,6 +730,11 @@ module OpenNebula @body['cooldown_duration'] = cooldown_duration.to_i end + # Updates the duration for the next cooldown with the default value + def set_default_cooldown_duration() + set_cooldown_duration(@body['cooldown'].to_i) + end + # Sets the cooldown end time from now + the duration set in set_cooldown_duration # @return [true, false] true if the cooldown duration is bigger than 0 def apply_cooldown_duration() diff --git a/flow/models/Service.rb b/flow/models/Service.rb index 081f6ce29f..1d0d56123b 100644 --- a/flow/models/Service.rb +++ b/flow/models/Service.rb @@ -345,6 +345,8 @@ module OpenNebula role.set_state(Role::STATE['SCALING']) + role.set_default_cooldown_duration() + self.set_state(Service::STATE['SCALING']) return self.update end From 18ddad5cc8e603407b2dcd5c9fe213fe7e9d6c46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Thu, 13 Jun 2013 12:22:55 +0200 Subject: [PATCH 5/7] Feature #2118: Default cooldown can be set in appflow-server.conf --- flow/config/appflow-server.conf | 3 +++ flow/controllers/appflow-server.rb | 5 +++++ flow/models/Role.rb | 11 +++++++++-- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/flow/config/appflow-server.conf b/flow/config/appflow-server.conf index 2f04058fc8..7f9f0ab5cf 100644 --- a/flow/config/appflow-server.conf +++ b/flow/config/appflow-server.conf @@ -30,6 +30,9 @@ :host: 127.0.0.1 :port: 2474 +# Default cooldown period after a scale operation, in seconds +:default_cooldown: 300 + ############################################################# # Auth ############################################################# diff --git a/flow/controllers/appflow-server.rb b/flow/controllers/appflow-server.rb index bdbf68ba13..02aa4d9a5a 100644 --- a/flow/controllers/appflow-server.rb +++ b/flow/controllers/appflow-server.rb @@ -60,6 +60,7 @@ end conf[:debug_level] ||= 2 conf[:lcm_interval] ||= 30 +conf[:default_cooldown] ||= 300 conf[:auth] = 'opennebula' @@ -110,7 +111,11 @@ before do end end +############################################################################## +# Cooldown config +############################################################################## +Role.init_default_cooldown(conf[:default_cooldown]) ############################################################################## # LCM thread diff --git a/flow/models/Role.rb b/flow/models/Role.rb index a665c71f20..5bd721309d 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -462,7 +462,7 @@ module OpenNebula diff, cooldown_duration = scale_attributes?(policy) if diff != 0 cooldown_duration = @body['cooldown'] if cooldown_duration.nil? - cooldown_duration = 0 if cooldown_duration.nil? + cooldown_duration = @@default_cooldown if cooldown_duration.nil? return [diff, cooldown_duration] end @@ -732,7 +732,10 @@ module OpenNebula # Updates the duration for the next cooldown with the default value def set_default_cooldown_duration() - set_cooldown_duration(@body['cooldown'].to_i) + cooldown_duration = @body['cooldown'] + cooldown_duration = @@default_cooldown if cooldown_duration.nil? + + set_cooldown_duration(cooldown_duration) end # Sets the cooldown end time from now + the duration set in set_cooldown_duration @@ -756,6 +759,10 @@ module OpenNebula return Time.now.to_i >= @body['cooldown_end'].to_i end + def self.init_default_cooldown(default_cooldown) + @@default_cooldown = default_cooldown + end + def update(template) set_cardinality(template["cardinality"]) end From 23e053499e16d14d57c2a014495cba50b58e6403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Thu, 13 Jun 2013 12:32:17 +0200 Subject: [PATCH 6/7] Feature #2118: Cooldown period can be finished manually with recover action --- flow/models/Service.rb | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/flow/models/Service.rb b/flow/models/Service.rb index 1d0d56123b..a267a5e0a0 100644 --- a/flow/models/Service.rb +++ b/flow/models/Service.rb @@ -225,6 +225,15 @@ module OpenNebula elsif self.state == Service::STATE['FAILED_UNDEPLOYING'] self.set_state(Service::STATE['UNDEPLOYING']) + elsif self.state == Service::STATE['COOLDOWN'] + @roles.each do |name, role| + if role.state == Role::STATE['COOLDOWN'] + role.set_state(Role::STATE['RUNNING']) + end + end + + self.set_state(Service::STATE['RUNNING']) + else return OpenNebula::Error.new("Action recover: Wrong state" \ " #{self.state_str()}") From 2649c3e843ecf400b8afc8d0605022cfd45ab964 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn?= Date: Thu, 13 Jun 2013 12:33:36 +0200 Subject: [PATCH 7/7] Feature #2118: New states in sunstone --- flow/sunstone/public/js/apptools-appflow-services.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/flow/sunstone/public/js/apptools-appflow-services.js b/flow/sunstone/public/js/apptools-appflow-services.js index f37b3f80f4..c2e96815ed 100644 --- a/flow/sunstone/public/js/apptools-appflow-services.js +++ b/flow/sunstone/public/js/apptools-appflow-services.js @@ -64,7 +64,10 @@ var Service = { tr("UNKNOWN"), tr("DONE"), tr("FAILED_UNDEPLOYING"), - tr("FAILED_DEPLOYING") + tr("FAILED_DEPLOYING"), + tr("SCALING"), + tr("FAILED_SCALING"), + tr("COOLDOWN") ][state_int] return state ? state : state_int; } @@ -84,7 +87,10 @@ var Role = { tr("UNKNOWN"), tr("DONE"), tr("FAILED_UNDEPLOYING"), - tr("FAILED_DEPLOYING") + tr("FAILED_DEPLOYING"), + tr("SCALING"), + tr("FAILED_SCALING"), + tr("COOLDOWN") ][state_int] return state ? state : state_int; },