diff --git a/flow/client/bin/appflow b/flow/client/bin/appflow index c626f608ca..820ecb5c23 100755 --- a/flow/client/bin/appflow +++ b/flow/client/bin/appflow @@ -223,6 +223,7 @@ def show_service(client, args, options) puts "ELASTICITY RULES" puts str % ["MIN VMS", role['min_vms']] puts str % ["MAX VMS", role['max_vms']] + puts str % ["COOLDOWN", "#{role['cooldown']}s"] if role['cooldown'] if role['elasticity_policies'] puts @@ -232,7 +233,7 @@ def show_service(client, args, options) adjust_str(d) end - column :EXPRESSION, "", :left, :size=>49 do |d| + column :EXPRESSION, "", :left, :size=>44 do |d| d['expression'] end @@ -244,7 +245,11 @@ def show_service(client, args, options) "#{d['period']}s" end - default :ADJUST, :EXPRESSION, :EVALUATION, :PERIOD + column :COOLDOWN, "", :size=>4 do |d| + d['cooldown'] ? "#{d['cooldown']}s" : '-' + end + + default :ADJUST, :EXPRESSION, :EVALUATION, :PERIOD, :COOLDOWN end.show([role['elasticity_policies']].flatten, {}) end diff --git a/flow/client/lib/appflow_client.rb b/flow/client/lib/appflow_client.rb index af32cdd268..6e5a0b2ddc 100644 --- a/flow/client/lib/appflow_client.rb +++ b/flow/client/lib/appflow_client.rb @@ -49,7 +49,8 @@ module Role 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -63,7 +64,8 @@ module Role 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] # Returns the string representation of the role state @@ -87,7 +89,8 @@ module Service 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -101,7 +104,8 @@ module Service 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] # Returns the string representation of the service state diff --git a/flow/config/appflow-server.conf b/flow/config/appflow-server.conf index 2f04058fc8..7f9f0ab5cf 100644 --- a/flow/config/appflow-server.conf +++ b/flow/config/appflow-server.conf @@ -30,6 +30,9 @@ :host: 127.0.0.1 :port: 2474 +# Default cooldown period after a scale operation, in seconds +:default_cooldown: 300 + ############################################################# # Auth ############################################################# diff --git a/flow/controllers/appflow-server.rb b/flow/controllers/appflow-server.rb index bdbf68ba13..02aa4d9a5a 100644 --- a/flow/controllers/appflow-server.rb +++ b/flow/controllers/appflow-server.rb @@ -60,6 +60,7 @@ end conf[:debug_level] ||= 2 conf[:lcm_interval] ||= 30 +conf[:default_cooldown] ||= 300 conf[:auth] = 'opennebula' @@ -110,7 +111,11 @@ before do end end +############################################################################## +# Cooldown config +############################################################################## +Role.init_default_cooldown(conf[:default_cooldown]) ############################################################################## # LCM thread diff --git a/flow/lcm/LifeCycleManager.rb b/flow/lcm/LifeCycleManager.rb index 0ed01695a0..dc14361949 100644 --- a/flow/lcm/LifeCycleManager.rb +++ b/flow/lcm/LifeCycleManager.rb @@ -93,11 +93,10 @@ class ServiceLCM when Service::STATE['SCALING'] strategy.monitor_step(service) - # TODO: Check scalability rules for roles that are - # not scaling now? - if service.any_role_failed_scaling? service.set_state(Service::STATE['FAILED_SCALING']) + elsif service.any_role_cooldown? + service.set_state(Service::STATE['COOLDOWN']) elsif !service.any_role_scaling? service.set_state(Service::STATE['RUNNING']) else @@ -106,6 +105,12 @@ class ServiceLCM service.set_state(Service::STATE['FAILED_SCALING']) end end + when Service::STATE['COOLDOWN'] + strategy.monitor_step(service) + + if !service.any_role_cooldown? + service.set_state(Service::STATE['RUNNING']) + end when Service::STATE['FAILED_SCALING'] strategy.monitor_step(service) diff --git a/flow/lcm/strategy.rb b/flow/lcm/strategy.rb index 3576dad7e3..36e346e46a 100644 --- a/flow/lcm/strategy.rb +++ b/flow/lcm/strategy.rb @@ -82,7 +82,7 @@ class Strategy Log.debug LOG_COMP, "Apply scaling policies", service.id() service.get_roles.each do |name, role| - diff = role.scale? + diff, cooldown_duration = role.scale? if diff != 0 Log.debug LOG_COMP, "Role #{name} needs to scale #{diff} nodes", service.id() @@ -90,6 +90,7 @@ class Strategy role.set_cardinality(role.cardinality() + diff) role.set_state(Role::STATE['SCALING']) + role.set_cooldown_duration(cooldown_duration) return true end @@ -153,10 +154,18 @@ class Strategy if OpenNebula.is_error?(rc) role.set_state(Role::STATE['FAILED_SCALING']) elsif role_finished_scaling?(role) - role.set_state(Role::STATE['RUNNING']) + if role.apply_cooldown_duration() + role.set_state(Role::STATE['COOLDOWN']) + else + role.set_state(Role::STATE['RUNNING']) + end elsif any_node_failed_scaling?(role) role.set_state(Role::STATE['FAILED_SCALING']) end + when Role::STATE['COOLDOWN'] + if role.cooldown_over? + role.set_state(Role::STATE['RUNNING']) + end when Role::STATE['UNKNOWN'] if role_nodes_running?(role) role.set_state(Role::STATE['RUNNING']) diff --git a/flow/models/Role.rb b/flow/models/Role.rb index 0a9407cc2b..5bd721309d 100644 --- a/flow/models/Role.rb +++ b/flow/models/Role.rb @@ -50,7 +50,8 @@ module OpenNebula 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -64,7 +65,8 @@ module OpenNebula 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] LOG_COMP = "ROL" @@ -442,7 +444,8 @@ module OpenNebula # Returns a positive, 0, or negative number of nodes to adjust, # according to the elasticity and scheduled policies - # @return [Integer] positive, 0, or negative number of nodes to adjust + # @return [Array] positive, 0, or negative number of nodes to + # adjust, plus the cooldown period duration def scale?() elasticity_pol = @body['elasticity_policies'] scheduled_pol = @body['scheduled_policies'] @@ -452,15 +455,20 @@ module OpenNebula scheduled_pol.each do |policy| diff = scale_time?(policy) - return diff if diff != 0 + return [diff, 0] if diff != 0 end elasticity_pol.each do |policy| - diff = scale_attributes?(policy) - return diff if diff != 0 + diff, cooldown_duration = scale_attributes?(policy) + if diff != 0 + cooldown_duration = @body['cooldown'] if cooldown_duration.nil? + cooldown_duration = @@default_cooldown if cooldown_duration.nil? + + return [diff, cooldown_duration] + end end - return 0 + return [0, 0] end # Returns a positive, 0, or negative number of nodes to adjust, @@ -529,7 +537,8 @@ module OpenNebula # Returns a positive, 0, or negative number of nodes to adjust, # according to a policy based on attributes # @param [Hash] A policy based on attributes - # @return [Integer] positive, 0, or negative number of nodes to adjust + # @return [Array] positive, 0, or negative number of nodes to + # adjust, plus the cooldown period duration def scale_attributes?(elasticity_pol) now = Time.now.to_i @@ -546,7 +555,7 @@ module OpenNebula if now < (last_eval + period_duration) Log.debug "ELAS", "Role #{name} expression '#{expression}' evaluation ignored, time < period" - return 0 + return [0, 0] end end @@ -570,7 +579,7 @@ module OpenNebula elasticity_pol['true_evals'] = new_evals - return new_cardinality - cardinality() + return [new_cardinality - cardinality(), elasticity_pol['cooldown']] end # Returns true if the scalability rule is triggered @@ -715,6 +724,45 @@ module OpenNebula set_cardinality( get_nodes.size() - n_dispose ) end + # Updates the duration for the next cooldown + # @param cooldown_duration [Integer] duration for the next cooldown + def set_cooldown_duration(cooldown_duration) + @body['cooldown_duration'] = cooldown_duration.to_i + end + + # Updates the duration for the next cooldown with the default value + def set_default_cooldown_duration() + cooldown_duration = @body['cooldown'] + cooldown_duration = @@default_cooldown if cooldown_duration.nil? + + set_cooldown_duration(cooldown_duration) + end + + # Sets the cooldown end time from now + the duration set in set_cooldown_duration + # @return [true, false] true if the cooldown duration is bigger than 0 + def apply_cooldown_duration() + cooldown_duration = @body['cooldown_duration'].to_i + + if cooldown_duration != 0 + @body['cooldown_end'] = Time.now.to_i + cooldown_duration + @body.delete('cooldown_duration') + + return true + end + + return false + end + + # Returns true if the cooldown period ended + # @return [true, false] true if the cooldown period ended + def cooldown_over?() + return Time.now.to_i >= @body['cooldown_end'].to_i + end + + def self.init_default_cooldown(default_cooldown) + @@default_cooldown = default_cooldown + end + def update(template) set_cardinality(template["cardinality"]) end diff --git a/flow/models/Service.rb b/flow/models/Service.rb index 8de2047a60..a267a5e0a0 100644 --- a/flow/models/Service.rb +++ b/flow/models/Service.rb @@ -30,7 +30,8 @@ module OpenNebula 'FAILED_UNDEPLOYING' => 7, 'FAILED_DEPLOYING' => 8, 'SCALING' => 9, - 'FAILED_SCALING' => 10 + 'FAILED_SCALING' => 10, + 'COOLDOWN' => 11 } STATE_STR = [ @@ -44,7 +45,8 @@ module OpenNebula 'FAILED_UNDEPLOYING', 'FAILED_DEPLOYING', 'SCALING', - 'FAILED_SCALING' + 'FAILED_SCALING', + 'COOLDOWN' ] LOG_COMP = "SER" @@ -163,6 +165,16 @@ module OpenNebula return false end + def any_role_cooldown?() + @roles.each do |name, role| + if role.state == Role::STATE['COOLDOWN'] + return true + end + end + + return false + end + # Create a new service based on the template provided # @param [String] template_json # @return [nil, OpenNebula::Error] nil in case of success, Error @@ -213,6 +225,15 @@ module OpenNebula elsif self.state == Service::STATE['FAILED_UNDEPLOYING'] self.set_state(Service::STATE['UNDEPLOYING']) + elsif self.state == Service::STATE['COOLDOWN'] + @roles.each do |name, role| + if role.state == Role::STATE['COOLDOWN'] + role.set_state(Role::STATE['RUNNING']) + end + end + + self.set_state(Service::STATE['RUNNING']) + else return OpenNebula::Error.new("Action recover: Wrong state" \ " #{self.state_str()}") @@ -333,6 +354,8 @@ module OpenNebula role.set_state(Role::STATE['SCALING']) + role.set_default_cooldown_duration() + self.set_state(Service::STATE['SCALING']) return self.update end diff --git a/flow/models/ServiceTemplate.rb b/flow/models/ServiceTemplate.rb index 285aa63276..f7d5eb3e81 100644 --- a/flow/models/ServiceTemplate.rb +++ b/flow/models/ServiceTemplate.rb @@ -42,13 +42,14 @@ module OpenNebula :type => :integer, :default => 1 }, - #'cooldown' => { - # :type => :integer - #} 'max_vms' => { :type => :integer, :required => false }, + 'cooldown' => { + :type => :integer, + :required => false + }, 'elasticity_policies' => { :type => :array, :items => { @@ -79,11 +80,11 @@ module OpenNebula 'expression' => { :type => :string, :required => false + }, + 'cooldown' => { + :type => :integer, + :required => false } - - #'cooldown' => { - # :type => :integer - #} #'statistic' => { # # SampleCount | Average | Sum | Minimum | Maximum # :type => :string @@ -117,10 +118,6 @@ module OpenNebula :type => :string, :required => false } - - #'cooldown' => { - # :type => :integer - #} } } } diff --git a/flow/sunstone/public/js/apptools-appflow-services.js b/flow/sunstone/public/js/apptools-appflow-services.js index f37b3f80f4..c2e96815ed 100644 --- a/flow/sunstone/public/js/apptools-appflow-services.js +++ b/flow/sunstone/public/js/apptools-appflow-services.js @@ -64,7 +64,10 @@ var Service = { tr("UNKNOWN"), tr("DONE"), tr("FAILED_UNDEPLOYING"), - tr("FAILED_DEPLOYING") + tr("FAILED_DEPLOYING"), + tr("SCALING"), + tr("FAILED_SCALING"), + tr("COOLDOWN") ][state_int] return state ? state : state_int; } @@ -84,7 +87,10 @@ var Role = { tr("UNKNOWN"), tr("DONE"), tr("FAILED_UNDEPLOYING"), - tr("FAILED_DEPLOYING") + tr("FAILED_DEPLOYING"), + tr("SCALING"), + tr("FAILED_SCALING"), + tr("COOLDOWN") ][state_int] return state ? state : state_int; },