mirror of
https://github.com/OpenNebula/one.git
synced 2026-11-22 20:54:49 +09:00
264 lines
8.3 KiB
Ruby
264 lines
8.3 KiB
Ruby
# rubocop:disable Naming/FileName
|
|
# -------------------------------------------------------------------------- #
|
|
# Copyright 2002-2020, OpenNebula Project, OpenNebula Systems #
|
|
# #
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
|
# not use this file except in compliance with the License. You may obtain #
|
|
# a copy of the License at #
|
|
# #
|
|
# http://www.apache.org/licenses/LICENSE-2.0 #
|
|
# #
|
|
# Unless required by applicable law or agreed to in writing, software #
|
|
# distributed under the License is distributed on an "AS IS" BASIS, #
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
|
# See the License for the specific language governing permissions and #
|
|
# limitations under the License. #
|
|
#--------------------------------------------------------------------------- #
|
|
|
|
require 'base64'
|
|
require 'ffi-rzmq'
|
|
require 'nokogiri'
|
|
require 'EventManager'
|
|
|
|
# Service watchdog class
|
|
class ServiceWD
|
|
|
|
############################################################################
|
|
# Default configuration options for the module
|
|
############################################################################
|
|
LOG_COMP = 'WD'
|
|
|
|
DEFAULT_CONF = {
|
|
:subscriber_endpoint => 'tcp://localhost:2101',
|
|
:concurrency => 10,
|
|
:cloud_auth => nil
|
|
}
|
|
|
|
############################################################################
|
|
# WARNING STATES
|
|
############################################################################
|
|
WARNING_STATES = %w[
|
|
POWEROFF
|
|
UNKNOWN
|
|
] + EventManager::FAILURE_STATES
|
|
|
|
# Class constructor
|
|
#
|
|
# @param options [Hash] event manager options
|
|
def initialize(client, options)
|
|
@conf = DEFAULT_CONF.merge(options)
|
|
|
|
@lcm = options[:lcm]
|
|
@context = ZMQ::Context.new(1)
|
|
@cloud_auth = @conf[:cloud_auth]
|
|
@wait_timeout = @cloud_auth.conf[:wait_timeout]
|
|
@client = client
|
|
|
|
# Array of running services to watch
|
|
@mutex = Mutex.new
|
|
@services = []
|
|
end
|
|
|
|
# Start services WD
|
|
#
|
|
# @param service_pool [ServicePool] All services to check
|
|
def start(service_pool)
|
|
Log.info LOG_COMP, 'Start watch dog'
|
|
|
|
service_pool.info_all
|
|
|
|
# check that all nodes are in RUNNING state, if not, notify
|
|
check_roles_state(client, service_pool)
|
|
|
|
# subscribe to all nodes
|
|
subscriber = gen_subscriber
|
|
|
|
subscribe(subscriber)
|
|
|
|
key = ''
|
|
content = ''
|
|
|
|
# wait for any STATE change
|
|
loop do
|
|
rc = subscriber.recv_string(key)
|
|
rc = subscriber.recv_string(content) if rc != -1
|
|
|
|
if rc == -1 && ZMQ::Util.errno != ZMQ::EAGAIN
|
|
Log.error LOG_COMP, 'Error reading from subscriber.'
|
|
end
|
|
|
|
# key format: EVENT SERVICE SERVICE_ID
|
|
next if key.nil?
|
|
|
|
split_key = key.split
|
|
|
|
# if there is no data skip
|
|
next if content.nil?
|
|
|
|
xml = Nokogiri::XML(Base64.decode64(content))
|
|
|
|
service_id = split_key[2].to_i
|
|
|
|
@mutex.synchronize do
|
|
next unless @services.include?(service_id)
|
|
end
|
|
|
|
node = xml.xpath('/HOOK_MESSAGE/VM/ID').text.to_i
|
|
state = xml.xpath('/HOOK_MESSAGE/STATE').text
|
|
lcm_state = xml.xpath('/HOOK_MESSAGE/LCM_STATE').text
|
|
role_name = xml.xpath('/HOOK_MESSAGE/VM//ROLE_NAME').text
|
|
|
|
# if the VM is not from the service skip
|
|
next if role_name.nil?
|
|
|
|
states = {}
|
|
states[:state] = state
|
|
states[:lcm] = lcm_state
|
|
|
|
check_role_state(client, service_id, role_name, node, states)
|
|
end
|
|
end
|
|
|
|
# Stop service WD thread
|
|
def stop
|
|
Log.info LOG_COMP, 'Stop watch dog'
|
|
|
|
# unsubscribe from all nodes
|
|
subscriber = gen_subscriber
|
|
|
|
unsubscribe(subscriber)
|
|
end
|
|
|
|
# Add service to watch dog
|
|
#
|
|
# @param service_id [String] Service ID
|
|
def add_service(service_id)
|
|
@mutex.synchronize do
|
|
@services << service_id.to_i
|
|
end
|
|
end
|
|
|
|
# Remove service from watch dog
|
|
#
|
|
# @param service_id [String] Service ID
|
|
def remove_service(service_id)
|
|
@mutex.synchronize do
|
|
@services.delete(service_id.to_i)
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
# Get OpenNebula client
|
|
def client
|
|
# If there's a client defined use it
|
|
return @client unless @client.nil?
|
|
|
|
# If not, get one via cloud_auth
|
|
@cloud_auth.client
|
|
end
|
|
|
|
# Get ZMQ subscriber object
|
|
def gen_subscriber
|
|
subscriber = @context.socket(ZMQ::SUB)
|
|
|
|
# Set timeout (TODO add option for customize timeout)
|
|
subscriber.setsockopt(ZMQ::RCVTIMEO, @wait_timeout * 10**3)
|
|
subscriber.connect(@conf[:subscriber_endpoint])
|
|
|
|
subscriber
|
|
end
|
|
|
|
# Subscribe to VM state changes
|
|
#
|
|
# @param subscriber [ZMQ] ZMQ subscriber object
|
|
def subscribe(subscriber)
|
|
subscriber.setsockopt(ZMQ::SUBSCRIBE, 'EVENT SERVICE')
|
|
end
|
|
|
|
# Unsubscribe from VM state changes
|
|
#
|
|
# @param subscriber [ZMQ] ZMQ subscriber object
|
|
def unsubscribe(subscriber)
|
|
subscriber.setsockopt(ZMQ::UNSUBSCRIBE, 'EVENT SERVICE')
|
|
end
|
|
|
|
# Check service roles state
|
|
#
|
|
# @param client [OpenNebula::Client] Client to make API calls
|
|
# @param service_pool [ServicePool] All services to check
|
|
def check_roles_state(client, service_pool)
|
|
service_pool.each do |service|
|
|
service.info
|
|
|
|
if service.state != Service::STATE['RUNNING'] &&
|
|
service.state != Service::STATE['WARNING']
|
|
next
|
|
end
|
|
|
|
@mutex.synchronize do
|
|
@services << service.id.to_i
|
|
end
|
|
|
|
service.roles.each do |name, role|
|
|
role.nodes_ids.each do |node|
|
|
check_role_state(client, service.id, name, node)
|
|
end
|
|
end
|
|
end
|
|
end
|
|
|
|
############################################################################
|
|
# HELPERS
|
|
############################################################################
|
|
|
|
# Check role state
|
|
#
|
|
# @param client [OpenNebula::Client] Client to make API calls
|
|
# @param service_id [Integer] Service ID to check
|
|
# @param role_name [String] Role to check
|
|
# @param node [Integer] VM ID
|
|
# @param states [Hash] node state and node lcm state
|
|
def check_role_state(client, service_id, role_name, node, states = nil)
|
|
# if don't have the state, query it by creating a VM object
|
|
if states.nil?
|
|
vm = OpenNebula::VirtualMachine.new_with_id(node, client)
|
|
vm.info
|
|
|
|
vm_state = OpenNebula::VirtualMachine::VM_STATE[vm.state]
|
|
vm_lcm_state = OpenNebula::VirtualMachine::LCM_STATE[vm.lcm_state]
|
|
else
|
|
vm_state = states[:state]
|
|
vm_lcm_state = states[:lcm]
|
|
end
|
|
|
|
if WARNING_STATES.include?(vm_lcm_state) ||
|
|
WARNING_STATES.include?(vm_state)
|
|
action = :error_wd_cb
|
|
action_msg = 'Warning'
|
|
elsif vm_state == 'DONE'
|
|
action = :done_wd_cb
|
|
action_msg = 'Warning'
|
|
elsif vm_lcm_state == 'RUNNING'
|
|
action = :running_wd_cb
|
|
action_msg = 'Running'
|
|
else
|
|
# in case there is other state, ignore it
|
|
return
|
|
end
|
|
|
|
Log.info LOG_COMP,
|
|
"#{action_msg} #{service_id}: #{role_name} is #{vm_state}"
|
|
|
|
# execute callback
|
|
@lcm.trigger_action(action,
|
|
service_id,
|
|
client,
|
|
service_id,
|
|
role_name,
|
|
node)
|
|
end
|
|
|
|
end
|
|
# rubocop:enable Naming/FileName
|