From f54e066949975ef83fd6ef2aea67100b451d856f Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Tue, 9 Oct 2018 11:05:08 +0200 Subject: [PATCH] F #2489: Improve performance of oned: * Reduce memory footprint * Improve connection handling in the RequestManager * Reduce the number of locking calls * Reduce the size of the VM pool --- include/Callbackable.h | 43 ++++++++- include/ClusterPool.h | 17 +++- include/DatastorePool.h | 15 ++- include/DocumentPool.h | 17 +++- include/GroupPool.h | 20 +++- include/History.h | 8 ++ include/HostPool.h | 73 ++++++++++++-- include/ImagePool.h | 28 +++++- include/MarketPlaceAppPool.h | 27 +++++- include/MarketPlacePool.h | 16 +++- include/Nebula.h | 4 +- include/PoolObjectSQL.h | 39 +++----- include/PoolSQL.h | 42 ++++---- include/PoolSQLCache.h | 25 +---- include/RequestManager.h | 23 ++--- include/RequestManagerClone.h | 2 +- include/RequestManagerConnection.h | 122 ++++++++++++++++++++++++ include/SecurityGroupPool.h | 15 ++- include/UserPool.h | 39 +++++++- include/VMGroupPool.h | 4 +- include/VMTemplatePool.h | 16 +++- include/VdcPool.h | 4 +- include/VirtualMachine.h | 8 ++ include/VirtualMachineDisk.h | 12 +++ include/VirtualMachineMonitorInfo.h | 30 +++++- include/VirtualMachineNic.h | 17 ++-- include/VirtualMachinePool.h | 45 +++++++-- include/VirtualMachineTemplate.h | 2 + include/VirtualNetworkPool.h | 33 ++++++- include/VirtualRouterPool.h | 19 +++- include/ZonePool.h | 4 +- src/cluster/Cluster.cc | 2 +- src/datastore/DatastorePool.cc | 2 +- src/dm/DispatchManagerActions.cc | 10 +- src/group/GroupPool.cc | 10 +- src/host/HostPool.cc | 2 +- src/im/MonitorThread.cc | 14 +-- src/image/ImageManager.cc | 2 +- src/image/ImageManagerActions.cc | 20 ++-- src/image/ImagePool.cc | 4 +- src/lcm/LifeCycleStates.cc | 12 +-- src/market/MarketPlaceAppPool.cc | 3 - src/market/MarketPlaceManagerActions.cc | 10 +- src/market/MarketPlacePool.cc | 2 +- src/onedb/database_schema.rb | 12 ++- src/parsers/expr_arith.cc | 73 +++++++------- src/parsers/expr_arith.h | 12 +-- src/parsers/expr_bool.cc | 105 ++++++++++---------- src/parsers/expr_bool.h | 12 +-- src/parsers/template_syntax.cc | 57 +++++------ src/parsers/template_syntax.h | 12 +-- src/parsers/vm_file_var_syntax.cc | 45 ++++----- src/parsers/vm_file_var_syntax.h | 12 +-- src/parsers/vm_file_var_syntax.y | 8 +- src/parsers/vm_var_syntax.cc | 51 +++++----- src/parsers/vm_var_syntax.h | 12 +-- src/parsers/vm_var_syntax.y | 6 +- src/pool/PoolSQL.cc | 83 ++++++++++++---- src/pool/PoolSQLCache.cc | 66 +++---------- src/rm/Request.cc | 2 +- src/rm/RequestManager.cc | 118 ++++++++++++++++++----- src/rm/RequestManagerAllocate.cc | 10 +- src/rm/RequestManagerChmod.cc | 4 +- src/rm/RequestManagerChown.cc | 2 +- src/rm/RequestManagerClone.cc | 2 +- src/rm/RequestManagerDelete.cc | 2 +- src/rm/RequestManagerHost.cc | 4 +- src/rm/RequestManagerImage.cc | 10 +- src/rm/RequestManagerInfo.cc | 6 +- src/rm/RequestManagerPoolInfoFilter.cc | 40 ++++---- src/rm/RequestManagerRename.cc | 12 +-- src/rm/RequestManagerUser.cc | 4 +- src/rm/RequestManagerVMTemplate.cc | 4 +- src/rm/RequestManagerVirtualMachine.cc | 36 +++---- src/rm/RequestManagerVirtualNetwork.cc | 6 +- src/rm/RequestManagerVirtualRouter.cc | 8 +- src/secgroup/SecurityGroupPool.cc | 4 +- src/tm/TransferManager.cc | 58 +++++------ src/um/Quotas.cc | 2 +- src/um/UserPool.cc | 22 +++-- src/vm/History.cc | 23 +++++ src/vm/VirtualMachine.cc | 117 ++++++++++++++++++++++- src/vm/VirtualMachineContext.cc | 2 +- src/vm/VirtualMachineDisk.cc | 44 ++++++++- src/vm/VirtualMachineNic.cc | 60 +++++++++++- src/vm/VirtualMachineParser.cc | 4 +- src/vm/VirtualMachinePool.cc | 19 ++-- src/vm/VirtualMachineTemplate.cc | 103 ++++++++++++++++++++ src/vmm/VirtualMachineManager.cc | 44 +++++---- src/vnm/VirtualNetworkPool.cc | 32 +++++-- 90 files changed, 1573 insertions(+), 659 deletions(-) create mode 100644 include/RequestManagerConnection.h diff --git a/include/Callbackable.h b/include/Callbackable.h index 7c5fedab4e..5405e829a2 100644 --- a/include/Callbackable.h +++ b/include/Callbackable.h @@ -22,6 +22,8 @@ #include #include +#include + using namespace std; /** @@ -275,6 +277,46 @@ private: /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ +class string_cb : public Callbackable +{ +public: + string_cb(int _total):total_values(_total){}; + + void set_callback(std::string * _str) + { + str = _str; + + Callbackable::set_callback( + static_cast(&string_cb::callback)); + }; + + int callback(void * nil, int num, char **values, char **names) + { + if ( (!values[0]) || (num != total_values) ) + { + return -1; + } + + for (int i=0; i < total_values; i++) + { + if ( values[i] != NULL ) + { + str->append(values[i]); + } + } + + return 0; + }; + +private: + int total_values; + + std::string * str; +}; + +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + class stream_cb : public Callbackable { public: @@ -327,5 +369,4 @@ public: }; }; - #endif /*CALLBACKABLE_H_*/ diff --git a/include/ClusterPool.h b/include/ClusterPool.h index 31853beca2..a950507d61 100644 --- a/include/ClusterPool.h +++ b/include/ClusterPool.h @@ -152,6 +152,19 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Function to get a read onlycluster from the pool, if the object is not + * in memory it is loaded from the DB + * @param oid cluster unique id + * + * @return a pointer to the cluster, 0 if the cluster could not be loaded + */ + + Cluster * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + } + /** * Drops the Cluster from the data base. The object mutex SHOULD be * locked. @@ -190,10 +203,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "CLUSTER_POOL", Cluster::table, where, + return PoolSQL::dump(oss, "CLUSTER_POOL", "body", Cluster::table, where, limit, desc); }; diff --git a/include/DatastorePool.h b/include/DatastorePool.h index de8040b599..f41c26e4ee 100644 --- a/include/DatastorePool.h +++ b/include/DatastorePool.h @@ -106,6 +106,17 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Function to get a read only Datastore from the pool, if the object is not in memory + * it is loaded from the DB + * @param oid Datastore unique id + * @return a pointer to the Datastore, 0 if the Datastore could not be loaded + */ + Datastore * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + } + /** * Drops the Datastore data in the data base. The object mutex SHOULD be * locked. @@ -135,10 +146,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "DATASTORE_POOL", Datastore::table, where, + return PoolSQL::dump(oss, "DATASTORE_POOL", "body", Datastore::table, where, limit, desc); }; diff --git a/include/DocumentPool.h b/include/DocumentPool.h index 0043c07891..ccda63229b 100644 --- a/include/DocumentPool.h +++ b/include/DocumentPool.h @@ -76,6 +76,19 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Gets a read only object from the pool (if needed the object is loaded from the + * database). + * @param oid the object unique identifier + * @param lock locks the object if true + * + * @return a pointer to the object, 0 in case of failure + */ + Document * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + }; + /** * Dumps the pool in XML format. A filter can be also added to the * query @@ -86,10 +99,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "DOCUMENT_POOL", Document::table, where, + return PoolSQL::dump(oss, "DOCUMENT_POOL", "body", Document::table, where, limit, desc); }; diff --git a/include/GroupPool.h b/include/GroupPool.h index 5a5b957fad..7231f04601 100644 --- a/include/GroupPool.h +++ b/include/GroupPool.h @@ -84,23 +84,35 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Function to get a read only group from the pool, if the object is not in memory + * it is loaded from the DB + * @param oid group unique id + * @param lock locks the group mutex + * @return a pointer to the group, 0 if the group could not be loaded + */ + Group * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + }; + /** * Returns the name of a group * @param id of the group * @return name of the group */ - const string& get_name(int gid) + const string get_name(int gid) { static string error_str = ""; - Group * group = get(gid); + Group * group = get_ro(gid); if ( group == 0 ) { return error_str; } - const string& gname = group->get_name(); + const string gname = group->get_name(); group->unlock(); @@ -152,7 +164,7 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc); private: diff --git a/include/History.h b/include/History.h index e1fbc81a47..63c713c1e6 100644 --- a/include/History.h +++ b/include/History.h @@ -111,6 +111,14 @@ public: */ string& to_xml(string& xml) const; + /** + * Function to print the History object into a string in + * XML format with reduce information + * @param xml the resulting XML string + * @return a reference to the generated string + */ + string& to_xml_short(string& xml) const; + private: friend class VirtualMachine; friend class VirtualMachinePool; diff --git a/include/HostPool.h b/include/HostPool.h index 65ccd8728d..89ed2ca50b 100644 --- a/include/HostPool.h +++ b/include/HostPool.h @@ -81,6 +81,44 @@ public: return h; }; + void update_prev_rediscovered_vms(int hoid, + const set& prev_rediscovered_vms) + { + + if (hoid < 0) + { + return; + } + + HostVM * hv = get_host_vm(hoid); + + hv->prev_rediscovered_vms = prev_rediscovered_vms; + } + + /** + * Function to get a read only Host from the pool, if the object is not in memory + * it is loaded from the DB + * @param oid Host unique id + * @return a pointer to the Host, 0 if the Host could not be loaded + */ + Host * get_ro( + int oid) + { + Host * h = static_cast(PoolSQL::get_ro(oid)); + + if ( h != 0 ) + { + HostVM * hv = get_host_vm(oid); + + h->tmp_lost_vms = &(hv->tmp_lost_vms); + h->tmp_zombie_vms = &(hv->tmp_zombie_vms); + + h->prev_rediscovered_vms = &(hv->prev_rediscovered_vms); + } + + return h; + }; + /** * Function to get a Host from the pool, if the object is not in memory * it is loaded from the DB @@ -106,6 +144,31 @@ public: return h; }; + /** + * Function to get a Host from the pool, if the object is not in memory + * it is loaded from the DB + * @param hostname + * @param lock locks the Host mutex + * @return a pointer to the Host, 0 if the Host could not be loaded + */ + Host * get_ro(string name) + { + // The owner is set to -1, because it is not used in the key() method + Host * h = static_cast(PoolSQL::get_ro(name,-1)); + + if ( h != 0 ) + { + HostVM * hv = get_host_vm(h->oid); + + h->tmp_lost_vms = &(hv->tmp_lost_vms); + h->tmp_zombie_vms = &(hv->tmp_zombie_vms); + + h->prev_rediscovered_vms = &(hv->prev_rediscovered_vms); + } + + return h; + }; + /** * Generate an index key for the object * @param name of the object @@ -225,10 +288,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "HOST_POOL", Host::table, where, limit, desc); + return PoolSQL::dump(oss, "HOST_POOL", "body", Host::table, where, limit, desc); }; /** @@ -253,8 +316,7 @@ public: * * @return 0 on success */ - int dump_monitoring(ostringstream& oss, - const string& where); + int dump_monitoring(string& oss, const string& where); /** * Dumps the HOST monitoring information for a single HOST @@ -264,8 +326,7 @@ public: * * @return 0 on success */ - int dump_monitoring(ostringstream& oss, - int hostid) + int dump_monitoring(string& oss, int hostid) { ostringstream filter; diff --git a/include/ImagePool.h b/include/ImagePool.h index 2da8850144..240620ee4b 100644 --- a/include/ImagePool.h +++ b/include/ImagePool.h @@ -101,6 +101,17 @@ public: * @return a pointer to the Image, 0 if the Image could not be loaded */ Image * get(int oid) + { + return static_cast(PoolSQL::get(oid)); + }; + + /** + ** Function to get a read only Image from the pool, if the object is not in memory + * it is loaded from the DB + * @param oid Image unique id + * @return a pointer to the Image, 0 if the Image could not be loaded + */ + Image * get_ro(int oid) { return static_cast(PoolSQL::get(oid)); }; @@ -119,6 +130,19 @@ public: return static_cast(PoolSQL::get(name,uid)); }; + /** + * Gets a read only object from the pool (if needed the object is loaded from the + * database). + * @param name of the object + * @param uid id of owner + * + * @return a pointer to the object, 0 in case of failure + */ + Image * get_ro(const string& name, int uid) + { + return static_cast(PoolSQL::get_ro(name,uid)); + }; + /** * Bootstraps the database table(s) associated to the Image pool * @return 0 on success @@ -138,10 +162,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "IMAGE_POOL", Image::table, where, limit, + return PoolSQL::dump(oss, "IMAGE_POOL", "body", Image::table, where, limit, desc); } diff --git a/include/MarketPlaceAppPool.h b/include/MarketPlaceAppPool.h index 9db95fff4e..2ece427901 100644 --- a/include/MarketPlaceAppPool.h +++ b/include/MarketPlaceAppPool.h @@ -93,6 +93,16 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Function to get a read only MarketPlaceApp from the pool + * @param oid MarketPlaceApp unique id + * @return a pointer to the MarketPlaceApp, 0 if not loaded + */ + MarketPlaceApp * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + }; + /** * Gets an object from the pool (if needed the object is loaded from the * database). @@ -107,6 +117,19 @@ public: return static_cast(PoolSQL::get(name, uid)); }; + /** + * Gets a read only object from the pool (if needed the object is loaded from the + * database). + * @param name of the object + * @param uid id of owner + * + * @return a pointer to the object, 0 in case of failure + */ + MarketPlaceApp * get_ro(const std::string& name, int uid) + { + return static_cast(PoolSQL::get_ro(name, uid)); + }; + /** * Bootstraps the database table(s) associated to the MarketPlace pool * @return 0 on success @@ -126,10 +149,10 @@ public: * * @return 0 on success */ - int dump(std::ostringstream& oss, const std::string& where, + int dump(std::string& oss, const std::string& where, const std::string& limit, bool desc) { - return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", MarketPlaceApp::table, + return PoolSQL::dump(oss, "MARKETPLACEAPP_POOL", "body", MarketPlaceApp::table, where, limit, desc); }; diff --git a/include/MarketPlacePool.h b/include/MarketPlacePool.h index 07826c9c3c..05cec25ab4 100644 --- a/include/MarketPlacePool.h +++ b/include/MarketPlacePool.h @@ -72,6 +72,18 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Function to get a read only MarketPlace from the pool, the object is loaded if not + * in memory + * @param oid MarketPlace unique id + * @param lock locks the MarketPlace mutex + * @return a pointer to the MarketPlace, 0 if not loaded + */ + MarketPlace * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + }; + /** Update a particular MarketPlace * @param objsql points to the market * @return 0 on success @@ -106,10 +118,10 @@ public: * * @return 0 on success */ - int dump(std::ostringstream& oss, const std::string& where, + int dump(std::string& oss, const std::string& where, const std::string& limit, bool desc) { - return PoolSQL::dump(oss, "MARKETPLACE_POOL", MarketPlace::table, where, + return PoolSQL::dump(oss, "MARKETPLACE_POOL", "body", MarketPlace::table, where, limit, desc); }; diff --git a/include/Nebula.h b/include/Nebula.h index 031d19039c..285a629029 100644 --- a/include/Nebula.h +++ b/include/Nebula.h @@ -458,7 +458,7 @@ public: { if ( uid != -1 ) { - User * user = upool->get(uid); + User * user = upool->get_ro(uid); if ( user == 0 ) { @@ -481,7 +481,7 @@ public: user->unlock(); } - Group * group = gpool->get(gid); + Group * group = gpool->get_ro(gid); if ( group == 0 ) { diff --git a/include/PoolObjectSQL.h b/include/PoolObjectSQL.h index bcce6b7a1e..3313acee91 100644 --- a/include/PoolObjectSQL.h +++ b/include/PoolObjectSQL.h @@ -177,18 +177,14 @@ public: lock_owner(-1), lock_req_id(-1), lock_time(0), + mutex(0), table(_table) { - pthread_mutex_init(&mutex,0); }; virtual ~PoolObjectSQL() { delete obj_template; - - pthread_mutex_unlock(&mutex); - - pthread_mutex_destroy(&mutex); }; /* --------------------------------------------------------------------- */ @@ -317,30 +313,18 @@ public: string& error_str); /* --------------------------------------------------------------------- */ - /** - * Function to lock the object - */ - void lock() - { - pthread_mutex_lock(&mutex); - }; - - /** - * Function to unlock the object + * Function to unlock the object. It also frees associated resources. Object + * cannot be access after unlocking it */ void unlock() { - pthread_mutex_unlock(&mutex); - }; + if (!ro && mutex != 0) + { + pthread_mutex_unlock(mutex); + } - /** - * Try to lock the object - * @return 0 on success or error_code - */ - int trylock() - { - return pthread_mutex_trylock(&mutex); + delete this; }; /** @@ -814,6 +798,11 @@ protected: */ time_t lock_time; + /** + * Attribute for check if is a read only object + */ + bool ro; + private: /** * Characters that can not be in a name @@ -834,7 +823,7 @@ private: * The mutex for the PoolObject. This implementation assumes that the mutex * IS LOCKED when the class destructor is called. */ - pthread_mutex_t mutex; + pthread_mutex_t * mutex; /** * Pointer to the SQL table for the PoolObjectSQL diff --git a/include/PoolSQL.h b/include/PoolSQL.h index e5f5e67f28..cea11fb131 100644 --- a/include/PoolSQL.h +++ b/include/PoolSQL.h @@ -62,12 +62,20 @@ public: * Gets an object from the pool (if needed the object is loaded from the * database). * @param oid the object unique identifier - * @param lock locks the object if true * * @return a pointer to the object, 0 in case of failure */ PoolObjectSQL * get(int oid); + /** + * Gets a read only object from the pool (if needed the object is loaded from the + * database). + * @param oid the object unique identifier + * + * @return a pointer to the object, 0 in case of failure + */ + PoolObjectSQL * get_ro(int oid); + /** * Check if there is an object with the same for a given user * @param name of object @@ -172,7 +180,7 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, bool desc) + int dump(string& oss, const string& where, bool desc) { return dump(oss, where, "", desc); } @@ -188,7 +196,7 @@ public: * @return 0 on success */ - virtual int dump(ostringstream& oss, const string& where, + virtual int dump(string& oss, const string& where, const string& limit, bool desc) = 0; // ------------------------------------------------------------------------- @@ -259,12 +267,21 @@ protected: * database). * @param name of the object * @param uid id of owner - * @param lock locks the object if true * * @return a pointer to the object, 0 in case of failure */ PoolObjectSQL * get(const string& name, int uid); + /** + * Gets a read only object from the pool (if needed the object is loaded from the + * database). + * @param name of the object + * @param uid id of owner + * + * @return a pointer to the object, 0 in case of failure + */ + PoolObjectSQL * get_ro(const string& name, int uid); + /** * Pointer to the database. */ @@ -282,8 +299,9 @@ protected: * * @return 0 on success */ - int dump(ostringstream& oss, + int dump(string& oss, const string& elem_name, + const string& column, const char * table, const string& where, const string& limit, @@ -300,13 +318,13 @@ protected: * * @return 0 on success */ - int dump(ostringstream& oss, + int dump(string& oss, const string& elem_name, const char * table, const string& where, bool desc) { - return dump(oss, elem_name, table, where, "", desc); + return dump(oss, elem_name, "body", table, where, "", desc); } /** @@ -318,18 +336,10 @@ protected: * * @return 0 on success */ - int dump(ostringstream& oss, + int dump(string& oss, const string& root_elem_name, ostringstream& sql_query); - /** - * Child classes can add extra elements to the dump xml, right after all the - * pool objects - * - * @param oss The output stream to dump the xml contents - */ - virtual void add_extra_xml(ostringstream& oss){}; - /* ---------------------------------------------------------------------- */ /* Interface to access the lastOID assigned by the pool */ /* ---------------------------------------------------------------------- */ diff --git a/include/PoolSQLCache.h b/include/PoolSQLCache.h index de828f32c3..c995861d9b 100644 --- a/include/PoolSQLCache.h +++ b/include/PoolSQLCache.h @@ -50,19 +50,7 @@ public: * * @param oid of the object */ - void lock_line(int oid); - - /** - * Sets the reference of the active object and unlocks the cache line. The - * mutex of the object MUST be locked - * - * @param oid of the object - * @param object to be inserted int the cache - * - * @return 0 on success - * - */ - int set_line(int oid, PoolObjectSQL * object); + pthread_mutex_t * lock_line(int oid); private: /** @@ -71,15 +59,13 @@ private: */ struct CacheLine { - CacheLine(PoolObjectSQL * o):active(0), object(o) + CacheLine():active(0) { pthread_mutex_init(&mutex, 0); }; ~CacheLine() { - delete object; - pthread_mutex_destroy(&mutex); } @@ -99,7 +85,7 @@ private: } /** - * Concurrent access to cache line + * Concurrent access to object */ pthread_mutex_t mutex; @@ -107,11 +93,6 @@ private: * Number of threads waiting on the line mutex */ int active; - - /** - * Reference to the object - */ - PoolObjectSQL * object; }; /** diff --git a/include/RequestManager.h b/include/RequestManager.h index 882e77e74c..9e7fdcca0b 100644 --- a/include/RequestManager.h +++ b/include/RequestManager.h @@ -74,13 +74,17 @@ public: }; /** - * + * Stops the main RM thread. */ void finalize() { am.finalize(); }; + /** + * @return an AbyssServer to run xmlrpc connections + */ + xmlrpc_c::serverAbyss * create_abyss(); private: @@ -95,12 +99,12 @@ private: /** * Thread id for the RequestManager */ - pthread_t rm_thread; + pthread_t rm_thread; /** * Thread id for the XML Server */ - pthread_t rm_xml_server_thread; + pthread_t rm_xml_server_thread; /** * Port number where the connection will be open @@ -150,18 +154,13 @@ private: /** * Action engine for the Manager */ - ActionManager am; + ActionManager am; /** * To register XML-RPC methods */ xmlrpc_c::registry RequestManagerRegistry; - /** - * The XML-RPC server - */ - xmlrpc_c::serverAbyss * AbyssServer; - /** * Register the XML-RPC API Calls */ @@ -182,8 +181,6 @@ private: NebulaLog::log("ReM",Log::INFO,"XML-RPC server stopped."); - delete AbyssServer; - if ( socket_fd != -1 ) { close(socket_fd); @@ -191,9 +188,5 @@ private: }; }; -/* -------------------------------------------------------------------------- */ -/* -------------------------------------------------------------------------- */ -/* -------------------------------------------------------------------------- */ - #endif diff --git a/include/RequestManagerClone.h b/include/RequestManagerClone.h index 4497209c27..7ea9f0dcca 100644 --- a/include/RequestManagerClone.h +++ b/include/RequestManagerClone.h @@ -153,7 +153,7 @@ protected: int pool_allocate(int sid, Template * tmpl, int& id, RequestAttributes& att) { DocumentPool * docpool = static_cast(pool); - Document * doc = docpool->get(sid); + Document * doc = docpool->get_ro(sid); if ( doc == 0 ) { diff --git a/include/RequestManagerConnection.h b/include/RequestManagerConnection.h new file mode 100644 index 0000000000..cf13eac09c --- /dev/null +++ b/include/RequestManagerConnection.h @@ -0,0 +1,122 @@ +/* -------------------------------------------------------------------------- */ +/* Copyright 2002-2018, OpenNebula Project, OpenNebula Systems */ +/* */ +/* Licensed under the Apache License, Version 2.0 (the "License"); you may */ +/* not use this file except in compliance with the License. You may obtain */ +/* a copy of the License at */ +/* */ +/* http://www.apache.org/licenses/LICENSE-2.0 */ +/* */ +/* Unless required by applicable law or agreed to in writing, software */ +/* distributed under the License is distributed on an "AS IS" BASIS, */ +/* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. */ +/* See the License for the specific language governing permissions and */ +/* limitations under the License. */ +/* -------------------------------------------------------------------------- */ + +#ifndef REQUEST_MANAGER_CONNECTION_H_ +#define REQUEST_MANAGER_CONNECTION_H_ + +#include "RequestManager.h" + + +/** + * The connection manager class synchronizes the connection and manager threads + */ +class ConnectionManager +{ +public: + ConnectionManager(RequestManager *_rm, int mc):rm(_rm), max_connections(mc) + { + pthread_mutex_init(&mutex,0); + + pthread_cond_init(&cond,0); + }; + + ~ConnectionManager() + { + pthread_mutex_destroy(&mutex); + + pthread_cond_destroy(&cond); + }; + + /** + * Increments number of active connections + */ + int add() + { + pthread_mutex_lock(&mutex); + + ++connections; + + pthread_mutex_unlock(&mutex); + + return connections; + }; + + /** + * Decrements number of active connections and signals management thread + */ + void del() + { + pthread_mutex_lock(&mutex); + + --connections; + + pthread_cond_signal(&cond); + + pthread_mutex_unlock(&mutex); + }; + + /** + * Waits for active connections to be under the max_connection threshold + */ + void wait() + { + pthread_mutex_lock(&mutex); + + while ( connections >= max_connections ) + { + pthread_cond_wait(&cond, &mutex); + } + + pthread_mutex_unlock(&mutex); + }; + + /** + * Run an xmlrpc connection + * @param fd connected socket + */ + void run_connection(int fd) + { + xmlrpc_c::serverAbyss * as = rm->create_abyss(); + + as->runConn(fd); + + delete as; + }; + +private: + /** + * Synchronization for connection threads and listener thread + */ + pthread_mutex_t mutex; + pthread_cond_t cond; + + /** + * RequestManager to create an AbyssSever class to handle each request + */ + RequestManager * rm; + + /** + * Number of active connections + */ + int connections; + + /** + * Max number of active connections + */ + int max_connections; +}; + +#endif diff --git a/include/SecurityGroupPool.h b/include/SecurityGroupPool.h index 243a046392..4df536d527 100644 --- a/include/SecurityGroupPool.h +++ b/include/SecurityGroupPool.h @@ -72,6 +72,17 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Function to get a read only SecurityGroup from the pool, if the object is not in memory + * it is loaded from the DB + * @param oid SecurityGroup unique id + * @return a pointer to the SecurityGroup, 0 if the SecurityGroup could not be loaded + */ + SecurityGroup * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + }; + /** Update a particular SecurityGroup * @param securitygroup pointer to SecurityGroup * @return 0 on success @@ -100,10 +111,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "SECURITY_GROUP_POOL", SecurityGroup::table, + return PoolSQL::dump(oss, "SECURITY_GROUP_POOL", "body", SecurityGroup::table, where, limit, desc); }; diff --git a/include/UserPool.h b/include/UserPool.h index d5ff328edb..5113eaf83b 100644 --- a/include/UserPool.h +++ b/include/UserPool.h @@ -92,6 +92,24 @@ public: return u; }; + /** + * Function to get a read only User from the pool, if the object is not in memory + * it is loaded from the DB + * @param oid User unique id + * @return a pointer to the User, 0 if the User could not be loaded + */ + User * get_ro(int oid) + { + User * u = static_cast(PoolSQL::get_ro(oid)); + + if ( u != 0 ) + { + u->session = get_session_token(oid); + } + + return u; + }; + /** * Function to get a User from the pool, if the object is not in memory * it is loaded from the DB @@ -112,6 +130,25 @@ public: return u; }; + /** + * Function to get a read only User from the pool, if the object is not in memory + * it is loaded from the DB + * @param username + * @return a pointer to the User, 0 if the User could not be loaded + */ + User * get_ro(string name) + { + // The owner is set to -1, because it is not used in the key() method + User * u = static_cast(PoolSQL::get_ro(name,-1)); + + if ( u != 0 ) + { + u->session = get_session_token(u->oid); + } + + return u; + }; + /** * Function to get the token password of an user from the pool * @param uid creator of the object @@ -182,7 +219,7 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc); /** diff --git a/include/VMGroupPool.h b/include/VMGroupPool.h index bb685a6db0..d28c226a69 100644 --- a/include/VMGroupPool.h +++ b/include/VMGroupPool.h @@ -104,10 +104,10 @@ public: * * @return 0 on success */ - int dump(std::ostringstream& os, const std::string& where, + int dump(std::string& os, const std::string& where, const std::string& limit, bool desc) { - return PoolSQL::dump(os, "VM_GROUP_POOL", VMGroup::table, where, limit, + return PoolSQL::dump(os, "VM_GROUP_POOL", "body", VMGroup::table, where, limit, desc); }; diff --git a/include/VMTemplatePool.h b/include/VMTemplatePool.h index 881647d00f..7623304568 100644 --- a/include/VMTemplatePool.h +++ b/include/VMTemplatePool.h @@ -67,6 +67,18 @@ public: return static_cast(PoolSQL::get(oid)); }; + /** + * Gets an object from the pool (if needed the object is loaded from the + * database). + * @param oid the object unique identifier + * + * @return a pointer to the object, 0 in case of failure + */ + VMTemplate * get_ro(int oid) + { + return static_cast(PoolSQL::get_ro(oid)); + }; + /** * Dumps the pool in XML format. A filter can be also added to the * query @@ -77,10 +89,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "VMTEMPLATE_POOL", VMTemplate::table, where, + return PoolSQL::dump(oss, "VMTEMPLATE_POOL", "body", VMTemplate::table, where, limit, desc); }; diff --git a/include/VdcPool.h b/include/VdcPool.h index c4818eaec2..6f05ea77ee 100644 --- a/include/VdcPool.h +++ b/include/VdcPool.h @@ -95,10 +95,10 @@ public: * * @return 0 on success */ - int dump(ostringstream& oss, const string& where, const string& limit, + int dump(string& oss, const string& where, const string& limit, bool desc) { - return PoolSQL::dump(oss, "VDC_POOL", Vdc::table, where, limit, desc); + return PoolSQL::dump(oss, "VDC_POOL", "body", Vdc::table, where, limit, desc); }; /** diff --git a/include/VirtualMachine.h b/include/VirtualMachine.h index c99f0e3679..d72b30274c 100644 --- a/include/VirtualMachine.h +++ b/include/VirtualMachine.h @@ -874,6 +874,14 @@ public: return to_xml_extended(xml, 1); } + /** + * Function to print the VirtualMachine object into a string in + * XML format, with reduced information + * @param xml the resulting XML string + * @return a reference to the generated string + */ + string& to_xml_short(string& xml); + /** * Function to print the VirtualMachine object into a string in * XML format, with extended information (full history records) diff --git a/include/VirtualMachineDisk.h b/include/VirtualMachineDisk.h index 8af6489ae5..120877d01b 100644 --- a/include/VirtualMachineDisk.h +++ b/include/VirtualMachineDisk.h @@ -333,6 +333,12 @@ public: */ void set_types(const string& ds_name); + /** + * Marshall disk attributes in XML format with just essential information + * @param stream to write the disk XML description + */ + void to_xml_short(std::ostringstream& oss) const; + private: Snapshots * snapshots; @@ -743,6 +749,12 @@ public: void delete_non_persistent_snapshots(Template **vm_quotas, vector