diff --git a/include/ClusterPool.h b/include/ClusterPool.h index f2314dd419..e98dd83d1f 100644 --- a/include/ClusterPool.h +++ b/include/ClusterPool.h @@ -135,12 +135,14 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "CLUSTER_POOL", Cluster::table, where); + return PoolSQL::dump(oss, "CLUSTER_POOL", Cluster::table, where, + limit); }; private: diff --git a/include/DatastorePool.h b/include/DatastorePool.h index 7f0033add5..43ba985554 100644 --- a/include/DatastorePool.h +++ b/include/DatastorePool.h @@ -168,12 +168,14 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "DATASTORE_POOL", Datastore::table, where); + return PoolSQL::dump(oss, "DATASTORE_POOL", Datastore::table, where, + limit); }; /** diff --git a/include/DocumentPool.h b/include/DocumentPool.h index 618061448a..e6b0c2695b 100644 --- a/include/DocumentPool.h +++ b/include/DocumentPool.h @@ -93,12 +93,14 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "DOCUMENT_POOL",Document::table,where); + return PoolSQL::dump(oss, "DOCUMENT_POOL", Document::table, where, + limit); }; /** diff --git a/include/GroupPool.h b/include/GroupPool.h index 3fbfce1c27..56fbf12629 100644 --- a/include/GroupPool.h +++ b/include/GroupPool.h @@ -147,12 +147,13 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "GROUP_POOL", Group::table, where); + return PoolSQL::dump(oss, "GROUP_POOL", Group::table, where, limit); }; protected: diff --git a/include/HostPool.h b/include/HostPool.h index 2bffaf4e87..177a99bf38 100644 --- a/include/HostPool.h +++ b/include/HostPool.h @@ -178,12 +178,13 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "HOST_POOL", Host::table, where); + return PoolSQL::dump(oss, "HOST_POOL", Host::table, where, limit); }; /** diff --git a/include/ImagePool.h b/include/ImagePool.h index 30d2eb95eb..234db68bd7 100644 --- a/include/ImagePool.h +++ b/include/ImagePool.h @@ -138,11 +138,13 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination + * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "IMAGE_POOL", Image::table, where); + return PoolSQL::dump(oss, "IMAGE_POOL", Image::table, where, limit); } /** diff --git a/include/PoolSQL.h b/include/PoolSQL.h index 0657a12a4e..f1e73b9205 100644 --- a/include/PoolSQL.h +++ b/include/PoolSQL.h @@ -171,7 +171,23 @@ public: * * @return 0 on success */ - virtual int dump(ostringstream& oss, const string& where) = 0; + int dump(ostringstream& oss, const string& where) + { + dump(oss, where, ""); + } + + /** + * Dumps the pool in XML format. A filter and limit can be also added + * to the query + * @param oss the output stream to dump the pool contents + * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination + * + * @return 0 on success + */ + + virtual int dump(ostringstream& oss, const string& where, + const string& limit) = 0; // ------------------------------------------------------------------------- // Function to generate dump filters @@ -244,6 +260,23 @@ protected: */ SqlDB * db; + /** + * Dumps the pool in XML format. A filter and limit can be also added + * to the query + * @param oss the output stream to dump the pool contents + * @param elem_name Name of the root xml pool name + * @param table Pool table name + * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination + * + * @return 0 on success + */ + int dump(ostringstream& oss, + const string& elem_name, + const char * table, + const string& where, + const string& limit); + /** * Dumps the pool in XML format. A filter can be also added to the * query @@ -257,7 +290,10 @@ protected: int dump(ostringstream& oss, const string& elem_name, const char * table, - const string& where); + const string& where) + { + dump(oss, elem_name, table, where, ""); + } /** * Dumps the output of the custom sql query into an xml diff --git a/include/UserPool.h b/include/UserPool.h index 51bb139bbc..caa624c0ac 100644 --- a/include/UserPool.h +++ b/include/UserPool.h @@ -147,12 +147,13 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "USER_POOL", User::table, where); + return PoolSQL::dump(oss, "USER_POOL", User::table, where, limit); }; /** diff --git a/include/VMTemplatePool.h b/include/VMTemplatePool.h index c6370e55df..026dca5458 100644 --- a/include/VMTemplatePool.h +++ b/include/VMTemplatePool.h @@ -98,12 +98,14 @@ public: * query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "VMTEMPLATE_POOL",VMTemplate::table,where); + return PoolSQL::dump(oss, "VMTEMPLATE_POOL", VMTemplate::table, where, + limit); }; /** diff --git a/include/VirtualMachinePool.h b/include/VirtualMachinePool.h index 972f302372..c83998551c 100644 --- a/include/VirtualMachinePool.h +++ b/include/VirtualMachinePool.h @@ -196,12 +196,14 @@ public: * pool * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "VM_POOL", VirtualMachine::table, where); + return PoolSQL::dump(oss, "VM_POOL", VirtualMachine::table, where, + limit); }; /** diff --git a/include/VirtualNetworkPool.h b/include/VirtualNetworkPool.h index 915cd3d2dc..772e8ceae6 100644 --- a/include/VirtualNetworkPool.h +++ b/include/VirtualNetworkPool.h @@ -138,12 +138,14 @@ public: * to the query * @param oss the output stream to dump the pool contents * @param where filter for the objects, defaults to all + * @param limit parameters used for pagination * * @return 0 on success */ - int dump(ostringstream& oss, const string& where) + int dump(ostringstream& oss, const string& where, const string& limit) { - return PoolSQL::dump(oss, "VNET_POOL", VirtualNetwork::table,where); + return PoolSQL::dump(oss, "VNET_POOL", VirtualNetwork::table, where, + limit); } /** diff --git a/share/install_gems/install_gems b/share/install_gems/install_gems index 18765a067e..784588d4d2 100755 --- a/share/install_gems/install_gems +++ b/share/install_gems/install_gems @@ -3,7 +3,7 @@ require 'pp' -DEFAULT=%w{sunstone quota cloud ozones_server auth_ldap vmware oneflow ec2_hybrid} +DEFAULT=%w{sunstone quota cloud ozones_server auth_ldap vmware oneflow ec2_hybrid oca} if defined?(RUBY_VERSION) && RUBY_VERSION>="1.8.7" SQLITE='sqlite3' @@ -30,7 +30,8 @@ GROUPS={ :auth_ldap => 'net-ldap', :vmware => %w{builder trollop}, :oneflow => %w{sinatra json treetop parse-cron}, - :ec2_hybrid => 'aws-sdk' + :ec2_hybrid => 'aws-sdk', + :oca => 'ox' } PACKAGES=GROUPS.keys diff --git a/src/cli/cli_helper.rb b/src/cli/cli_helper.rb index 5ffbe579a7..5034b8b3a5 100644 --- a/src/cli/cli_helper.rb +++ b/src/cli/cli_helper.rb @@ -176,11 +176,11 @@ module CLIHelper begin while true + data = block.call + CLIHelper.scr_cls CLIHelper.scr_move(0,0) - data = block.call - show(data, options) sleep delay end @@ -216,8 +216,8 @@ module CLIHelper end begin - print res_data.collect{|l| - (0..ncolumns-1).collect{ |i| + res_data.each{|l| + puts (0..ncolumns-1).collect{ |i| dat=l[i] col=@default_columns[i] @@ -225,8 +225,8 @@ module CLIHelper str=CLIHelper.color_state(str) if i==stat_column str - }.join(' ') - }.join("\n").gsub(/ *$/, '') + }.join(' ').rstrip + } rescue Errno::EPIPE end diff --git a/src/cli/one_helper.rb b/src/cli/one_helper.rb index 2f925ae904..16cb97d67c 100644 --- a/src/cli/one_helper.rb +++ b/src/cli/one_helper.rb @@ -357,21 +357,26 @@ EOT pool = factory_pool(filter_flag) - rc = pool.info - return -1, rc.message if OpenNebula.is_error?(rc) - if options[:xml] + # TODO: use paginated functions + rc=pool.info + return -1, rc.message if OpenNebula.is_error?(rc) return 0, pool.to_xml(true) else table = format_pool(options) if top table.top(options) { - pool.info - pool_to_array(pool) + array=pool_to_array(pool.get_hash) + if OpenNebula.is_error?(array) + STDERR.puts array.mesage + exit(-1) + end + array } else - array=pool_to_array(pool) + array=pool_to_array(pool.get_hash) + return -1, array.message if OpenNebula.is_error?(array) if options[:ids] array=array.select do |element| diff --git a/src/oca/ruby/opennebula/client.rb b/src/oca/ruby/opennebula/client.rb index e89e831643..16a9ca8bbb 100644 --- a/src/oca/ruby/opennebula/client.rb +++ b/src/oca/ruby/opennebula/client.rb @@ -15,9 +15,34 @@ #--------------------------------------------------------------------------- # require 'xmlrpc/client' +require 'bigdecimal' +require 'stringio' + module OpenNebula - if OpenNebula::NOKOGIRI + attr_accessor :pool_page_size + + if OpenNebula::OX + class OxStreamParser < XMLRPC::XMLParser::AbstractStreamParser + def initialize + @parser_class = OxParser + end + + class OxParser < Ox::Sax + include XMLRPC::XMLParser::StreamParserMixin + + alias :text :character + alias :end_element :endElement + alias :start_element :startElement + + def parse(str) + Ox.sax_parse(self, StringIO.new(str), + :symbolize => false, + :convert_special => true) + end + end + end + elsif OpenNebula::NOKOGIRI class NokogiriStreamParser < XMLRPC::XMLParser::AbstractStreamParser def initialize @parser_class = NokogiriParser @@ -39,6 +64,19 @@ module OpenNebula end end + DEFAULT_POOL_PAGE_SIZE = 2000 + + if size=ENV['ONE_POOL_PAGE_SIZE'] + if size.strip.match(/^\d+$/) && size.to_i >= 2 + @pool_page_size = size.to_i + else + @pool_page_size = nil + end + else + @pool_page_size = DEFAULT_POOL_PAGE_SIZE + end + + # The client class, represents the connection with the core and handles the # xml-rpc calls. class Client @@ -91,7 +129,9 @@ module OpenNebula @server = XMLRPC::Client.new2(@one_endpoint, nil, timeout) - if OpenNebula::NOKOGIRI + if defined?(OxStreamParser) + @server.set_parser(OxStreamParser.new) + elsif OpenNebula::NOKOGIRI @server.set_parser(NokogiriStreamParser.new) elsif XMLPARSER @server.set_parser(XMLRPC::XMLParser::XMLStreamParser.new) diff --git a/src/oca/ruby/opennebula/pool.rb b/src/oca/ruby/opennebula/pool.rb index 95f4c9b7c6..340dbee00c 100644 --- a/src/oca/ruby/opennebula/pool.rb +++ b/src/oca/ruby/opennebula/pool.rb @@ -24,6 +24,9 @@ module OpenNebula include Enumerable alias_method :each_with_xpath, :each + PAGINATED_POOLS=%w{VM_POOL IMAGE_POOL TEMPLATE_POOL VN_POOL + DOCUMENT_POOL} + protected #pool:: _String_ XML name of the root element #element:: _String_ XML name of the Pool elements @@ -156,5 +159,55 @@ module OpenNebula return str end + + # Gets a hash from a pool + # + # size:: nil => default page size + # < 2 => not paginated + # >=2 => page size + # + # The default page size can be changed with the environment variable + # ONE_POOL_PAGE_SIZE. Any value > 2 will set a page size, a non + # numeric value disables pagination. + def get_hash(size=nil) + allow_paginated = PAGINATED_POOLS.include?(@pool_name) + + if OpenNebula.pool_page_size && allow_paginated && + ( ( size && size >= 2 ) || !size ) + size = OpenNebula.pool_page_size if !size + { @pool_name => { @element_name => info_paginated(size) } } + else + rc=info + return rc if OpenNebula.is_error?(rc) + to_hash + end + end + + # Gets a pool in hash form using pagination + # + # size:: _Integer_ size of each page + def info_paginated(size) + array=Array.new + current=0 + + parser=ParsePoolSax.new(@pool_name, @element_name) + + while true + a=@client.call("#{@pool_name.delete('_').downcase}.info", + -2, current, -size, -1) + return a if OpenNebula.is_error?(a) + + a_array=parser.parse(a) + + array += a_array + current += size + break if !a || a_array.length false, + :convert_special => true) + end + + class PoolSax < Ox::Sax + include ParsePoolBase + + alias :text :characters + end + end + elsif NOKOGIRI + class ParsePoolSax < ParsePoolSaxBase + def initialize(pool_name, elem_name) + super(pool_name, elem_name) + @parser = Nokogiri::XML::SAX::Parser.new(@pool_sax) + end + + def sax_parse(str) + @parser.parse(str) + end + + class PoolSax < Nokogiri::XML::SAX::Document + include ParsePoolBase + + alias :cdata_block :characters + end + end + end end diff --git a/src/pool/PoolSQL.cc b/src/pool/PoolSQL.cc index 93733e1a09..b3c300525e 100644 --- a/src/pool/PoolSQL.cc +++ b/src/pool/PoolSQL.cc @@ -488,7 +488,8 @@ int PoolSQL::dump_cb(void * _oss, int num, char **values, char **names) int PoolSQL::dump(ostringstream& oss, const string& elem_name, const char * table, - const string& where) + const string& where, + const string& limit) { ostringstream cmd; @@ -501,6 +502,11 @@ int PoolSQL::dump(ostringstream& oss, cmd << " ORDER BY oid"; + if ( !limit.empty() ) + { + cmd << " LIMIT " << limit; + } + return dump(oss, elem_name, cmd); } @@ -698,7 +704,7 @@ void PoolSQL::oid_filter(int start_id, { ostringstream idfilter; - if ( start_id != -1 ) + if ( end_id >= -1 && start_id != -1 ) { idfilter << "oid >= " << start_id; diff --git a/src/rm/RequestManagerPoolInfoFilter.cc b/src/rm/RequestManagerPoolInfoFilter.cc index de92aaa4d3..bd38a5d7ed 100644 --- a/src/rm/RequestManagerPoolInfoFilter.cc +++ b/src/rm/RequestManagerPoolInfoFilter.cc @@ -343,7 +343,7 @@ void RequestManagerPoolInfoFilter::dump( const string& or_clause) { ostringstream oss; - string where_string; + string where_string, limit_clause; int rc; if ( filter_flag < MINE ) @@ -362,7 +362,14 @@ void RequestManagerPoolInfoFilter::dump( or_clause, where_string); - rc = pool->dump(oss, where_string); + if ( end_id < -1 ) + { + oss << start_id << "," << -end_id; + limit_clause = oss.str(); + oss.str(""); + } + + rc = pool->dump(oss, where_string, limit_clause); if ( rc != 0 ) { diff --git a/src/sunstone/models/SunstoneServer.rb b/src/sunstone/models/SunstoneServer.rb index 7efe602abe..7ac2b3eed9 100644 --- a/src/sunstone/models/SunstoneServer.rb +++ b/src/sunstone/models/SunstoneServer.rb @@ -65,12 +65,12 @@ class SunstoneServer < CloudServer return [404, error.to_json] end - rc = pool.info + rc = pool.get_hash if OpenNebula.is_error?(rc) return [500, rc.to_json] else - return [200, pool.to_json] + return [200, rc.to_json] end end