From 97f957460fa2ae1ba3522d93bd344e5f9674dff2 Mon Sep 17 00:00:00 2001 From: "Ruben S. Montero" Date: Mon, 8 Jan 2018 00:32:52 +0100 Subject: [PATCH] B #1555: Replicate log on leadership changes (cherry picked from commit 5c975532678b57f6e5a5b7b9612170510f7b9eed) --- include/LogDB.h | 9 ++++++ include/RaftManager.h | 24 +++++++++++++- src/pool/PoolSQL.cc | 11 +++++-- src/raft/RaftManager.cc | 59 ++++++++++++++++++++++++++++++++-- src/rm/Request.cc | 5 +++ src/sql/LogDB.cc | 70 ++++++++++++++++++++++++----------------- 6 files changed, 142 insertions(+), 36 deletions(-) diff --git a/include/LogDB.h b/include/LogDB.h index 93a36c6642..cc5ab7abd4 100644 --- a/include/LogDB.h +++ b/include/LogDB.h @@ -125,6 +125,15 @@ public: int insert_log_record(unsigned int index, unsigned int term, std::ostringstream& sql, time_t timestamp, int fed_index); + /** + * Replicate a log record on followers. It will also replicate any missing + * previous records + * @param rindex of the record to replicate + * + * @return 0 on success, -1 in case of failure + */ + int replicate(int rindex); + //-------------------------------------------------------------------------- // Functions to manage the Raft state. Log record 0, term -1 // ------------------------------------------------------------------------- diff --git a/include/RaftManager.h b/include/RaftManager.h index 24951218fb..8bfef4457b 100644 --- a/include/RaftManager.h +++ b/include/RaftManager.h @@ -25,6 +25,8 @@ extern "C" void * raft_manager_loop(void *arg); +extern "C" void * reconciling_thread(void *arg); + /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ @@ -87,7 +89,6 @@ public: */ void replicate_log(ReplicaRequest * rr); - /** * Finalizes the Raft Consensus Manager */ @@ -194,6 +195,19 @@ public: return test_state(SOLO); } + bool is_reconciling() + { + bool _reconciling; + + pthread_mutex_lock(&mutex); + + _reconciling = reconciling; + + pthread_mutex_unlock(&mutex); + + return _reconciling; + } + /** * Get next index to send to the follower * @param follower server id @@ -275,6 +289,8 @@ public: private: friend void * raft_manager_loop(void *arg); + friend void * reconciling_thread(void *arg); + /** * Thread id of the main event loop */ @@ -337,6 +353,12 @@ private: */ Template raft_state; + /** + * After becoming a leader it is replicating and applying any pending + * log entry. + */ + bool reconciling; + //-------------------------------------------------------------------------- // Timers // - timer_period_ms. Base timer to wake up the manager (10ms) diff --git a/src/pool/PoolSQL.cc b/src/pool/PoolSQL.cc index 41595fd8b2..1d3db5a229 100644 --- a/src/pool/PoolSQL.cc +++ b/src/pool/PoolSQL.cc @@ -65,14 +65,14 @@ int PoolSQL::get_lastOID() /* -------------------------------------------------------------------------- */ -static void _set_lastOID(int _last_oid, SqlDB * db, const string& table) +static int _set_lastOID(int _last_oid, SqlDB * db, const string& table) { ostringstream oss; oss << "REPLACE INTO pool_control (tablename, last_oid) VALUES ('" << table << "'," << _last_oid << ")"; - db->exec_wr(oss); + return db->exec_wr(oss); } void PoolSQL::set_lastOID(int _last_oid) @@ -137,7 +137,12 @@ int PoolSQL::allocate(PoolObjectSQL *objsql, string& error_str) objsql->oid = ++lastOID; - _set_lastOID(lastOID, db, table); + if ( _set_lastOID(lastOID, db, table) == -1 ) + { + unlock(); + + return -1; + } rc = objsql->insert(db, error_str); diff --git a/src/raft/RaftManager.cc b/src/raft/RaftManager.cc index 5214926283..ba769261c8 100644 --- a/src/raft/RaftManager.cc +++ b/src/raft/RaftManager.cc @@ -49,7 +49,7 @@ RaftManager::RaftManager(int id, const VectorAttribute * leader_hook_mad, const VectorAttribute * follower_hook_mad, time_t log_purge, long long bcast, long long elect, time_t xmlrpc, const string& remotes_location):server_id(id), term(0), num_servers(0), - commit(0), leader_hook(0), follower_hook(0) + reconciling(false), commit(0), leader_hook(0), follower_hook(0) { Nebula& nd = Nebula::instance(); LogDB * logdb = nd.get_logdb(); @@ -371,6 +371,32 @@ void RaftManager::delete_server(int follower_id) /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ +extern "C" void * reconciling_thread(void *arg) +{ + Nebula& nd = Nebula::instance(); + + LogDB * logdb = nd.get_logdb(); + RaftManager * rm = nd.get_raftm(); + + int * index = static_cast(arg); + + NebulaLog::log("RCM", Log::INFO, "Replicating log to followers"); + + logdb->replicate(*index); + + NebulaLog::log("RCM", Log::INFO, "Leader log replicated"); + + pthread_mutex_lock(&(rm->mutex)); + + rm->reconciling = false; + + pthread_mutex_unlock(&(rm->mutex)); + + free(index); + + return 0; +} + void RaftManager::leader() { Nebula& nd = Nebula::instance(); @@ -383,7 +409,7 @@ void RaftManager::leader() std::map::iterator it; std::vector _follower_ids; - int index, _applied; + int index, _applied, _next_index; std::map _servers; @@ -420,6 +446,16 @@ void RaftManager::leader() leader_id = server_id; + if ( _applied < index ) + { + reconciling = true; + _next_index = index; + } + else + { + _next_index = index + 1; + } + for (it = servers.begin(); it != servers.end() ; ++it ) { if ( it->first == server_id ) @@ -427,7 +463,7 @@ void RaftManager::leader() continue; } - next.insert(std::make_pair(it->first, index + 1)); + next.insert(std::make_pair(it->first, _next_index)); match.insert(std::make_pair(it->first, 0)); @@ -446,6 +482,23 @@ void RaftManager::leader() frm->start_replica_threads(); } + if ( _applied < index ) + { + pthread_attr_t pattr; + pthread_t thid; + + pthread_attr_init (&pattr); + pthread_attr_setdetachstate(&pattr, PTHREAD_CREATE_DETACHED); + + int * _index = (int *) malloc(sizeof(int)); + + *_index = _next_index; + + pthread_create(&thid, &pattr, reconciling_thread, (void *) _index); + + pthread_attr_destroy(&pattr); + } + NebulaLog::log("RCM", Log::INFO, "oned is now the leader of the zone"); } diff --git a/src/rm/Request.cc b/src/rm/Request.cc index 095369d714..b7833d33a9 100644 --- a/src/rm/Request.cc +++ b/src/rm/Request.cc @@ -329,6 +329,11 @@ void Request::execute( att.resp_msg = "Cannot process request, oned cluster in election mode"; failure_response(INTERNAL, att); } + else if ( raftm->is_reconciling() && leader_only) + { + att.resp_msg = "Cannot process request, oned cluster is replicating log"; + failure_response(INTERNAL, att); + } else //leader or solo or !leader_only { request_execute(_paramList, att); diff --git a/src/sql/LogDB.cc b/src/sql/LogDB.cc index 981deb0146..a322cb44f6 100644 --- a/src/sql/LogDB.cc +++ b/src/sql/LogDB.cc @@ -487,35 +487,7 @@ int LogDB::_exec_wr(ostringstream& cmd, int federated_index) return -1; } - ReplicaRequest rr(rindex); - - raftm->replicate_log(&rr); - - // Wait for completion - rr.wait(); - - if ( !raftm->is_leader() ) // Check we are still leaders before applying - { - NebulaLog::log("DBM", Log::ERROR, "Not applying log record, oned is" - " now a follower"); - rc = -1; - } - else if ( rr.result == true ) //Record replicated on majority of followers - { - rc = apply_log_records(rindex); - } - else - { - std::ostringstream oss; - - oss << "Cannot replicate log record on followers: " << rr.message; - - NebulaLog::log("DBM", Log::ERROR, oss); - - rc = -1; - } - - return rc; + return replicate(rindex); } /* -------------------------------------------------------------------------- */ @@ -608,6 +580,46 @@ int LogDB::purge_log() return rc; } +/* -------------------------------------------------------------------------- */ +/* -------------------------------------------------------------------------- */ + +int LogDB::replicate(int rindex) +{ + int rc; + + RaftManager * raftm = Nebula::instance().get_raftm(); + + ReplicaRequest rr(rindex); + + raftm->replicate_log(&rr); + + // Wait for completion + rr.wait(); + + if ( !raftm->is_leader() ) // Check we are still leaders before applying + { + NebulaLog::log("DBM", Log::ERROR, "Not applying log record, oned is" + " now a follower"); + rc = -1; + } + else if ( rr.result == true ) //Record replicated on majority of followers + { + rc = apply_log_records(rindex); + } + else + { + std::ostringstream oss; + + oss << "Cannot replicate log record on followers: " << rr.message; + + NebulaLog::log("DBM", Log::ERROR, oss); + + rc = -1; + } + + return rc; +} + /* -------------------------------------------------------------------------- */ /* -------------------------------------------------------------------------- */ int LogDB::index_cb(void *null, int num, char **values, char **names)