tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor network stack

Browse Source 
Summary:
Previously, the network stack `communication::Server` accepted connections and
assigned them statically in a round-robin fashion to `communication::Worker`.
That meant that if two compute intensive connections were assigned to the same
worker they would block each other while the other workers would do nothing.

This implementation replaces `communication::Worker` with
`communication::Listener` which holds all accepted connections in one pool and
ensures that all workers execute all connections.

Reviewers: buda, florijan, teon.banek

Reviewed By: buda

Subscribers: pullbot

Differential Revision: https://phabricator.memgraph.io/D1220
This commit is contained in:
Matej Ferencevic
2018-02-22 16:17:45 +01:00
parent fc75fadee3
commit 017e8004e8
20 changed files with 462 additions and 563 deletions
Download Patch File Download Diff File Expand all files Collapse all files

131
src/communication/server.hpp
View File

@@ -10,24 +10,21 @@
#include <fmt/format.h>
#include <glog/logging.h>
#include "communication/worker.hpp"
#include "communication/listener.hpp"
#include "io/network/socket.hpp"
#include "io/network/socket_event_dispatcher.hpp"
namespace communication {
/**
* TODO (mferencevic): document methods
*/
/**
* Communication server.
* Listens for incomming connections on the server port and assigns them in a
* round-robin manner to it's workers. Started automatically on constructor, and
* stopped at destructor.
*
* Listens for incoming connections on the server port and assigns them to the
* connection listener. The listener processes the events with a thread pool
* that has `num_workers` threads. It is started automatically on constructor,
* and stopped at destructor.
*
* Current Server achitecture:
* incomming connection -> server -> worker -> session
* incoming connection -> server -> listener -> session
*
* @tparam TSession the server can handle different Sessions, each session
* represents a different protocol so the same network infrastructure
@@ -38,7 +35,6 @@ namespace communication {
template <typename TSession, typename TSessionData>
class Server {
public:
using WorkerT = Worker<TSession, TSessionData>;
using Socket = io::network::Socket;
/**
@@ -46,43 +42,35 @@ class Server {
* invokes workers_count workers
*/
Server(const io::network::Endpoint &endpoint, TSessionData &session_data,
bool check_for_timeouts,
size_t workers_count = std::thread::hardware_concurrency())
: session_data_(session_data) {
: listener_(session_data, check_for_timeouts) {
// Without server we can't continue with application so we can just
// terminate here.
if (!socket_.Bind(endpoint)) {
LOG(FATAL) << "Cannot bind to socket on " << endpoint.address() << " at "
<< endpoint.port();
LOG(FATAL) << "Cannot bind to socket on " << endpoint;
}
socket_.SetNonBlocking();
socket_.SetTimeout(1, 0);
if (!socket_.Listen(1024)) {
LOG(FATAL) << "Cannot listen on socket!";
}
working_thread_ = std::thread([this, workers_count]() {
thread_ = std::thread([this, workers_count]() {
std::cout << fmt::format("Starting {} workers", workers_count)
<< std::endl;
workers_.reserve(workers_count);
for (size_t i = 0; i < workers_count; ++i) {
workers_.push_back(std::make_unique<WorkerT>(session_data_));
worker_threads_.emplace_back(
[this](WorkerT &worker) -> void { worker.Start(alive_); },
std::ref(*workers_.back()));
worker_threads_.emplace_back([this]() {
while (alive_) {
listener_.WaitAndProcessEvents();
}
});
}
std::cout << "Server is fully armed and operational" << std::endl;
std::cout << fmt::format("Listening on {} at {}",
socket_.endpoint().address(),
socket_.endpoint().port())
<< std::endl;
std::vector<std::unique_ptr<ConnectionAcceptor>> acceptors;
acceptors.emplace_back(
std::make_unique<ConnectionAcceptor>(socket_, *this));
auto &acceptor = *acceptors.back().get();
io::network::SocketEventDispatcher<ConnectionAcceptor> dispatcher{
acceptors};
dispatcher.AddListener(socket_.fd(), acceptor, EPOLLIN);
std::cout << "Server is fully armed and operational" << std::endl;
std::cout << "Listening on " << socket_.endpoint() << std::endl;
while (alive_) {
dispatcher.WaitAndProcessEvents();
AcceptConnection();
}
std::cout << "Shutting down..." << std::endl;
@@ -97,6 +85,11 @@ class Server {
AwaitShutdown();
}
Server(const Server &) = delete;
Server(Server &&) = delete;
Server &operator=(const Server &) = delete;
Server &operator=(Server &&) = delete;
const auto &endpoint() const { return socket_.endpoint(); }
/// Stops server manually
@@ -104,73 +97,33 @@ class Server {
// This should be as simple as possible, so that it can be called inside a
// signal handler.
alive_.store(false);
// Shutdown the socket to return from any waiting `Accept` calls.
socket_.Shutdown();
}
/// Waits for the server to be signaled to shutdown
void AwaitShutdown() {
if (working_thread_.joinable()) working_thread_.join();
if (thread_.joinable()) thread_.join();
}
private:
class ConnectionAcceptor {
public:
ConnectionAcceptor(Socket &socket, Server<TSession, TSessionData> &server)
: socket_(socket), server_(server) {}
void OnData() {
DCHECK(server_.idx_ < server_.workers_.size()) << "Invalid worker id.";
DLOG(INFO) << "On connect";
auto connection = AcceptConnection();
if (!connection) {
// Connection is not available anymore or configuration failed.
return;
}
server_.workers_[server_.idx_]->AddConnection(std::move(*connection));
server_.idx_ = (server_.idx_ + 1) % server_.workers_.size();
void AcceptConnection() {
// Accept a connection from a socket.
auto s = socket_.Accept();
if (!s) {
// Connection is not available anymore or configuration failed.
return;
}
LOG(INFO) << "Accepted a connection from " << s->endpoint();
listener_.AddConnection(std::move(*s));
}
void OnClose() { socket_.Close(); }
void OnException(const std::exception &e) {
LOG(FATAL) << "Exception was thrown while processing event on socket "
<< socket_.fd() << " with message: " << e.what();
}
void OnError() { LOG(FATAL) << "Error on server side occured in epoll"; }
private:
// Accepts connection on socket_ and configures new connections. If done
// successfuly new socket (connection) is returner, nullopt otherwise.
std::experimental::optional<Socket> AcceptConnection() {
DLOG(INFO) << "Accept new connection on socket: " << socket_.fd();
// Accept a connection from a socket.
auto s = socket_.Accept();
if (!s) return std::experimental::nullopt;
DLOG(INFO) << fmt::format(
"Accepted a connection: socket {}, address '{}', family {}, port {}",
s->fd(), s->endpoint().address(), s->endpoint().family(),
s->endpoint().port());
s->SetTimeout(1, 0);
s->SetKeepAlive();
s->SetNoDelay();
return s;
}
Socket &socket_;
Server<TSession, TSessionData> &server_;
};
std::vector<std::unique_ptr<WorkerT>> workers_;
std::vector<std::thread> worker_threads_;
std::thread working_thread_;
std::atomic<bool> alive_{true};
int idx_{0};
std::thread thread_;
std::vector<std::thread> worker_threads_;
Socket socket_;
TSessionData &session_data_;
Listener<TSession, TSessionData> listener_;
};
} // namespace communication