tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Refactor memgraph.cpp (#1164)

Browse Source
This commit is contained in:
Gareth Andrew Lloyd
2023-08-18 17:23:15 +01:00
committed by GitHub
parent 3bf2cf65ab
commit adb65b2fff
36 changed files with 1467 additions and 932 deletions
Download Patch File Download Diff File Expand all files Collapse all files

887
src/memgraph.cpp
View File

@@ -9,465 +9,36 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <algorithm>
#include <atomic>
#include <chrono>
#include <csignal>
#include <cstdint>
#include <exception>
#include <filesystem>
#include <functional>
#include <limits>
#include <map>
#include <memory>
#include <optional>
#include <regex>
#include <string>
#include <string_view>
#include <thread>
#include <fmt/core.h>
#include <fmt/format.h>
#include <gflags/gflags.h>
#include <spdlog/common.h>
#include <spdlog/sinks/daily_file_sink.h>
#include <spdlog/sinks/dist_sink.h>
#include <spdlog/sinks/stdout_color_sinks.h>
#ifndef MG_ENTERPRISE
#include "dbms/session_context_handler.hpp"
#endif
#include "audit/log.hpp"
#include "auth/models.hpp"
#include "communication/bolt/v1/constants.hpp"
#include "communication/http/server.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "dbms/constants.hpp"
#include "dbms/global.hpp"
#include "dbms/session_context.hpp"
#include "flags/all.hpp"
#include "glue/MonitoringServerT.hpp"
#include "glue/ServerT.hpp"
#include "glue/auth_checker.hpp"
#include "glue/auth_handler.hpp"
#include "helpers.hpp"
#include "http_handlers/metrics.hpp"
#include "license/license.hpp"
#include "license/license_sender.hpp"
#include "py/py.hpp"
#include "query/auth_checker.hpp"
#include "query/discard_value_stream.hpp"
#include "query/exceptions.hpp"
#include "query/frontend/ast/ast.hpp"
#include "query/interpreter.hpp"
#include "query/plan/operator.hpp"
#include "query/procedure/callable_alias_mapper.hpp"
#include "query/procedure/module.hpp"
#include "query/procedure/py_module.hpp"
#include "requests/requests.hpp"
#include "storage/v2/config.hpp"
#include "storage/v2/disk/storage.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/view.hpp"
#include "telemetry/telemetry.hpp"
#include "utils/enum.hpp"
#include "utils/event_counter.hpp"
#include "utils/file.hpp"
#include "utils/flag_validation.hpp"
#include "utils/logging.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/message.hpp"
#include "utils/readable_size.hpp"
#include "utils/rw_lock.hpp"
#include "utils/settings.hpp"
#include "utils/signals.hpp"
#include "utils/string.hpp"
#include "utils/synchronized.hpp"
#include "utils/sysinfo/memory.hpp"
#include "utils/system_info.hpp"
#include "utils/terminate_handler.hpp"
#include "version.hpp"
// Communication libraries must be included after query libraries are included.
// This is to enable compilation of the binary when linking with old OpenSSL
// libraries (as on CentOS 7).
//
// The OpenSSL library available on CentOS 7 is v1.0.0, that version includes
// `libkrb5` in its public API headers (that we include in our communication
// stack). The `libkrb5` library has `#define`s for `TRUE` and `FALSE`. Those
// defines clash with Antlr's usage of `TRUE` and `FALSE` as enumeration keys.
// Because of that the definitions of `TRUE` and `FALSE` that are inherited
// from `libkrb5` must be included after the Antlr includes. Hence,
// communication headers must be included after query headers.
#include "communication/bolt/v1/exceptions.hpp"
#include "communication/bolt/v1/session.hpp"
#include "communication/init.hpp"
#include "communication/v2/server.hpp"
#include "communication/v2/session.hpp"
#include "dbms/session_context_handler.hpp"
#include "glue/communication.hpp"
#include "auth/auth.hpp"
#include "glue/auth.hpp"
constexpr const char *kMgUser = "MEMGRAPH_USER";
constexpr const char *kMgPassword = "MEMGRAPH_PASSWORD";
constexpr const char *kMgPassfile = "MEMGRAPH_PASSFILE";
// Short help flag.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(h, false, "Print usage and exit.");
// Bolt server flags.
DEFINE_string(bolt_address, "0.0.0.0", "IP address on which the Bolt server should listen.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(monitoring_address, "0.0.0.0",
"IP address on which the websocket server for Memgraph monitoring should listen.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(metrics_address, "0.0.0.0",
"IP address on which the Memgraph server for exposing metrics should listen.");
DEFINE_VALIDATED_int32(bolt_port, 7687, "Port on which the Bolt server should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(monitoring_port, 7444,
"Port on which the websocket server for Memgraph monitoring should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(metrics_port, 9091, "Port on which the Memgraph server for exposing metrics should listen.",
FLAG_IN_RANGE(0, std::numeric_limits<uint16_t>::max()));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(bolt_num_workers, std::max(std::thread::hardware_concurrency(), 1U),
"Number of workers used by the Bolt server. By default, this will be the "
"number of processing units available on the machine.",
FLAG_IN_RANGE(1, INT32_MAX));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(bolt_session_inactivity_timeout, 1800,
"Time in seconds after which inactive Bolt sessions will be "
"closed.",
FLAG_IN_RANGE(1, INT32_MAX));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(bolt_cert_file, "", "Certificate file which should be used for the Bolt server.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(bolt_key_file, "", "Key file which should be used for the Bolt server.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(bolt_server_name_for_init, "",
"Server name which the database should send to the client in the "
"Bolt INIT message.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(init_file, "",
"Path to cypherl file that is used for configuring users and database schema before server starts.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(init_data_file, "", "Path to cypherl file that is used for creating data after server starts.");
// General purpose flags.
// NOTE: The `data_directory` flag must be the same here and in
// `mg_import_csv`. If you change it, make sure to change it there as well.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(data_directory, "mg_data", "Path to directory in which to save all permanent data.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(data_recovery_on_startup, false, "Controls whether the database recovers persisted data on startup.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(memory_warning_threshold, 1024,
"Memory warning threshold, in MB. If Memgraph detects there is "
"less available RAM it will log a warning. Set to 0 to "
"disable.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(allow_load_csv, true, "Controls whether LOAD CSV clause is allowed in queries.");
// Storage flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).",
FLAG_IN_RANGE(1, 24 * 3600));
// NOTE: The `storage_properties_on_edges` flag must be the same here and in
// `mg_import_csv`. If you change it, make sure to change it there as well.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_properties_on_edges, false, "Controls whether edges have properties.");
// storage_recover_on_startup deprecated; use data_recovery_on_startup instead
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(storage_recover_on_startup, false,
"Controls whether the storage recovers persisted data on startup.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_uint64(storage_snapshot_interval_sec, 0,
"Storage snapshot creation interval (in seconds). Set "
"to 0 to disable periodic snapshot creation.",
FLAG_IN_RANGE(0, 7 * 24 * 3600));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_wal_enabled, false,
"Controls whether the storage uses write-ahead-logging. To enable "
"WAL periodic snapshots must be enabled.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_uint64(storage_snapshot_retention_count, 3, "The number of snapshots that should always be kept.",
FLAG_IN_RANGE(1, 1000000));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_uint64(storage_wal_file_size_kib, memgraph::storage::Config::Durability().wal_file_size_kibibytes,
"Minimum file size of each WAL file.",
FLAG_IN_RANGE(1, static_cast<unsigned long>(1000) * 1024));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_uint64(storage_wal_file_flush_every_n_tx,
memgraph::storage::Config::Durability().wal_file_flush_every_n_tx,
"Issue a 'fsync' call after this amount of transactions are written to the "
"WAL file. Set to 1 for fully synchronous operation.",
FLAG_IN_RANGE(1, 1000000));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_snapshot_on_exit, false, "Controls whether the storage creates another snapshot on exit.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(storage_items_per_batch, memgraph::storage::Config::Durability().items_per_batch,
"The number of edges and vertices stored in a batch in a snapshot file.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_parallel_index_recovery, false,
"Controls whether the index creation can be done in a multithreaded fashion.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(storage_recovery_thread_count,
std::max(static_cast<uint64_t>(std::thread::hardware_concurrency()),
memgraph::storage::Config::Durability().recovery_thread_count),
"The number of threads used to recover persisted data from disk.");
#ifdef MG_ENTERPRISE
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(storage_delete_on_drop, true,
"If set to true the query 'DROP DATABASE x' will delete the underlying storage as well.");
#endif
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(telemetry_enabled, false,
"Set to true to enable telemetry. We collect information about the "
"running system (CPU and memory information) and information about "
"the database runtime (vertex and edge counts and resource usage) "
"to allow for easier improvement of the product.");
// Streams flags
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(
stream_transaction_conflict_retries, 30,
"Number of times to retry when a stream transformation fails to commit because of conflicting transactions");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint32(
stream_transaction_retry_interval, 500,
"Retry interval in milliseconds when a stream transformation fails to commit because of conflicting transactions");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(kafka_bootstrap_servers, "",
"List of default Kafka brokers as a comma separated list of broker host or host:port.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(pulsar_service_url, "", "Default URL used while connecting to Pulsar brokers.");
// Audit logging flags.
#ifdef MG_ENTERPRISE
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(audit_enabled, false, "Set to true to enable audit logging.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(audit_buffer_size, memgraph::audit::kBufferSizeDefault,
"Maximum number of items in the audit log buffer.", FLAG_IN_RANGE(1, INT32_MAX));
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(audit_buffer_flush_interval_ms, memgraph::audit::kBufferFlushIntervalMillisDefault,
"Interval (in milliseconds) used for flushing the audit log buffer.",
FLAG_IN_RANGE(10, INT32_MAX));
#endif
// Query flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_double(query_execution_timeout_sec, 600,
"Maximum allowed query execution time. Queries exceeding this "
"limit will be aborted. Value of 0 means no limit.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(replication_replica_check_frequency_sec, 1,
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: "
"The MAIN instance allocates a new thread for each REPLICA.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(replication_restore_state_on_startup, false, "Restore replication state on startup, e.g. recover replica");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(
memory_limit, 0,
"Total memory limit in MiB. Set to 0 to use the default values which are 100\% of the phyisical memory if the swap "
"is enabled and 90\% of the physical memory otherwise.");
namespace {
using namespace std::literals;
inline constexpr std::array isolation_level_mappings{
std::pair{"SNAPSHOT_ISOLATION"sv, memgraph::storage::IsolationLevel::SNAPSHOT_ISOLATION},
std::pair{"READ_COMMITTED"sv, memgraph::storage::IsolationLevel::READ_COMMITTED},
std::pair{"READ_UNCOMMITTED"sv, memgraph::storage::IsolationLevel::READ_UNCOMMITTED}};
const std::string isolation_level_help_string =
fmt::format("Default isolation level used for the transactions. Allowed values: {}",
memgraph::utils::GetAllowedEnumValuesString(isolation_level_mappings));
} // namespace
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_string(isolation_level, "SNAPSHOT_ISOLATION", isolation_level_help_string.c_str(), {
if (const auto result = memgraph::utils::IsValidEnumValueString(value, isolation_level_mappings); result.HasError()) {
const auto error = result.GetError();
switch (error) {
case memgraph::utils::ValidationError::EmptyValue: {
std::cout << "Isolation level cannot be empty." << std::endl;
break;
}
case memgraph::utils::ValidationError::InvalidValue: {
std::cout << "Invalid value for isolation level. Allowed values: "
<< memgraph::utils::GetAllowedEnumValuesString(isolation_level_mappings) << std::endl;
break;
}
}
return false;
}
return true;
});
namespace {
memgraph::storage::IsolationLevel ParseIsolationLevel() {
const auto isolation_level =
memgraph::utils::StringToEnum<memgraph::storage::IsolationLevel>(FLAGS_isolation_level, isolation_level_mappings);
MG_ASSERT(isolation_level, "Invalid isolation level");
return *isolation_level;
}
int64_t GetMemoryLimit() {
if (FLAGS_memory_limit == 0) {
auto maybe_total_memory = memgraph::utils::sysinfo::TotalMemory();
MG_ASSERT(maybe_total_memory, "Failed to fetch the total physical memory");
const auto maybe_swap_memory = memgraph::utils::sysinfo::SwapTotalMemory();
MG_ASSERT(maybe_swap_memory, "Failed to fetch the total swap memory");
if (*maybe_swap_memory == 0) {
// take only 90% of the total memory
*maybe_total_memory *= 9;
*maybe_total_memory /= 10;
}
return *maybe_total_memory * 1024;
}
// We parse the memory as MiB every time
return FLAGS_memory_limit * 1024 * 1024;
}
} // namespace
namespace {
std::vector<std::filesystem::path> query_modules_directories;
} // namespace
DEFINE_VALIDATED_string(query_modules_directory, "",
"Directory where modules with custom query procedures are stored. "
"NOTE: Multiple comma-separated directories can be defined.",
{
query_modules_directories.clear();
if (value.empty()) return true;
const auto directories = memgraph::utils::Split(value, ",");
for (const auto &dir : directories) {
if (!memgraph::utils::DirExists(dir)) {
std::cout << "Expected --" << flagname << " to point to directories." << std::endl;
std::cout << dir << " is not a directory." << std::endl;
return false;
}
}
query_modules_directories.reserve(directories.size());
std::transform(directories.begin(), directories.end(),
std::back_inserter(query_modules_directories),
[](const auto &dir) { return dir; });
return true;
});
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(query_callable_mappings_path, "",
"The path to mappings that describes aliases to callables in cypher queries in the form of key-value "
"pairs in a json file. With this option query module procedures that do not exist in memgraph can be "
"mapped to ones that exist.");
// Logging flags
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(also_log_to_stderr, false, "Log messages go to stderr in addition to logfiles");
DEFINE_string(log_file, "", "Path to where the log should be stored.");
namespace {
inline constexpr std::array log_level_mappings{
std::pair{"TRACE"sv, spdlog::level::trace}, std::pair{"DEBUG"sv, spdlog::level::debug},
std::pair{"INFO"sv, spdlog::level::info}, std::pair{"WARNING"sv, spdlog::level::warn},
std::pair{"ERROR"sv, spdlog::level::err}, std::pair{"CRITICAL"sv, spdlog::level::critical}};
const std::string log_level_help_string = fmt::format("Minimum log level. Allowed values: {}",
memgraph::utils::GetAllowedEnumValuesString(log_level_mappings));
} // namespace
DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
if (const auto result = memgraph::utils::IsValidEnumValueString(value, log_level_mappings); result.HasError()) {
const auto error = result.GetError();
switch (error) {
case memgraph::utils::ValidationError::EmptyValue: {
std::cout << "Log level cannot be empty." << std::endl;
break;
}
case memgraph::utils::ValidationError::InvalidValue: {
std::cout << "Invalid value for log level. Allowed values: "
<< memgraph::utils::GetAllowedEnumValuesString(log_level_mappings) << std::endl;
break;
}
}
return false;
}
return true;
});
namespace {
spdlog::level::level_enum ParseLogLevel() {
const auto log_level = memgraph::utils::StringToEnum<spdlog::level::level_enum>(FLAGS_log_level, log_level_mappings);
MG_ASSERT(log_level, "Invalid log level");
return *log_level;
}
// 5 weeks * 7 days
inline constexpr auto log_retention_count = 35;
void CreateLoggerFromSink(const auto &sinks, const auto log_level) {
auto logger = std::make_shared<spdlog::logger>("memgraph_log", sinks.begin(), sinks.end());
logger->set_level(log_level);
logger->flush_on(spdlog::level::trace);
spdlog::set_default_logger(std::move(logger));
}
void InitializeLogger() {
std::vector<spdlog::sink_ptr> sinks;
if (FLAGS_also_log_to_stderr) {
sinks.emplace_back(std::make_shared<spdlog::sinks::stderr_color_sink_mt>());
}
if (!FLAGS_log_file.empty()) {
// get local time
time_t current_time{0};
struct tm *local_time{nullptr};
time(&current_time);
local_time = localtime(&current_time);
sinks.emplace_back(std::make_shared<spdlog::sinks::daily_file_sink_mt>(
FLAGS_log_file, local_time->tm_hour, local_time->tm_min, false, log_retention_count));
}
CreateLoggerFromSink(sinks, ParseLogLevel());
}
void AddLoggerSink(spdlog::sink_ptr new_sink) {
auto default_logger = spdlog::default_logger();
auto sinks = default_logger->sinks();
sinks.push_back(new_sink);
CreateLoggerFromSink(sinks, default_logger->level());
}
} // namespace
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_string(license_key, "", "License key for Memgraph Enterprise.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_string(organization_name, "", "Organization name.");
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_string(auth_user_or_role_name_regex, memgraph::glue::kDefaultUserRoleRegex.data(),
"Set to the regular expression that each user or role name must fulfill.");
void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, std::string cypherl_file_path,
memgraph::audit::Log *audit_log = nullptr) {
memgraph::query::Interpreter interpreter(&ctx);
@@ -494,430 +65,6 @@ void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, std::string c
file.close();
}
namespace memgraph::metrics {
extern const Event ActiveBoltSessions;
} // namespace memgraph::metrics
auto ToQueryExtras(memgraph::communication::bolt::Value const &extra) -> memgraph::query::QueryExtras {
auto const &as_map = extra.ValueMap();
auto metadata_pv = std::map<std::string, memgraph::storage::PropertyValue>{};
if (auto const it = as_map.find("tx_metadata"); it != as_map.cend() && it->second.IsMap()) {
for (const auto &[key, bolt_md] : it->second.ValueMap()) {
metadata_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_md));
}
}
auto tx_timeout = std::optional<int64_t>{};
if (auto const it = as_map.find("tx_timeout"); it != as_map.cend() && it->second.IsInt()) {
tx_timeout = it->second.ValueInt();
}
return memgraph::query::QueryExtras{std::move(metadata_pv), tx_timeout};
}
class SessionHL final : public memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream> {
public:
struct ContextWrapper {
explicit ContextWrapper(memgraph::dbms::SessionContext sc)
: session_context(sc),
interpreter(std::make_unique<memgraph::query::Interpreter>(session_context.interpreter_context.get())),
defunct_(false) {
session_context.interpreter_context->interpreters.WithLock(
[this](auto &interpreters) { interpreters.insert(interpreter.get()); });
}
~ContextWrapper() { Defunct(); }
void Defunct() {
if (!defunct_) {
session_context.interpreter_context->interpreters.WithLock(
[this](auto &interpreters) { interpreters.erase(interpreter.get()); });
defunct_ = true;
}
}
ContextWrapper(const ContextWrapper &) = delete;
ContextWrapper &operator=(const ContextWrapper &) = delete;
ContextWrapper(ContextWrapper &&in) noexcept
: session_context(std::move(in.session_context)),
interpreter(std::move(in.interpreter)),
defunct_(in.defunct_) {
in.defunct_ = true;
}
ContextWrapper &operator=(ContextWrapper &&in) noexcept {
if (this != &in) {
Defunct();
session_context = std::move(in.session_context);
interpreter = std::move(in.interpreter);
defunct_ = in.defunct_;
in.defunct_ = true;
}
return *this;
}
memgraph::query::InterpreterContext *interpreter_context() { return session_context.interpreter_context.get(); }
memgraph::query::Interpreter *interp() { return interpreter.get(); }
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth() const {
return session_context.auth;
}
#ifdef MG_ENTERPRISE
memgraph::audit::Log *audit_log() const { return session_context.audit_log; }
#endif
std::string run_id() const { return session_context.run_id; }
bool defunct() const { return defunct_; }
private:
memgraph::dbms::SessionContext session_context;
std::unique_ptr<memgraph::query::Interpreter> interpreter;
bool defunct_;
};
SessionHL(
#ifdef MG_ENTERPRISE
memgraph::dbms::SessionContextHandler &sc_handler,
#else
memgraph::dbms::SessionContext sc,
#endif
const memgraph::communication::v2::ServerEndpoint &endpoint,
memgraph::communication::v2::InputStream *input_stream, memgraph::communication::v2::OutputStream *output_stream,
const std::string &default_db = memgraph::dbms::kDefaultDB) // NOLINT
: memgraph::communication::bolt::Session<memgraph::communication::v2::InputStream,
memgraph::communication::v2::OutputStream>(input_stream, output_stream),
#ifdef MG_ENTERPRISE
sc_handler_(sc_handler),
current_(sc_handler_.Get(default_db)),
#else
current_(sc),
#endif
interpreter_context_(current_.interpreter_context()),
interpreter_(current_.interp()),
auth_(current_.auth()),
#ifdef MG_ENTERPRISE
audit_log_(current_.audit_log()),
#endif
endpoint_(endpoint),
run_id_(current_.run_id()) {
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions);
}
~SessionHL() override { memgraph::metrics::DecrementCounter(memgraph::metrics::ActiveBoltSessions); }
SessionHL(const SessionHL &) = delete;
SessionHL &operator=(const SessionHL &) = delete;
SessionHL(SessionHL &&) = delete;
SessionHL &operator=(SessionHL &&) = delete;
void Configure(const std::map<std::string, memgraph::communication::bolt::Value> &run_time_info) override {
#ifdef MG_ENTERPRISE
std::string db;
bool update = false;
// Check if user explicitly defined the database to use
if (run_time_info.contains("db")) {
const auto &db_info = run_time_info.at("db");
if (!db_info.IsString()) {
throw memgraph::communication::bolt::ClientError("Malformed database name.");
}
db = db_info.ValueString();
update = db != current_.interpreter_context()->db->id();
in_explicit_db_ = true;
// NOTE: Once in a transaction, the drivers stop explicitly sending the db and count on using it until commit
} else if (in_explicit_db_ && !interpreter_->in_explicit_transaction_) { // Just on a switch
db = GetDefaultDB();
update = db != current_.interpreter_context()->db->id();
in_explicit_db_ = false;
}
// Check if the underlying database needs to be updated
if (update) {
sc_handler_.SetInPlace(db, [this](auto new_sc) mutable {
const auto &db_name = new_sc.interpreter_context->db->id();
MultiDatabaseAuth(db_name);
try {
Update(ContextWrapper(new_sc));
return memgraph::dbms::SetForResult::SUCCESS;
} catch (memgraph::dbms::UnknownDatabaseException &e) {
throw memgraph::communication::bolt::ClientError("No database named \"{}\" found!", db_name);
}
});
}
#endif
}
using TEncoder = memgraph::communication::bolt::Encoder<
memgraph::communication::bolt::ChunkedEncoderBuffer<memgraph::communication::v2::OutputStream>>;
void BeginTransaction(const std::map<std::string, memgraph::communication::bolt::Value> &extra) override {
interpreter_->BeginTransaction(ToQueryExtras(extra));
}
void CommitTransaction() override { interpreter_->CommitTransaction(); }
void RollbackTransaction() override { interpreter_->RollbackTransaction(); }
std::pair<std::vector<std::string>, std::optional<int>> Interpret(
const std::string &query, const std::map<std::string, memgraph::communication::bolt::Value> &params,
const std::map<std::string, memgraph::communication::bolt::Value> &extra) override {
std::map<std::string, memgraph::storage::PropertyValue> params_pv;
for (const auto &[key, bolt_param] : params) {
params_pv.emplace(key, memgraph::glue::ToPropertyValue(bolt_param));
}
const std::string *username{nullptr};
if (user_) {
username = &user_->username();
}
#ifdef MG_ENTERPRISE
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
audit_log_->Record(endpoint_.address().to_string(), user_ ? *username : "", query,
memgraph::storage::PropertyValue(params_pv), interpreter_context_->db->id());
}
#endif
try {
auto result = interpreter_->Prepare(query, params_pv, username, ToQueryExtras(extra), UUID());
const std::string db_name = result.db ? *result.db : "";
if (user_ && !memgraph::glue::AuthChecker::IsUserAuthorized(*user_, result.privileges, db_name)) {
interpreter_->Abort();
if (db_name.empty()) {
throw memgraph::communication::bolt::ClientError(
"You are not authorized to execute this query! Please contact your database administrator.");
}
throw memgraph::communication::bolt::ClientError(
"You are not authorized to execute this query on database \"{}\"! Please contact your database "
"administrator.",
db_name);
}
return {result.headers, result.qid};
} catch (const memgraph::query::QueryException &e) {
// Wrap QueryException into ClientError, because we want to allow the
// client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what());
} catch (const memgraph::query::ReplicationException &e) {
throw memgraph::communication::bolt::ClientError(e.what());
}
}
std::map<std::string, memgraph::communication::bolt::Value> Pull(TEncoder *encoder, std::optional<int> n,
std::optional<int> qid) override {
TypedValueResultStream stream(encoder, interpreter_context_);
return PullResults(stream, n, qid);
}
std::map<std::string, memgraph::communication::bolt::Value> Discard(std::optional<int> n,
std::optional<int> qid) override {
memgraph::query::DiscardValueResultStream stream;
return PullResults(stream, n, qid);
}
void Abort() override { interpreter_->Abort(); }
// Called during Init
// During Init, the user cannot choose the landing DB (switch is done during query execution)
bool Authenticate(const std::string &username, const std::string &password) override {
auto locked_auth = auth_->Lock();
if (!locked_auth->HasUsers()) {
return true;
}
user_ = locked_auth->Authenticate(username, password);
#ifdef MG_ENTERPRISE
if (user_.has_value()) {
const auto &db = user_->db_access().GetDefault();
// Check if the underlying database needs to be updated
if (db != current_.interpreter_context()->db->id()) {
const auto &res = sc_handler_.SetFor(UUID(), db);
return res == memgraph::dbms::SetForResult::SUCCESS || res == memgraph::dbms::SetForResult::ALREADY_SET;
}
}
#endif
return user_.has_value();
}
std::optional<std::string> GetServerNameForInit() override {
if (FLAGS_bolt_server_name_for_init.empty()) return std::nullopt;
return FLAGS_bolt_server_name_for_init;
}
#ifdef MG_ENTERPRISE
memgraph::dbms::SetForResult OnChange(const std::string &db_name) override {
MultiDatabaseAuth(db_name);
if (db_name != current_.interpreter_context()->db->id()) {
UpdateAndDefunct(db_name); // Done during Pull, so we cannot just replace the current db
return memgraph::dbms::SetForResult::SUCCESS;
}
return memgraph::dbms::SetForResult::ALREADY_SET;
}
bool OnDelete(const std::string &db_name) override {
MG_ASSERT(current_.interpreter_context()->db->id() != db_name && (!defunct_ || defunct_->defunct()),
"Trying to delete a database while still in use.");
return true;
}
#endif
std::string GetDatabaseName() const override { return interpreter_context_->db->id(); }
private:
template <typename TStream>
std::map<std::string, memgraph::communication::bolt::Value> PullResults(TStream &stream, std::optional<int> n,
std::optional<int> qid) {
try {
const auto &summary = interpreter_->Pull(&stream, n, qid);
std::map<std::string, memgraph::communication::bolt::Value> decoded_summary;
for (const auto &kv : summary) {
auto maybe_value =
memgraph::glue::ToBoltValue(kv.second, *interpreter_context_->db, memgraph::storage::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:
case memgraph::storage::Error::SERIALIZATION_ERROR:
case memgraph::storage::Error::VERTEX_HAS_EDGES:
case memgraph::storage::Error::PROPERTIES_DISABLED:
case memgraph::storage::Error::NONEXISTENT_OBJECT:
throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming summary.");
}
}
decoded_summary.emplace(kv.first, std::move(*maybe_value));
}
// Add this memgraph instance run_id, received from telemetry
// This is sent with every query, instead of only on bolt init inside
// communication/bolt/v1/states/init.hpp because neo4jdriver does not
// read the init message.
if (auto run_id = run_id_; run_id) {
decoded_summary.emplace("run_id", *run_id);
}
// Clean up previous session (session gets defunct when switching between databases)
if (defunct_) {
defunct_.reset();
}
return decoded_summary;
} catch (const memgraph::query::QueryException &e) {
// Wrap QueryException into ClientError, because we want to allow the
// client to fix their query.
throw memgraph::communication::bolt::ClientError(e.what());
}
}
#ifdef MG_ENTERPRISE
/**
* @brief Update setup to the new database.
*
* @param db_name name of the target database
* @throws UnknownDatabaseException if handler cannot get it
*/
void UpdateAndDefunct(const std::string &db_name) { UpdateAndDefunct(ContextWrapper(sc_handler_.Get(db_name))); }
void UpdateAndDefunct(ContextWrapper &&cntxt) {
defunct_.emplace(std::move(current_));
Update(std::forward<ContextWrapper>(cntxt));
defunct_->Defunct();
}
void Update(const std::string &db_name) {
ContextWrapper tmp(sc_handler_.Get(db_name));
Update(std::move(tmp));
}
void Update(ContextWrapper &&cntxt) {
current_ = std::move(cntxt);
interpreter_ = current_.interp();
interpreter_->in_explicit_db_ = in_explicit_db_;
interpreter_context_ = current_.interpreter_context();
}
/**
* @brief Authenticate user on passed database.
*
* @param db database to check against
* @throws bolt::ClientError when user is not authorized
*/
void MultiDatabaseAuth(const std::string &db) {
if (user_ && !memgraph::glue::AuthChecker::IsUserAuthorized(*user_, {}, db)) {
throw memgraph::communication::bolt::ClientError(
"You are not authorized on the database \"{}\"! Please contact your database administrator.", db);
}
}
/**
* @brief Get the user's default database
*
* @return std::string
*/
std::string GetDefaultDB() {
if (user_.has_value()) {
return user_->db_access().GetDefault();
}
return memgraph::dbms::kDefaultDB;
}
#endif
/// Wrapper around TEncoder which converts TypedValue to Value
/// before forwarding the calls to original TEncoder.
class TypedValueResultStream {
public:
TypedValueResultStream(TEncoder *encoder, memgraph::query::InterpreterContext *ic)
: encoder_(encoder), interpreter_context_(ic) {}
void Result(const std::vector<memgraph::query::TypedValue> &values) {
std::vector<memgraph::communication::bolt::Value> decoded_values;
decoded_values.reserve(values.size());
for (const auto &v : values) {
auto maybe_value = memgraph::glue::ToBoltValue(v, *interpreter_context_->db, memgraph::storage::View::NEW);
if (maybe_value.HasError()) {
switch (maybe_value.GetError()) {
case memgraph::storage::Error::DELETED_OBJECT:
throw memgraph::communication::bolt::ClientError("Returning a deleted object as a result.");
case memgraph::storage::Error::NONEXISTENT_OBJECT:
throw memgraph::communication::bolt::ClientError("Returning a nonexistent object as a result.");
case memgraph::storage::Error::VERTEX_HAS_EDGES:
case memgraph::storage::Error::SERIALIZATION_ERROR:
case memgraph::storage::Error::PROPERTIES_DISABLED:
throw memgraph::communication::bolt::ClientError("Unexpected storage error when streaming results.");
}
}
decoded_values.emplace_back(std::move(*maybe_value));
}
encoder_->MessageRecord(decoded_values);
}
private:
TEncoder *encoder_;
// NOTE: Needed only for ToBoltValue conversions
memgraph::query::InterpreterContext *interpreter_context_;
};
#ifdef MG_ENTERPRISE
memgraph::dbms::SessionContextHandler &sc_handler_;
#endif
ContextWrapper current_;
std::optional<ContextWrapper> defunct_;
memgraph::query::InterpreterContext *interpreter_context_;
memgraph::query::Interpreter *interpreter_;
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth_;
std::optional<memgraph::auth::User> user_;
#ifdef MG_ENTERPRISE
memgraph::audit::Log *audit_log_;
bool in_explicit_db_{false}; //!< If true, the user has defined the database to use via metadata
#endif
memgraph::communication::v2::ServerEndpoint endpoint_;
// NOTE: run_id should be const but that complicates code a lot.
std::optional<std::string> run_id_;
};
#ifdef MG_ENTERPRISE
using ServerT = memgraph::communication::v2::Server<SessionHL, memgraph::dbms::SessionContextHandler>;
#else
using ServerT = memgraph::communication::v2::Server<SessionHL, memgraph::dbms::SessionContext>;
#endif
using MonitoringServerT =
memgraph::communication::http::Server<memgraph::http::MetricsRequestHandler<memgraph::dbms::SessionContext>,
memgraph::dbms::SessionContext>;
using memgraph::communication::ServerContext;
// Needed to correctly handle memgraph destruction from a signal handler.
@@ -925,6 +72,7 @@ using memgraph::communication::ServerContext;
// when we are exiting main, inside destructors of database::GraphDb and
// similar. The signal handler may then initiate another shutdown on memgraph
// which is in half destructed state, causing invalid memory access and crash.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
volatile sig_atomic_t is_shutting_down = 0;
void InitSignalHandlers(const std::function<void()> &shutdown_fun) {
@@ -965,7 +113,7 @@ int main(int argc, char **argv) {
exit(1);
}
InitializeLogger();
memgraph::flags::InitializeLogger();
// Unhandled exception handler init.
std::set_terminate(&memgraph::utils::TerminateHandler);
@@ -1049,7 +197,7 @@ int main(int argc, char **argv) {
auto data_directory = std::filesystem::path(FLAGS_data_directory);
const auto memory_limit = GetMemoryLimit();
const auto memory_limit = memgraph::flags::GetMemoryLimit();
// NOLINTNEXTLINE(bugprone-narrowing-conversions,cppcoreguidelines-narrowing-conversions)
spdlog::info("Memory limit in config is set to {}", memgraph::utils::GetReadableSize(memory_limit));
memgraph::utils::total_memory_tracker.SetMaximumHardLimit(memory_limit);
@@ -1112,7 +260,7 @@ int main(int argc, char **argv) {
.items_per_batch = FLAGS_storage_items_per_batch,
.recovery_thread_count = FLAGS_storage_recovery_thread_count,
.allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery},
.transaction = {.isolation_level = ParseIsolationLevel()},
.transaction = {.isolation_level = memgraph::flags::ParseIsolationLevel()},
.disk = {.main_storage_directory = FLAGS_data_directory + "/rocksdb_main_storage",
.label_index_directory = FLAGS_data_directory + "/rocksdb_label_index",
.label_property_index_directory = FLAGS_data_directory + "/rocksdb_label_property_index",
@@ -1192,7 +340,8 @@ int main(int argc, char **argv) {
auto *auth = session_context.auth;
auto &interpreter_context = *session_context.interpreter_context; // TODO remove
memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(query_modules_directories, FLAGS_data_directory);
memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(memgraph::flags::ParseQueryModulesDirectory(),
FLAGS_data_directory);
memgraph::query::procedure::gModuleRegistry.UnloadAndLoadModulesFromDirectories();
memgraph::query::procedure::gCallableAliasMapper.LoadMapping(FLAGS_query_callable_mappings_path);
@@ -1239,11 +388,11 @@ int main(int argc, char **argv) {
auto server_endpoint = memgraph::communication::v2::ServerEndpoint{
boost::asio::ip::address::from_string(FLAGS_bolt_address), static_cast<uint16_t>(FLAGS_bolt_port)};
#ifdef MG_ENTERPRISE
ServerT server(server_endpoint, &sc_handler, &context, FLAGS_bolt_session_inactivity_timeout, service_name,
FLAGS_bolt_num_workers);
memgraph::glue::ServerT server(server_endpoint, &sc_handler, &context, FLAGS_bolt_session_inactivity_timeout,
service_name, FLAGS_bolt_num_workers);
#else
ServerT server(server_endpoint, &session_context, &context, FLAGS_bolt_session_inactivity_timeout, service_name,
FLAGS_bolt_num_workers);
memgraph::glue::ServerT server(server_endpoint, &session_context, &context, FLAGS_bolt_session_inactivity_timeout,
service_name, FLAGS_bolt_num_workers);
#endif
const auto machine_id = memgraph::utils::GetMachineId();
@@ -1283,9 +432,9 @@ int main(int argc, char **argv) {
memgraph::communication::websocket::SafeAuth websocket_auth{auth};
memgraph::communication::websocket::Server websocket_server{
{FLAGS_monitoring_address, static_cast<uint16_t>(FLAGS_monitoring_port)}, &context, websocket_auth};
AddLoggerSink(websocket_server.GetLoggingSink());
memgraph::flags::AddLoggerSink(websocket_server.GetLoggingSink());
MonitoringServerT metrics_server{
memgraph::glue::MonitoringServerT metrics_server{
{FLAGS_metrics_address, static_cast<uint16_t>(FLAGS_metrics_port)}, &session_context, &context};
#ifdef MG_ENTERPRISE