From f63a5e9de5d142c0aed65937c1fc7fec81be5d4e Mon Sep 17 00:00:00 2001 From: Teon Banek Date: Wed, 25 Sep 2019 15:03:39 +0200 Subject: [PATCH] Rename PluginRegistry to ModuleRegistry Summary: The registry is now in the `query::procedure` namespace, this makes the naming more consistent. I.e. we are dealing with custom procedure which are contained in modules. This naming convention is similar to Python source code where each file represents a module and each module provides multiple functions (or procedures in our case). At the moment we only support exactly 1 procedure per module, but the openCypher syntax allows for more. Reviewers: mferencevic, ipaljak, dsantl Differential Revision: https://phabricator.memgraph.io/D2454 --- src/CMakeLists.txt | 4 +- src/query/plugin/plugin.cpp | 142 --------------------------------- src/query/plugin/plugin.hpp | 89 --------------------- src/query/procedure/module.cpp | 142 +++++++++++++++++++++++++++++++++ src/query/procedure/module.hpp | 89 +++++++++++++++++++++ 5 files changed, 233 insertions(+), 233 deletions(-) delete mode 100644 src/query/plugin/plugin.cpp delete mode 100644 src/query/plugin/plugin.hpp create mode 100644 src/query/procedure/module.cpp create mode 100644 src/query/procedure/module.hpp diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 875c3def0..6bf666887 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -63,7 +63,7 @@ set(mg_single_node_sources query/plan/rewrite/index_lookup.cpp query/plan/rule_based_planner.cpp query/plan/variable_start_planner.cpp - query/plugin/plugin.cpp + query/procedure/module.cpp query/repl.cpp query/typed_value.cpp storage/common/constraints/record.cpp @@ -134,7 +134,7 @@ set(mg_single_node_v2_sources query/plan/rewrite/index_lookup.cpp query/plan/rule_based_planner.cpp query/plan/variable_start_planner.cpp - query/plugin/plugin.cpp + query/procedure/module.cpp query/typed_value.cpp memgraph_init.cpp ) diff --git a/src/query/plugin/plugin.cpp b/src/query/plugin/plugin.cpp deleted file mode 100644 index 0462012f4..000000000 --- a/src/query/plugin/plugin.cpp +++ /dev/null @@ -1,142 +0,0 @@ -#include "query/plugin/plugin.hpp" - -extern "C" { -#include -} - -#include - -namespace query::plugin { - -PluginRegistry gPluginRegistry; - -namespace { - -std::optional LoadPluginFromSharedLibrary(std::filesystem::path path) { - LOG(INFO) << "Loading plugin " << path << " ..."; - Plugin plugin{path}; - dlerror(); // Clear any existing error. - plugin.handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); - if (!plugin.handle) { - LOG(ERROR) << "Unable to load plugin " << path << "; " << dlerror(); - return std::nullopt; - } - // Get required mg_main - plugin.main_fn = - reinterpret_cast(dlsym(plugin.handle, "mg_main")); - const char *error = dlerror(); - if (!plugin.main_fn || error) { - LOG(ERROR) << "Unable to load plugin " << path << "; " << error; - dlclose(plugin.handle); - return std::nullopt; - } - // Get optional mg_init_module - plugin.init_fn = - reinterpret_cast(dlsym(plugin.handle, "mg_init_module")); - error = dlerror(); - if (error) LOG(WARNING) << "When loading plugin " << path << "; " << error; - // Run mg_init_module which must succeed. - if (plugin.init_fn) { - int init_res = plugin.init_fn(); - if (init_res != 0) { - LOG(ERROR) << "Unable to load plugin " << path - << "; mg_init_module returned " << init_res; - dlclose(plugin.handle); - return std::nullopt; - } - } - // Get optional mg_shutdown_module - plugin.shutdown_fn = - reinterpret_cast(dlsym(plugin.handle, "mg_shutdown_module")); - error = dlerror(); - if (error) LOG(WARNING) << "When loading plugin " << path << "; " << error; - LOG(INFO) << "Loaded plugin " << path; - return plugin; -} - -bool ClosePlugin(Plugin *plugin) { - LOG(INFO) << "Closing plugin " << plugin->file_path << " ..."; - if (plugin->shutdown_fn) { - int shutdown_res = plugin->shutdown_fn(); - if (shutdown_res != 0) { - LOG(WARNING) << "When closing plugin " << plugin->file_path - << "; mg_shutdown_module returned " << shutdown_res; - } - } - if (dlclose(plugin->handle) != 0) { - LOG(ERROR) << "Failed to close plugin " << plugin->file_path << "; " - << dlerror(); - return false; - } - LOG(INFO) << "Closed plugin " << plugin->file_path; - return true; -} - -} // namespace - -bool PluginRegistry::LoadPluginLibrary(std::filesystem::path path) { - std::unique_lock guard(lock_); - std::string plugin_name(path.stem()); - if (plugins_.find(plugin_name) != plugins_.end()) return true; - auto maybe_plugin = LoadPluginFromSharedLibrary(path); - if (!maybe_plugin) return false; - plugins_[plugin_name] = std::move(*maybe_plugin); - return true; -} - -PluginPtr PluginRegistry::GetPluginNamed(const std::string_view &name) { - std::shared_lock guard(lock_); - // NOTE: std::unordered_map::find cannot work with std::string_view :( - auto found_it = plugins_.find(std::string(name)); - if (found_it == plugins_.end()) return nullptr; - return PluginPtr(&found_it->second, std::move(guard)); -} - -bool PluginRegistry::ReloadPluginNamed(const std::string_view &name) { - std::unique_lock guard(lock_); - // NOTE: std::unordered_map::find cannot work with std::string_view :( - auto found_it = plugins_.find(std::string(name)); - if (found_it == plugins_.end()) { - LOG(ERROR) << "Trying to reload plugin '" << name - << "' which is not loaded."; - return false; - } - auto &plugin = found_it->second; - if (!ClosePlugin(&plugin)) { - plugins_.erase(found_it); - return false; - } - auto maybe_plugin = LoadPluginFromSharedLibrary(plugin.file_path); - if (!maybe_plugin) { - plugins_.erase(found_it); - return false; - } - plugin = std::move(*maybe_plugin); - return true; -} - -bool PluginRegistry::ReloadAllPlugins() { - std::unique_lock guard(lock_); - for (auto &[name, plugin] : plugins_) { - if (!ClosePlugin(&plugin)) { - plugins_.erase(name); - return false; - } - auto maybe_plugin = LoadPluginFromSharedLibrary(plugin.file_path); - if (!maybe_plugin) { - plugins_.erase(name); - return false; - } - plugin = std::move(*maybe_plugin); - } - return true; -} - -void PluginRegistry::UnloadAllPlugins() { - std::unique_lock guard(lock_); - for (auto &name_and_plugin : plugins_) ClosePlugin(&name_and_plugin.second); - plugins_.clear(); -} - -} // namespace query::plugin diff --git a/src/query/plugin/plugin.hpp b/src/query/plugin/plugin.hpp deleted file mode 100644 index 5e4643846..000000000 --- a/src/query/plugin/plugin.hpp +++ /dev/null @@ -1,89 +0,0 @@ -/// @file API for loading and registering plugins providing custom oC procedures -#pragma once - -#include -#include -#include -#include -#include -#include - -#include "utils/rw_lock.hpp" - -struct mg_value; -struct mg_graph; -struct mg_result; - -namespace query::plugin { - -struct Plugin final { - /// Path as requested for loading the plugin from a library. - std::filesystem::path file_path; - /// System handle to shared library. - void *handle; - /// Entry-point for plugin's custom procedure. - std::function - main_fn; - /// Optional initialization function called on plugin load. - std::function init_fn; - /// Optional shutdown function called on plugin unload. - std::function shutdown_fn; -}; - - -/// Proxy for a registered Plugin, acquires a read lock from PluginRegistry. -class PluginPtr final { - const Plugin *plugin_{nullptr}; - std::shared_lock lock_; - - public: - PluginPtr() = default; - PluginPtr(std::nullptr_t) {} - PluginPtr(const Plugin *plugin, std::shared_lock lock) - : plugin_(plugin), lock_(std::move(lock)) {} - - explicit operator bool() const { return static_cast(plugin_); } - - const Plugin &operator*() const { return *plugin_; } - const Plugin *operator->() const { return plugin_; } -}; - -/// Thread-safe registration of plugins from libraries, uses utils::RWLock. -class PluginRegistry final { - std::unordered_map plugins_; - utils::RWLock lock_{utils::RWLock::Priority::WRITE}; - - public: - /// Load a plugin from the given path and return true if successful. - /// - /// A write lock is taken during the execution of this method. Loading a - /// plugin is done through `dlopen` facility and path is resolved accordingly. - /// The plugin is registered using the filename part of the path, with the - /// extension removed. If a plugin with the same name already exists, the - /// function does nothing. - bool LoadPluginLibrary(std::filesystem::path path); - - /// Find a plugin with given name or return nullptr. - /// Takes a read lock. - PluginPtr GetPluginNamed(const std::string_view &name); - - /// Reload a plugin with given name and return true if successful. - /// Takes a write lock. If false was returned, then the plugin is no longer - /// registered. - bool ReloadPluginNamed(const std::string_view &name); - - /// Reload all loaded plugins and return true if successful. - /// Takes a write lock. If false was returned, the plugin which failed to - /// reload is no longer registered. Remaining plugins may or may not be - /// reloaded, but are valid and registered. - bool ReloadAllPlugins(); - - /// Remove all loaded plugins. - /// Takes a write lock. - void UnloadAllPlugins(); -}; - -/// Single, global plugin registry. -extern PluginRegistry gPluginRegistry; - -} // namespace query::plugin diff --git a/src/query/procedure/module.cpp b/src/query/procedure/module.cpp new file mode 100644 index 000000000..69582a22e --- /dev/null +++ b/src/query/procedure/module.cpp @@ -0,0 +1,142 @@ +#include "query/procedure/module.hpp" + +extern "C" { +#include +} + +#include + +namespace query::procedure { + +ModuleRegistry gModuleRegistry; + +namespace { + +std::optional LoadModuleFromSharedLibrary(std::filesystem::path path) { + LOG(INFO) << "Loading module " << path << " ..."; + Module module{path}; + dlerror(); // Clear any existing error. + module.handle = dlopen(path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (!module.handle) { + LOG(ERROR) << "Unable to load module " << path << "; " << dlerror(); + return std::nullopt; + } + // Get required mgp_main + module.main_fn = reinterpret_cast( + dlsym(module.handle, "mgp_main")); + const char *error = dlerror(); + if (!module.main_fn || error) { + LOG(ERROR) << "Unable to load module " << path << "; " << error; + dlclose(module.handle); + return std::nullopt; + } + // Get optional mgp_init_module + module.init_fn = + reinterpret_cast(dlsym(module.handle, "mgp_init_module")); + error = dlerror(); + if (error) LOG(WARNING) << "When loading module " << path << "; " << error; + // Run mgp_init_module which must succeed. + if (module.init_fn) { + int init_res = module.init_fn(); + if (init_res != 0) { + LOG(ERROR) << "Unable to load module " << path + << "; mgp_init_module returned " << init_res; + dlclose(module.handle); + return std::nullopt; + } + } + // Get optional mgp_shutdown_module + module.shutdown_fn = + reinterpret_cast(dlsym(module.handle, "mgp_shutdown_module")); + error = dlerror(); + if (error) LOG(WARNING) << "When loading module " << path << "; " << error; + LOG(INFO) << "Loaded module " << path; + return module; +} + +bool CloseModule(Module *module) { + LOG(INFO) << "Closing module " << module->file_path << " ..."; + if (module->shutdown_fn) { + int shutdown_res = module->shutdown_fn(); + if (shutdown_res != 0) { + LOG(WARNING) << "When closing module " << module->file_path + << "; mgp_shutdown_module returned " << shutdown_res; + } + } + if (dlclose(module->handle) != 0) { + LOG(ERROR) << "Failed to close module " << module->file_path << "; " + << dlerror(); + return false; + } + LOG(INFO) << "Closed module " << module->file_path; + return true; +} + +} // namespace + +bool ModuleRegistry::LoadModuleLibrary(std::filesystem::path path) { + std::unique_lock guard(lock_); + std::string module_name(path.stem()); + if (modules_.find(module_name) != modules_.end()) return true; + auto maybe_module = LoadModuleFromSharedLibrary(path); + if (!maybe_module) return false; + modules_[module_name] = std::move(*maybe_module); + return true; +} + +ModulePtr ModuleRegistry::GetModuleNamed(const std::string_view &name) { + std::shared_lock guard(lock_); + // NOTE: std::unordered_map::find cannot work with std::string_view :( + auto found_it = modules_.find(std::string(name)); + if (found_it == modules_.end()) return nullptr; + return ModulePtr(&found_it->second, std::move(guard)); +} + +bool ModuleRegistry::ReloadModuleNamed(const std::string_view &name) { + std::unique_lock guard(lock_); + // NOTE: std::unordered_map::find cannot work with std::string_view :( + auto found_it = modules_.find(std::string(name)); + if (found_it == modules_.end()) { + LOG(ERROR) << "Trying to reload module '" << name + << "' which is not loaded."; + return false; + } + auto &module = found_it->second; + if (!CloseModule(&module)) { + modules_.erase(found_it); + return false; + } + auto maybe_module = LoadModuleFromSharedLibrary(module.file_path); + if (!maybe_module) { + modules_.erase(found_it); + return false; + } + module = std::move(*maybe_module); + return true; +} + +bool ModuleRegistry::ReloadAllModules() { + std::unique_lock guard(lock_); + for (auto &[name, module] : modules_) { + if (!CloseModule(&module)) { + modules_.erase(name); + return false; + } + auto maybe_module = LoadModuleFromSharedLibrary(module.file_path); + if (!maybe_module) { + modules_.erase(name); + return false; + } + module = std::move(*maybe_module); + } + return true; +} + +void ModuleRegistry::UnloadAllModules() { + std::unique_lock guard(lock_); + for (auto &name_and_module : modules_) CloseModule(&name_and_module.second); + modules_.clear(); +} + +} // namespace query::procedure diff --git a/src/query/procedure/module.hpp b/src/query/procedure/module.hpp new file mode 100644 index 000000000..82cd51da5 --- /dev/null +++ b/src/query/procedure/module.hpp @@ -0,0 +1,89 @@ +/// @file +/// API for loading and registering modules providing custom oC procedures +#pragma once + +#include +#include +#include +#include +#include +#include + +#include "utils/rw_lock.hpp" + +struct mgp_graph; +struct mgp_list; +struct mgp_result; + +namespace query::procedure { + +struct Module final { + /// Path as requested for loading the module from a library. + std::filesystem::path file_path; + /// System handle to shared library. + void *handle; + /// Entry-point for module's custom procedure. + std::function + main_fn; + /// Optional initialization function called on module load. + std::function init_fn; + /// Optional shutdown function called on module unload. + std::function shutdown_fn; +}; + +/// Proxy for a registered Module, acquires a read lock from ModuleRegistry. +class ModulePtr final { + const Module *module_{nullptr}; + std::shared_lock lock_; + + public: + ModulePtr() = default; + ModulePtr(std::nullptr_t) {} + ModulePtr(const Module *module, std::shared_lock lock) + : module_(module), lock_(std::move(lock)) {} + + explicit operator bool() const { return static_cast(module_); } + + const Module &operator*() const { return *module_; } + const Module *operator->() const { return module_; } +}; + +/// Thread-safe registration of modules from libraries, uses utils::RWLock. +class ModuleRegistry final { + std::unordered_map modules_; + utils::RWLock lock_{utils::RWLock::Priority::WRITE}; + + public: + /// Load a module from the given path and return true if successful. + /// + /// A write lock is taken during the execution of this method. Loading a + /// module is done through `dlopen` facility and path is resolved accordingly. + /// The module is registered using the filename part of the path, with the + /// extension removed. If a module with the same name already exists, the + /// function does nothing. + bool LoadModuleLibrary(std::filesystem::path path); + + /// Find a module with given name or return nullptr. + /// Takes a read lock. + ModulePtr GetModuleNamed(const std::string_view &name); + + /// Reload a module with given name and return true if successful. + /// Takes a write lock. If false was returned, then the module is no longer + /// registered. + bool ReloadModuleNamed(const std::string_view &name); + + /// Reload all loaded modules and return true if successful. + /// Takes a write lock. If false was returned, the module which failed to + /// reload is no longer registered. Remaining modules may or may not be + /// reloaded, but are valid and registered. + bool ReloadAllModules(); + + /// Remove all loaded modules. + /// Takes a write lock. + void UnloadAllModules(); +}; + +/// Single, global module registry. +extern ModuleRegistry gModuleRegistry; + +} // namespace query::procedure