Procedures for handling modules (#330)

This commit is contained in:
Antonio Andelic
2022-02-11 11:29:41 +01:00
committed by GitHub
parent 5aeaad198b
commit 69eca9b043
33 changed files with 938 additions and 228 deletions

View File

@@ -13,5 +13,6 @@ add_subdirectory(isolation_levels)
add_subdirectory(streams)
add_subdirectory(temporal_types)
add_subdirectory(write_procedures)
add_subdirectory(module_file_manager)
copy_e2e_python_files(pytest_runner pytest_runner.sh "")

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -42,10 +42,11 @@ int main(int argc, char **argv) {
LOG_FATAL("The test timed out");
}
client->Execute(create_query);
if (!client->FetchOne()) {
try {
client->DiscardAll();
} catch (const mg::TransientException & /*unused*/) {
break;
}
client->DiscardAll();
}
spdlog::info("Memgraph is out of memory");

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -10,8 +10,8 @@
// licenses/APL.txt.
#include <gflags/gflags.h>
#include <mgclient.hpp>
#include <algorithm>
#include <mgclient.hpp>
#include "utils/logging.hpp"
#include "utils/timer.hpp"
@@ -31,11 +31,20 @@ int main(int argc, char **argv) {
if (!client) {
LOG_FATAL("Failed to connect!");
}
bool result = client->Execute("CALL libglobal_memory_limit_proc.error() YIELD *");
auto result1 = client->FetchAll();
MG_ASSERT(result1 != std::nullopt && result1->size() == 0);
MG_ASSERT(client->Execute("CALL libglobal_memory_limit_proc.error() YIELD *"));
MG_ASSERT(std::invoke([&] {
try {
auto result1 = client->FetchAll();
} catch (const mg::ClientException &e) {
MG_ASSERT(e.what() == std::string_view{"libglobal_memory_limit_proc.error: Out of memory"},
"Invalid message received");
return true;
}
return false;
}),
"Procedure didn't throw the expected `mg::ClientException`");
result = client->Execute("CALL libglobal_memory_limit_proc.success() YIELD *");
MG_ASSERT(client->Execute("CALL libglobal_memory_limit_proc.success() YIELD *"));
auto result2 = client->FetchAll();
MG_ASSERT(result2 != std::nullopt && result2->size() > 0);
return 0;

View File

@@ -0,0 +1,4 @@
find_package(gflags REQUIRED)
add_executable(memgraph__e2e__module_file_manager module_file_manager.cpp)
target_link_libraries(memgraph__e2e__module_file_manager gflags mgclient mg-utils mg-io Threads::Threads)

View File

@@ -0,0 +1,270 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <filesystem>
#include <fstream>
#include <gflags/gflags.h>
#include <mgclient.hpp>
#include "utils/file.hpp"
#include "utils/logging.hpp"
#include "utils/timer.hpp"
DEFINE_uint64(bolt_port, 7687, "Bolt port");
DEFINE_uint64(timeout, 120, "Timeout seconds");
namespace {
auto GetClient() {
auto client =
mg::Client::Connect({.host = "127.0.0.1", .port = static_cast<uint16_t>(FLAGS_bolt_port), .use_ssl = false});
MG_ASSERT(client, "Failed to connect!");
return client;
}
std::vector<std::filesystem::path> GetModuleFiles(auto &client) {
MG_ASSERT(client->Execute("CALL mg.get_module_files() YIELD path"));
const auto result_rows = client->FetchAll();
MG_ASSERT(result_rows, "Failed to get results");
std::vector<std::filesystem::path> result;
result.reserve(result_rows->size());
for (const auto &row : *result_rows) {
MG_ASSERT(row.size() == 1, "Invalid result received from mg.get_module_files");
result.emplace_back(row[0].ValueString());
}
return result;
}
bool ModuleFileExists(auto &client, const auto &path) {
const auto module_files = GetModuleFiles(client);
return std::any_of(module_files.begin(), module_files.end(),
[&](const auto &module_file) { return module_file == path; });
}
void AssertModuleFileExists(auto &client, const auto &path) {
MG_ASSERT(ModuleFileExists(client, path), "Module file {} is missing", path);
}
void AssertModuleFileNotExists(auto &client, const auto &path) {
MG_ASSERT(!ModuleFileExists(client, path), "Invalid module file {} is present", path);
}
bool ProcedureExists(auto &client, const std::string_view procedure_name,
std::optional<std::filesystem::path> path = std::nullopt) {
MG_ASSERT(client->Execute("CALL mg.procedures() YIELD name, path"));
const auto result_rows = client->FetchAll();
MG_ASSERT(result_rows, "Failed to get results for mg.procedures()");
return std::find_if(result_rows->begin(), result_rows->end(), [&, procedure_name](const auto &row) {
MG_ASSERT(row.size() == 2, "Invalid result received from mg.procedures()");
if (row[0].ValueString() == procedure_name) {
if (path) {
return row[1].ValueString() == std::filesystem::canonical(*path).generic_string();
}
return true;
}
return false;
}) != result_rows->end();
}
void AssertProcedureExists(auto &client, const std::string_view procedure_name,
std::optional<std::filesystem::path> path = std::nullopt) {
MG_ASSERT(ProcedureExists(client, procedure_name, path), "Procedure {} is missing", procedure_name);
}
void AssertProcedureNotExists(auto &client, const std::string_view procedure_name) {
MG_ASSERT(!ProcedureExists(client, procedure_name), "Invalid procedure ('{}') is present", procedure_name);
}
template <typename TException>
void AssertQueryFails(auto &client, const std::string &query, std::optional<std::string> expected_message) {
spdlog::info("Asserting query '{}' fails", query);
MG_ASSERT(client->Execute(query));
try {
client->FetchAll();
} catch (const TException &exception) {
if (expected_message) {
MG_ASSERT(*expected_message == exception.what(),
"Exception with a different message was thrown.\n\t\tExpected: {}\n\t\tActual: {}", *expected_message,
exception.what());
}
return;
}
LOG_FATAL("Didn't receive expected exception");
}
std::string CreateModuleFileQuery(const std::string_view filename, const std::string_view content) {
return fmt::format("CALL mg.create_module_file('{}', '{}') YIELD path", filename, content);
}
std::filesystem::path CreateModuleFile(auto &client, const std::string_view filename, const std::string_view content) {
spdlog::info("Creating module file '{}' with content:\n{}", filename, content);
MG_ASSERT(client->Execute(CreateModuleFileQuery(filename, content)));
const auto result_row = client->FetchOne();
MG_ASSERT(result_row && result_row->size() == 1, "Received invalid result from mg.create_module_file");
MG_ASSERT(!client->FetchOne().has_value(), "Too many results received from mg.create_module_file");
return result_row->at(0).ValueString();
}
std::string GetModuleFileQuery(const std::filesystem::path &path) {
return fmt::format("CALL mg.get_module_file({}) YIELD content", path);
}
std::string GetModuleFile(auto &client, const std::filesystem::path &path) {
spdlog::info("Getting content of module file '{}'", path);
MG_ASSERT(client->Execute(GetModuleFileQuery(path)));
const auto result_row = client->FetchOne();
MG_ASSERT(result_row && result_row->size() == 1, "Received invalid result from mg.get_module_file");
MG_ASSERT(!client->FetchOne().has_value(), "Too many results received from mg.get_module_file");
return std::string{result_row->at(0).ValueString()};
}
std::string UpdateModuleFileQuery(const std::filesystem::path &path, const std::string_view content) {
return fmt::format("CALL mg.update_module_file({}, '{}')", path, content);
}
void UpdateModuleFile(auto &client, const std::filesystem::path &path, const std::string_view content) {
spdlog::info("Updating module file {} with content:\n{}", path, content);
MG_ASSERT(client->Execute(UpdateModuleFileQuery(path, content)));
MG_ASSERT(client->FetchAll().has_value());
}
std::string DeleteModuleFileQuery(const std::filesystem::path &path) {
return fmt::format("CALL mg.delete_module_file({})", path);
}
void DeleteModuleFile(auto &client, const std::filesystem::path &path) {
spdlog::info("Deleting module file {}", path);
MG_ASSERT(client->Execute(DeleteModuleFileQuery(path)));
MG_ASSERT(client->FetchAll().has_value());
}
constexpr std::string_view module_content1 = R"(import mgp
@mgp.read_proc
def simple1(ctx: mgp.ProcCtx) -> mgp.Record(result=bool):
return mgp.Record(mutable=True))";
constexpr std::string_view module_content2 = R"(import mgp
@mgp.read_proc
def simple2(ctx: mgp.ProcCtx) -> mgp.Record(result=bool):
return mgp.Record(mutable=True))";
} // namespace
int main(int argc, char **argv) {
google::SetUsageMessage("Memgraph E2E Isolation Levels");
gflags::ParseCommandLineFlags(&argc, &argv, true);
logging::RedirectToStderr();
mg::Client::Init();
auto client = GetClient();
AssertQueryFails<mg::ClientException>(client, CreateModuleFileQuery("some.cpp", "some content"),
"mg.create_module_file: The specified file isn't in the supported format.");
AssertQueryFails<mg::ClientException>(client, CreateModuleFileQuery("../some.cpp", "some content"),
"mg.create_module_file: Invalid relative path defined. The module file cannot "
"be define outside the internal modules directory.");
AssertProcedureNotExists(client, "some.simple1");
const auto module_path = CreateModuleFile(client, "some.py", module_content1);
AssertQueryFails<mg::ClientException>(client, CreateModuleFileQuery("some.py", "some content"),
"mg.create_module_file: File with the same name already exists!");
AssertProcedureExists(client, "some.simple1", module_path);
AssertModuleFileExists(client, module_path);
MG_ASSERT(GetModuleFile(client, module_path) == module_content1,
"Content received from mg.get_module_file is incorrect");
AssertQueryFails<mg::ClientException>(client, GetModuleFileQuery("some.py"),
"mg.get_module_file: The path should be an absolute path.");
AssertQueryFails<mg::ClientException>(client, GetModuleFileQuery(module_path.parent_path() / "some.cpp"),
"mg.get_module_file: The specified file isn't in the supported format.");
AssertQueryFails<mg::ClientException>(client, GetModuleFileQuery(module_path.parent_path() / "some2.py"),
"mg.get_module_file: The specified file doesn't exist.");
AssertQueryFails<mg::ClientException>(client, UpdateModuleFileQuery("some.py", "some content"),
"mg.update_module_file: The path should be an absolute path.");
AssertQueryFails<mg::ClientException>(client,
UpdateModuleFileQuery(module_path.parent_path() / "some.cpp", "some content"),
"mg.update_module_file: The specified file isn't in the supported format.");
AssertQueryFails<mg::ClientException>(client,
UpdateModuleFileQuery(module_path.parent_path() / "some2.py", "some content"),
"mg.update_module_file: The specified file doesn't exist.");
UpdateModuleFile(client, module_path, module_content2);
AssertProcedureNotExists(client, "some.simple1");
AssertProcedureExists(client, "some.simple2", module_path);
AssertModuleFileExists(client, module_path);
MG_ASSERT(GetModuleFile(client, module_path) == module_content2,
"Content received from mg.get_module_file is incorrect");
AssertQueryFails<mg::ClientException>(client, DeleteModuleFileQuery("some.py"),
"mg.delete_module_file: The path should be an absolute path.");
AssertQueryFails<mg::ClientException>(client, DeleteModuleFileQuery(module_path.parent_path() / "some.cpp"),
"mg.delete_module_file: The specified file isn't in the supported format.");
AssertQueryFails<mg::ClientException>(client, DeleteModuleFileQuery(module_path.parent_path() / "some2.py"),
"mg.delete_module_file: The specified file doesn't exist.");
DeleteModuleFile(client, module_path);
AssertProcedureNotExists(client, "some.simple1");
AssertProcedureNotExists(client, "some.simple2");
AssertModuleFileNotExists(client, module_path);
const auto non_module_directory =
std::filesystem::temp_directory_path() / "module_file_manager_e2e_non_module_directory";
utils::EnsureDirOrDie(non_module_directory);
const auto non_module_file_path{non_module_directory / "something.py"};
{
std::ofstream non_module_file{non_module_file_path};
MG_ASSERT(non_module_file.is_open(), "Failed to open {} for writing", non_module_file_path);
constexpr std::string_view content = "import mgp";
non_module_file.write(content.data(), content.size());
non_module_file.flush();
}
AssertQueryFails<mg::ClientException>(
client, GetModuleFileQuery(non_module_file_path),
"mg.get_module_file: The specified file isn't contained in any of the module directories.");
AssertQueryFails<mg::ClientException>(
client, UpdateModuleFileQuery(non_module_file_path, "some content"),
"mg.update_module_file: The specified file isn't contained in any of the module directories.");
AssertQueryFails<mg::ClientException>(
client, DeleteModuleFileQuery(non_module_file_path),
"mg.delete_module_file: The specified file isn't contained in any of the module directories.");
MG_ASSERT(std::filesystem::remove_all(non_module_directory), "Failed to cleanup directories");
return 0;
}

View File

@@ -0,0 +1,14 @@
bolt_port: &bolt_port "7687"
template_cluster: &template_cluster
cluster:
main:
args: ["--bolt-port", *bolt_port, "--log-level=TRACE"]
log_file: "module-file-manager-e2e.log"
setup_queries: []
validation_queries: []
workloads:
- name: "Module File Manager"
binary: "tests/e2e/module_file_manager/memgraph__e2e__module_file_manager"
args: ["--bolt-port", *bolt_port]
<<: *template_cluster

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -101,14 +101,17 @@ void DropOnDeleteTriggers(mg::Client &client, const std::unordered_set<AllowedTr
case AllowedTriggerType::VERTEX: {
client.Execute("DROP TRIGGER DeletedVerticesTrigger");
client.DiscardAll();
break;
}
case AllowedTriggerType::EDGE: {
client.Execute("DROP TRIGGER DeletedEdgesTrigger");
client.DiscardAll();
break;
}
case AllowedTriggerType::OBJECT: {
client.Execute("DROP TRIGGER DeletedObjectsTrigger");
client.DiscardAll();
break;
}
}
}

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -174,7 +174,7 @@ void DropOnUpdateTriggers(mg::Client &client) {
client.DiscardAll();
client.Execute("DROP TRIGGER SetVertexPropertiesTrigger");
client.DiscardAll();
client.Execute("DROP TRIGGER RemoveVertexPropertiesTrigger");
client.Execute("DROP TRIGGER RemovedVertexPropertiesTrigger");
client.DiscardAll();
client.Execute("DROP TRIGGER SetVertexLabelsTrigger");
client.DiscardAll();

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,6 +20,16 @@
constexpr std::string_view kTriggerPrefix{"CreatedVerticesTrigger"};
template <typename TException>
bool FunctionThrows(const auto &function) {
try {
function();
} catch (const TException & /*unused*/) {
return true;
}
return false;
}
int main(int argc, char **argv) {
gflags::SetUsageMessage("Memgraph E2E Triggers privilege check");
gflags::ParseCommandLineFlags(&argc, &argv, true);
@@ -52,7 +62,10 @@ int main(int argc, char **argv) {
"UNWIND createdVertices as createdVertex "
"CREATE (n: {} {{ id: createdVertex.id }})",
kTriggerPrefix, vertexLabel, vertexLabel));
client.DiscardAll();
const bool succeeded = !FunctionThrows<mg::TransientException>([&] { client.DiscardAll(); });
MG_ASSERT(succeeded == should_succeed, "Unexpected outcome from creating triggers: expected {}, actual {}",
should_succeed, succeeded);
const auto number_of_triggers_after = get_number_of_triggers();
if (should_succeed) {
MG_ASSERT(number_of_triggers_after == number_of_triggers_before + 1);
@@ -162,10 +175,12 @@ int main(int argc, char **argv) {
"CREATE (n: {} {{ id: createdVertex.id }})",
kTriggerPrefix, kUserWithoutCreate, kUserWithoutCreate));
client_without_create->DiscardAll();
userless_client->Execute(fmt::format("REVOKE CREATE FROM {};", kUserWithoutCreate));
userless_client->DiscardAll();
CreateVertex(*userless_client, kVertexId);
MG_ASSERT(FunctionThrows<mg::TransientException>([&] { CreateVertex(*userless_client, kVertexId); }),
"Create should have thrown because user doesn't have privilege for CREATE");
CheckNumberOfAllVertices(*userless_client, 0);
return 0;

View File

@@ -211,7 +211,7 @@ class MockModule : public procedure::Module {
const std::map<std::string, mgp_proc, std::less<>> *Procedures() const override { return &procedures; }
const std::map<std::string, mgp_trans, std::less<>> *Transformations() const override { return &transformations; }
std::optional<std::filesystem::path> Path() const override { return std::nullopt; };
std::optional<std::filesystem::path> Path() const override { return std::nullopt; }
std::map<std::string, mgp_proc, std::less<>> procedures{};
std::map<std::string, mgp_trans, std::less<>> transformations{};
@@ -248,7 +248,7 @@ class CypherMainVisitorTest : public ::testing::TestWithParam<std::shared_ptr<Ba
const std::vector<std::string_view> &results, const ProcedureType type) {
utils::MemoryResource *memory = utils::NewDeleteResource();
const bool is_write = type == ProcedureType::WRITE;
mgp_proc proc(name, DummyProcCallback, memory, is_write);
mgp_proc proc(name, DummyProcCallback, memory, {.is_write = is_write});
for (const auto arg : args) {
proc.args.emplace_back(utils::pmr::string{arg, memory}, &any_type);
}

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -451,6 +451,14 @@ auto GetMerge(AstStorage &storage, Pattern *pattern, OnMatch on_match, OnCreate
return merge;
}
auto GetCallProcedure(AstStorage &storage, std::string procedure_name,
std::vector<query::Expression *> arguments = {}) {
auto *call_procedure = storage.Create<query::CallProcedure>();
call_procedure->procedure_name_ = std::move(procedure_name);
call_procedure->arguments_ = std::move(arguments);
return call_procedure;
}
} // namespace test_common
} // namespace query
@@ -558,3 +566,4 @@ auto GetMerge(AstStorage &storage, Pattern *pattern, OnMatch on_match, OnCreate
#define AUTH_QUERY(action, user, role, user_or_role, password, privileges) \
storage.Create<query::AuthQuery>((action), (user), (role), (user_or_role), password, (privileges))
#define DROP_USER(usernames) storage.Create<query::DropUser>((usernames))
#define CALL_PROCEDURE(...) query::test_common::GetCallProcedure(storage, __VA_ARGS__)

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -125,12 +125,12 @@ TEST(Module, ProcedureSignatureOnlyOptArg) {
TEST(Module, ReadWriteProcedures) {
mgp_module module(utils::NewDeleteResource());
auto *read_proc = EXPECT_MGP_NO_ERROR(mgp_proc *, mgp_module_add_read_procedure, &module, "read", &DummyCallback);
EXPECT_FALSE(read_proc->is_write_procedure);
EXPECT_FALSE(read_proc->info.is_write);
auto *write_proc = EXPECT_MGP_NO_ERROR(mgp_proc *, mgp_module_add_write_procedure, &module, "write", &DummyCallback);
EXPECT_TRUE(write_proc->is_write_procedure);
EXPECT_TRUE(write_proc->info.is_write);
mgp_proc read_proc_with_function{"dummy_name",
std::function<void(mgp_list *, mgp_graph *, mgp_result *, mgp_memory *)>{
[](mgp_list *, mgp_graph *, mgp_result *, mgp_memory *) {}},
utils::NewDeleteResource(), false};
EXPECT_FALSE(read_proc_with_function.is_write_procedure);
utils::NewDeleteResource()};
EXPECT_FALSE(read_proc_with_function.info.is_write);
}

View File

@@ -191,3 +191,27 @@ TEST_F(TestPrivilegeExtractor, ShowVersion) {
auto *query = storage.Create<VersionQuery>();
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::STATS));
}
TEST_F(TestPrivilegeExtractor, CallProcedureQuery) {
{
auto *query = QUERY(SINGLE_QUERY(CALL_PROCEDURE("mg.get_module_files")));
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::MODULE_READ));
}
{
auto *query = QUERY(SINGLE_QUERY(CALL_PROCEDURE("mg.create_module_file", {LITERAL("some_name.py")})));
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::MODULE_WRITE));
}
{
auto *query = QUERY(
SINGLE_QUERY(CALL_PROCEDURE("mg.update_module_file", {LITERAL("some_name.py"), LITERAL("some content")})));
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::MODULE_WRITE));
}
{
auto *query = QUERY(SINGLE_QUERY(CALL_PROCEDURE("mg.get_module_file", {LITERAL("some_name.py")})));
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::MODULE_READ));
}
{
auto *query = QUERY(SINGLE_QUERY(CALL_PROCEDURE("mg.delete_module_file", {LITERAL("some_name.py")})));
EXPECT_THAT(GetRequiredPrivileges(query), UnorderedElementsAre(AuthQuery::Privilege::MODULE_WRITE));
}
}