Compare commits

..

19 Commits

Author SHA1 Message Date
Andreja Tonev
ed29168723 Bugfix: Using reference in a callback 2024-02-26 19:43:07 +01:00
andrejtonev
c2e9df309a Correctly call driver v1 tests (#1630) 2024-02-26 17:28:13 +00:00
andrejtonev
82c47ee80d GetInfo simplification (#1621)
* Removed force dir in the GetInfo functions
2024-02-26 14:55:45 +00:00
andrejtonev
6a4ef55e90 Better auth user/role handling (#1699)
* Stop auth module from creating users
* Explicit about auth policy (check if no users defined OR auth module used)
* Role supports database access definition
* Authenticate() returns user or role
* AuthChecker generates QueryUserOrRole (can be empty)
* QueryUserOrRole actually authorizes
* Add auth cache invalidation
* Better database access queries (GRANT, DENY, REVOKE DATABASE)
2024-02-22 14:00:39 +00:00
Marko Budiselić
98727e0fa0 Update operating systems (#1371) 2024-02-22 11:14:48 +01:00
Aidar Samerkhanov
9a20ac494d In BFS expansion filter by path we should shrink path to restore state prior to expansion only if the path was changed. (#1745) 2024-02-22 05:34:08 +00:00
Marko Barišić
e302be98a2 Push successful RC builds to S3 (#1741)
* Add new workflow which calls release build workflows

* Make the workflow build packages only on RC tags

* Change artifact names to include OS name
2024-02-21 17:08:14 +01:00
Marko Budiselić
61b9bb0f59 Add toolchain-v5 compatibility Revert to C++20 (#587)
* Upgrade cppitertools, spdlog, fmt, rapidcheck
* Make compilation work on both v4 and v5 toolchains
2024-02-19 21:09:54 +01:00
Andi
7ec648b4ce Add --experimental-enabled=high-availability (#1720) 2024-02-19 16:28:15 +00:00
Marko Budiselić
f098a9d5e3 Patch NuRaft for clang-17 compilation (#1733) 2024-02-19 14:50:37 +01:00
Josipmrden
bae3e8a6d3 Add function for property sizes (#1557)
Add function for property sizes
2024-02-19 13:56:01 +01:00
Andi
f3574012c5 Add cpp23 support (#1726) 2024-02-19 10:36:51 +00:00
Gareth Andrew Lloyd
33c400fcc1 Fixup memory e2e tests (#1715)
- Remove the e2e that did concurrent mgp_* calls on the same transaction
  (ATM this is unsupported)
- Fix up the concurrent mgp_global_alloc test to be testing it more precisely
- Reduce the memory limit on detach delete test due to recent memory
  optimizations around deltas.
- No longer throw from hook, through jemalloc C, to our C++ on other
  side. This cause mutex unlocks to not happen.
- No longer allocate error messages while inside the hook. This caused
  recursive entry back inside jamalloc which would try to relock a
  non-recursive mutex.
2024-02-16 15:35:08 +00:00
Marko Budiselić
5ac938a6c9 Remove default assignees from issue-bug template (#1730) 2024-02-16 14:41:53 +01:00
Andi
3e3224f0a2 Forbid having multiple mains in the cluster (#1727) 2024-02-16 11:41:15 +00:00
Antonio Filipovic
bfc756c092 HA: Polish flow for replicas from coordinator (#1711) 2024-02-16 10:58:01 +01:00
Marko Barišić
5f2e3f01d0 Turn e2e tests back on for release build workflows (#1725) 2024-02-15 16:20:04 +01:00
Marko Barišić
2c774ff09b Add rules for rc workflows (#1722) 2024-02-15 15:33:14 +01:00
Andi
20b47845f0 Forbid writing to cluster-managed main on restart (#1717) 2024-02-15 14:07:04 +01:00
17 changed files with 84 additions and 79 deletions

View File

@@ -3,7 +3,6 @@ name: Bug report
about: Create a report to help us improve
title: ""
labels: bug
assignees: gitbuda
---
**Memgraph version**

View File

@@ -64,11 +64,11 @@ option(MG_ENTERPRISE "Build Memgraph Enterprise Edition" ON)
# Set the current version here to override the automatic version detection. The
# version must be specified as `X.Y.Z`. Primarily used when building new patch
# versions.
set(MEMGRAPH_OVERRIDE_VERSION "2.15.0")
set(MEMGRAPH_OVERRIDE_VERSION "")
# Custom suffix that this version should have. The suffix can be any arbitrary
# string. Primarily used when building a version for a specific customer.
set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "rc4")
set(MEMGRAPH_OVERRIDE_VERSION_SUFFIX "")
# Variables used to generate the versions.
if (MG_ENTERPRISE)

View File

@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
3. using the Licensed Work to create a work or solution
which competes (or might reasonably be expected to
compete) with the Licensed Work.
CHANGE DATE: 2028-28-02
CHANGE DATE: 2028-21-01
CHANGE LICENSE: Apache License, Version 2.0
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.

View File

@@ -110,9 +110,9 @@ class Database {
* @param force_directory Use the configured directory, do not try to decipher the multi-db version
* @return DatabaseInfo
*/
DatabaseInfo GetInfo(bool force_directory, replication_coordination_glue::ReplicationRole replication_role) const {
DatabaseInfo GetInfo(replication_coordination_glue::ReplicationRole replication_role) const {
DatabaseInfo info;
info.storage_info = storage_->GetInfo(force_directory, replication_role);
info.storage_info = storage_->GetInfo(replication_role);
info.triggers = trigger_store_.GetTriggerInfo().size();
info.streams = streams_.GetStreamInfo().size();
return info;

View File

@@ -302,7 +302,7 @@ class DbmsHandler {
auto db_acc_opt = db_gk.access();
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
const auto &info = db_acc->GetInfo(false, replication_role);
const auto &info = db_acc->GetInfo(replication_role);
const auto &storage_info = info.storage_info;
stats.num_vertex += storage_info.vertex_count;
stats.num_edges += storage_info.edge_count;
@@ -338,7 +338,7 @@ class DbmsHandler {
auto db_acc_opt = db_gk.access();
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
res.push_back(db_acc->GetInfo(false, replication_role));
res.push_back(db_acc->GetInfo(replication_role));
}
}
return res;

View File

@@ -825,7 +825,7 @@ uint64_t DiskStorage::GetDiskSpaceUsage() const {
durability_disk_storage_size;
}
StorageInfo DiskStorage::GetBaseInfo(bool /* unused */) {
StorageInfo DiskStorage::GetBaseInfo() {
StorageInfo info{};
info.vertex_count = vertex_count_;
info.edge_count = edge_count_.load(std::memory_order_acquire);
@@ -838,9 +838,8 @@ StorageInfo DiskStorage::GetBaseInfo(bool /* unused */) {
return info;
}
StorageInfo DiskStorage::GetInfo(bool force_dir,
memgraph::replication_coordination_glue::ReplicationRole replication_role) {
StorageInfo info = GetBaseInfo(force_dir);
StorageInfo DiskStorage::GetInfo(memgraph::replication_coordination_glue::ReplicationRole replication_role) {
StorageInfo info = GetBaseInfo();
{
auto access = Access(replication_role);
const auto &lbl = access->ListAllIndices();

View File

@@ -307,9 +307,8 @@ class DiskStorage final : public Storage {
std::vector<std::pair<std::string, std::string>> SerializeVerticesForLabelPropertyIndex(LabelId label,
PropertyId property);
StorageInfo GetBaseInfo(bool force_directory) override;
StorageInfo GetInfo(bool force_directory,
memgraph::replication_coordination_glue::ReplicationRole replication_role) override;
StorageInfo GetBaseInfo() override;
StorageInfo GetInfo(memgraph::replication_coordination_glue::ReplicationRole replication_role) override;
void FreeMemory(std::unique_lock<utils::ResourceLock> /*lock*/) override {}

View File

@@ -11,6 +11,7 @@
#include "storage/v2/inmemory/storage.hpp"
#include <algorithm>
#include <filesystem>
#include <functional>
#include <optional>
#include "dbms/constants.hpp"
@@ -1758,7 +1759,7 @@ void InMemoryStorage::CollectGarbage(std::unique_lock<utils::ResourceLock> main_
template void InMemoryStorage::CollectGarbage<true>(std::unique_lock<utils::ResourceLock>);
template void InMemoryStorage::CollectGarbage<false>(std::unique_lock<utils::ResourceLock>);
StorageInfo InMemoryStorage::GetBaseInfo(bool force_directory) {
StorageInfo InMemoryStorage::GetBaseInfo() {
StorageInfo info{};
info.vertex_count = vertices_.size();
info.edge_count = edge_count_.load(std::memory_order_acquire);
@@ -1769,27 +1770,23 @@ StorageInfo InMemoryStorage::GetBaseInfo(bool force_directory) {
info.memory_res = utils::GetMemoryRES();
// Special case for the default database
auto update_path = [&](const std::filesystem::path &dir) {
if (!force_directory && std::filesystem::is_directory(dir) && dir.has_filename()) {
const auto end = dir.end();
auto it = end;
--it;
if (it != end) {
--it;
if (it != end && *it != "databases") {
// Default DB points to the root (for back-compatibility); update to the "database" dir
return dir / dbms::kMultiTenantDir / dbms::kDefaultDB;
}
#ifdef MG_ENTERPRISE
if (config_.salient.name == dbms::kDefaultDB) {
// Default DB points to the root (for back-compatibility); update to the "database" dir
std::filesystem::path new_dir = dir / "databases" / dbms::kDefaultDB;
if (std::filesystem::exists(new_dir) && std::filesystem::is_directory(new_dir)) {
return new_dir;
}
}
#endif
return dir;
};
info.disk_usage = utils::GetDirDiskUsage<false>(update_path(config_.durability.storage_directory));
return info;
}
StorageInfo InMemoryStorage::GetInfo(bool force_directory,
memgraph::replication_coordination_glue::ReplicationRole replication_role) {
StorageInfo info = GetBaseInfo(force_directory);
StorageInfo InMemoryStorage::GetInfo(memgraph::replication_coordination_glue::ReplicationRole replication_role) {
StorageInfo info = GetBaseInfo();
{
auto access = Access(replication_role); // TODO: override isolation level?
const auto &lbl = access->ListAllIndices();

View File

@@ -368,9 +368,8 @@ class InMemoryStorage final : public Storage {
bool InitializeWalFile(memgraph::replication::ReplicationEpoch &epoch);
void FinalizeWalFile();
StorageInfo GetBaseInfo(bool force_directory) override;
StorageInfo GetInfo(bool force_directory,
memgraph::replication_coordination_glue::ReplicationRole replication_role) override;
StorageInfo GetBaseInfo() override;
StorageInfo GetInfo(memgraph::replication_coordination_glue::ReplicationRole replication_role) override;
/// Return true in all cases excepted if any sync replicas have not sent confirmation.
[[nodiscard]] bool AppendToWal(const Transaction &transaction, uint64_t final_commit_timestamp,

View File

@@ -215,11 +215,11 @@ bool ReplicationStorageClient::FinalizeTransactionReplication(Storage *storage,
MG_ASSERT(replica_stream_, "Missing stream for transaction deltas");
try {
auto response = replica_stream_->Finalize();
return replica_state_.WithLock([storage, &response, db_acc = std::move(db_acc), this](auto &state) mutable {
return replica_state_.WithLock([storage, response, db_acc = std::move(db_acc), this](auto &state) mutable {
replica_stream_.reset();
if (!response.success || state == replication::ReplicaState::RECOVERY) {
state = replication::ReplicaState::RECOVERY;
client_.thread_pool_.AddTask([storage, &response, db_acc = std::move(db_acc), this] {
client_.thread_pool_.AddTask([storage, response, db_acc = std::move(db_acc), this] {
this->RecoverReplica(response.current_commit_timestamp, storage);
});
return false;

View File

@@ -359,18 +359,9 @@ class Storage {
utils::BasicResult<SetIsolationLevelError> SetIsolationLevel(IsolationLevel isolation_level);
IsolationLevel GetIsolationLevel() const noexcept;
virtual StorageInfo GetBaseInfo(bool force_directory) = 0;
StorageInfo GetBaseInfo() {
#if MG_ENTERPRISE
const bool force_dir = false;
#else
const bool force_dir = true; //!< Use the configured directory (multi-tenancy reroutes to another dir)
#endif
return GetBaseInfo(force_dir);
}
virtual StorageInfo GetBaseInfo() = 0;
virtual StorageInfo GetInfo(bool force_directory,
memgraph::replication_coordination_glue::ReplicationRole replication_role) = 0;
virtual StorageInfo GetInfo(memgraph::replication_coordination_glue::ReplicationRole replication_role) = 0;
virtual Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode,
memgraph::replication_coordination_glue::ReplicationRole replication_role) = 0;

View File

@@ -1,7 +1,7 @@
#!/bin/bash
# Old v1 tests
run_v1.sh
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR"
# New tests
pushd () { command pushd "$@" > /dev/null; }
@@ -15,8 +15,9 @@ function wait_for_server {
sleep 1
}
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$DIR"
# Old v1 tests
tests_v1="$DIR/run_v1.sh"
$tests_v1
# Create a temporary directory.
tmpdir=/tmp/memgraph_drivers

View File

@@ -15,6 +15,7 @@
#include <optional>
#include "dbms/database.hpp"
#include "dbms/dbms_handler.hpp"
#include "disk_test_utils.hpp"
#include "query/interpret/awesome_memgraph_functions.hpp"
#include "query/interpreter_context.hpp"
@@ -30,15 +31,31 @@ using namespace memgraph::storage;
constexpr auto testSuite = "database_v2_get_info";
const std::filesystem::path storage_directory{std::filesystem::temp_directory_path() / testSuite};
template <typename StorageType>
struct TestConfig {};
struct DefaultConfig : TestConfig {};
struct TenantConfig : TestConfig {};
template <typename TestType>
class InfoTest : public testing::Test {
using StorageType = typename TestType::first_type;
using ConfigType = typename TestType::second_type;
protected:
void SetUp() {
repl_state.emplace(memgraph::storage::ReplicationStateRootPath(config));
db_gk.emplace(config, *repl_state);
auto db_acc_opt = db_gk->access();
MG_ASSERT(db_acc_opt, "Failed to access db");
auto &db_acc = *db_acc_opt;
repl_state_.emplace(ReplicationStateRootPath(config));
#ifdef MG_ENTERPRISE
dbms_handler_.emplace(config, *repl_state_, auth_, false);
auto db_acc = dbms_handler_->Get(); // Default db
if (std::is_same_v<ConfigType, TenantConfig>) {
constexpr std::string_view db_name = "test_db";
MG_ASSERT(dbms_handler_->New(std::string{db_name}).HasValue(), "Failed to create database.");
db_acc = dbms_handler_->Get(db_name);
}
#else
dbms_handler_.emplace(config, *repl_state_);
auto db_acc = dbms_handler_->Get();
#endif
MG_ASSERT(db_acc, "Failed to access db");
MG_ASSERT(db_acc->GetStorageMode() == (std::is_same_v<StorageType, memgraph::storage::DiskStorage>
? memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL
: memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL),
@@ -48,8 +65,8 @@ class InfoTest : public testing::Test {
void TearDown() {
db_acc_.reset();
db_gk.reset();
repl_state.reset();
dbms_handler_.reset();
repl_state_.reset();
if (std::is_same<StorageType, memgraph::storage::DiskStorage>::value) {
disk_test_utils::RemoveRocksDbDirs(testSuite);
}
@@ -59,9 +76,9 @@ class InfoTest : public testing::Test {
StorageMode mode{std::is_same_v<StorageType, DiskStorage> ? StorageMode::ON_DISK_TRANSACTIONAL
: StorageMode::IN_MEMORY_TRANSACTIONAL};
std::optional<memgraph::replication::ReplicationState> repl_state;
std::optional<memgraph::dbms::DatabaseAccess> db_acc_;
std::optional<memgraph::utils::Gatekeeper<memgraph::dbms::Database>> db_gk;
#ifdef MG_ENTERPRISE
memgraph::auth::SynchedAuth auth_{storage_directory, memgraph::auth::Auth::Config {}};
#endif
memgraph::storage::Config config{
[&]() {
memgraph::storage::Config config{};
@@ -69,18 +86,27 @@ class InfoTest : public testing::Test {
config.durability.snapshot_wal_mode =
memgraph::storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL;
if constexpr (std::is_same_v<StorageType, memgraph::storage::DiskStorage>) {
config.disk = disk_test_utils::GenerateOnDiskConfig(testSuite).disk;
config.force_on_disk = true;
}
return config;
}() // iile
};
std::optional<memgraph::replication::ReplicationState> repl_state_;
std::optional<memgraph::dbms::DbmsHandler> dbms_handler_;
std::optional<memgraph::dbms::DatabaseAccess> db_acc_;
};
using StorageTypes = ::testing::Types<memgraph::storage::InMemoryStorage, memgraph::storage::DiskStorage>;
using TestTypes = ::testing::Types<std::pair<memgraph::storage::InMemoryStorage, DefaultConfig>,
std::pair<memgraph::storage::DiskStorage, DefaultConfig>
TYPED_TEST_CASE(InfoTest, StorageTypes);
// TYPED_TEST_CASE(IndexTest, InMemoryStorageType);
#ifdef MG_ENTERPRISE
,
std::pair<memgraph::storage::InMemoryStorage, TenantConfig>,
std::pair<memgraph::storage::DiskStorage, TenantConfig>
#endif
>;
TYPED_TEST_CASE(InfoTest, TestTypes);
// NOLINTNEXTLINE(hicpp-special-member-functions)
TYPED_TEST(InfoTest, InfoCheck) {
@@ -166,13 +192,13 @@ TYPED_TEST(InfoTest, InfoCheck) {
}
const auto &info = db_acc->GetInfo(
true, memgraph::replication_coordination_glue::ReplicationRole::MAIN); // force to use configured directory
memgraph::replication_coordination_glue::ReplicationRole::MAIN); // force to use configured directory
ASSERT_EQ(info.storage_info.vertex_count, 5);
ASSERT_EQ(info.storage_info.edge_count, 2);
ASSERT_EQ(info.storage_info.average_degree, 0.8);
ASSERT_GT(info.storage_info.memory_res, 10'000'000); // 200MB < > 10MB
ASSERT_LT(info.storage_info.memory_res, 200'000'000);
ASSERT_GT(info.storage_info.memory_res, 10'000'000); // 250MB < > 10MB
ASSERT_LT(info.storage_info.memory_res, 250'000'000);
ASSERT_GT(info.storage_info.disk_usage, 100); // 1MB < > 100B
ASSERT_LT(info.storage_info.disk_usage, 1000'000);
ASSERT_EQ(info.storage_info.label_indices, 1);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -113,7 +113,6 @@ struct ConsumerTest : public ::testing::Test {
void SeedTopicWithInt(const std::string &topic_name, int value) {
std::array<char, sizeof(int)> int_as_char{};
std::memcpy(int_as_char.data(), &value, int_as_char.size());
cluster.SeedTopic(topic_name, int_as_char);
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -78,10 +78,6 @@ void KafkaClusterMock::CreateTopic(const std::string &topic_name) {
}
}
void KafkaClusterMock::SeedTopic(const std::string &topic_name, std::string_view message) {
SeedTopic(topic_name, std::span{message.data(), message.size()});
}
void KafkaClusterMock::SeedTopic(const std::string &topic_name, std::span<const char> message) {
char errstr[256] = {'\0'};
std::string bootstraps_servers = Bootstraps();

View File

@@ -1,4 +1,4 @@
// Copyright 2021 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -41,7 +41,6 @@ class KafkaClusterMock {
std::string Bootstraps() const;
void CreateTopic(const std::string &topic_name);
void SeedTopic(const std::string &topic_name, std::span<const char> message);
void SeedTopic(const std::string &topic_name, std::string_view message);
private:
RdKafkaUniquePtr rk_{nullptr};

View File

@@ -13,12 +13,11 @@
#include <gtest/gtest.h>
#include <filesystem>
#include "disk_test_utils.hpp"
#include "dbms/constants.hpp"
#include "storage/v2/disk/storage.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/isolation_level.hpp"
#include "storage/v2/storage.hpp"
#include "storage/v2/storage_error.hpp"
// NOLINTNEXTLINE(google-build-using-namespace)
using namespace memgraph::storage;
@@ -31,6 +30,7 @@ class InfoTest : public testing::Test {
protected:
void SetUp() override {
std::filesystem::remove_all(storage_directory);
config_.salient.name = memgraph::dbms::kDefaultDB;
memgraph::storage::UpdatePaths(config_, storage_directory);
config_.durability.snapshot_wal_mode =
memgraph::storage::Config::Durability::SnapshotWalMode::PERIODIC_SNAPSHOT_WITH_WAL;
@@ -135,7 +135,7 @@ TYPED_TEST(InfoTest, InfoCheck) {
ASSERT_FALSE(unique_acc->Commit().HasError());
}
StorageInfo info = this->storage->GetInfo(true, ReplicationRole::MAIN); // force to use configured directory
StorageInfo info = this->storage->GetInfo(ReplicationRole::MAIN);
ASSERT_EQ(info.vertex_count, 5);
ASSERT_EQ(info.edge_count, 2);