Compare commits

...

16 Commits

Author SHA1 Message Date
Josip Mrden
8e1d86eea1 Delete trigger address comments 2024-03-22 13:54:17 +01:00
Josip Mrden
e99c7dd45b Added edge type indices to be cleaned up inside drop graph 2024-03-22 13:47:27 +01:00
Josip Mrden
84df6dcd02 Fix 2024-03-22 12:29:35 +01:00
Josip Mrden
15e7f1fa02 Add creation of unique database transaction 2024-03-22 12:14:39 +01:00
Josip Mrden
1e7dad000f Merge branch 'master' into drop-graph 2024-03-13 14:12:50 +01:00
Andi
24f8a14b43 Improve registration queries in HA environment(#1809) 2024-03-13 13:04:27 +00:00
Josipmrden
dca71d3f7b Merge branch 'master' into drop-graph 2024-03-13 13:20:49 +01:00
Josipmrden
2cab07429e Add new PR template (#1798) 2024-03-13 10:09:22 +01:00
Josip Mrden
95bcb9c040 Clang tidy fix 2024-03-12 17:39:04 +01:00
Josip Mrden
4a6f062ef1 Add e2e tests and fix clearing of constriants 2024-03-12 14:07:14 +01:00
Josip Mrden
57794828e3 Merge branch 'master' into drop-graph 2024-03-12 12:45:45 +01:00
Josip Mrden
fe9f6f291b Add deletion of triggers and streams 2024-03-11 15:33:58 +01:00
Josip Mrden
d055495b44 Take just main lock over the storage 2024-02-27 13:22:47 +01:00
Josip Mrden
ffb5c751c8 Merge branch 'master' into drop-graph 2024-02-27 13:04:45 +01:00
Josip Mrden
6f47a860a4 Typo 2024-02-16 17:31:02 +01:00
Josip Mrden
d89193dd6d Drop graph functionality 2024-02-15 15:57:16 +01:00
55 changed files with 748 additions and 189 deletions

View File

@@ -1,14 +1,28 @@
### Description
Please briefly explain the changes you made here.
Please delete either the [master < EPIC] or [master < Task] part, depending on what are your needs.
[master < Epic] PR
- [ ] Check, and update documentation if necessary
- [ ] Write E2E tests
- [ ] Compare the [benchmarking results](https://bench-graph.memgraph.com/) between the master branch and the Epic branch
- [ ] Provide the full content or a guide for the final git message
- [FINAL GIT MESSAGE]
[master < Task] PR
- [ ] Check, and update documentation if necessary
- [ ] Provide the full content or a guide for the final git message
- **[FINAL GIT MESSAGE]**
To keep docs changelog up to date, one more thing to do:
- [ ] Write a release note here, including added/changed clauses
### Documentation checklist
- [ ] Add the documentation label tag
- [ ] Add the bug / feature label tag
- [ ] Add the milestone for which this feature is intended
- If not known, set for a later milestone
- [ ] Write a release note, including added/changed clauses
- **[Release note text]**
- [ ] Link the documentation PR here
- **[Documentation PR link]**
- [ ] Tag someone from docs team in the comments

View File

@@ -736,6 +736,8 @@ class DbAccessor final {
const std::set<storage::PropertyId> &properties) {
return accessor_->DropUniqueConstraint(label, properties);
}
void DropGraph() { return accessor_->DropGraph(); }
};
class SubgraphDbAccessor final {

View File

@@ -433,4 +433,11 @@ class MultiDatabaseQueryInMulticommandTxException : public QueryException {
SPECIALIZE_GET_EXCEPTION_NAME(MultiDatabaseQueryInMulticommandTxException)
};
class DropGraphInMulticommandTxException : public QueryException {
public:
DropGraphInMulticommandTxException()
: QueryException("Drop graph can not be executed in multicommand transactions.") {}
SPECIALIZE_GET_EXCEPTION_NAME(DropGraphInMulticommandTxException)
};
} // namespace memgraph::query

View File

@@ -245,6 +245,9 @@ constexpr utils::TypeInfo query::ReplicationQuery::kType{utils::TypeId::AST_REPL
constexpr utils::TypeInfo query::CoordinatorQuery::kType{utils::TypeId::AST_COORDINATOR_QUERY, "CoordinatorQuery",
&query::Query::kType};
constexpr utils::TypeInfo query::DropGraphQuery::kType{utils::TypeId::AST_DROP_GRAPH_QUERY, "DropGraphQuery",
&query::Query::kType};
constexpr utils::TypeInfo query::LockPathQuery::kType{utils::TypeId::AST_LOCK_PATH_QUERY, "LockPathQuery",
&query::Query::kType};

View File

@@ -26,6 +26,11 @@
namespace memgraph::query {
constexpr std::string_view kBoltServer = "bolt_server";
constexpr std::string_view kReplicationServer = "replication_server";
constexpr std::string_view kCoordinatorServer = "coordinator_server";
constexpr std::string_view kManagementServer = "management_server";
struct LabelIx {
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const { return kType; }
@@ -3140,24 +3145,21 @@ class CoordinatorQuery : public memgraph::query::Query {
DEFVISITABLE(QueryVisitor<void>);
memgraph::query::CoordinatorQuery::Action action_;
std::string instance_name_;
memgraph::query::Expression *replication_socket_address_{nullptr};
memgraph::query::Expression *coordinator_socket_address_{nullptr};
memgraph::query::Expression *raft_socket_address_{nullptr};
memgraph::query::Expression *raft_server_id_{nullptr};
std::string instance_name_{};
std::unordered_map<memgraph::query::Expression *, memgraph::query::Expression *> configs_;
memgraph::query::Expression *coordinator_server_id_{nullptr};
memgraph::query::CoordinatorQuery::SyncMode sync_mode_;
CoordinatorQuery *Clone(AstStorage *storage) const override {
auto *object = storage->Create<CoordinatorQuery>();
object->action_ = action_;
object->instance_name_ = instance_name_;
object->replication_socket_address_ =
replication_socket_address_ ? replication_socket_address_->Clone(storage) : nullptr;
object->coordinator_server_id_ = coordinator_server_id_ ? coordinator_server_id_->Clone(storage) : nullptr;
object->sync_mode_ = sync_mode_;
object->coordinator_socket_address_ =
coordinator_socket_address_ ? coordinator_socket_address_->Clone(storage) : nullptr;
object->raft_socket_address_ = raft_socket_address_ ? raft_socket_address_->Clone(storage) : nullptr;
object->raft_server_id_ = raft_server_id_ ? raft_server_id_->Clone(storage) : nullptr;
for (const auto &[key, value] : configs_) {
object->configs_[key->Clone(storage)] = value->Clone(storage);
}
return object;
}
@@ -3166,6 +3168,24 @@ class CoordinatorQuery : public memgraph::query::Query {
friend class AstStorage;
};
class DropGraphQuery : public memgraph::query::Query {
public:
static const utils::TypeInfo kType;
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
DropGraphQuery() = default;
DEFVISITABLE(QueryVisitor<void>);
DropGraphQuery *Clone(AstStorage *storage) const override {
auto *object = storage->Create<DropGraphQuery>();
return object;
}
private:
friend class AstStorage;
};
class EdgeImportModeQuery : public memgraph::query::Query {
public:
static const utils::TypeInfo kType;

View File

@@ -110,6 +110,7 @@ class ShowDatabasesQuery;
class EdgeImportModeQuery;
class PatternComprehension;
class CoordinatorQuery;
class DropGraphQuery;
using TreeCompositeVisitor = utils::CompositeVisitor<
SingleQuery, CypherUnion, NamedExpression, OrOperator, XorOperator, AndOperator, NotOperator, AdditionOperator,
@@ -149,6 +150,6 @@ class QueryVisitor
LockPathQuery, FreeMemoryQuery, TriggerQuery, IsolationLevelQuery, CreateSnapshotQuery,
StreamQuery, SettingQuery, VersionQuery, ShowConfigQuery, TransactionQueueQuery,
StorageModeQuery, AnalyzeGraphQuery, MultiDatabaseQuery, ShowDatabasesQuery,
EdgeImportModeQuery, CoordinatorQuery> {};
EdgeImportModeQuery, CoordinatorQuery, DropGraphQuery> {};
} // namespace memgraph::query

View File

@@ -341,6 +341,12 @@ antlrcpp::Any CypherMainVisitor::visitEdgeImportModeQuery(MemgraphCypher::EdgeIm
return edge_import_mode_query;
}
antlrcpp::Any CypherMainVisitor::visitDropGraphQuery(MemgraphCypher::DropGraphQueryContext * /*ctx*/) {
auto *drop_graph_query = storage_->Create<DropGraphQuery>();
query_ = drop_graph_query;
return drop_graph_query;
}
antlrcpp::Any CypherMainVisitor::visitSetReplicationRole(MemgraphCypher::SetReplicationRoleContext *ctx) {
auto *replication_query = storage_->Create<ReplicationQuery>();
replication_query->action_ = ReplicationQuery::Action::SET_REPLICATION_ROLE;
@@ -398,24 +404,17 @@ antlrcpp::Any CypherMainVisitor::visitRegisterReplica(MemgraphCypher::RegisterRe
antlrcpp::Any CypherMainVisitor::visitRegisterInstanceOnCoordinator(
MemgraphCypher::RegisterInstanceOnCoordinatorContext *ctx) {
auto *coordinator_query = storage_->Create<CoordinatorQuery>();
if (!ctx->replicationSocketAddress()->literal()->StringLiteral()) {
throw SemanticException("Replication socket address should be a string literal!");
}
if (!ctx->coordinatorSocketAddress()->literal()->StringLiteral()) {
throw SemanticException("Coordinator socket address should be a string literal!");
}
coordinator_query->action_ = CoordinatorQuery::Action::REGISTER_INSTANCE;
coordinator_query->replication_socket_address_ =
std::any_cast<Expression *>(ctx->replicationSocketAddress()->accept(this));
coordinator_query->coordinator_socket_address_ =
std::any_cast<Expression *>(ctx->coordinatorSocketAddress()->accept(this));
coordinator_query->instance_name_ = std::any_cast<std::string>(ctx->instanceName()->symbolicName()->accept(this));
if (ctx->ASYNC()) {
coordinator_query->sync_mode_ = memgraph::query::CoordinatorQuery::SyncMode::ASYNC;
} else {
coordinator_query->sync_mode_ = memgraph::query::CoordinatorQuery::SyncMode::SYNC;
}
coordinator_query->configs_ =
std::any_cast<std::unordered_map<Expression *, Expression *>>(ctx->configsMap->accept(this));
coordinator_query->sync_mode_ = [ctx]() {
if (ctx->ASYNC()) {
return CoordinatorQuery::SyncMode::ASYNC;
}
return CoordinatorQuery::SyncMode::SYNC;
}();
return coordinator_query;
}
@@ -431,17 +430,10 @@ antlrcpp::Any CypherMainVisitor::visitUnregisterInstanceOnCoordinator(
antlrcpp::Any CypherMainVisitor::visitAddCoordinatorInstance(MemgraphCypher::AddCoordinatorInstanceContext *ctx) {
auto *coordinator_query = storage_->Create<CoordinatorQuery>();
if (!ctx->raftSocketAddress()->literal()->StringLiteral()) {
throw SemanticException("Raft socket address should be a string literal!");
}
if (!ctx->raftServerId()->literal()->numberLiteral()) {
throw SemanticException("Raft server id should be a number literal!");
}
coordinator_query->action_ = CoordinatorQuery::Action::ADD_COORDINATOR_INSTANCE;
coordinator_query->raft_socket_address_ = std::any_cast<Expression *>(ctx->raftSocketAddress()->accept(this));
coordinator_query->raft_server_id_ = std::any_cast<Expression *>(ctx->raftServerId()->accept(this));
coordinator_query->coordinator_server_id_ = std::any_cast<Expression *>(ctx->coordinatorServerId()->accept(this));
coordinator_query->configs_ =
std::any_cast<std::unordered_map<Expression *, Expression *>>(ctx->configsMap->accept(this));
return coordinator_query;
}

View File

@@ -243,6 +243,11 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
*/
antlrcpp::Any visitCoordinatorQuery(MemgraphCypher::CoordinatorQueryContext *ctx) override;
/**
* @return DropGraphQuery*
*/
antlrcpp::Any visitDropGraphQuery(MemgraphCypher::DropGraphQueryContext *ctx) override;
/**
* @return CoordinatorQuery*
*/

View File

@@ -158,6 +158,7 @@ query : cypherQuery
| showDatabases
| edgeImportModeQuery
| coordinatorQuery
| dropGraphQuery
;
cypherQuery : ( indexHints )? singleQuery ( cypherUnion )* ( queryMemoryLimit )? ;
@@ -388,22 +389,22 @@ instanceName : symbolicName ;
socketAddress : literal ;
coordinatorSocketAddress : literal ;
replicationSocketAddress : literal ;
raftSocketAddress : literal ;
registerReplica : REGISTER REPLICA instanceName ( SYNC | ASYNC )
TO socketAddress ;
registerInstanceOnCoordinator : REGISTER INSTANCE instanceName ON coordinatorSocketAddress ( AS ASYNC ) ? WITH replicationSocketAddress ;
configKeyValuePair : literal ':' literal ;
configMap : '{' ( configKeyValuePair ( ',' configKeyValuePair )* )? '}' ;
registerInstanceOnCoordinator : REGISTER INSTANCE instanceName ( AS ASYNC ) ? WITH CONFIG configsMap=configMap ;
unregisterInstanceOnCoordinator : UNREGISTER INSTANCE instanceName ;
setInstanceToMain : SET INSTANCE instanceName TO MAIN ;
raftServerId : literal ;
coordinatorServerId : literal ;
addCoordinatorInstance : ADD COORDINATOR raftServerId ON raftSocketAddress ;
addCoordinatorInstance : ADD COORDINATOR coordinatorServerId WITH CONFIG configsMap=configMap ;
dropReplica : DROP REPLICA instanceName ;
@@ -457,10 +458,6 @@ commonCreateStreamConfig : TRANSFORM transformationName=procedureName
createStream : kafkaCreateStream | pulsarCreateStream ;
configKeyValuePair : literal ':' literal ;
configMap : '{' ( configKeyValuePair ( ',' configKeyValuePair )* )? '}' ;
kafkaCreateStreamConfig : TOPICS topicNames
| CONSUMER_GROUP consumerGroup=symbolicNameWithDotsAndMinus
| BOOTSTRAP_SERVERS bootstrapServers=literal
@@ -534,3 +531,5 @@ createEdgeIndex : CREATE EDGE INDEX ON ':' labelName ;
dropEdgeIndex : DROP EDGE INDEX ON ':' labelName ;
edgeIndexQuery : createEdgeIndex | dropEdgeIndex ;
dropGraphQuery : DROP GRAPH ;

View File

@@ -99,6 +99,8 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
void Visit(EdgeImportModeQuery & /*edge_import_mode_query*/) override {}
void Visit(DropGraphQuery & /*drop_graph_query*/) override {}
void Visit(VersionQuery & /*version_query*/) override { AddPrivilege(AuthQuery::Privilege::STATS); }
void Visit(MultiDatabaseQuery &query) override {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1146,6 +1146,27 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
}
#ifdef MG_ENTERPRISE
auto ParseConfigMap(std::unordered_map<Expression *, Expression *> const &config_map,
ExpressionVisitor<TypedValue> &evaluator)
-> std::optional<std::map<std::string, std::string, std::less<>>> {
if (std::ranges::any_of(config_map, [&evaluator](const auto &entry) {
auto key_expr = entry.first->Accept(evaluator);
auto value_expr = entry.second->Accept(evaluator);
return !key_expr.IsString() || !value_expr.IsString();
})) {
spdlog::error("Config map must contain only string keys and values!");
return std::nullopt;
}
return ranges::views::all(config_map) | ranges::views::transform([&evaluator](const auto &entry) {
auto key_expr = entry.first->Accept(evaluator);
auto value_expr = entry.second->Accept(evaluator);
return std::pair{key_expr.ValueString(), value_expr.ValueString()};
}) |
ranges::to<std::map<std::string, std::string, std::less<>>>;
}
Callback HandleCoordinatorQuery(CoordinatorQuery *coordinator_query, const Parameters &parameters,
coordination::CoordinatorState *coordinator_state,
const query::InterpreterConfig &config, std::vector<Notification> *notifications) {
@@ -1173,17 +1194,37 @@ Callback HandleCoordinatorQuery(CoordinatorQuery *coordinator_query, const Param
EvaluationContext evaluation_context{.timestamp = QueryTimestamp(), .parameters = parameters};
auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context};
auto raft_socket_address_tv = coordinator_query->raft_socket_address_->Accept(evaluator);
auto raft_server_id_tv = coordinator_query->raft_server_id_->Accept(evaluator);
callback.fn = [handler = CoordQueryHandler{*coordinator_state}, raft_socket_address_tv,
raft_server_id_tv]() mutable {
handler.AddCoordinatorInstance(raft_server_id_tv.ValueInt(), std::string(raft_socket_address_tv.ValueString()));
auto config_map = ParseConfigMap(coordinator_query->configs_, evaluator);
if (!config_map) {
throw QueryRuntimeException("Failed to parse config map!");
}
if (config_map->size() != 2) {
throw QueryRuntimeException("Config map must contain exactly 2 entries: {} and !", kCoordinatorServer,
kBoltServer);
}
auto const &coordinator_server_it = config_map->find(kCoordinatorServer);
if (coordinator_server_it == config_map->end()) {
throw QueryRuntimeException("Config map must contain {} entry!", kCoordinatorServer);
}
auto const &bolt_server_it = config_map->find(kBoltServer);
if (bolt_server_it == config_map->end()) {
throw QueryRuntimeException("Config map must contain {} entry!", kBoltServer);
}
auto coord_server_id = coordinator_query->coordinator_server_id_->Accept(evaluator).ValueInt();
callback.fn = [handler = CoordQueryHandler{*coordinator_state}, coord_server_id,
coordinator_server = coordinator_server_it->second]() mutable {
handler.AddCoordinatorInstance(coord_server_id, coordinator_server);
return std::vector<std::vector<TypedValue>>();
};
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::ADD_COORDINATOR_INSTANCE,
fmt::format("Coordinator has added instance {} on coordinator server {}.",
coordinator_query->instance_name_, raft_socket_address_tv.ValueString()));
coordinator_query->instance_name_, coordinator_server_it->second));
return callback;
}
case CoordinatorQuery::Action::REGISTER_INSTANCE: {
@@ -1194,27 +1235,49 @@ Callback HandleCoordinatorQuery(CoordinatorQuery *coordinator_query, const Param
// the argument to Callback.
EvaluationContext evaluation_context{.timestamp = QueryTimestamp(), .parameters = parameters};
auto evaluator = PrimitiveLiteralExpressionEvaluator{evaluation_context};
auto config_map = ParseConfigMap(coordinator_query->configs_, evaluator);
auto coordinator_socket_address_tv = coordinator_query->coordinator_socket_address_->Accept(evaluator);
auto replication_socket_address_tv = coordinator_query->replication_socket_address_->Accept(evaluator);
callback.fn = [handler = CoordQueryHandler{*coordinator_state}, coordinator_socket_address_tv,
replication_socket_address_tv,
if (!config_map) {
throw QueryRuntimeException("Failed to parse config map!");
}
if (config_map->size() != 3) {
throw QueryRuntimeException("Config map must contain exactly 3 entries: {}, {} and {}!", kBoltServer,
kManagementServer, kReplicationServer);
}
auto const &replication_server_it = config_map->find(kReplicationServer);
if (replication_server_it == config_map->end()) {
throw QueryRuntimeException("Config map must contain {} entry!", kReplicationServer);
}
auto const &management_server_it = config_map->find(kManagementServer);
if (management_server_it == config_map->end()) {
throw QueryRuntimeException("Config map must contain {} entry!", kManagementServer);
}
auto const &bolt_server_it = config_map->find(kBoltServer);
if (bolt_server_it == config_map->end()) {
throw QueryRuntimeException("Config map must contain {} entry!", kBoltServer);
}
callback.fn = [handler = CoordQueryHandler{*coordinator_state},
instance_health_check_frequency_sec = config.instance_health_check_frequency_sec,
management_server = management_server_it->second,
replication_server = replication_server_it->second, bolt_server = bolt_server_it->second,
instance_name = coordinator_query->instance_name_,
instance_down_timeout_sec = config.instance_down_timeout_sec,
instance_get_uuid_frequency_sec = config.instance_get_uuid_frequency_sec,
sync_mode = coordinator_query->sync_mode_]() mutable {
handler.RegisterReplicationInstance(std::string(coordinator_socket_address_tv.ValueString()),
std::string(replication_socket_address_tv.ValueString()),
instance_health_check_frequency_sec, instance_down_timeout_sec,
instance_get_uuid_frequency_sec, instance_name, sync_mode);
handler.RegisterReplicationInstance(management_server, replication_server, instance_health_check_frequency_sec,
instance_down_timeout_sec, instance_get_uuid_frequency_sec, instance_name,
sync_mode);
return std::vector<std::vector<TypedValue>>();
};
notifications->emplace_back(
SeverityLevel::INFO, NotificationCode::REGISTER_REPLICATION_INSTANCE,
fmt::format("Coordinator has registered coordinator server on {} for instance {}.",
coordinator_socket_address_tv.ValueString(), coordinator_query->instance_name_));
notifications->emplace_back(SeverityLevel::INFO, NotificationCode::REGISTER_REPLICATION_INSTANCE,
fmt::format("Coordinator has registered replication instance on {} for instance {}.",
bolt_server_it->second, coordinator_query->instance_name_));
return callback;
}
case CoordinatorQuery::Action::UNREGISTER_INSTANCE:
@@ -3258,6 +3321,27 @@ Callback SwitchMemoryDevice(storage::StorageMode current_mode, storage::StorageM
return callback;
}
Callback DropGraph(memgraph::dbms::DatabaseAccess &db, DbAccessor *dba) {
Callback callback;
callback.fn = [&db, dba]() mutable {
auto storage_mode = db->GetStorageMode();
if (storage_mode != storage::StorageMode::IN_MEMORY_ANALYTICAL) {
throw utils::BasicException("Drop graph can not be used without IN_MEMORY_ANALYTICAL storage mode!");
}
dba->DropGraph();
auto *trigger_store = db->trigger_store();
trigger_store->DropAll();
auto *streams = db->streams();
streams->DropAll();
return std::vector<std::vector<TypedValue>>();
};
return callback;
}
bool ActiveTransactionsExist(InterpreterContext *interpreter_context) {
bool exists_active_transaction = interpreter_context->interpreters.WithLock([](const auto &interpreters_) {
return std::any_of(interpreters_.begin(), interpreters_.end(), [](const auto &interpreter) {
@@ -3310,6 +3394,28 @@ PreparedQuery PrepareStorageModeQuery(ParsedQuery parsed_query, const bool in_ex
RWType::NONE};
}
PreparedQuery PrepareDropGraphQuery(ParsedQuery parsed_query, CurrentDB &current_db) {
MG_ASSERT(current_db.db_acc_, "Drop graph query expects a current DB");
memgraph::dbms::DatabaseAccess &db_acc = *current_db.db_acc_;
MG_ASSERT(current_db.db_transactional_accessor_, "Drop graph query expects a current DB transaction");
auto *dba = &*current_db.execution_db_accessor_;
auto *drop_graph_query = utils::Downcast<DropGraphQuery>(parsed_query.query);
MG_ASSERT(drop_graph_query);
std::function<void()> callback = DropGraph(db_acc, dba).fn;
return PreparedQuery{{},
std::move(parsed_query.required_privileges),
[callback = std::move(callback)](AnyStream * /*stream*/,
std::optional<int> /*n*/) -> std::optional<QueryHandlerResult> {
callback();
return QueryHandlerResult::COMMIT;
},
RWType::NONE};
}
PreparedQuery PrepareEdgeImportModeQuery(ParsedQuery parsed_query, CurrentDB &current_db) {
MG_ASSERT(current_db.db_acc_, "Edge Import query expects a current DB");
storage::Storage *storage = current_db.db_acc_->get()->storage();
@@ -4358,7 +4464,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
utils::Downcast<ProfileQuery>(parsed_query.query) || utils::Downcast<DumpQuery>(parsed_query.query) ||
utils::Downcast<TriggerQuery>(parsed_query.query) || utils::Downcast<AnalyzeGraphQuery>(parsed_query.query) ||
utils::Downcast<IndexQuery>(parsed_query.query) || utils::Downcast<EdgeIndexQuery>(parsed_query.query) ||
utils::Downcast<DatabaseInfoQuery>(parsed_query.query) || utils::Downcast<ConstraintQuery>(parsed_query.query);
utils::Downcast<DatabaseInfoQuery>(parsed_query.query) ||
utils::Downcast<ConstraintQuery>(parsed_query.query) || utils::Downcast<DropGraphQuery>(parsed_query.query);
if (!in_explicit_transaction_ && requires_db_transaction) {
// TODO: ATM only a single database, will change when we have multiple database transactions
@@ -4366,6 +4473,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
bool unique = utils::Downcast<IndexQuery>(parsed_query.query) != nullptr ||
utils::Downcast<EdgeIndexQuery>(parsed_query.query) != nullptr ||
utils::Downcast<ConstraintQuery>(parsed_query.query) != nullptr ||
utils::Downcast<DropGraphQuery>(parsed_query.query) != nullptr ||
upper_case_query.find(kSchemaAssert) != std::string::npos;
SetupDatabaseTransaction(could_commit, unique);
}
@@ -4482,6 +4590,11 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
throw EdgeImportModeModificationInMulticommandTxException();
}
prepared_query = PrepareEdgeImportModeQuery(std::move(parsed_query), current_db_);
} else if (utils::Downcast<DropGraphQuery>(parsed_query.query)) {
if (in_explicit_transaction_) {
throw DropGraphInMulticommandTxException();
}
prepared_query = PrepareDropGraphQuery(std::move(parsed_query), current_db_);
} else {
LOG_FATAL("Should not get here -- unknown query type!");
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -432,6 +432,14 @@ void TriggerStore::DropTrigger(const std::string &name) {
storage_.Delete(name);
}
void TriggerStore::DropAll() {
std::unique_lock store_guard{store_lock_};
storage_.DeletePrefix("");
before_commit_triggers_.clear();
after_commit_triggers_.clear();
}
std::vector<TriggerStore::TriggerInfo> TriggerStore::GetTriggerInfo() const {
std::vector<TriggerInfo> info;
info.reserve(before_commit_triggers_.size() + after_commit_triggers_.size());

View File

@@ -90,6 +90,7 @@ struct TriggerStore {
const InterpreterConfig::Query &query_config, std::shared_ptr<QueryUserOrRole> owner);
void DropTrigger(const std::string &name);
void DropAll();
struct TriggerInfo {
std::string name;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -33,4 +33,10 @@ Constraints::Constraints(const Config &config, StorageMode storage_mode) {
void Constraints::AbortEntries(std::span<Vertex const *const> vertices, uint64_t exact_start_timestamp) const {
static_cast<InMemoryUniqueConstraints *>(unique_constraints_.get())->AbortEntries(vertices, exact_start_timestamp);
}
void Constraints::DropGraphClearConstraints() const {
static_cast<InMemoryUniqueConstraints *>(unique_constraints_.get())->DropGraphClearConstraints();
existence_constraints_->DropGraphClearConstraints();
}
} // namespace memgraph::storage

View File

@@ -30,6 +30,7 @@ struct Constraints {
~Constraints() = default;
void AbortEntries(std::span<Vertex const *const> vertices, uint64_t exact_start_timestamp) const;
void DropGraphClearConstraints() const;
std::unique_ptr<ExistenceConstraints> existence_constraints_;
std::unique_ptr<UniqueConstraints> unique_constraints_;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -122,4 +122,6 @@ std::optional<ConstraintViolation> ExistenceConstraints::SingleThreadConstraintV
return std::nullopt;
}
void ExistenceConstraints::DropGraphClearConstraints() { constraints_.clear(); }
} // namespace memgraph::storage

View File

@@ -67,6 +67,8 @@ class ExistenceConstraints {
std::vector<std::pair<LabelId, PropertyId>> ListConstraints() const;
void LoadExistenceConstraints(const std::vector<std::string> &keys);
void DropGraphClearConstraints();
};
} // namespace memgraph::storage

View File

@@ -46,4 +46,8 @@ void DiskEdgeTypeIndex::UpdateOnEdgeModification(Vertex * /*old_from*/, Vertex *
spdlog::warn("Edge-type index related operations are not yet supported using on-disk storage mode.");
}
void DiskEdgeTypeIndex::DropGraphClearIndices() {
spdlog::warn("Edge-type index related operations are not yet supported using on-disk storage mode.");
}
} // namespace memgraph::storage

View File

@@ -30,6 +30,8 @@ class DiskEdgeTypeIndex : public storage::EdgeTypeIndex {
void UpdateOnEdgeModification(Vertex *old_from, Vertex *old_to, Vertex *new_from, Vertex *new_to, EdgeRef edge_ref,
EdgeTypeId edge_type, const Transaction &tx) override;
void DropGraphClearIndices() override;
};
} // namespace memgraph::storage

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -58,6 +58,8 @@ class DiskLabelIndex : public storage::LabelIndex {
std::unordered_set<LabelId> GetInfo() const;
void DropGraphClearIndices() override{};
private:
utils::Synchronized<std::map<uint64_t, std::map<Gid, std::vector<LabelId>>>> entries_for_deletion;
std::unordered_set<LabelId> index_;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -62,6 +62,8 @@ class DiskLabelPropertyIndex : public storage::LabelPropertyIndex {
std::set<std::pair<LabelId, PropertyId>> GetInfo() const;
void DropGraphClearIndices() override{};
private:
utils::Synchronized<std::map<uint64_t, std::map<Gid, std::vector<std::pair<LabelId, PropertyId>>>>>
entries_for_deletion;

View File

@@ -183,6 +183,8 @@ class DiskStorage final : public Storage {
UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label,
const std::set<PropertyId> &properties) override;
void DropGraph() override{};
};
using Storage::Access;

View File

@@ -41,6 +41,8 @@ class EdgeTypeIndex {
virtual void UpdateOnEdgeModification(Vertex *old_from, Vertex *old_to, Vertex *new_from, Vertex *new_to,
EdgeRef edge_ref, EdgeTypeId edge_type, const Transaction &tx) = 0;
virtual void DropGraphClearIndices() = 0;
};
} // namespace memgraph::storage

View File

@@ -42,6 +42,12 @@ void Indices::RemoveObsoleteEntries(uint64_t oldest_active_start_timestamp, std:
->RemoveObsoleteEntries(oldest_active_start_timestamp, std::move(token));
}
void Indices::DropGraphClearIndices() const {
static_cast<InMemoryLabelIndex *>(label_index_.get())->DropGraphClearIndices();
static_cast<InMemoryLabelPropertyIndex *>(label_property_index_.get())->DropGraphClearIndices();
static_cast<InMemoryEdgeTypeIndex *>(edge_type_index_.get())->DropGraphClearIndices();
}
void Indices::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) const {
label_index_->UpdateOnAddLabel(label, vertex, tx);
label_property_index_->UpdateOnAddLabel(label, vertex, tx);

View File

@@ -44,6 +44,8 @@ struct Indices {
void AbortEntries(LabelId label, std::span<std::pair<PropertyValue, Vertex *> const> vertices,
uint64_t exact_start_timestamp) const;
void DropGraphClearIndices() const;
struct IndexStats {
std::vector<LabelId> label;
LabelPropertyIndex::IndexStats property_label;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -40,6 +40,8 @@ class LabelIndex {
virtual std::vector<LabelId> ListIndices() const = 0;
virtual uint64_t ApproximateVertexCount(LabelId label) const = 0;
virtual void DropGraphClearIndices() = 0;
};
} // namespace memgraph::storage

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -53,6 +53,8 @@ class LabelPropertyIndex {
virtual uint64_t ApproximateVertexCount(LabelId label, PropertyId property,
const std::optional<utils::Bound<PropertyValue>> &lower,
const std::optional<utils::Bound<PropertyValue>> &upper) const = 0;
virtual void DropGraphClearIndices() = 0;
};
} // namespace memgraph::storage

View File

@@ -215,6 +215,8 @@ void InMemoryEdgeTypeIndex::UpdateOnEdgeModification(Vertex *old_from, Vertex *o
acc.insert(Entry{new_from, new_to, edge_ref.ptr, tx.start_timestamp});
}
void InMemoryEdgeTypeIndex::DropGraphClearIndices() { index_.clear(); }
InMemoryEdgeTypeIndex::Iterable::Iterable(utils::SkipList<Entry>::Accessor index_accessor, EdgeTypeId edge_type,
View view, Storage *storage, Transaction *transaction)
: index_accessor_(std::move(index_accessor)),

View File

@@ -61,6 +61,8 @@ class InMemoryEdgeTypeIndex : public storage::EdgeTypeIndex {
void UpdateOnEdgeModification(Vertex *old_from, Vertex *old_to, Vertex *new_from, Vertex *new_to, EdgeRef edge_ref,
EdgeTypeId edge_type, const Transaction &tx) override;
void DropGraphClearIndices() override;
static constexpr std::size_t kEdgeTypeIdPos = 0U;
static constexpr std::size_t kVertexPos = 1U;
static constexpr std::size_t kEdgeRefPos = 2U;

View File

@@ -235,4 +235,10 @@ std::vector<LabelId> InMemoryLabelIndex::Analysis() const {
}
return res;
}
void InMemoryLabelIndex::DropGraphClearIndices() {
index_.clear();
stats_->clear();
}
} // namespace memgraph::storage

View File

@@ -115,6 +115,8 @@ class InMemoryLabelIndex : public storage::LabelIndex {
bool DeleteIndexStats(const storage::LabelId &label);
void DropGraphClearIndices() override;
private:
std::map<LabelId, utils::SkipList<Entry>> index_;
utils::Synchronized<std::map<LabelId, storage::LabelIndexStats>, utils::ReadPrioritizedRWLock> stats_;

View File

@@ -498,4 +498,11 @@ void InMemoryLabelPropertyIndex::AbortEntries(LabelId label,
}
}
}
void InMemoryLabelPropertyIndex::DropGraphClearIndices() {
index_.clear();
indices_by_property_.clear();
stats_->clear();
}
} // namespace memgraph::storage

View File

@@ -154,6 +154,8 @@ class InMemoryLabelPropertyIndex : public storage::LabelPropertyIndex {
const std::optional<utils::Bound<PropertyValue>> &upper_bound, View view, Storage *storage,
Transaction *transaction);
void DropGraphClearIndices() override;
private:
std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>> index_;
std::unordered_map<PropertyId, std::unordered_map<LabelId, utils::SkipList<Entry> *>> indices_by_property_;

View File

@@ -2214,6 +2214,66 @@ void InMemoryStorage::FreeMemory(std::unique_lock<utils::ResourceLock> main_guar
edges_.run_gc();
}
void InMemoryStorage::DropGraph() {
auto const unlink_remove_clear = [&](std::deque<Delta> &deltas) {
for (auto &delta : deltas) {
auto prev = delta.prev.Get();
switch (prev.type) {
case PreviousPtr::Type::NULLPTR:
case PreviousPtr::Type::DELTA:
break;
case PreviousPtr::Type::VERTEX: {
// safe because no other txn can be reading this while we have engine lock
auto &vertex = *prev.vertex;
vertex.delta = nullptr;
if (vertex.deleted) {
DMG_ASSERT(delta.action == Delta::Action::RECREATE_OBJECT);
}
break;
}
case PreviousPtr::Type::EDGE: {
// safe because no other txn can be reading this while we have engine lock
auto &edge = *prev.edge;
edge.delta = nullptr;
if (edge.deleted) {
DMG_ASSERT(delta.action == Delta::Action::RECREATE_OBJECT);
}
break;
}
}
}
// delete deltas
deltas.clear();
};
// STEP 1) ensure everything in GC is gone
// 1.a) old garbage_undo_buffers are safe to remove
// we are the only transaction, no one is reading those unlinked deltas
garbage_undo_buffers_.WithLock([&](auto &garbage_undo_buffers) { garbage_undo_buffers.clear(); });
// 1.b.0) old committed_transactions_ need mininal unlinking + remove + clear
// must be done before this transactions delta unlinking
auto linked_undo_buffers = std::list<GCDeltas>{};
committed_transactions_.WithLock(
[&](auto &committed_transactions) { committed_transactions.swap(linked_undo_buffers); });
// 1.b.1) unlink, gathering the removals
for (auto &gc_deltas : linked_undo_buffers) {
unlink_remove_clear(gc_deltas.deltas_);
}
// 1.b.2) clear the list of deltas deques
linked_undo_buffers.clear();
indices_.DropGraphClearIndices();
constraints_.DropGraphClearConstraints();
vertices_.clear();
edges_.clear();
memory::PurgeUnusedMemory();
}
uint64_t InMemoryStorage::CommitTimestamp(const std::optional<uint64_t> desired_commit_timestamp) {
if (!desired_commit_timestamp) {
return timestamp_++;
@@ -2337,4 +2397,9 @@ std::vector<std::pair<LabelId, PropertyId>> InMemoryStorage::InMemoryAccessor::D
return res;
}
void InMemoryStorage::InMemoryAccessor::DropGraph() {
auto *mem_storage = static_cast<InMemoryStorage *>(storage_);
mem_storage->DropGraph();
}
} // namespace memgraph::storage

View File

@@ -322,6 +322,8 @@ class InMemoryStorage final : public Storage {
UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label,
const std::set<PropertyId> &properties) override;
void DropGraph() override;
protected:
// TODO Better naming
/// @throw std::bad_alloc
@@ -377,6 +379,8 @@ class InMemoryStorage final : public Storage {
const durability::Recovery &GetRecovery() const noexcept { return recovery_; }
void DropGraph();
private:
/// The force parameter determines the behaviour of the garbage collector.
/// If it's set to true, it will behave as a global operation, i.e. it can't

View File

@@ -524,4 +524,8 @@ void InMemoryUniqueConstraints::Clear() {
}
bool InMemoryUniqueConstraints::empty() const { return constraints_.empty() && constraints_by_label_.empty(); }
void InMemoryUniqueConstraints::DropGraphClearConstraints() {
constraints_.clear();
constraints_by_label_.clear();
}
} // namespace memgraph::storage

View File

@@ -129,6 +129,8 @@ class InMemoryUniqueConstraints : public UniqueConstraints {
void Clear() override;
void DropGraphClearConstraints();
static std::variant<MultipleThreadsConstraintValidation, SingleThreadConstraintValidation> GetCreationFunction(
const std::optional<durability::ParallelizedSchemaCreationInfo> &);

View File

@@ -296,6 +296,7 @@ class Storage {
virtual UniqueConstraints::DeletionStatus DropUniqueConstraint(LabelId label,
const std::set<PropertyId> &properties) = 0;
virtual void DropGraph() = 0;
auto GetTransaction() -> Transaction * { return std::addressof(transaction_); }
protected:

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2024 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -230,6 +230,7 @@ enum class TypeId : uint64_t {
AST_EDGE_IMPORT_MODE_QUERY,
AST_PATTERN_COMPREHENSION,
AST_COORDINATOR_QUERY,
AST_DROP_GRAPH_QUERY,
// Symbol
SYMBOL = 4000,

View File

@@ -77,6 +77,7 @@ add_subdirectory(garbage_collection)
add_subdirectory(query_planning)
add_subdirectory(awesome_functions)
add_subdirectory(high_availability)
add_subdirectory(drop_graph)
add_subdirectory(replication_experimental)

View File

@@ -0,0 +1,6 @@
function(copy_drop_graph_e2e_python_files FILE_NAME)
copy_e2e_python_files(drop_graph ${FILE_NAME})
endfunction()
copy_drop_graph_e2e_python_files(common.py)
copy_drop_graph_e2e_python_files(drop_graph.py)

View File

@@ -0,0 +1,40 @@
# Copyright 2023 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import pytest
from gqlalchemy import Memgraph
def get_results_length(memgraph, query):
return len(list(memgraph.execute_and_fetch(query)))
@pytest.fixture
def memgraph(**kwargs) -> Memgraph:
memgraph = Memgraph()
memgraph.execute("STORAGE MODE IN_MEMORY_TRANSACTIONAL")
memgraph.drop_indexes()
memgraph.ensure_constraints([])
trigger_names = [x["trigger name"] for x in list(memgraph.execute_and_fetch("SHOW TRIGGERS"))]
for trigger_name in trigger_names:
memgraph.execute(f"DROP TRIGGER {trigger_name}")
yield memgraph
memgraph.execute("STORAGE MODE IN_MEMORY_TRANSACTIONAL")
memgraph.drop_indexes()
memgraph.ensure_constraints([])
trigger_names = [x["trigger name"] for x in list(memgraph.execute_and_fetch("SHOW TRIGGERS"))]
for trigger_name in trigger_names:
memgraph.execute(f"DROP TRIGGER {trigger_name}")
memgraph.drop_database()

View File

@@ -0,0 +1,48 @@
# Copyright 2023 Memgraph Ltd.
#
# Use of this software is governed by the Business Source License
# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
# License, and you may not use this file except in compliance with the Business Source License.
#
# As of the Change Date specified in that file, in accordance with
# the Business Source License, use of this software will be governed
# by the Apache License, Version 2.0, included in the file
# licenses/APL.txt.
import sys
import pytest
from common import get_results_length, memgraph
from gqlalchemy import GQLAlchemyError
def test_create_everything_then_drop_graph(memgraph):
memgraph.execute("CREATE (:Node {id:1})-[:TYPE {id:2}]->(:Node {id:3})")
memgraph.execute("CREATE INDEX ON :Node")
memgraph.execute("CREATE INDEX ON :Node(id)")
memgraph.execute("CREATE CONSTRAINT ON (n:Node) ASSERT n.id IS UNIQUE;")
memgraph.execute("CREATE CONSTRAINT ON (n:Node) ASSERT EXISTS (n.id);")
memgraph.execute("CREATE TRIGGER t1 ON () UPDATE BEFORE COMMIT EXECUTE RETURN 1")
memgraph.execute("CREATE TRIGGER t2 ON () UPDATE AFTER COMMIT EXECUTE RETURN 1")
assert get_results_length(memgraph, "MATCH (n) RETURN n") == 2
assert get_results_length(memgraph, "MATCH (n)-[r]->(m) RETURN r") == 1
assert get_results_length(memgraph, "SHOW INDEX INFO") == 2
assert get_results_length(memgraph, "SHOW CONSTRAINT INFO") == 2
assert get_results_length(memgraph, "SHOW TRIGGERS") == 2
with pytest.raises(GQLAlchemyError):
memgraph.execute("DROP GRAPH")
memgraph.execute("STORAGE MODE IN_MEMORY_ANALYTICAL")
memgraph.execute("DROP GRAPH")
assert get_results_length(memgraph, "MATCH (n) RETURN n") == 0
assert get_results_length(memgraph, "MATCH (n)-[r]->(m) RETURN r") == 0
assert get_results_length(memgraph, "SHOW INDEX INFO") == 0
assert get_results_length(memgraph, "SHOW CONSTRAINT INFO") == 0
assert get_results_length(memgraph, "SHOW TRIGGERS") == 0
if __name__ == "__main__":
sys.exit(pytest.main([__file__, "-rA"]))

View File

@@ -0,0 +1,14 @@
drop_graph_cluster: &drop_graph_cluster
cluster:
main:
args: ["--bolt-port", "7687", "--log-level=TRACE"]
log_file: "drop_graph.log"
setup_queries: []
validation_queries: []
workloads:
- name: "Drop graph"
binary: "tests/e2e/pytest_runner.sh"
args: ["drop_graph/drop_graph.py"]
<<: *drop_graph_cluster

View File

@@ -117,17 +117,26 @@ def test_register_repl_instances_then_coordinators():
coordinator3_cursor = connect(host="localhost", port=7692).cursor()
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'"
coordinator3_cursor,
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'"
coordinator3_cursor,
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'"
coordinator3_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(coordinator3_cursor, "SET INSTANCE instance_3 TO MAIN")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 1 ON '127.0.0.1:10111'")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 2 ON '127.0.0.1:10112'")
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
)
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
)
def check_coordinator3():
return sorted(list(execute_and_fetch_all(coordinator3_cursor, "SHOW INSTANCES")))
@@ -172,16 +181,25 @@ def test_register_coordinator_then_repl_instances():
coordinator3_cursor = connect(host="localhost", port=7692).cursor()
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 1 ON '127.0.0.1:10111'")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 2 ON '127.0.0.1:10112'")
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'"
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
)
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'"
coordinator3_cursor,
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'"
coordinator3_cursor,
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
)
execute_and_fetch_all(
coordinator3_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(coordinator3_cursor, "SET INSTANCE instance_3 TO MAIN")
@@ -228,16 +246,25 @@ def test_coordinators_communication_with_restarts():
coordinator3_cursor = connect(host="localhost", port=7692).cursor()
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 1 ON '127.0.0.1:10111'")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 2 ON '127.0.0.1:10112'")
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'"
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
)
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'"
coordinator3_cursor,
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'"
coordinator3_cursor,
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
)
execute_and_fetch_all(
coordinator3_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(coordinator3_cursor, "SET INSTANCE instance_3 TO MAIN")
@@ -295,16 +322,25 @@ def test_unregister_replicas(kill_instance):
coordinator2_cursor = connect(host="localhost", port=7691).cursor()
coordinator3_cursor = connect(host="localhost", port=7692).cursor()
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 1 ON '127.0.0.1:10111'")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 2 ON '127.0.0.1:10112'")
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'"
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
)
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'"
coordinator3_cursor,
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'"
coordinator3_cursor,
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
)
execute_and_fetch_all(
coordinator3_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(coordinator3_cursor, "SET INSTANCE instance_3 TO MAIN")
@@ -429,16 +465,26 @@ def test_unregister_main():
coordinator1_cursor = connect(host="localhost", port=7690).cursor()
coordinator2_cursor = connect(host="localhost", port=7691).cursor()
coordinator3_cursor = connect(host="localhost", port=7692).cursor()
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 1 ON '127.0.0.1:10111'")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 2 ON '127.0.0.1:10112'")
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'"
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
)
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'"
coordinator3_cursor,
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'"
coordinator3_cursor,
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
)
execute_and_fetch_all(
coordinator3_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(coordinator3_cursor, "SET INSTANCE instance_3 TO MAIN")

View File

@@ -79,7 +79,7 @@ def test_main_and_replicas_cannot_register_coord_server(port):
with pytest.raises(Exception) as e:
execute_and_fetch_all(
cursor,
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10001' WITH '127.0.0.1:10011';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
assert str(e.value) == "Only coordinator can register coordinator server!"

View File

@@ -133,11 +133,18 @@ def test_writing_disabled_on_main_restart():
coordinator3_cursor = connect(host="localhost", port=7692).cursor()
execute_and_fetch_all(
coordinator3_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'"
coordinator3_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(coordinator3_cursor, "SET INSTANCE instance_3 TO MAIN")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 1 ON '127.0.0.1:10111'")
assert add_coordinator(coordinator3_cursor, "ADD COORDINATOR 2 ON '127.0.0.1:10112'")
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
)
assert add_coordinator(
coordinator3_cursor,
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
)
def check_coordinator3():
return sorted(list(execute_and_fetch_all(coordinator3_cursor, "SHOW INSTANCES")))

View File

@@ -110,11 +110,11 @@ MEMGRAPH_INSTANCES_DESCRIPTION = {
],
"log_file": "coordinator3.log",
"setup_queries": [
"ADD COORDINATOR 1 ON '127.0.0.1:10111'",
"ADD COORDINATOR 2 ON '127.0.0.1:10112'",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -221,11 +221,11 @@ def test_old_main_comes_back_on_new_leader_as_replica():
interactive_mg_runner.start_all(inner_instances_description)
setup_queries = [
"ADD COORDINATOR 1 ON '127.0.0.1:10111'",
"ADD COORDINATOR 2 ON '127.0.0.1:10112'",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
]
coord_cursor_3 = connect(host="localhost", port=7692).cursor()
@@ -416,11 +416,11 @@ def test_distributed_automatic_failover_with_leadership_change():
interactive_mg_runner.start_all(inner_instances_description)
setup_queries = [
"ADD COORDINATOR 1 ON '127.0.0.1:10111'",
"ADD COORDINATOR 2 ON '127.0.0.1:10112'",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
]
coord_cursor_3 = connect(host="localhost", port=7692).cursor()
@@ -522,7 +522,10 @@ def test_no_leader_after_leader_and_follower_die():
coord_cursor_1 = connect(host="localhost", port=7690).cursor()
with pytest.raises(Exception) as e:
execute_and_fetch_all(coord_cursor_1, "REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.10001'")
execute_and_fetch_all(
coord_cursor_1,
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
assert str(e) == "Couldn't register replica instance since coordinator is not a leader!"
@@ -541,11 +544,11 @@ def test_old_main_comes_back_on_new_leader_as_main():
coord_cursor_3 = connect(host="localhost", port=7692).cursor()
setup_queries = [
"ADD COORDINATOR 1 ON '127.0.0.1:10111'",
"ADD COORDINATOR 2 ON '127.0.0.1:10112'",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
]
@@ -719,12 +722,12 @@ def test_registering_4_coords():
],
"log_file": "coordinator4.log",
"setup_queries": [
"ADD COORDINATOR 1 ON '127.0.0.1:10111';",
"ADD COORDINATOR 2 ON '127.0.0.1:10112';",
"ADD COORDINATOR 3 ON '127.0.0.1:10113';",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"ADD COORDINATOR 3 WITH CONFIG {'bolt_server': '127.0.0.1:7692', 'coordinator_server': '127.0.0.1:10113'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -854,12 +857,12 @@ def test_registering_coord_log_store():
],
"log_file": "coordinator4.log",
"setup_queries": [
"ADD COORDINATOR 1 ON '127.0.0.1:10111';",
"ADD COORDINATOR 2 ON '127.0.0.1:10112';",
"ADD COORDINATOR 3 ON '127.0.0.1:10113';",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"ADD COORDINATOR 3 WITH CONFIG {'bolt_server': '127.0.0.1:7692', 'coordinator_server': '127.0.0.1:10113'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
],
},
}
@@ -896,7 +899,7 @@ def test_registering_coord_log_store():
# 3
instances_ports_added = [10011, 10012, 10013]
bolt_port_id = 7700
coord_port_id = 10014
manag_port_id = 10014
additional_instances = []
for i in range(4, 7):
@@ -908,10 +911,10 @@ def test_registering_coord_log_store():
bolt_port = f"--bolt-port={bolt_port_id}"
coord_server_port = f"--coordinator-server-port={coord_port_id}"
manag_server_port = f"--coordinator-server-port={manag_port_id}"
args_desc.append(bolt_port)
args_desc.append(coord_server_port)
args_desc.append(manag_server_port)
instance_description = {
"args": args_desc,
@@ -922,17 +925,23 @@ def test_registering_coord_log_store():
full_instance_desc = {instance_name: instance_description}
interactive_mg_runner.start(full_instance_desc, instance_name)
repl_port_id = coord_port_id - 10
repl_port_id = manag_port_id - 10
assert repl_port_id < 10011, "Wrong test setup, repl port must be smaller than smallest coord port id"
bolt_server = f"127.0.0.1:{bolt_port_id}"
management_server = f"127.0.0.1:{manag_port_id}"
repl_server = f"127.0.0.1:{repl_port_id}"
config_str = f"{{'bolt_server': '{bolt_server}', 'management_server': '{management_server}', 'replication_server': '{repl_server}'}}"
execute_and_fetch_all(
coord_cursor,
f"REGISTER INSTANCE {instance_name} ON '127.0.0.1:{coord_port_id}' WITH '127.0.0.1:{repl_port_id}'",
f"REGISTER INSTANCE {instance_name} WITH CONFIG {config_str}",
)
additional_instances.append((f"{instance_name}", "", f"127.0.0.1:{coord_port_id}", "up", "replica"))
instances_ports_added.append(coord_port_id)
coord_port_id += 1
additional_instances.append((f"{instance_name}", "", management_server, "up", "replica"))
instances_ports_added.append(manag_port_id)
manag_port_id += 1
bolt_port_id += 1
# 4
@@ -1004,11 +1013,11 @@ def test_multiple_failovers_in_row_no_leadership_change():
coord_cursor_3 = connect(host="localhost", port=7692).cursor()
setup_queries = [
"ADD COORDINATOR 1 ON '127.0.0.1:10111'",
"ADD COORDINATOR 2 ON '127.0.0.1:10112'",
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001'",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002'",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003'",
"ADD COORDINATOR 1 WITH CONFIG {'bolt_server': '127.0.0.1:7690', 'coordinator_server': '127.0.0.1:10111'}",
"ADD COORDINATOR 2 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'coordinator_server': '127.0.0.1:10112'}",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
]

View File

@@ -185,8 +185,8 @@ def test_not_replicate_old_main_register_new_cluster():
],
"log_file": "coordinator.log",
"setup_queries": [
"REGISTER INSTANCE shared_instance ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE shared_instance WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"SET INSTANCE instance_2 TO MAIN",
],
},
@@ -244,10 +244,12 @@ def test_not_replicate_old_main_register_new_cluster():
interactive_mg_runner.start_all_keep_others(MEMGRAPH_SECOND_COORD_CLUSTER_DESCRIPTION)
second_cluster_coord_cursor = connect(host="localhost", port=7691).cursor()
execute_and_fetch_all(
second_cluster_coord_cursor, "REGISTER INSTANCE shared_instance ON '127.0.0.1:10011' WITH '127.0.0.1:10001';"
second_cluster_coord_cursor,
"REGISTER INSTANCE shared_instance WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
)
execute_and_fetch_all(
second_cluster_coord_cursor, "REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';"
second_cluster_coord_cursor,
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
)
execute_and_fetch_all(second_cluster_coord_cursor, "SET INSTANCE instance_3 TO MAIN")

View File

@@ -90,9 +90,9 @@ MEMGRAPH_INSTANCES_DESCRIPTION = {
],
"log_file": "coordinator.log",
"setup_queries": [
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -185,9 +185,9 @@ def test_replication_works_on_failover_replica_1_epoch_2_commits_away(data_recov
],
"log_file": "coordinator.log",
"setup_queries": [
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -415,10 +415,10 @@ def test_replication_works_on_failover_replica_2_epochs_more_commits_away(data_r
],
"log_file": "coordinator.log",
"setup_queries": [
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';",
"REGISTER INSTANCE instance_4 ON '127.0.0.1:10014' WITH '127.0.0.1:10004';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"REGISTER INSTANCE instance_4 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'management_server': '127.0.0.1:10014', 'replication_server': '127.0.0.1:10004'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -702,10 +702,10 @@ def test_replication_forcefully_works_on_failover_replica_misses_epoch(data_reco
],
"log_file": "coordinator.log",
"setup_queries": [
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';",
"REGISTER INSTANCE instance_4 ON '127.0.0.1:10014' WITH '127.0.0.1:10004';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"REGISTER INSTANCE instance_4 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'management_server': '127.0.0.1:10014', 'replication_server': '127.0.0.1:10004'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -989,10 +989,10 @@ def test_replication_correct_replica_chosen_up_to_date_data(data_recovery):
],
"log_file": "coordinator.log",
"setup_queries": [
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';",
"REGISTER INSTANCE instance_4 ON '127.0.0.1:10014' WITH '127.0.0.1:10004';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"REGISTER INSTANCE instance_4 WITH CONFIG {'bolt_server': '127.0.0.1:7691', 'management_server': '127.0.0.1:10014', 'replication_server': '127.0.0.1:10004'};",
"SET INSTANCE instance_3 TO MAIN",
],
},
@@ -1559,7 +1559,7 @@ def test_registering_replica_fails_name_exists():
with pytest.raises(Exception) as e:
execute_and_fetch_all(
coord_cursor,
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10051' WITH '127.0.0.1:10111';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7693', 'management_server': '127.0.0.1:10051', 'replication_server': '127.0.0.1:10111'};",
)
assert str(e.value) == "Couldn't register replica instance since instance with such name already exists!"
shutil.rmtree(TEMP_DIR)
@@ -1573,7 +1573,7 @@ def test_registering_replica_fails_endpoint_exists():
with pytest.raises(Exception) as e:
execute_and_fetch_all(
coord_cursor,
"REGISTER INSTANCE instance_5 ON '127.0.0.1:10011' WITH '127.0.0.1:10005';",
"REGISTER INSTANCE instance_5 WITH CONFIG {'bolt_server': '127.0.0.1:7693', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10005'};",
)
assert (
str(e.value)

View File

@@ -16,9 +16,9 @@ ha_cluster: &ha_cluster
args: ["--experimental-enabled=high-availability", "--bolt-port", "7690", "--log-level=TRACE", "--raft-server-id=1", "--raft-server-port=10111"]
log_file: "replication-e2e-coordinator.log"
setup_queries: [
"REGISTER INSTANCE instance_1 ON '127.0.0.1:10011' WITH '127.0.0.1:10001';",
"REGISTER INSTANCE instance_2 ON '127.0.0.1:10012' WITH '127.0.0.1:10002';",
"REGISTER INSTANCE instance_3 ON '127.0.0.1:10013' WITH '127.0.0.1:10003';",
"REGISTER INSTANCE instance_1 WITH CONFIG {'bolt_server': '127.0.0.1:7688', 'management_server': '127.0.0.1:10011', 'replication_server': '127.0.0.1:10001'};",
"REGISTER INSTANCE instance_2 WITH CONFIG {'bolt_server': '127.0.0.1:7689', 'management_server': '127.0.0.1:10012', 'replication_server': '127.0.0.1:10002'};",
"REGISTER INSTANCE instance_3 WITH CONFIG {'bolt_server': '127.0.0.1:7687', 'management_server': '127.0.0.1:10013', 'replication_server': '127.0.0.1:10003'};",
"SET INSTANCE instance_3 TO MAIN;"
]

View File

@@ -2633,15 +2633,99 @@ TEST_P(CypherMainVisitorTest, TestRegisterReplicationQuery) {
}
#ifdef MG_ENTERPRISE
TEST_P(CypherMainVisitorTest, TestRegisterSyncInstance) {
auto &ast_generator = *GetParam();
std::string const sync_instance = R"(REGISTER INSTANCE instance_1 WITH CONFIG {"bolt_server": "127.0.0.1:7688",
"replication_server": "127.0.0.1:10001", "management_server": "127.0.0.1:10011"
})";
auto *parsed_query = dynamic_cast<CoordinatorQuery *>(ast_generator.ParseQuery(sync_instance));
EXPECT_EQ(parsed_query->action_, CoordinatorQuery::Action::REGISTER_INSTANCE);
EXPECT_EQ(parsed_query->sync_mode_, CoordinatorQuery::SyncMode::SYNC);
auto const evaluate_config_map = [&ast_generator](std::unordered_map<Expression *, Expression *> const &config_map)
-> std::unordered_map<std::string, std::string> {
auto const expr_to_str = [&ast_generator](Expression *expression) {
return std::string{ast_generator.GetLiteral(expression, ast_generator.context_.is_query_cached).ValueString()};
};
return ranges::views::transform(config_map,
[&expr_to_str](auto const &expr_pair) {
return std::pair{expr_to_str(expr_pair.first), expr_to_str(expr_pair.second)};
}) |
ranges::to<std::unordered_map<std::string, std::string>>;
};
auto const config_map = evaluate_config_map(parsed_query->configs_);
ASSERT_EQ(config_map.size(), 3);
EXPECT_EQ(config_map.at("bolt_server"), "127.0.0.1:7688");
EXPECT_EQ(config_map.at("management_server"), "127.0.0.1:10011");
EXPECT_EQ(config_map.at("replication_server"), "127.0.0.1:10001");
}
TEST_P(CypherMainVisitorTest, TestRegisterAsyncInstance) {
auto &ast_generator = *GetParam();
std::string const async_instance =
R"(REGISTER INSTANCE instance_1 AS ASYNC WITH CONFIG {"bolt_server": "127.0.0.1:7688",
"replication_server": "127.0.0.1:10001",
"management_server": "127.0.0.1:10011"})";
auto *parsed_query = dynamic_cast<CoordinatorQuery *>(ast_generator.ParseQuery(async_instance));
EXPECT_EQ(parsed_query->action_, CoordinatorQuery::Action::REGISTER_INSTANCE);
EXPECT_EQ(parsed_query->sync_mode_, CoordinatorQuery::SyncMode::ASYNC);
auto const evaluate_config_map = [&ast_generator](std::unordered_map<Expression *, Expression *> const &config_map)
-> std::map<std::string, std::string, std::less<>> {
auto const expr_to_str = [&ast_generator](Expression *expression) {
return std::string{ast_generator.GetLiteral(expression, ast_generator.context_.is_query_cached).ValueString()};
};
return ranges::views::transform(config_map,
[&expr_to_str](auto const &expr_pair) {
return std::pair{expr_to_str(expr_pair.first), expr_to_str(expr_pair.second)};
}) |
ranges::to<std::map<std::string, std::string, std::less<>>>;
};
auto const config_map = evaluate_config_map(parsed_query->configs_);
ASSERT_EQ(config_map.size(), 3);
EXPECT_EQ(config_map.find(memgraph::query::kBoltServer)->second, "127.0.0.1:7688");
EXPECT_EQ(config_map.find(memgraph::query::kManagementServer)->second, "127.0.0.1:10011");
EXPECT_EQ(config_map.find(memgraph::query::kReplicationServer)->second, "127.0.0.1:10001");
}
TEST_P(CypherMainVisitorTest, TestAddCoordinatorInstance) {
auto &ast_generator = *GetParam();
std::string const correct_query = R"(ADD COORDINATOR 1 ON "127.0.0.1:10111")";
std::string const correct_query =
R"(ADD COORDINATOR 1 WITH CONFIG {"bolt_server": "127.0.0.1:7688", "coordinator_server": "127.0.0.1:10111"})";
auto *parsed_query = dynamic_cast<CoordinatorQuery *>(ast_generator.ParseQuery(correct_query));
EXPECT_EQ(parsed_query->action_, CoordinatorQuery::Action::ADD_COORDINATOR_INSTANCE);
ast_generator.CheckLiteral(parsed_query->raft_socket_address_, TypedValue("127.0.0.1:10111"));
ast_generator.CheckLiteral(parsed_query->raft_server_id_, TypedValue(1));
ast_generator.CheckLiteral(parsed_query->coordinator_server_id_, TypedValue(1));
auto const evaluate_config_map = [&ast_generator](std::unordered_map<Expression *, Expression *> const &config_map)
-> std::map<std::string, std::string, std::less<>> {
auto const expr_to_str = [&ast_generator](Expression *expression) {
return std::string{ast_generator.GetLiteral(expression, ast_generator.context_.is_query_cached).ValueString()};
};
return ranges::views::transform(config_map,
[&expr_to_str](auto const &expr_pair) {
return std::pair{expr_to_str(expr_pair.first), expr_to_str(expr_pair.second)};
}) |
ranges::to<std::map<std::string, std::string, std::less<>>>;
};
auto const config_map = evaluate_config_map(parsed_query->configs_);
ASSERT_EQ(config_map.size(), 2);
EXPECT_EQ(config_map.find(kBoltServer)->second, "127.0.0.1:7688");
EXPECT_EQ(config_map.find(kCoordinatorServer)->second, "127.0.0.1:10111");
}
#endif