From 05cc35bf93b3c492c0ea1f0d2f469f5d504b1d4c Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Wed, 21 Jun 2023 14:50:46 +0200 Subject: [PATCH 1/9] Add command NULLIF for identifying nulls in LOAD CSV (#914) Add NULLIF command which turns all row values corresponding to the string to the nullif character sequence. --- src/query/frontend/ast/ast.hpp | 5 +- .../frontend/ast/cypher_main_visitor.cpp | 5 ++ .../opencypher/grammar/MemgraphCypher.g4 | 4 ++ .../opencypher/grammar/MemgraphCypherLexer.g4 | 1 + src/query/plan/operator.cpp | 35 +++++++++--- src/query/plan/operator.hpp | 4 +- src/query/plan/pretty_print.cpp | 4 ++ src/query/plan/rule_based_planner.hpp | 7 ++- tests/e2e/load_csv/CMakeLists.txt | 3 ++ tests/e2e/load_csv/load_csv_nullif.py | 53 +++++++++++++++++++ tests/e2e/load_csv/nullif.csv | 5 ++ tests/e2e/load_csv/workloads.yaml | 11 ++++ 12 files changed, 124 insertions(+), 13 deletions(-) create mode 100644 tests/e2e/load_csv/load_csv_nullif.py create mode 100644 tests/e2e/load_csv/nullif.csv diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index 3bdf704ce..caee103a2 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -3022,6 +3022,7 @@ class LoadCsv : public memgraph::query::Clause { bool ignore_bad_; memgraph::query::Expression *delimiter_{nullptr}; memgraph::query::Expression *quote_{nullptr}; + memgraph::query::Expression *nullif_{nullptr}; memgraph::query::Identifier *row_var_{nullptr}; LoadCsv *Clone(AstStorage *storage) const override { @@ -3031,18 +3032,20 @@ class LoadCsv : public memgraph::query::Clause { object->ignore_bad_ = ignore_bad_; object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr; object->quote_ = quote_ ? quote_->Clone(storage) : nullptr; + object->nullif_ = nullif_; object->row_var_ = row_var_ ? row_var_->Clone(storage) : nullptr; return object; } protected: explicit LoadCsv(Expression *file, bool with_header, bool ignore_bad, Expression *delimiter, Expression *quote, - Identifier *row_var) + Expression *nullif, Identifier *row_var) : file_(file), with_header_(with_header), ignore_bad_(ignore_bad), delimiter_(delimiter), quote_(quote), + nullif_(nullif), row_var_(row_var) { DMG_ASSERT(row_var, "LoadCsv cannot take nullptr for identifier"); } diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index 8b772ca0d..f2e037172 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -362,6 +362,11 @@ antlrcpp::Any CypherMainVisitor::visitLoadCsv(MemgraphCypher::LoadCsvContext *ct // handle skip bad row option load_csv->ignore_bad_ = ctx->IGNORE() && ctx->BAD(); + // handle character sequence which will correspond to nulls + if (ctx->NULLIF()) { + load_csv->nullif_ = std::any_cast(ctx->nullif()->accept(this)); + } + // handle delimiter if (ctx->DELIMITER()) { if (ctx->delimiter()->literal()->StringLiteral()) { diff --git a/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 b/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 index 9bb1bfabc..f7ffe8803 100644 --- a/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 +++ b/src/query/frontend/opencypher/grammar/MemgraphCypher.g4 @@ -59,6 +59,7 @@ memgraphCypherKeyword : cypherKeyword | GRANT | HEADER | IDENTIFIED + | NULLIF | ISOLATION | IN_MEMORY_ANALYTICAL | IN_MEMORY_TRANSACTIONAL @@ -224,6 +225,7 @@ loadCsv : LOAD CSV FROM csvFile ( WITH | NO ) HEADER ( IGNORE BAD ) ? ( DELIMITER delimiter ) ? ( QUOTE quote ) ? + ( NULLIF nullif ) ? AS rowVar ; csvFile : literal ; @@ -232,6 +234,8 @@ delimiter : literal ; quote : literal ; +nullif : literal ; + rowVar : variable ; userOrRoleName : symbolicName ; diff --git a/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 b/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 index 862682d6e..674a5f61d 100644 --- a/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 +++ b/src/query/frontend/opencypher/grammar/MemgraphCypherLexer.g4 @@ -85,6 +85,7 @@ MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ; NEXT : N E X T ; NO : N O ; NOTHING : N O T H I N G ; +NULLIF : N U L L I F ; PASSWORD : P A S S W O R D ; PORT : P O R T ; PRIVILEGES : P R I V I L E G E S ; diff --git a/src/query/plan/operator.cpp b/src/query/plan/operator.cpp index a922e92bf..d50ef68e4 100644 --- a/src/query/plan/operator.cpp +++ b/src/query/plan/operator.cpp @@ -4637,13 +4637,14 @@ UniqueCursorPtr CallProcedure::MakeCursor(utils::MemoryResource *mem) const { } LoadCsv::LoadCsv(std::shared_ptr input, Expression *file, bool with_header, bool ignore_bad, - Expression *delimiter, Expression *quote, Symbol row_var) + Expression *delimiter, Expression *quote, Expression *nullif, Symbol row_var) : input_(input ? input : (std::make_shared())), file_(file), with_header_(with_header), ignore_bad_(ignore_bad), delimiter_(delimiter), quote_(quote), + nullif_(nullif), row_var_(row_var) { MG_ASSERT(file_, "Something went wrong - '{}' member file_ shouldn't be a nullptr", __func__); } @@ -4674,22 +4675,31 @@ auto ToOptionalString(ExpressionEvaluator *evaluator, Expression *expression) -> return std::nullopt; }; -TypedValue CsvRowToTypedList(csv::Reader::Row &row) { +TypedValue CsvRowToTypedList(csv::Reader::Row &row, std::optional &nullif) { auto *mem = row.get_allocator().GetMemoryResource(); auto typed_columns = utils::pmr::vector(mem); typed_columns.reserve(row.size()); for (auto &column : row) { - typed_columns.emplace_back(std::move(column)); + if (!nullif.has_value() || column != nullif.value()) { + typed_columns.emplace_back(std::move(column)); + } else { + typed_columns.emplace_back(); + } } return {std::move(typed_columns), mem}; } -TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header) { +TypedValue CsvRowToTypedMap(csv::Reader::Row &row, csv::Reader::Header header, + std::optional &nullif) { // a valid row has the same number of elements as the header auto *mem = row.get_allocator().GetMemoryResource(); utils::pmr::map m(mem); for (auto i = 0; i < row.size(); ++i) { - m.emplace(std::move(header[i]), std::move(row[i])); + if (!nullif.has_value() || row[i] != nullif.value()) { + m.emplace(std::move(header[i]), std::move(row[i])); + } else { + m.emplace(std::piecewise_construct, std::forward_as_tuple(std::move(header[i])), std::forward_as_tuple()); + } } return {std::move(m), mem}; } @@ -4701,6 +4711,7 @@ class LoadCsvCursor : public Cursor { const UniqueCursorPtr input_cursor_; bool did_pull_; std::optional reader_{}; + std::optional nullif_; public: LoadCsvCursor(const LoadCsv *self, utils::MemoryResource *mem) @@ -4718,6 +4729,7 @@ class LoadCsvCursor : public Cursor { // without massacring the code even worse than I did here if (UNLIKELY(!reader_)) { reader_ = MakeReader(&context.evaluation_context); + nullif_ = ParseNullif(&context.evaluation_context); } if (input_cursor_->Pull(frame, context)) { @@ -4733,10 +4745,10 @@ class LoadCsvCursor : public Cursor { return false; } if (!reader_->HasHeader()) { - frame[self_->row_var_] = CsvRowToTypedList(*row); + frame[self_->row_var_] = CsvRowToTypedList(*row, nullif_); } else { frame[self_->row_var_] = - CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory)); + CsvRowToTypedMap(*row, csv::Reader::Header(reader_->GetHeader(), context.evaluation_context.memory), nullif_); } if (context.frame_change_collector && context.frame_change_collector->IsKeyTracked(self_->row_var_.name())) { context.frame_change_collector->ResetTrackingValue(self_->row_var_.name()); @@ -4768,6 +4780,15 @@ class LoadCsvCursor : public Cursor { csv::Reader::Config(self_->with_header_, self_->ignore_bad_, std::move(maybe_delim), std::move(maybe_quote)), utils::NewDeleteResource()); } + + std::optional ParseNullif(EvaluationContext *eval_context) { + Frame frame(0); + SymbolTable symbol_table; + DbAccessor *dba = nullptr; + auto evaluator = ExpressionEvaluator(&frame, symbol_table, *eval_context, dba, storage::View::OLD); + + return ToOptionalString(&evaluator, self_->nullif_); + } }; UniqueCursorPtr LoadCsv::MakeCursor(utils::MemoryResource *mem) const { diff --git a/src/query/plan/operator.hpp b/src/query/plan/operator.hpp index 9c0a0c831..8cbbccc9c 100644 --- a/src/query/plan/operator.hpp +++ b/src/query/plan/operator.hpp @@ -2227,7 +2227,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator { LoadCsv() = default; LoadCsv(std::shared_ptr input, Expression *file, bool with_header, bool ignore_bad, - Expression *delimiter, Expression *quote, Symbol row_var); + Expression *delimiter, Expression *quote, Expression *nullif, Symbol row_var); bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override; UniqueCursorPtr MakeCursor(utils::MemoryResource *) const override; std::vector OutputSymbols(const SymbolTable &) const override; @@ -2243,6 +2243,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator { bool ignore_bad_; Expression *delimiter_{nullptr}; Expression *quote_{nullptr}; + Expression *nullif_{nullptr}; Symbol row_var_; std::unique_ptr Clone(AstStorage *storage) const override { @@ -2253,6 +2254,7 @@ class LoadCsv : public memgraph::query::plan::LogicalOperator { object->ignore_bad_ = ignore_bad_; object->delimiter_ = delimiter_ ? delimiter_->Clone(storage) : nullptr; object->quote_ = quote_ ? quote_->Clone(storage) : nullptr; + object->nullif_ = nullif_; object->row_var_ = row_var_; return object; } diff --git a/src/query/plan/pretty_print.cpp b/src/query/plan/pretty_print.cpp index 3b5c2303b..3c23a2e8c 100644 --- a/src/query/plan/pretty_print.cpp +++ b/src/query/plan/pretty_print.cpp @@ -895,6 +895,10 @@ bool PlanToJsonVisitor::PreVisit(query::plan::LoadCsv &op) { self["quote"] = ToJson(op.quote_); } + if (op.nullif_) { + self["nullif"] = ToJson(op.nullif_); + } + self["row_variable"] = ToJson(op.row_var_); op.input_->Accept(*this); diff --git a/src/query/plan/rule_based_planner.hpp b/src/query/plan/rule_based_planner.hpp index b05b8d06e..09a53cf29 100644 --- a/src/query/plan/rule_based_planner.hpp +++ b/src/query/plan/rule_based_planner.hpp @@ -226,10 +226,9 @@ class RuleBasedPlanner { const auto &row_sym = context.symbol_table->at(*load_csv->row_var_); context.bound_symbols.insert(row_sym); - input_op = - std::make_unique(std::move(input_op), load_csv->file_, load_csv->with_header_, - load_csv->ignore_bad_, load_csv->delimiter_, load_csv->quote_, row_sym); - + input_op = std::make_unique(std::move(input_op), load_csv->file_, load_csv->with_header_, + load_csv->ignore_bad_, load_csv->delimiter_, load_csv->quote_, + load_csv->nullif_, row_sym); } else if (auto *foreach = utils::Downcast(clause)) { context.is_write_query = true; input_op = HandleForeachClause(foreach, std::move(input_op), *context.symbol_table, context.bound_symbols, diff --git a/tests/e2e/load_csv/CMakeLists.txt b/tests/e2e/load_csv/CMakeLists.txt index 06e6d6e33..368915dbe 100644 --- a/tests/e2e/load_csv/CMakeLists.txt +++ b/tests/e2e/load_csv/CMakeLists.txt @@ -8,3 +8,6 @@ endfunction() copy_load_csv_e2e_python_files(load_csv.py) copy_load_csv_e2e_files(simple.csv) + +copy_load_csv_e2e_python_files(load_csv_nullif.py) +copy_load_csv_e2e_files(nullif.csv) diff --git a/tests/e2e/load_csv/load_csv_nullif.py b/tests/e2e/load_csv/load_csv_nullif.py new file mode 100644 index 000000000..018781683 --- /dev/null +++ b/tests/e2e/load_csv/load_csv_nullif.py @@ -0,0 +1,53 @@ +# Copyright 2022 Memgraph Ltd. +# +# Use of this software is governed by the Business Source License +# included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +# License, and you may not use this file except in compliance with the Business Source License. +# +# As of the Change Date specified in that file, in accordance with +# the Business Source License, use of this software will be governed +# by the Apache License, Version 2.0, included in the file +# licenses/APL.txt. + +import os +import sys +from pathlib import Path + +import pytest +from gqlalchemy import Memgraph + +NULLIF_CSV_FILE = "nullif.csv" + + +def get_file_path(file: str) -> str: + parent_path = Path(__file__).parent.absolute() + return os.path.join(parent_path, file) + + +def test_given_csv_when_nullif_then_all_identical_rows_are_null(): + memgraph = Memgraph("localhost", 7687) + + results = list( + memgraph.execute_and_fetch( + f"""LOAD CSV FROM '{get_file_path(NULLIF_CSV_FILE)}' + WITH HEADER NULLIF 'N/A' AS row + CREATE (n:Person {{name: row.name, age: row.age, + percentage: row.percentage, works_in_IT: row.works_in_IT}}) + RETURN n + """ + ) + ) + + expected_properties = [ + {"age": "10", "percentage": "15.0", "works_in_IT": "false"}, + {"name": "John", "percentage": "35.4", "works_in_IT": "false"}, + {"name": "Milewa", "age": "34", "works_in_IT": "false"}, + {"name": "Lucas", "age": "50", "percentage": "12.5"}, + ] + properties = [result["n"]._properties for result in results] + + assert expected_properties == properties + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-rA"])) diff --git a/tests/e2e/load_csv/nullif.csv b/tests/e2e/load_csv/nullif.csv new file mode 100644 index 000000000..3a38bf775 --- /dev/null +++ b/tests/e2e/load_csv/nullif.csv @@ -0,0 +1,5 @@ +name,age,percentage,works_in_IT +N/A,10,15.0,false +John,N/A,35.4,false +Milewa,34,N/A,false +Lucas,50,12.5,N/A diff --git a/tests/e2e/load_csv/workloads.yaml b/tests/e2e/load_csv/workloads.yaml index e54fa728f..d07609699 100644 --- a/tests/e2e/load_csv/workloads.yaml +++ b/tests/e2e/load_csv/workloads.yaml @@ -1,3 +1,10 @@ +nullif_cluster: &nullif_cluster + cluster: + main: + args: ["--bolt-port", "7687", "--log-level=TRACE"] + log_file: "load_csv_log_file.txt" + validation_queries: [] + load_csv_cluster: &load_csv_cluster cluster: main: @@ -9,6 +16,10 @@ load_csv_cluster: &load_csv_cluster validation_queries: [] workloads: + - name: "LOAD CSV nullif" + binary: "tests/e2e/pytest_runner.sh" + args: ["load_csv/load_csv_nullif.py"] + <<: *nullif_cluster - name: "MATCH + LOAD CSV" binary: "tests/e2e/pytest_runner.sh" args: ["load_csv/load_csv.py"] From b875649270e51e13245a1c7e6c8f5fa81b8bbf05 Mon Sep 17 00:00:00 2001 From: Josipmrden Date: Wed, 21 Jun 2023 19:08:58 +0200 Subject: [PATCH 2/9] Add restoring of replication roles upon database startup (#791) Fix replica node restoration on startup so it is restored as replica and not as main. --- src/memgraph.cpp | 2 +- src/query/constants.hpp | 2 - src/query/interpreter.cpp | 20 ++- src/storage/v2/config.hpp | 2 +- src/storage/v2/replication/enums.hpp | 4 +- .../replication_persistence_helper.cpp | 34 ++-- .../replication_persistence_helper.hpp | 15 +- src/storage/v2/storage.cpp | 165 ++++++++++++++---- src/storage/v2/storage.hpp | 11 +- src/utils/frame_change_id.hpp | 11 ++ .../show_while_creating_invalid_state.py | 143 ++++++++++++++- tests/unit/replication_persistence_helper.cpp | 85 +++++---- tests/unit/storage_v2_replication.cpp | 12 +- 13 files changed, 385 insertions(+), 121 deletions(-) diff --git a/src/memgraph.cpp b/src/memgraph.cpp index 3ad5556c1..6d730a137 100644 --- a/src/memgraph.cpp +++ b/src/memgraph.cpp @@ -891,7 +891,7 @@ int main(int argc, char **argv) { .wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib, .wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx, .snapshot_on_exit = FLAGS_storage_snapshot_on_exit, - .restore_replicas_on_startup = true, + .restore_replication_state_on_startup = true, .items_per_batch = FLAGS_storage_items_per_batch, .recovery_thread_count = FLAGS_storage_recovery_thread_count, .allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery}, diff --git a/src/query/constants.hpp b/src/query/constants.hpp index 5a563524d..55b1eebea 100644 --- a/src/query/constants.hpp +++ b/src/query/constants.hpp @@ -14,8 +14,6 @@ #include namespace memgraph::query { -inline constexpr uint16_t kDefaultReplicationPort = 10000; -inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0"; inline const std::string kAsterisk = "*"; inline constexpr uint16_t kDeleteStatisticsNumResults = 6; } // namespace memgraph::query diff --git a/src/query/interpreter.cpp b/src/query/interpreter.cpp index e3753923f..f7d45b3b5 100644 --- a/src/query/interpreter.cpp +++ b/src/query/interpreter.cpp @@ -176,7 +176,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { throw QueryRuntimeException("Port number invalid!"); } if (!db_->SetReplicaRole( - io::network::Endpoint(query::kDefaultReplicationServerIp, static_cast(*port)))) { + io::network::Endpoint(storage::replication::kDefaultReplicationServerIp, static_cast(*port)))) { throw QueryRuntimeException("Couldn't set role to replica!"); } } @@ -185,9 +185,9 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { /// @throw QueryRuntimeException if an error ocurred. ReplicationQuery::ReplicationRole ShowReplicationRole() const override { switch (db_->GetReplicationRole()) { - case storage::ReplicationRole::MAIN: + case storage::replication::ReplicationRole::MAIN: return ReplicationQuery::ReplicationRole::MAIN; - case storage::ReplicationRole::REPLICA: + case storage::replication::ReplicationRole::REPLICA: return ReplicationQuery::ReplicationRole::REPLICA; } throw QueryRuntimeException("Couldn't show replication role - invalid role set!"); @@ -197,11 +197,15 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { void RegisterReplica(const std::string &name, const std::string &socket_address, const ReplicationQuery::SyncMode sync_mode, const std::chrono::seconds replica_check_frequency) override { - if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) { + if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { // replica can't register another replica throw QueryRuntimeException("Replica can't register another replica!"); } + if (name == storage::replication::kReservedReplicationRoleName) { + throw QueryRuntimeException("This replica name is reserved and can not be used as replica name!"); + } + storage::replication::ReplicationMode repl_mode; switch (sync_mode) { case ReplicationQuery::SyncMode::ASYNC: { @@ -215,7 +219,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { } auto maybe_ip_and_port = - io::network::Endpoint::ParseSocketOrIpAddress(socket_address, query::kDefaultReplicationPort); + io::network::Endpoint::ParseSocketOrIpAddress(socket_address, storage::replication::kDefaultReplicationPort); if (maybe_ip_and_port) { auto [ip, port] = *maybe_ip_and_port; auto ret = db_->RegisterReplica(name, {std::move(ip), port}, repl_mode, @@ -231,7 +235,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { /// @throw QueryRuntimeException if an error ocurred. void DropReplica(const std::string &replica_name) override { - if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) { + if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { // replica can't unregister a replica throw QueryRuntimeException("Replica can't unregister a replica!"); } @@ -242,7 +246,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler { using Replica = ReplicationQueryHandler::Replica; std::vector ShowReplicas() const override { - if (db_->GetReplicationRole() == storage::ReplicationRole::REPLICA) { + if (db_->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA) { // replica can't show registered replicas (it shouldn't have any) throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!"); } @@ -2982,7 +2986,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string, UpdateTypeCount(rw_type); if (const auto query_type = query_execution->prepared_query->rw_type; - interpreter_context_->db->GetReplicationRole() == storage::ReplicationRole::REPLICA && + interpreter_context_->db->GetReplicationRole() == storage::replication::ReplicationRole::REPLICA && (query_type == RWType::W || query_type == RWType::RW)) { query_execution = nullptr; throw QueryException("Write query forbidden on the replica!"); diff --git a/src/storage/v2/config.hpp b/src/storage/v2/config.hpp index 126cbc10c..3b1beec92 100644 --- a/src/storage/v2/config.hpp +++ b/src/storage/v2/config.hpp @@ -49,7 +49,7 @@ struct Config { uint64_t wal_file_flush_every_n_tx{100000}; bool snapshot_on_exit{false}; - bool restore_replicas_on_startup{false}; + bool restore_replication_state_on_startup{false}; uint64_t items_per_batch{1'000'000}; uint64_t recovery_thread_count{8}; diff --git a/src/storage/v2/replication/enums.hpp b/src/storage/v2/replication/enums.hpp index 133fd9b74..bbe8ffc62 100644 --- a/src/storage/v2/replication/enums.hpp +++ b/src/storage/v2/replication/enums.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -13,6 +13,8 @@ #include namespace memgraph::storage::replication { +enum class ReplicationRole : uint8_t { MAIN, REPLICA }; + enum class ReplicationMode : std::uint8_t { SYNC, ASYNC }; enum class ReplicaState : std::uint8_t { READY, REPLICATING, RECOVERY, INVALID }; diff --git a/src/storage/v2/replication/replication_persistence_helper.cpp b/src/storage/v2/replication/replication_persistence_helper.cpp index f05848cea..262938a82 100644 --- a/src/storage/v2/replication/replication_persistence_helper.cpp +++ b/src/storage/v2/replication/replication_persistence_helper.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -10,21 +10,24 @@ // licenses/APL.txt. #include "storage/v2/replication/replication_persistence_helper.hpp" + +#include "storage/v2/replication/enums.hpp" #include "utils/logging.hpp" namespace { -const std::string kReplicaName = "replica_name"; -const std::string kIpAddress = "replica_ip_address"; -const std::string kPort = "replica_port"; -const std::string kSyncMode = "replica_sync_mode"; -const std::string kCheckFrequency = "replica_check_frequency"; -const std::string kSSLKeyFile = "replica_ssl_key_file"; -const std::string kSSLCertFile = "replica_ssl_cert_file"; +inline constexpr auto *kReplicaName = "replica_name"; +inline constexpr auto *kIpAddress = "replica_ip_address"; +inline constexpr auto *kPort = "replica_port"; +inline constexpr auto *kSyncMode = "replica_sync_mode"; +inline constexpr auto *kCheckFrequency = "replica_check_frequency"; +inline constexpr auto *kSSLKeyFile = "replica_ssl_key_file"; +inline constexpr auto *kSSLCertFile = "replica_ssl_cert_file"; +inline constexpr auto *kReplicationRole = "replication_role"; } // namespace namespace memgraph::storage::replication { -nlohmann::json ReplicaStatusToJSON(ReplicaStatus &&status) { +nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status) { auto data = nlohmann::json::object(); data[kReplicaName] = std::move(status.name); @@ -42,11 +45,15 @@ nlohmann::json ReplicaStatusToJSON(ReplicaStatus &&status) { data[kSSLCertFile] = nullptr; } + if (status.role.has_value()) { + data[kReplicationRole] = *status.role; + } + return data; } -std::optional JSONToReplicaStatus(nlohmann::json &&data) { - ReplicaStatus replica_status; +std::optional JSONToReplicationStatus(nlohmann::json &&data) { + ReplicationStatus replica_status; const auto get_failed_message = [](const std::string_view message, const std::string_view nested_message) { return fmt::format("Failed to deserialize replica's configuration: {} : {}", message, nested_message); @@ -70,6 +77,11 @@ std::optional JSONToReplicaStatus(nlohmann::json &&data) { data.at(kSSLKeyFile).get_to(replica_status.ssl->key_file); data.at(kSSLCertFile).get_to(replica_status.ssl->cert_file); } + + if (data.find(kReplicationRole) != data.end()) { + replica_status.role = replication::ReplicationRole::MAIN; + data.at(kReplicationRole).get_to(replica_status.role.value()); + } } catch (const nlohmann::json::type_error &exception) { spdlog::error(get_failed_message("Invalid type conversion", exception.what())); return std::nullopt; diff --git a/src/storage/v2/replication/replication_persistence_helper.hpp b/src/storage/v2/replication/replication_persistence_helper.hpp index c22164e33..df6cb03b9 100644 --- a/src/storage/v2/replication/replication_persistence_helper.hpp +++ b/src/storage/v2/replication/replication_persistence_helper.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -23,18 +23,23 @@ namespace memgraph::storage::replication { -struct ReplicaStatus { +inline constexpr auto *kReservedReplicationRoleName{"__replication_role"}; +inline constexpr uint16_t kDefaultReplicationPort = 10000; +inline constexpr auto *kDefaultReplicationServerIp = "0.0.0.0"; + +struct ReplicationStatus { std::string name; std::string ip_address; uint16_t port; ReplicationMode sync_mode; std::chrono::seconds replica_check_frequency; std::optional ssl; + std::optional role; - friend bool operator==(const ReplicaStatus &, const ReplicaStatus &) = default; + friend bool operator==(const ReplicationStatus &, const ReplicationStatus &) = default; }; -nlohmann::json ReplicaStatusToJSON(ReplicaStatus &&status); +nlohmann::json ReplicationStatusToJSON(ReplicationStatus &&status); -std::optional JSONToReplicaStatus(nlohmann::json &&data); +std::optional JSONToReplicationStatus(nlohmann::json &&data); } // namespace memgraph::storage::replication diff --git a/src/storage/v2/storage.cpp b/src/storage/v2/storage.cpp index 11ea723ea..b450891e3 100644 --- a/src/storage/v2/storage.cpp +++ b/src/storage/v2/storage.cpp @@ -335,13 +335,6 @@ Storage::Storage(Config config) uuid_(utils::GenerateUUID()), epoch_id_(utils::GenerateUUID()), global_locker_(file_retainer_.AddLocker()) { - if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED && - replication_role_ == ReplicationRole::MAIN) { - spdlog::warn( - "The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider " - "enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because " - "without write-ahead logs this instance is not replicating any data."); - } if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED || config_.durability.snapshot_on_exit || config_.durability.recover_on_startup) { // Create the directory initially to crash the database in case of @@ -437,14 +430,29 @@ Storage::Storage(Config config) commit_log_.emplace(timestamp_); } - if (config_.durability.restore_replicas_on_startup) { - spdlog::info("Replica's configuration will be stored and will be automatically restored in case of a crash."); + if (config_.durability.restore_replication_state_on_startup) { + spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash."); utils::EnsureDirOrDie(config_.durability.storage_directory / durability::kReplicationDirectory); storage_ = std::make_unique(config_.durability.storage_directory / durability::kReplicationDirectory); - RestoreReplicas(); + + RestoreReplicationRole(); + + if (replication_role_ == replication::ReplicationRole::MAIN) { + RestoreReplicas(); + } } else { - spdlog::warn("Replicas' configuration will NOT be stored. When the server restarts, replicas will be forgotten."); + spdlog::warn( + "Replicastion configuration will NOT be stored. When the server restarts, replication state will be " + "forgotten."); + } + + if (config_.durability.snapshot_wal_mode == Config::Durability::SnapshotWalMode::DISABLED && + replication_role_ == replication::ReplicationRole::MAIN) { + spdlog::warn( + "The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider " + "enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because " + "without write-ahead logs this instance is not replicating any data."); } } @@ -968,7 +976,7 @@ utils::BasicResult Storage::Accessor::Commit // modifications before they are written to disk. // Replica can log only the write transaction received from Main // so the Wal files are consistent - if (storage_->replication_role_ == ReplicationRole::MAIN || desired_commit_timestamp.has_value()) { + if (storage_->replication_role_ == replication::ReplicationRole::MAIN || desired_commit_timestamp.has_value()) { could_replicate_all_sync_replicas = storage_->AppendToWalDataManipulation(transaction_, *commit_timestamp_); } @@ -982,7 +990,8 @@ utils::BasicResult Storage::Accessor::Commit transaction_.commit_timestamp->store(*commit_timestamp_, std::memory_order_release); // Replica can only update the last commit timestamp with // the commits received from main. - if (storage_->replication_role_ == ReplicationRole::MAIN || desired_commit_timestamp.has_value()) { + if (storage_->replication_role_ == replication::ReplicationRole::MAIN || + desired_commit_timestamp.has_value()) { // Update the last commit timestamp storage_->last_commit_timestamp_.store(*commit_timestamp_); } @@ -1447,7 +1456,7 @@ Transaction Storage::CreateTransaction(IsolationLevel isolation_level, StorageMo // of any query on replica to the last commited transaction // which is timestamp_ as only commit of transaction with writes // can change the value of it. - if (replication_role_ == ReplicationRole::REPLICA) { + if (replication_role_ == replication::ReplicationRole::REPLICA) { start_timestamp = timestamp_; } else { start_timestamp = timestamp_++; @@ -1752,7 +1761,7 @@ bool Storage::AppendToWalDataManipulation(const Transaction &transaction, uint64 // A single transaction will always be contained in a single WAL file. auto current_commit_timestamp = transaction.commit_timestamp->load(std::memory_order_acquire); - if (replication_role_.load() == ReplicationRole::MAIN) { + if (replication_role_.load() == replication::ReplicationRole::MAIN) { replication_clients_.WithLock([&](auto &clients) { for (auto &client : clients) { client->StartTransactionReplication(wal_file_->SequenceNumber()); @@ -1940,7 +1949,7 @@ bool Storage::AppendToWalDataDefinition(durability::StorageGlobalOperation opera auto finalized_on_all_replicas = true; wal_file_->AppendOperation(operation, label, properties, final_commit_timestamp); { - if (replication_role_.load() == ReplicationRole::MAIN) { + if (replication_role_.load() == replication::ReplicationRole::MAIN) { replication_clients_.WithLock([&](auto &clients) { for (auto &client : clients) { client->StartTransactionReplication(wal_file_->SequenceNumber()); @@ -1960,7 +1969,7 @@ bool Storage::AppendToWalDataDefinition(durability::StorageGlobalOperation opera } utils::BasicResult Storage::CreateSnapshot(std::optional is_periodic) { - if (replication_role_.load() != ReplicationRole::MAIN) { + if (replication_role_.load() != replication::ReplicationRole::MAIN) { return CreateSnapshotError::DisabledForReplica; } @@ -2054,20 +2063,38 @@ uint64_t Storage::CommitTimestamp(const std::optional desired_commit_t bool Storage::SetReplicaRole(io::network::Endpoint endpoint, const replication::ReplicationServerConfig &config) { // We don't want to restart the server if we're already a REPLICA - if (replication_role_ == ReplicationRole::REPLICA) { + if (replication_role_ == replication::ReplicationRole::REPLICA) { return false; } + auto port = endpoint.port; // assigning because we will move the endpoint replication_server_ = std::make_unique(this, std::move(endpoint), config); - replication_role_.store(ReplicationRole::REPLICA); + if (ShouldStoreAndRestoreReplicationState()) { + // Only thing that matters here is the role saved as REPLICA and the listening port + auto data = replication::ReplicationStatusToJSON( + replication::ReplicationStatus{.name = replication::kReservedReplicationRoleName, + .ip_address = "", + .port = port, + .sync_mode = replication::ReplicationMode::SYNC, + .replica_check_frequency = std::chrono::seconds(0), + .ssl = std::nullopt, + .role = replication::ReplicationRole::REPLICA}); + + if (!storage_->Put(replication::kReservedReplicationRoleName, data.dump())) { + spdlog::error("Error when saving REPLICA replication role in settings."); + return false; + } + } + + replication_role_.store(replication::ReplicationRole::REPLICA); return true; } bool Storage::SetMainReplicationRole() { // We don't want to generate new epoch_id and do the // cleanup if we're already a MAIN - if (replication_role_ == ReplicationRole::MAIN) { + if (replication_role_ == replication::ReplicationRole::MAIN) { return false; } @@ -2090,14 +2117,33 @@ bool Storage::SetMainReplicationRole() { epoch_id_ = utils::GenerateUUID(); } - replication_role_.store(ReplicationRole::MAIN); + if (ShouldStoreAndRestoreReplicationState()) { + // Only thing that matters here is the role saved as MAIN + auto data = replication::ReplicationStatusToJSON( + replication::ReplicationStatus{.name = replication::kReservedReplicationRoleName, + .ip_address = "", + .port = 0, + .sync_mode = replication::ReplicationMode::SYNC, + .replica_check_frequency = std::chrono::seconds(0), + .ssl = std::nullopt, + .role = replication::ReplicationRole::MAIN}); + + if (!storage_->Put(replication::kReservedReplicationRoleName, data.dump())) { + spdlog::error("Error when saving MAIN replication role in settings."); + return false; + } + } + + replication_role_.store(replication::ReplicationRole::MAIN); + return true; } utils::BasicResult Storage::RegisterReplica( std::string name, io::network::Endpoint endpoint, const replication::ReplicationMode replication_mode, const replication::RegistrationMode registration_mode, const replication::ReplicationClientConfig &config) { - MG_ASSERT(replication_role_.load() == ReplicationRole::MAIN, "Only main instance can register a replica!"); + MG_ASSERT(replication_role_.load() == replication::ReplicationRole::MAIN, + "Only main instance can register a replica!"); const bool name_exists = replication_clients_.WithLock([&](auto &clients) { return std::any_of(clients.begin(), clients.end(), [&name](const auto &client) { return client->Name() == name; }); @@ -2116,14 +2162,15 @@ utils::BasicResult Storage::RegisterReplica( return RegisterReplicaError::END_POINT_EXISTS; } - if (ShouldStoreAndRestoreReplicas()) { - auto data = replication::ReplicaStatusToJSON( - replication::ReplicaStatus{.name = name, - .ip_address = endpoint.address, - .port = endpoint.port, - .sync_mode = replication_mode, - .replica_check_frequency = config.replica_check_frequency, - .ssl = config.ssl}); + if (ShouldStoreAndRestoreReplicationState()) { + auto data = replication::ReplicationStatusToJSON( + replication::ReplicationStatus{.name = name, + .ip_address = endpoint.address, + .port = endpoint.port, + .sync_mode = replication_mode, + .replica_check_frequency = config.replica_check_frequency, + .ssl = config.ssl, + .role = replication::ReplicationRole::REPLICA}); if (!storage_->Put(name, data.dump())) { spdlog::error("Error when saving replica {} in settings.", name); return RegisterReplicaError::COULD_NOT_BE_PERSISTED; @@ -2159,8 +2206,9 @@ utils::BasicResult Storage::RegisterReplica( } bool Storage::UnregisterReplica(const std::string &name) { - MG_ASSERT(replication_role_.load() == ReplicationRole::MAIN, "Only main instance can unregister a replica!"); - if (ShouldStoreAndRestoreReplicas()) { + MG_ASSERT(replication_role_.load() == replication::ReplicationRole::MAIN, + "Only main instance can unregister a replica!"); + if (ShouldStoreAndRestoreReplicationState()) { if (!storage_->Delete(name)) { spdlog::error("Error when removing replica {} from settings.", name); return false; @@ -2183,7 +2231,7 @@ std::optional Storage::GetReplicaState(const std::str }); } -ReplicationRole Storage::GetReplicationRole() const { return replication_role_; } +replication::ReplicationRole Storage::GetReplicationRole() const { return replication_role_; } std::vector Storage::ReplicasInfo() { return replication_clients_.WithLock([](auto &clients) { @@ -2207,6 +2255,46 @@ utils::BasicResult Storage::SetIsolationLevel(I return {}; } +void Storage::RestoreReplicationRole() { + if (!ShouldStoreAndRestoreReplicationState()) { + return; + } + + spdlog::info("Restoring replication role."); + + uint16_t port = replication::kDefaultReplicationPort; + for (const auto &[replica_name, replica_data] : *storage_) { + const auto maybe_replica_status = replication::JSONToReplicationStatus(nlohmann::json::parse(replica_data)); + if (!maybe_replica_status.has_value()) { + LOG_FATAL("Cannot parse previously saved configuration of replica {}.", replica_name); + } + + if (replica_name != replication::kReservedReplicationRoleName) { + continue; + } + + auto replica_status = *maybe_replica_status; + + if (!replica_status.role.has_value()) { + replication_role_.store(replication::ReplicationRole::MAIN); + } else { + replication_role_.store(*replica_status.role); + port = replica_status.port; + } + + break; + } + + if (replication_role_ == replication::ReplicationRole::REPLICA) { + io::network::Endpoint endpoint(replication::kDefaultReplicationServerIp, port); + replication_server_ = + std::make_unique(this, std::move(endpoint), replication::ReplicationServerConfig{}); + } + + spdlog::info("Replication role restored to {}.", + replication_role_ == replication::ReplicationRole::MAIN ? "MAIN" : "REPLICA"); +} + IsolationLevel Storage::GetIsolationLevel() const noexcept { return isolation_level_; } void Storage::SetStorageMode(StorageMode storage_mode) { @@ -2217,8 +2305,7 @@ void Storage::SetStorageMode(StorageMode storage_mode) { StorageMode Storage::GetStorageMode() { return storage_mode_; } void Storage::RestoreReplicas() { - MG_ASSERT(memgraph::storage::ReplicationRole::MAIN == GetReplicationRole()); - if (!ShouldStoreAndRestoreReplicas()) { + if (!ShouldStoreAndRestoreReplicationState()) { return; } spdlog::info("Restoring replicas."); @@ -2226,7 +2313,7 @@ void Storage::RestoreReplicas() { for (const auto &[replica_name, replica_data] : *storage_) { spdlog::info("Restoring replica {}.", replica_name); - const auto maybe_replica_status = replication::JSONToReplicaStatus(nlohmann::json::parse(replica_data)); + const auto maybe_replica_status = replication::JSONToReplicationStatus(nlohmann::json::parse(replica_data)); if (!maybe_replica_status.has_value()) { LOG_FATAL("Cannot parse previously saved configuration of replica {}.", replica_name); } @@ -2235,6 +2322,10 @@ void Storage::RestoreReplicas() { MG_ASSERT(replica_status.name == replica_name, "Expected replica name is '{}', but got '{}'", replica_status.name, replica_name); + if (replica_name == replication::kReservedReplicationRoleName) { + continue; + } + auto ret = RegisterReplica(std::move(replica_status.name), {std::move(replica_status.ip_address), replica_status.port}, replica_status.sync_mode, replication::RegistrationMode::CAN_BE_INVALID, @@ -2251,6 +2342,6 @@ void Storage::RestoreReplicas() { } } -bool Storage::ShouldStoreAndRestoreReplicas() const { return nullptr != storage_; } +bool Storage::ShouldStoreAndRestoreReplicationState() const { return nullptr != storage_; } } // namespace memgraph::storage diff --git a/src/storage/v2/storage.hpp b/src/storage/v2/storage.hpp index 7fd816530..9a9bc29e9 100644 --- a/src/storage/v2/storage.hpp +++ b/src/storage/v2/storage.hpp @@ -50,6 +50,7 @@ #include "rpc/server.hpp" #include "storage/v2/replication/config.hpp" #include "storage/v2/replication/enums.hpp" +#include "storage/v2/replication/replication_persistence_helper.hpp" #include "storage/v2/replication/rpc.hpp" #include "storage/v2/replication/serialization.hpp" #include "storage/v2/storage_error.hpp" @@ -188,8 +189,6 @@ struct StorageInfo { uint64_t disk_usage; }; -enum class ReplicationRole : uint8_t { MAIN, REPLICA }; - class Storage final { public: /// @throw std::system_error @@ -493,7 +492,7 @@ class Storage final { std::optional GetReplicaState(std::string_view name); - ReplicationRole GetReplicationRole() const; + replication::ReplicationRole GetReplicationRole() const; struct TimestampInfo { uint64_t current_timestamp_of_replica; @@ -557,9 +556,11 @@ class Storage final { uint64_t CommitTimestamp(std::optional desired_commit_timestamp = {}); + void RestoreReplicationRole(); + void RestoreReplicas(); - bool ShouldStoreAndRestoreReplicas() const; + bool ShouldStoreAndRestoreReplicationState() const; // Main storage lock. // @@ -680,7 +681,7 @@ class Storage final { using ReplicationClientList = utils::Synchronized>, utils::SpinLock>; ReplicationClientList replication_clients_; - std::atomic replication_role_{ReplicationRole::MAIN}; + std::atomic replication_role_{replication::ReplicationRole::MAIN}; }; } // namespace memgraph::storage diff --git a/src/utils/frame_change_id.hpp b/src/utils/frame_change_id.hpp index 525af4e92..e3facfa5e 100644 --- a/src/utils/frame_change_id.hpp +++ b/src/utils/frame_change_id.hpp @@ -1,3 +1,14 @@ +// Copyright 2023 Memgraph Ltd. +// +// Use of this software is governed by the Business Source License +// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source +// License, and you may not use this file except in compliance with the Business Source License. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0, included in the file +// licenses/APL.txt. + #include #include "query/frontend/ast/ast.hpp" diff --git a/tests/e2e/replication/show_while_creating_invalid_state.py b/tests/e2e/replication/show_while_creating_invalid_state.py index d57f556f2..6bd36dc83 100644 --- a/tests/e2e/replication/show_while_creating_invalid_state.py +++ b/tests/e2e/replication/show_while_creating_invalid_state.py @@ -9,17 +9,16 @@ # by the Apache License, Version 2.0, included in the file # licenses/APL.txt. -import sys - import os -import pytest import random +import sys +import tempfile -from common import execute_and_fetch_all -from mg_utils import mg_sleep_and_assert import interactive_mg_runner import mgclient -import tempfile +import pytest +from common import execute_and_fetch_all +from mg_utils import mg_sleep_and_assert interactive_mg_runner.SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) interactive_mg_runner.PROJECT_DIR = os.path.normpath( @@ -340,6 +339,138 @@ def test_basic_recovery(connection): assert interactive_mg_runner.MEMGRAPH_INSTANCES[f"replica_{index}"].query(QUERY_TO_CHECK) == res_from_main +def test_replication_role_recovery(connection): + # Goal of this test is to check the recovery of main and replica role. + # 0/ We start all replicas manually: we want to be able to kill them ourselves without relying on external tooling to kill processes. + # 1/ We try to add a replica with reserved name which results in an exception + # 2/ We check that all replicas have the correct state: they should all be ready. + # 3/ We kill main. + # 4/ We re-start main. We check that main indeed has the role main and replicas still have the correct state. + # 5/ We kill the replica. + # 6/ We observed that the replica result is in invalid state. + # 7/ We start the replica again. We observe that indeed the replica has the replica state. + # 8/ We observe that main has the replica ready. + # 9/ We kill the replica again. + # 10/ We add data to main. + # 11/ We start the replica again. We observe that the replica has the same + # data as main because it synced and added lost data. + + # 0/ + data_directory = tempfile.TemporaryDirectory() + CONFIGURATION = { + "replica": { + "args": ["--bolt-port", "7688", "--log-level=TRACE"], + "log_file": "replica.log", + "setup_queries": ["SET REPLICATION ROLE TO REPLICA WITH PORT 10001;"], + "data_directory": f"{data_directory.name}/replica", + }, + "main": { + "args": ["--bolt-port", "7687", "--log-level=TRACE", "--storage-recover-on-startup=true"], + "log_file": "main.log", + "setup_queries": [], + "data_directory": f"{data_directory.name}/main", + }, + } + + interactive_mg_runner.start_all(CONFIGURATION) + cursor = connection(7687, "main").cursor() + + # We want to execute manually and not via the configuration, otherwise re-starting main would also execute these registration. + execute_and_fetch_all(cursor, "REGISTER REPLICA replica SYNC TO '127.0.0.1:10001';") + + # When we restart the replica, it does not need this query anymore since it needs to remember state + CONFIGURATION = { + "replica": { + "args": ["--bolt-port", "7688", "--log-level=TRACE"], + "log_file": "replica.log", + "setup_queries": [], + "data_directory": f"{data_directory.name}/replica", + }, + "main": { + "args": ["--bolt-port", "7687", "--log-level=TRACE", "--storage-recover-on-startup=true"], + "log_file": "main.log", + "setup_queries": [], + "data_directory": f"{data_directory.name}/main", + }, + } + # 1/ + with pytest.raises(mgclient.DatabaseError): + execute_and_fetch_all(cursor, "REGISTER REPLICA __replication_role SYNC TO '127.0.0.1:10002';") + + # 2/ + expected_data = { + ("replica", "127.0.0.1:10001", "sync", 0, 0, "ready"), + } + actual_data = set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + + assert actual_data == expected_data + + def check_roles(): + assert "main" == interactive_mg_runner.MEMGRAPH_INSTANCES["main"].query("SHOW REPLICATION ROLE;")[0][0] + assert "replica" == interactive_mg_runner.MEMGRAPH_INSTANCES["replica"].query("SHOW REPLICATION ROLE;")[0][0] + + check_roles() + + # 3/ + interactive_mg_runner.kill(CONFIGURATION, "main") + + # 4/ + interactive_mg_runner.start(CONFIGURATION, "main") + cursor = connection(7687, "main").cursor() + check_roles() + + def retrieve_data(): + return set(execute_and_fetch_all(cursor, "SHOW REPLICAS;")) + + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + assert actual_data == expected_data + + # 5/ + interactive_mg_runner.kill(CONFIGURATION, "replica") + + # 6/ + expected_data = { + ("replica", "127.0.0.1:10001", "sync", 0, 0, "invalid"), + } + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + + assert actual_data == expected_data + + # 7/ + interactive_mg_runner.start(CONFIGURATION, "replica") + check_roles() + + # 8/ + expected_data = { + ("replica", "127.0.0.1:10001", "sync", 0, 0, "ready"), + } + + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + assert actual_data == expected_data + + # 9/ + interactive_mg_runner.kill(CONFIGURATION, "replica") + + # 10/ + with pytest.raises(mgclient.DatabaseError): + execute_and_fetch_all(cursor, "CREATE (n:First)") + + # 11/ + interactive_mg_runner.start(CONFIGURATION, "replica") + check_roles() + + expected_data = { + ("replica", "127.0.0.1:10001", "sync", 2, 0, "ready"), + } + actual_data = mg_sleep_and_assert(expected_data, retrieve_data) + assert actual_data == expected_data + + QUERY_TO_CHECK = "MATCH (node) return node;" + res_from_main = execute_and_fetch_all(cursor, QUERY_TO_CHECK) + assert len(res_from_main) == 1 + assert res_from_main == interactive_mg_runner.MEMGRAPH_INSTANCES["replica"].query(QUERY_TO_CHECK) + + def test_conflict_at_startup(connection): # Goal of this test is to check starting up several instance with different replicas' configuration directory works as expected. # main_1 and main_2 have different directory. diff --git a/tests/unit/replication_persistence_helper.cpp b/tests/unit/replication_persistence_helper.cpp index ffe125cd5..7915d315c 100644 --- a/tests/unit/replication_persistence_helper.cpp +++ b/tests/unit/replication_persistence_helper.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -19,73 +19,82 @@ #include #include -class ReplicationPersistanceHelperTest : public ::testing::Test { +using namespace memgraph::storage::replication; + +class ReplicationPersistanceHelperTest : public testing::Test { protected: void SetUp() override {} void TearDown() override {} - memgraph::storage::replication::ReplicaStatus CreateReplicaStatus( - std::string name, std::string ip_address, uint16_t port, - memgraph::storage::replication::ReplicationMode sync_mode, std::chrono::seconds replica_check_frequency, - std::optional ssl) const { - return memgraph::storage::replication::ReplicaStatus{.name = name, - .ip_address = ip_address, - .port = port, - .sync_mode = sync_mode, - .replica_check_frequency = replica_check_frequency, - .ssl = ssl}; + ReplicationStatus CreateReplicationStatus(std::string name, std::string ip_address, uint16_t port, + ReplicationMode sync_mode, std::chrono::seconds replica_check_frequency, + std::optional ssl, + std::optional role) const { + return ReplicationStatus{.name = name, + .ip_address = ip_address, + .port = port, + .sync_mode = sync_mode, + .replica_check_frequency = replica_check_frequency, + .ssl = ssl, + .role = role}; } static_assert( - sizeof(memgraph::storage::replication::ReplicaStatus) == 152, - "Most likely you modified ReplicaStatus without updating the tests. Please modify CreateReplicaStatus. "); + sizeof(ReplicationStatus) == 160, + "Most likely you modified ReplicationStatus without updating the tests. Please modify CreateReplicationStatus. "); }; TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesInitialized) { - auto replicas_status = CreateReplicaStatus( - "name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC, std::chrono::seconds(1), - memgraph::storage::replication::ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}); + auto replicas_status = CreateReplicationStatus( + "name", "ip_address", 0, ReplicationMode::SYNC, std::chrono::seconds(1), + ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}, ReplicationRole::REPLICA); - auto json_status = memgraph::storage::replication::ReplicaStatusToJSON( - memgraph::storage::replication::ReplicaStatus(replicas_status)); - auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status)); + auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status)); + auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status)); ASSERT_EQ(replicas_status, *replicas_status_converted); } TEST_F(ReplicationPersistanceHelperTest, BasicTestOnlyMandatoryAttributesInitialized) { - auto replicas_status = - CreateReplicaStatus("name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC, - std::chrono::seconds(1), std::nullopt); + auto replicas_status = CreateReplicationStatus("name", "ip_address", 0, ReplicationMode::SYNC, + std::chrono::seconds(1), std::nullopt, std::nullopt); - auto json_status = memgraph::storage::replication::ReplicaStatusToJSON( - memgraph::storage::replication::ReplicaStatus(replicas_status)); - auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status)); + auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status)); + auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status)); ASSERT_EQ(replicas_status, *replicas_status_converted); } TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesButSSLInitialized) { - auto replicas_status = - CreateReplicaStatus("name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC, - std::chrono::seconds(1), std::nullopt); + auto replicas_status = CreateReplicationStatus("name", "ip_address", 0, ReplicationMode::SYNC, + std::chrono::seconds(1), std::nullopt, ReplicationRole::MAIN); - auto json_status = memgraph::storage::replication::ReplicaStatusToJSON( - memgraph::storage::replication::ReplicaStatus(replicas_status)); - auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status)); + auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status)); + auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status)); ASSERT_EQ(replicas_status, *replicas_status_converted); } TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesButTimeoutInitialized) { - auto replicas_status = CreateReplicaStatus( - "name", "ip_address", 0, memgraph::storage::replication::ReplicationMode::SYNC, std::chrono::seconds(1), - memgraph::storage::replication::ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}); + auto replicas_status = CreateReplicationStatus( + "name", "ip_address", 0, ReplicationMode::SYNC, std::chrono::seconds(1), + ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}, ReplicationRole::REPLICA); - auto json_status = memgraph::storage::replication::ReplicaStatusToJSON( - memgraph::storage::replication::ReplicaStatus(replicas_status)); - auto replicas_status_converted = memgraph::storage::replication::JSONToReplicaStatus(std::move(json_status)); + auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status)); + auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status)); + + ASSERT_EQ(replicas_status, *replicas_status_converted); +} + +TEST_F(ReplicationPersistanceHelperTest, BasicTestAllAttributesButReplicationRoleInitialized) { + // this one is importand for backwards compatibility + auto replicas_status = CreateReplicationStatus( + "name", "ip_address", 0, ReplicationMode::SYNC, std::chrono::seconds(1), + ReplicationClientConfig::SSL{.key_file = "key_file", .cert_file = "cert_file"}, std::nullopt); + + auto json_status = ReplicationStatusToJSON(ReplicationStatus(replicas_status)); + auto replicas_status_converted = JSONToReplicationStatus(std::move(json_status)); ASSERT_EQ(replicas_status, *replicas_status_converted); } diff --git a/tests/unit/storage_v2_replication.cpp b/tests/unit/storage_v2_replication.cpp index 555b7ab27..54baa2a65 100644 --- a/tests/unit/storage_v2_replication.cpp +++ b/tests/unit/storage_v2_replication.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -652,9 +652,9 @@ TEST_F(ReplicationTest, ReplicationInformation) { memgraph::storage::replication::RegistrationMode::MUST_BE_INSTANTLY_VALID) .HasError()); - ASSERT_EQ(main_store.GetReplicationRole(), memgraph::storage::ReplicationRole::MAIN); - ASSERT_EQ(replica_store1.GetReplicationRole(), memgraph::storage::ReplicationRole::REPLICA); - ASSERT_EQ(replica_store2.GetReplicationRole(), memgraph::storage::ReplicationRole::REPLICA); + ASSERT_EQ(main_store.GetReplicationRole(), memgraph::storage::replication::ReplicationRole::MAIN); + ASSERT_EQ(replica_store1.GetReplicationRole(), memgraph::storage::replication::ReplicationRole::REPLICA); + ASSERT_EQ(replica_store2.GetReplicationRole(), memgraph::storage::replication::ReplicationRole::REPLICA); const auto replicas_info = main_store.ReplicasInfo(); ASSERT_EQ(replicas_info.size(), 2); @@ -730,7 +730,7 @@ TEST_F(ReplicationTest, ReplicationReplicaWithExistingEndPoint) { TEST_F(ReplicationTest, RestoringReplicationAtStartupAftgerDroppingReplica) { auto main_config = configuration; - main_config.durability.restore_replicas_on_startup = true; + main_config.durability.restore_replication_state_on_startup = true; auto main_store = std::make_unique(main_config); memgraph::storage::Storage replica_store1(configuration); @@ -773,7 +773,7 @@ TEST_F(ReplicationTest, RestoringReplicationAtStartupAftgerDroppingReplica) { TEST_F(ReplicationTest, RestoringReplicationAtStartup) { auto main_config = configuration; - main_config.durability.restore_replicas_on_startup = true; + main_config.durability.restore_replication_state_on_startup = true; auto main_store = std::make_unique(main_config); memgraph::storage::Storage replica_store1(configuration); replica_store1.SetReplicaRole(memgraph::io::network::Endpoint{local_host, ports[0]}); From d51a61fc5f7ad73bc81e4fc02f4ebe031943c7f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Wed, 21 Jun 2023 23:14:37 +0200 Subject: [PATCH 3/9] Update dependencies under `environment/os` (#862) --- environment/README.md | 10 ++ environment/os/.gitignore | 3 + environment/os/amzn-2.sh | 44 ++++++++- environment/os/centos-7.sh | 44 ++++++++- environment/os/centos-9.sh | 35 ++++++- environment/os/debian-10.sh | 65 ++++++++++++- environment/os/debian-11-arm.sh | 54 ++++++++++- environment/os/debian-11.sh | 56 ++++++++++- environment/os/fedora-36.sh | 60 ++++++++++-- environment/os/run.sh | 145 +++++++++++++++++++++++++++++ environment/os/template.sh | 11 +++ environment/os/ubuntu-18.04.sh | 60 +++++++++++- environment/os/ubuntu-20.04.sh | 61 ++++++++++-- environment/os/ubuntu-22.04-arm.sh | 56 ++++++++++- environment/os/ubuntu-22.04.sh | 60 ++++++++++-- environment/util.sh | 23 +++++ init | 4 +- release/package/run.sh | 18 +++- 18 files changed, 759 insertions(+), 50 deletions(-) create mode 100644 environment/README.md create mode 100755 environment/os/run.sh diff --git a/environment/README.md b/environment/README.md new file mode 100644 index 000000000..cc9d90073 --- /dev/null +++ b/environment/README.md @@ -0,0 +1,10 @@ +# Memgraph Operating Environments + +## os + +Under the `os` directory, you can find scripts to install all required system +dependencies on operating systems where Memgraph natively builds. The testing +script helps to see how to install all packages (in the case of a new package), +or make any adjustments in the overall system setup. Also, the testing script +helps check if Memgraph runs on a freshly installed operating system (with no +packages installed). diff --git a/environment/os/.gitignore b/environment/os/.gitignore index 122d85988..d5b44d8f0 100644 --- a/environment/os/.gitignore +++ b/environment/os/.gitignore @@ -1,3 +1,6 @@ *.deb +*.deb.* *.rpm +*.rpm.* *.tar.gz +*.tar.gz.* diff --git a/environment/os/amzn-2.sh b/environment/os/amzn-2.sh index 6df12312c..f2dcc4cc0 100755 --- a/environment/os/amzn-2.sh +++ b/environment/os/amzn-2.sh @@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make # build system + make cmake # build system wget # for downloading libs libuuid-devel java-11-openjdk # required by antlr readline-devel # for memgraph console @@ -57,9 +57,18 @@ MEMGRAPH_BUILD_DEPS=( libcurl-devel # mg-requests rpm-build rpmlint # for RPM package building doxygen graphviz # source documentation generators - which nodejs golang zip unzip java-11-openjdk-devel # for driver tests + which nodejs golang custom-golang1.18.9 zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.2 # for driver tests autoconf # for jemalloc code generation libtool # for protobuf code generation + cyrus-sasl-devel +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -73,6 +82,18 @@ check() { local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""} LD_LIBRARY_PATH="" for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi if [ "$pkg" == "PyYAML" ]; then if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then missing="$pkg $missing" @@ -103,8 +124,27 @@ install() { else echo "NOTE: export LANG=en_US.utf8" fi + yum update -y for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == jdk-17 ]; then + if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then + wget --no-check-certificate -c --header "Cookie: oraclelicense=accept-securebackup-cookie" https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm + rpm -Uvh jdk-17_linux-x64_bin.rpm + # NOTE: Set Java 11 as default. + update-alternatives --set java java-11-openjdk.x86_64 + update-alternatives --set javac java-11-openjdk.x86_64 + fi + continue + fi if [ "$pkg" == libipt ]; then if ! yum list installed libipt >/dev/null 2>/dev/null; then yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm diff --git a/environment/os/centos-7.sh b/environment/os/centos-7.sh index 04ba00197..aa6cce77c 100755 --- a/environment/os/centos-7.sh +++ b/environment/os/centos-7.sh @@ -39,7 +39,7 @@ TOOLCHAIN_RUN_DEPS=( ) MEMGRAPH_BUILD_DEPS=( - make pkgconfig # build system + make cmake pkgconfig # build system curl wget # for downloading libs libuuid-devel java-11-openjdk # required by antlr readline-devel # for memgraph console @@ -56,9 +56,19 @@ MEMGRAPH_BUILD_DEPS=( sbcl # for custom Lisp C++ preprocessing rpm-build rpmlint # for RPM package building doxygen graphviz # source documentation generators - which mono-complete dotnet-sdk-3.1 golang nodejs zip unzip java-11-openjdk-devel # for driver tests + which mono-complete dotnet-sdk-3.1 golang custom-golang1.18.9 # for driver tests + nodejs zip unzip java-11-openjdk-devel jdk-17 custom-maven3.9.2 # for driver tests autoconf # for jemalloc code generation libtool # for protobuf code generation + cyrus-sasl-devel +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -68,6 +78,18 @@ list() { check() { local missing="" for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi if [ "$pkg" == git ]; then if ! which "git" >/dev/null; then missing="git $missing" @@ -110,7 +132,25 @@ install() { yum update -y yum install -y wget python3 python3-pip yum install -y git + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == jdk-17 ]; then + if ! yum list installed jdk-17 >/dev/null 2>/dev/null; then + wget https://download.oracle.com/java/17/latest/jdk-17_linux-x64_bin.rpm + rpm -ivh jdk-17_linux-x64_bin.rpm + update-alternatives --set java java-11-openjdk.x86_64 + update-alternatives --set javac java-11-openjdk.x86_64 + fi + continue + fi if [ "$pkg" == libipt ]; then if ! yum list installed libipt >/dev/null 2>/dev/null; then yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm diff --git a/environment/os/centos-9.sh b/environment/os/centos-9.sh index e8fd65f2e..c1499f909 100755 --- a/environment/os/centos-9.sh +++ b/environment/os/centos-9.sh @@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkgconf-pkg-config # build system + make cmake pkgconf-pkg-config # build system wget # for downloading libs libuuid-devel java-11-openjdk # required by antlr readline-devel # for memgraph console @@ -56,10 +56,20 @@ MEMGRAPH_BUILD_DEPS=( libcurl-devel # mg-requests rpm-build rpmlint # for RPM package building doxygen graphviz # source documentation generators - which nodejs golang zip unzip java-11-openjdk-devel # for driver tests + which nodejs golang custom-golang1.18.9 # for driver tests + zip unzip java-11-openjdk-devel java-17-openjdk java-17-openjdk-devel custom-maven3.9.2 # for driver tests sbcl # for custom Lisp C++ preprocessing autoconf # for jemalloc code generation libtool # for protobuf code generation + cyrus-sasl-devel +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -69,6 +79,18 @@ list() { check() { local missing="" for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi if [ "$pkg" == "PyYAML" ]; then if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then missing="$pkg $missing" @@ -103,7 +125,16 @@ install() { fi yum update -y yum install -y wget git python3 python3-pip + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi # Since there is no support for libipt-devel for CentOS 9 we install # Fedoras version of same libs, they are the same version but released # for different OS diff --git a/environment/os/debian-10.sh b/environment/os/debian-10.sh index 5888e3e93..9318ac5df 100755 --- a/environment/os/debian-10.sh +++ b/environment/os/debian-10.sh @@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkg-config # build system + make cmake pkg-config # build system curl wget # for downloading libs uuid-dev default-jre-headless # required by antlr libreadline-dev # for memgraph console @@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # for custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests - dotnet-sdk-3.1 golang nodejs npm + mono-runtime mono-mcs zip unzip default-jdk-headless oracle-java17-installer custom-maven3.9.2 # for driver tests + dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -64,7 +73,28 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { @@ -75,8 +105,15 @@ deb http://deb.debian.org/debian/ buster-updates main contrib non-free deb-src http://deb.debian.org/debian/ buster-updates main contrib non-free deb http://security.debian.org/debian-security buster/updates main contrib non-free deb-src http://security.debian.org/debian-security buster/updates main contrib non-free +EOF + apt --allow-releaseinfo-change update + cat >/etc/apt/sources.list.d/java.list << EOF +deb http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main +deb-src http://ppa.launchpad.net/linuxuprising/java/ubuntu bionic main EOF cd "$DIR" + apt install -y gnupg + apt-key adv --keyserver keyserver.ubuntu.com --recv-keys EA8CACC073C3DB2A apt --allow-releaseinfo-change update # If GitHub Actions runner is installed, append LANG to the environment. # Python related tests doesn't work the LANG export. @@ -85,8 +122,26 @@ EOF else echo "NOTE: export LANG=en_US.utf8" fi - apt install -y wget + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == oracle-java17-installer ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + echo oracle-java17-installer shared/accepted-oracle-license-v1-3 select true | /usr/bin/debconf-set-selections + echo oracle-java17-installer shared/accepted-oracle-license-v1-3 seen true | /usr/bin/debconf-set-selections + apt install -y "$pkg" + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi if [ "$pkg" == dotnet-sdk-3.1 ]; then if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb diff --git a/environment/os/debian-11-arm.sh b/environment/os/debian-11-arm.sh index a5a5dfda2..9225ebb11 100755 --- a/environment/os/debian-11-arm.sh +++ b/environment/os/debian-11-arm.sh @@ -54,10 +54,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # for custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests - golang nodejs npm + mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.2 # for driver tests + golang custom-golang1.18.9 nodejs npm autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -65,7 +74,28 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { @@ -89,7 +119,25 @@ EOF echo "NOTE: export LANG=en_US.utf8" fi apt install -y wget + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == openjdk-17-jdk ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + apt install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi apt install -y "$pkg" done } diff --git a/environment/os/debian-11.sh b/environment/os/debian-11.sh index d319f58e8..3bd9bde20 100755 --- a/environment/os/debian-11.sh +++ b/environment/os/debian-11.sh @@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkg-config # build system + make cmake pkg-config # build system curl wget # for downloading libs uuid-dev default-jre-headless # required by antlr libreadline-dev # for memgraph console @@ -54,10 +54,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # for custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests - dotnet-sdk-3.1 golang nodejs npm + mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk custom-maven3.9.2 # for driver tests + dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -65,7 +74,28 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { @@ -89,7 +119,25 @@ EOF echo "NOTE: export LANG=en_US.utf8" fi apt install -y wget + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == openjdk-17-jdk ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + apt install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi if [ "$pkg" == dotnet-sdk-3.1 ]; then if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then wget -nv https://packages.microsoft.com/config/debian/10/packages-microsoft-prod.deb -O packages-microsoft-prod.deb diff --git a/environment/os/fedora-36.sh b/environment/os/fedora-36.sh index c59881f65..ef7021afc 100755 --- a/environment/os/fedora-36.sh +++ b/environment/os/fedora-36.sh @@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkgconf-pkg-config # build system + make cmake pkgconf-pkg-config # build system wget # for downloading libs libuuid-devel java-11-openjdk # required by antlr readline-devel # for memgraph console @@ -52,10 +52,21 @@ MEMGRAPH_BUILD_DEPS=( libcurl-devel # mg-requests rpm-build rpmlint # for RPM package building doxygen graphviz # source documentation generators - which nodejs golang zip unzip java-11-openjdk-devel # for driver tests + java-11-openjdk-devel java-17-openjdk-devel custom-maven3.9.2 # for driver tests + which zip unzip + nodejs golang custom-golang1.18.9 # for driver tests sbcl # for custom Lisp C++ preprocessing autoconf # for jemalloc code generation libtool # for protobuf code generation + cyrus-sasl-devel +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -64,11 +75,25 @@ list() { check() { local missing="" - # On Fedora yum/dnf and python10 use newer glibc which is not compatible - # with ours, so we need to momentarely disable env - local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} - LD_LIBRARY_PATH="" + if [ -v LD_LIBRARY_PATH ]; then + # On Fedora yum/dnf and python10 use newer glibc which is not compatible + # with ours, so we need to momentarely disable env + local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH} + LD_LIBRARY_PATH="" + fi for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then missing="$pkg $missing" fi @@ -77,7 +102,10 @@ check() { echo "MISSING PACKAGES: $missing" exit 1 fi - LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH} + if [ -v OLD_LD_LIBRARY_PATH ]; then + echo "Restoring LD_LIBRARY_PATH..." + LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH} + fi } install() { @@ -94,7 +122,25 @@ install() { echo "NOTE: export LANG=en_US.utf8" fi dnf update -y + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == java-17-openjdk-devel ]; then + if ! dnf list installed "$pkg" >/dev/null 2>/dev/null; then + dnf install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java java-11-openjdk.x86_64 + update-alternatives --set javac java-11-openjdk.x86_64 + fi + continue + fi dnf install -y "$pkg" done } diff --git a/environment/os/run.sh b/environment/os/run.sh new file mode 100755 index 000000000..e7c370f62 --- /dev/null +++ b/environment/os/run.sh @@ -0,0 +1,145 @@ +#!/bin/bash +set -Eeuo pipefail +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +IFS=' ' +# NOTE: docker_image_name could be local image build based on release/package images. +# NOTE: each line has to be under quotes, docker_container_type, script_name and docker_image_name separate with a space. +# "docker_container_type script_name docker_image_name" +OPERATING_SYSTEMS=( + "mgrun amzn-2 amazonlinux:2" + "mgrun centos-7 centos:7" + "mgrun centos-9 dokken/centos-stream-9" + "mgrun debian-10 debian:10" + "mgrun debian-11 debian:11" + "mgrun fedora-36 fedora:36" + "mgrun ubuntu-18.04 ubuntu:18.04" + "mgrun ubuntu-20.04 ubuntu:20.04" + "mgrun ubuntu-22.04 ubuntu:22.04" + # "mgbuild centos-7 package-mgbuild_centos-7" +) + +if [ ! "$(docker info)" ]; then + echo "ERROR: Docker is required" + exit 1 +fi +print_help () { + echo -e "$0 all\t\t\t\t => start + init all containers in the background" + echo -e "$0 check\t\t\t\t => check all containers" + echo -e "$0 delete\t\t\t\t => stop + remove all containers" + echo -e "$0 copy src_container dst_container => copy build package from src to dst container" + exit 1 +} + +# NOTE: This is an idempotent operation! +# TODO(gitbuda): Consider making docker_run always delete + start a new container or add a new function. +docker_run () { + cnt_name="$1" + cnt_image="$2" + if [ ! "$(docker ps -q -f name=$cnt_name)" ]; then + if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then + echo "Cleanup of the old exited container..." + docker rm $cnt_name + fi + docker run -d --volume "$SCRIPT_DIR/../../:/memgraph" --network host --name "$cnt_name" "$cnt_image" sleep infinity + fi + echo "The $cnt_image container is active under $cnt_name name!" +} + +docker_exec () { + cnt_name="$1" + cnt_cmd="$2" + docker exec -it "$cnt_name" bash -c "$cnt_cmd" +} + +docker_stop_and_rm () { + cnt_name="$1" + if [ "$(docker ps -q -f name=$cnt_name)" ]; then + docker stop "$1" + if [ "$(docker ps -aq -f status=exited -f name=$cnt_name)" ]; then + docker rm "$1" + fi + fi +} + +# TODO(gitbuda): Make the call to `install NEW_DEPS` configurable, the question what else is useful? +start_all () { + for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do + read -a script_docker <<< "$script_docker_pair" + docker_container_type="${script_docker[0]}" + script_name="${script_docker[1]}" + docker_image="${script_docker[2]}" + docker_name="${docker_container_type}_$script_name" + echo "" + echo "~~~~ OPERATING ON $docker_image as $docker_name..." + docker_run "$docker_name" "$docker_image" + docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh install NEW_DEPS" + echo "---- DONE EVERYHING FOR $docker_image as $docker_name..." + echo "" + done +} + +check_all () { + for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do + read -a script_docker <<< "$script_docker_pair" + docker_container_type="${script_docker[0]}" + script_name="${script_docker[1]}" + docker_image="${script_docker[2]}" + docker_name="${docker_container_type}_$script_name" + echo "" + echo "~~~~ OPERATING ON $docker_image as $docker_name..." + docker_exec "$docker_name" "/memgraph/environment/os/$script_name.sh check NEW_DEPS" + echo "---- DONE EVERYHING FOR $docker_image as $docker_name..." + echo "" + done +} + +delete_all () { + for script_docker_pair in "${OPERATING_SYSTEMS[@]}"; do + read -a script_docker <<< "$script_docker_pair" + docker_container_type="${script_docker[0]}" + script_name="${script_docker[1]}" + docker_image="${script_docker[2]}" + docker_name="${docker_container_type}_$script_name" + docker_stop_and_rm "$docker_name" + echo "~~~~ $docker_image as $docker_name DELETED" + done +} + +# TODO(gitbuda): Copy file between containers is a useful util, also delete, + consider copying of a whole folder. +# TODO(gitbuda): Add args: src_cnt dst_cnt abs_path; both file and recursive folder, always delete + copy. +copy_build_package () { + src_container="$1" + dst_container="$2" + src="$src_container:/memgraph/build/output" + tmp_dst="$SCRIPT_DIR/../../build" + mkdir -p "$tmp_dst" + rm -rf "$tmp_dst/output" + dst="$dst_container:/" + docker cp "$src" "$tmp_dst" + docker cp "$tmp_dst/output" "$dst" +} + +if [ "$#" -eq 0 ]; then + print_help +else + case $1 in + all) + start_all + ;; + check) + check_all + ;; + delete) + delete_all + ;; + copy) # src_container dst_container + if [ "$#" -ne 3 ]; then + print_help + fi + copy_build_package "$2" "$3" + ;; + *) + print_help + ;; + esac +fi diff --git a/environment/os/template.sh b/environment/os/template.sh index 47c9725fd..b1f2f8fe4 100755 --- a/environment/os/template.sh +++ b/environment/os/template.sh @@ -6,6 +6,7 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" source "$DIR/../util.sh" check_operating_system "todo-os-name" +check_architecture "todo-arch-name" TOOLCHAIN_BUILD_DEPS=( pkg @@ -19,6 +20,16 @@ MEMGRAPH_BUILD_DEPS=( pkg ) +MEMGRAPH_RUN_DEPS=( + pkg +) + +# NEW_DEPS is useful when you won't to test the installation of a new package. +# During the test you can put here packages like wget curl tar gzip +NEW_DEPS=( + pkg +) + list() { echo "$1" } diff --git a/environment/os/ubuntu-18.04.sh b/environment/os/ubuntu-18.04.sh index dfda2534e..e4b20ae1e 100755 --- a/environment/os/ubuntu-18.04.sh +++ b/environment/os/ubuntu-18.04.sh @@ -41,7 +41,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkg-config # build system + make cmake pkg-config # build system curl wget # downloading libs uuid-dev default-jre-headless # required by antlr libreadline-dev # memgraph console @@ -53,9 +53,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs nodejs zip unzip default-jdk-headless # driver tests + mono-runtime mono-mcs nodejs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # driver tests + custom-golang1.18.9 # for driver tests autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp2 +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -63,11 +73,53 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { - apt install -y $1 + apt update -y + + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi + if [ "$pkg" == openjdk-17-jdk-headless ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + apt install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi + apt install -y "$pkg" + done } deps=$2"[*]" diff --git a/environment/os/ubuntu-20.04.sh b/environment/os/ubuntu-20.04.sh index f440aff98..5b3fcfcf5 100755 --- a/environment/os/ubuntu-20.04.sh +++ b/environment/os/ubuntu-20.04.sh @@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkg-config # build system + make cmake pkg-config # build system curl wget # for downloading libs uuid-dev default-jre-headless # required by antlr libreadline-dev # for memgraph console @@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # for custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests - dotnet-sdk-3.1 golang nodejs npm + mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # for driver tests + dotnet-sdk-3.1 golang custom-golang1.18.9 nodejs npm # for driver tests autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp2 +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -64,12 +73,35 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { cd "$DIR" - apt update + export DEBIAN_FRONTEND=noninteractive + apt update -y + apt install -y wget # If GitHub Actions runner is installed, append LANG to the environment. # Python related tests doesn't work the LANG export. if [ -d "/home/gh/actions-runner" ]; then @@ -77,8 +109,16 @@ install() { else echo "NOTE: export LANG=en_US.utf8" fi - apt install -y wget + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi if [ "$pkg" == dotnet-sdk-3.1 ]; then if ! dpkg -s dotnet-sdk-3.1 2>/dev/null >/dev/null; then wget -nv https://packages.microsoft.com/config/ubuntu/20.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb @@ -88,6 +128,15 @@ install() { fi continue fi + if [ "$pkg" == openjdk-17-jdk-headless ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + apt install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi apt install -y "$pkg" done } diff --git a/environment/os/ubuntu-22.04-arm.sh b/environment/os/ubuntu-22.04-arm.sh index d3bf8f040..d1e8856e0 100755 --- a/environment/os/ubuntu-22.04-arm.sh +++ b/environment/os/ubuntu-22.04-arm.sh @@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkg-config # build system + make cmake pkg-config # build system curl wget # for downloading libs uuid-dev default-jre-headless # required by antlr libreadline-dev # for memgraph console @@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # for custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests - dotnet-sdk-6.0 golang nodejs npm + mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # for driver tests + dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp2 +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -64,7 +73,28 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { @@ -78,7 +108,16 @@ install() { echo "NOTE: export LANG=en_US.utf8" fi apt install -y wget + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi if [ "$pkg" == dotnet-sdk-6.0 ]; then if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb @@ -88,6 +127,15 @@ install() { fi continue fi + if [ "$pkg" == openjdk-17-jdk-headless ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + apt install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi apt install -y "$pkg" done } diff --git a/environment/os/ubuntu-22.04.sh b/environment/os/ubuntu-22.04.sh index 75fffae29..02b659d1d 100755 --- a/environment/os/ubuntu-22.04.sh +++ b/environment/os/ubuntu-22.04.sh @@ -40,7 +40,7 @@ TOOLCHAIN_RUN_DEPS=( MEMGRAPH_BUILD_DEPS=( git # source code control - make pkg-config # build system + make cmake pkg-config # build system curl wget # for downloading libs uuid-dev default-jre-headless # required by antlr libreadline-dev # for memgraph console @@ -53,10 +53,19 @@ MEMGRAPH_BUILD_DEPS=( libcurl4-openssl-dev # mg-requests sbcl # for custom Lisp C++ preprocessing doxygen graphviz # source documentation generators - mono-runtime mono-mcs zip unzip default-jdk-headless # for driver tests - dotnet-sdk-6.0 golang nodejs npm + mono-runtime mono-mcs zip unzip default-jdk-headless openjdk-17-jdk-headless custom-maven3.9.2 # for driver tests + dotnet-sdk-6.0 golang custom-golang1.18.9 nodejs npm # for driver tests autoconf # for jemalloc code generation libtool # for protobuf code generation + libsasl2-dev +) + +MEMGRAPH_RUN_DEPS=( + logrotate openssl python3 libseccomp2 +) + +NEW_DEPS=( + wget curl tar gzip ) list() { @@ -64,12 +73,34 @@ list() { } check() { - check_all_dpkg "$1" + local missing="" + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + if [ ! -f "/opt/apache-maven-3.9.2/bin/mvn" ]; then + missing="$pkg $missing" + fi + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + if [ ! -f "/opt/go1.18.9/go/bin/go" ]; then + missing="$pkg $missing" + fi + continue + fi + if ! dpkg -s "$pkg" >/dev/null 2>/dev/null; then + missing="$pkg $missing" + fi + done + if [ "$missing" != "" ]; then + echo "MISSING PACKAGES: $missing" + exit 1 + fi } install() { cd "$DIR" - apt update + apt update -y + apt install -y wget # If GitHub Actions runner is installed, append LANG to the environment. # Python related tests doesn't work the LANG export. if [ -d "/home/gh/actions-runner" ]; then @@ -77,8 +108,16 @@ install() { else echo "NOTE: export LANG=en_US.utf8" fi - apt install -y wget + for pkg in $1; do + if [ "$pkg" == custom-maven3.9.2 ]; then + install_custom_maven "3.9.2" + continue + fi + if [ "$pkg" == custom-golang1.18.9 ]; then + install_custom_golang "1.18.9" + continue + fi if [ "$pkg" == dotnet-sdk-6.0 ]; then if ! dpkg -s dotnet-sdk-6.0 2>/dev/null >/dev/null; then wget -nv https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb @@ -88,6 +127,15 @@ install() { fi continue fi + if [ "$pkg" == openjdk-17-jdk-headless ]; then + if ! dpkg -s "$pkg" 2>/dev/null >/dev/null; then + apt install -y "$pkg" + # The default Java version should be Java 11 + update-alternatives --set java /usr/lib/jvm/java-11-openjdk-amd64/bin/java + update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-amd64/bin/javac + fi + continue + fi apt install -y "$pkg" done } diff --git a/environment/util.sh b/environment/util.sh index e2ecf67cf..c6c48cc68 100644 --- a/environment/util.sh +++ b/environment/util.sh @@ -76,3 +76,26 @@ function install_all_apt() { apt install -y "$pkg" done } + +function install_custom_golang() { + # NOTE: The official https://go.dev/doc/manage-install doesn't seem to be working. + GOVERSION="$1" + GOINSTALLDIR="/opt/go$GOVERSION" + GOROOT="$GOINSTALLDIR/go" # GOPATH=$HOME/go + if [ ! -f "$GOROOT/bin/go" ]; then + curl -LO https://go.dev/dl/go$GOVERSION.linux-amd64.tar.gz + mkdir -p "$GOINSTALLDIR" + tar -C "$GOINSTALLDIR" -xzf go$GOVERSION.linux-amd64.tar.gz + fi + echo "go $GOVERSION installed under $GOROOT" +} + +function install_custom_maven() { + MVNVERSION="$1" + MVNINSTALLDIR="/opt/apache-maven-$MVNVERSION" + if [ ! -f "$MVNINSTALLDIR/bin/mvn" ]; then + curl -LO "https://dlcdn.apache.org/maven/maven-3/$MVNVERSION/binaries/apache-maven-$MVNVERSION-bin.tar.gz" + tar -C "/opt" -xzf "apache-maven-$MVNVERSION-bin.tar.gz" + fi + echo "maven $MVNVERSION installed under $MVNINSTALLDIR" +} diff --git a/init b/init index bbcd116a3..029160f88 100755 --- a/init +++ b/init @@ -77,7 +77,9 @@ fi # Fix for centos 7 during release if [ "${DISTRO}" = "centos-7" ] || [ "${DISTRO}" = "debian-11" ] || [ "${DISTRO}" = "amzn-2" ]; then - python3 -m pip uninstall -y virtualenv + if python3 -m pip show virtualenv >/dev/null 2>/dev/null; then + python3 -m pip uninstall -y virtualenv + fi python3 -m pip install virtualenv fi diff --git a/release/package/run.sh b/release/package/run.sh index cdf95466a..baaca8f34 100755 --- a/release/package/run.sh +++ b/release/package/run.sh @@ -17,7 +17,8 @@ ACTIVATE_TOOLCHAIN="source /opt/${TOOLCHAIN_VERSION}/activate" HOST_OUTPUT_DIR="$PROJECT_ROOT/build/output" print_help () { - echo "$0 init|package {os} [--for-docker|--for-platform]|docker|test" + # TODO(gitbuda): Update the release/package/run.sh help + echo "$0 init|package|docker|test {os} [--for-docker|--for-platform]" echo "" echo " OSs: ${SUPPORTED_OS[*]}" exit 1 @@ -35,7 +36,7 @@ make_package () { package_command=" cpack -G RPM --config ../CPackConfig.cmake && rpmlint --file='../../release/rpm/rpmlintrc' memgraph*.rpm " fi if [[ "$os" =~ ^"debian".* ]]; then - docker exec "$build_container" bash -c "apt update" + docker exec "$build_container" bash -c "apt --allow-releaseinfo-change -y update" package_command=" cpack -G DEB --config ../CPackConfig.cmake " fi if [[ "$os" =~ ^"ubuntu".* ]]; then @@ -64,6 +65,7 @@ make_package () { git fetch origin master:master fi docker exec "$build_container" mkdir -p /memgraph + # TODO(gitbuda): Revisit copying the whole repo -> makese sense under CI. docker cp "$PROJECT_ROOT/." "$build_container:/memgraph/" container_build_dir="/memgraph/build" @@ -74,6 +76,8 @@ make_package () { # environment/os/{os}.sh does not come within the toolchain package. When # migrating to the next version of toolchain do that, and remove the # TOOLCHAIN_RUN_DEPS installation from here. + # TODO(gitbuda): On the other side, having this here allows updating deps + # wihout reruning the build containers. echo "Installing dependencies using '/memgraph/environment/os/$os.sh' script..." docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install TOOLCHAIN_RUN_DEPS" docker exec "$build_container" bash -c "/memgraph/environment/os/$os.sh install MEMGRAPH_BUILD_DEPS" @@ -83,6 +87,7 @@ make_package () { docker exec "$build_container" bash -c "cd /memgraph && git config --global --add safe.directory '*'" docker exec "$build_container" bash -c "cd /memgraph && $ACTIVATE_TOOLCHAIN && ./init" docker exec "$build_container" bash -c "cd $container_build_dir && rm -rf ./*" + # TODO(gitbuda): cmake fails locally if remote is clone via ssh because of the key -> FIX if [[ "$os" =~ "-arm" ]]; then docker exec "$build_container" bash -c "cd $container_build_dir && $ACTIVATE_TOOLCHAIN && cmake -DCMAKE_BUILD_TYPE=release -DMG_ARCH="ARM64" $telemetry_id_override_flag .." else @@ -108,8 +113,13 @@ make_package () { case "$1" in init) cd "$SCRIPT_DIR" - docker-compose build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" - docker-compose up -d + if ! which "docker-compose" >/dev/null; then + docker_compose_cmd="docker compose" + else + docker_compose_cmd="docker-compose" + fi + $docker_compose_cmd build --build-arg TOOLCHAIN_VERSION="${TOOLCHAIN_VERSION}" + $docker_compose_cmd up -d ;; docker) From e73eac77a95636e86ea7439315c5d1fea3d86e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Thu, 22 Jun 2023 10:14:59 +0200 Subject: [PATCH 4/9] Improve libstdc++ dependency on RPM systems (#863) --- config/generate.py | 8 ++++++++ libs/CMakeLists.txt | 1 + release/CMakeLists.txt | 16 ++++++++-------- src/query/procedure/module.hpp | 16 ++++++++++++---- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/config/generate.py b/config/generate.py index 05e9c2339..ae323ee34 100755 --- a/config/generate.py +++ b/config/generate.py @@ -23,6 +23,14 @@ def wrap_text(s, initial_indent="# "): def extract_flags(binary_path): ret = {} data = subprocess.run([binary_path, "--help-xml"], stdout=subprocess.PIPE).stdout.decode("utf-8") + # If something is printed out before the help output, it will break the the + # XML parsing -> filter out if something is not XML line because something + # can be logged before gflags output (e.g. during the global objects init). + # This gets called during memgraph build phase to generate default config + # file later installed under /etc/memgraph/memgraph.conf + # NOTE: Don't use \n in the gflags description strings. + # NOTE: Check here if gflags version changes because of the XML format. + data = "\n".join([line for line in data.split("\n") if line.startswith("<")]) root = ET.fromstring(data) for child in root: if child.tag == "usage" and child.text.lower().count("warning"): diff --git a/libs/CMakeLists.txt b/libs/CMakeLists.txt index 618c0ce5c..4c5367961 100644 --- a/libs/CMakeLists.txt +++ b/libs/CMakeLists.txt @@ -12,6 +12,7 @@ find_package(Boost 1.78 REQUIRED) find_package(BZip2 1.0.6 REQUIRED) find_package(Threads REQUIRED) set(GFLAGS_NOTHREADS OFF) +# NOTE: config/generate.py depends on the gflags help XML format. find_package(gflags REQUIRED) find_package(fmt 8.0.1) find_package(Jemalloc REQUIRED) diff --git a/release/CMakeLists.txt b/release/CMakeLists.txt index 4bdb4bf69..7055f4543 100644 --- a/release/CMakeLists.txt +++ b/release/CMakeLists.txt @@ -48,7 +48,7 @@ set(CPACK_DEBIAN_PACKAGE_DESCRIPTION "${CPACK_PACKAGE_DESCRIPTION_SUMMARY} # Add `openssl` package to dependencies list. Used to generate SSL certificates. # We also depend on `python3` because we embed it in Memgraph. -set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0), libstdc++6") +set(CPACK_DEBIAN_PACKAGE_DEPENDS "openssl (>= 1.1.0), python3 (>= 3.5.0)") # Setting arhitecture extension for rpm packages set(MG_ARCH_EXTENSION_RPM "noarch") @@ -69,6 +69,12 @@ set(CPACK_RPM_EXCLUDE_FROM_AUTO_FILELIST_ADDITION set(CPACK_RPM_PACKAGE_REQUIRES_PRE "shadow-utils") set(CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_SOURCE_DIR}/rpm/memgraph.spec.in") set(CPACK_RPM_PACKAGE_LICENSE "Memgraph License") +# CPACK deduces dependency to libstdc++ which: +# * can't be easily installed on Centos 7 (the one from the toolchain, +# only required to avoid printing issue within query modules) +# * it causes issues with glibcxx 2.4 +# `if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2")` +set(CPACK_RPM_PACKAGE_AUTOREQ " no") # Description formatting is important, no line must be greater than 80 characters. set(CPACK_RPM_PACKAGE_DESCRIPTION "Contains Memgraph, the graph database. @@ -77,13 +83,7 @@ the next generation of applications driver by real-time connected data.") # Add `openssl` package to dependencies list. Used to generate SSL certificates. # We also depend on `python3` because we embed it in Memgraph. -set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, libstdc++ >= 3.4.29, logrotate") - -# If amzn-2 -if(DISTRO STREQUAL "Amazon Linux" AND DISTRO_VERSION STREQUAL "2") - # It causes issues with glibcxx 2.4 - set(CPACK_RPM_PACKAGE_AUTOREQ " no") -endif() +set(CPACK_RPM_PACKAGE_REQUIRES "openssl >= 1.0.0, curl >= 7.29.0, python3 >= 3.5.0, logrotate") # All variables must be set before including. include(CPack) diff --git a/src/query/procedure/module.hpp b/src/query/procedure/module.hpp index 82478257b..ccb56b2fc 100644 --- a/src/query/procedure/module.hpp +++ b/src/query/procedure/module.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -131,7 +131,12 @@ class ModuleRegistry final { private: class SharedLibraryHandle { public: - SharedLibraryHandle(const std::string &shared_library, int mode) : handle_{dlopen(shared_library.c_str(), mode)} {} + SharedLibraryHandle(const std::string &shared_library, int mode, const std::string &hint_message = "") + : handle_{dlopen(shared_library.c_str(), mode)} { + if (!handle_) { + spdlog::warn("Unable to load {}. {}", shared_library, hint_message); + } + } SharedLibraryHandle(const SharedLibraryHandle &) = delete; SharedLibraryHandle(SharedLibraryHandle &&) = delete; SharedLibraryHandle operator=(const SharedLibraryHandle &) = delete; @@ -147,10 +152,13 @@ class ModuleRegistry final { void *handle_; }; + inline static const std::string kLibstdcppWarning = + "Query modules might not work as expected. Printing non-string values from query modules might not work. Please " + "install libstdc++ or compile from source with the recent toolchain (all included)."; #if __has_feature(address_sanitizer) // This is why we need RTLD_NODELETE and we must not use RTLD_DEEPBIND with // ASAN: https://github.com/google/sanitizers/issues/89 - SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE}; + SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_NODELETE, kLibstdcppWarning}; #else // The reason behind opening share library during runtime is to avoid issues // with loading symbols from stdlib. We have encounter issues with locale @@ -161,7 +169,7 @@ class ModuleRegistry final { // mentioned library will be first performed in the already existing binded // libraries and then the global namespace. // RTLD_DEEPBIND => https://linux.die.net/man/3/dlopen - SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND}; + SharedLibraryHandle libstd_handle{"libstdc++.so.6", RTLD_NOW | RTLD_LOCAL | RTLD_DEEPBIND, kLibstdcppWarning}; #endif std::vector modules_dirs_; std::filesystem::path internal_module_dir_; From da17fe92d63c53a8dfc4dbf6a85dc2dbe6572270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Budiseli=C4=87?= Date: Thu, 22 Jun 2023 12:12:31 +0200 Subject: [PATCH 5/9] Update `package_docker` by adding --pull (#1032) Each release will pull the latest base image because we want to include any new security patches. --- release/docker/package_docker | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/release/docker/package_docker b/release/docker/package_docker index b921a4c19..b12c4ca61 100755 --- a/release/docker/package_docker +++ b/release/docker/package_docker @@ -55,7 +55,8 @@ image_name="memgraph:${version}" image_package_name="memgraph-${version}-docker.tar.gz" # Build docker image. -docker build -t ${image_name} ${tag_latest} -f ${dockerfile_path} \ +# NOTE: --pull is here to always pull that latest base image because of security patches. +docker build --pull -t ${image_name} ${tag_latest} -f ${dockerfile_path} \ --build-arg BINARY_NAME=${package_name} \ --build-arg EXTENSION=${extension} \ --build-arg TARGETARCH="" . From 0ea96663bac68382fb0ae4e9931b1096ef199a6d Mon Sep 17 00:00:00 2001 From: Ante Javor Date: Thu, 22 Jun 2023 13:29:49 +0200 Subject: [PATCH 6/9] Add check for opening snapshots (#966) --- src/storage/v2/durability/durability.cpp | 6 ++++++ src/storage/v2/durability/snapshot.cpp | 1 + src/utils/file.cpp | 4 +++- src/utils/file.hpp | 5 ++++- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/storage/v2/durability/durability.cpp b/src/storage/v2/durability/durability.cpp index 4bd4e9d39..1f1052528 100644 --- a/src/storage/v2/durability/durability.cpp +++ b/src/storage/v2/durability/durability.cpp @@ -74,6 +74,12 @@ std::vector GetSnapshotFiles(const std::filesystem::path if (utils::DirExists(snapshot_directory)) { for (const auto &item : std::filesystem::directory_iterator(snapshot_directory, error_code)) { if (!item.is_regular_file()) continue; + if (!utils::HasReadAccess(item.path())) { + spdlog::warn( + "Skipping snapshot file '{}' because it is not readable, check file ownership and read permissions!", + item.path()); + continue; + } try { auto info = ReadSnapshotInfo(item.path()); if (uuid.empty() || info.uuid == uuid) { diff --git a/src/storage/v2/durability/snapshot.cpp b/src/storage/v2/durability/snapshot.cpp index 4a65731db..61c159c99 100644 --- a/src/storage/v2/durability/snapshot.cpp +++ b/src/storage/v2/durability/snapshot.cpp @@ -26,6 +26,7 @@ #include "storage/v2/vertex.hpp" #include "storage/v2/vertex_accessor.hpp" #include "utils/concepts.hpp" +#include "utils/file.hpp" #include "utils/file_locker.hpp" #include "utils/logging.hpp" #include "utils/message.hpp" diff --git a/src/utils/file.cpp b/src/utils/file.cpp index 96fa0763e..7be907428 100644 --- a/src/utils/file.cpp +++ b/src/utils/file.cpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -82,6 +82,8 @@ bool RenamePath(const std::filesystem::path &src, const std::filesystem::path &d return !error_code; } +bool HasReadAccess(const std::filesystem::path &path) { return access(path.c_str(), R_OK) == 0; } + static_assert(std::is_same_v, "off_t must fit into ssize_t!"); InputFile::~InputFile() { Close(); } diff --git a/src/utils/file.hpp b/src/utils/file.hpp index 5e1345742..f40e1f3da 100644 --- a/src/utils/file.hpp +++ b/src/utils/file.hpp @@ -1,4 +1,4 @@ -// Copyright 2022 Memgraph Ltd. +// Copyright 2023 Memgraph Ltd. // // Use of this software is governed by the Business Source License // included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source @@ -63,6 +63,9 @@ bool CopyFile(const std::filesystem::path &src, const std::filesystem::path &dst /// don't exist, the renaming fails. Symlinks are not followed. bool RenamePath(const std::filesystem::path &src, const std::filesystem::path &dst); +/// Checks if process has read access to the file. +bool HasReadAccess(const std::filesystem::path &path); + /// Buffer size used for `InputFile` and `OutputFile` implementations. Using /// system calls is very expensive and we can't afford to call either `read` or /// `write` for each of our (very small) logical reads/writes. Because of that, From 68e56105664936494b272c49e3212d136ca695f3 Mon Sep 17 00:00:00 2001 From: Katarina Supe <61758502+katarinasupe@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:41:59 +0200 Subject: [PATCH 7/9] Fix replica exception message (#930) --- src/query/exceptions.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/exceptions.hpp b/src/query/exceptions.hpp index d82e3a062..874a102f6 100644 --- a/src/query/exceptions.hpp +++ b/src/query/exceptions.hpp @@ -254,7 +254,7 @@ class ReplicationException : public utils::BasicException { public: using utils::BasicException::BasicException; explicit ReplicationException(const std::string &message) - : utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICA' query.", + : utils::BasicException("Replication Exception: {} Check the status of the replicas using 'SHOW REPLICAS' query.", message) {} }; From bcd23fe3cb1705f963a0b5c8c2365ea722a6eb6c Mon Sep 17 00:00:00 2001 From: Vlasta <95473291+vpavicic@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:44:46 +0200 Subject: [PATCH 8/9] Update CSV import tool error docs --- docs/csv-import-tool/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/csv-import-tool/README.md b/docs/csv-import-tool/README.md index 89957aa16..1a8489b30 100644 --- a/docs/csv-import-tool/README.md +++ b/docs/csv-import-tool/README.md @@ -202,3 +202,29 @@ for row in csv.reader(stream, delimiter=',', doublequote=True, For more information about the meaning of the above values, see: https://docs.python.org/3/library/csv.html#csv.Dialect +## Errors + +1. [Skipping duplicate node with ID '{}'. For more details, visit: + memgr.ph/csv-import-tool.](#error-1) +2. [Skipping bad relationship with START_ID '{}'. For more details, visit: + memgr.ph/csv-import-tool.](#error-2) +3. [Skipping bad relationship with END_ID '{}'. For more details, visit: + memgr.ph/csv-import-tool.](#error-3) + +## Skipping duplicate node with ID {} {#error-1} + +Duplicate nodes are nodes that have an ID that is the same as another node that +was already imported. You can instruct the importer to ignore all duplicate +nodes (instead of raising an error) by using the `--skip-duplicate-nodes` flag. + +## Skipping bad relationship with START_ID {} {#error-2} + +A node with the id `START_ID` doesn't exist. You can instruct the importer to +ignore all bad relationships (instead of raising an error) that refer to nodes +that don't exist in the node files by using the `--skip-bad-relationships` flag. + +## Skipping bad relationship with END_ID {} {#error-3} + +A node with the id `END_ID` doesn't exist. You can instruct the importer to +ignore all bad relationships (instead of raising an error) that refer to nodes +that don't exist in the node files by using the `--skip-bad-relationships` flag. From b25e9968ee462d0d81452b31c19983c25a438af9 Mon Sep 17 00:00:00 2001 From: Vlasta <95473291+vpavicic@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:00:22 +0200 Subject: [PATCH 9/9] Update links inside CSV import tool (#834) --- src/mg_import_csv.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mg_import_csv.cpp b/src/mg_import_csv.cpp index 138e1e6d1..3b9717ae6 100644 --- a/src/mg_import_csv.cpp +++ b/src/mg_import_csv.cpp @@ -436,7 +436,7 @@ void ProcessNodeRow(memgraph::storage::Storage *store, const std::vectorend()) { if (FLAGS_skip_duplicate_nodes) { spdlog::warn(memgraph::utils::MessageWithLink("Skipping duplicate node with ID '{}'.", node_id, - "https://memgr.ph/csv")); + "https://memgr.ph/csv-import-tool")); return; } else { throw LoadException("Node with ID '{}' already exists", node_id); @@ -529,7 +529,7 @@ void ProcessRelationshipsRow(memgraph::storage::Storage *store, const std::vecto if (it == node_id_map.end()) { if (FLAGS_skip_bad_relationships) { spdlog::warn(memgraph::utils::MessageWithLink("Skipping bad relationship with START_ID '{}'.", node_id, - "https://memgr.ph/csv")); + "https://memgr.ph/csv-import-tool")); return; } else { throw LoadException("Node with ID '{}' does not exist", node_id); @@ -547,7 +547,7 @@ void ProcessRelationshipsRow(memgraph::storage::Storage *store, const std::vecto if (it == node_id_map.end()) { if (FLAGS_skip_bad_relationships) { spdlog::warn(memgraph::utils::MessageWithLink("Skipping bad relationship with END_ID '{}'.", node_id, - "https://memgr.ph/csv")); + "https://memgr.ph/csv-import-tool")); return; } else { throw LoadException("Node with ID '{}' does not exist", node_id);