diff --git a/config/memgraph.yaml b/config/memgraph.yaml index cb0c90bd7..a815b8ff1 100644 --- a/config/memgraph.yaml +++ b/config/memgraph.yaml @@ -14,8 +14,9 @@ template_cpp_path: "./template/plan_template_cpp" # path to the folder with snapshots snapshots_path: "snapshots" -# leaning cycle interval -cleaning_cycle_sec: "300" +# cleaning cycle interval +# if set to -1 the GC will not run +cleaning_cycle_sec: "30" # snapshot cycle interval snapshot_cycle_sec: "60" diff --git a/src/database/graph_db.cpp b/src/database/graph_db.cpp index d225a969b..1db9f10d0 100644 --- a/src/database/graph_db.cpp +++ b/src/database/graph_db.cpp @@ -1,10 +1,22 @@ - #include "database/graph_db.hpp" -#include +#include "config/config.hpp" #include "database/creation_exception.hpp" +#include "logging/logger.hpp" +#include "storage/edge.hpp" +#include "storage/garbage_collector.hpp" //#include "snapshot/snapshoter.hpp" -GraphDb::GraphDb(const std::string &name, bool import_snapshot) : name_(name) { +const int DEFAULT_CLEANING_CYCLE_SEC = 30; // 30 seconds + +GraphDb::GraphDb(const std::string &name, bool import_snapshot) + : name_(name), + gc_vertices_(&vertices_, &tx_engine), + gc_edges_(&edges_, &tx_engine) { + const std::string timeStr = CONFIG(config::CLEANING_CYCLE_SEC); + int pause = DEFAULT_CLEANING_CYCLE_SEC; + if (!timeStr.empty()) pause = stoll(timeStr); + this->gc_edges_.Run(std::chrono::seconds(pause)); + this->gc_vertices_.Run(std::chrono::seconds(pause)); // if (import_snapshot) // snap_engine.import(); } diff --git a/src/database/graph_db.hpp b/src/database/graph_db.hpp index ee37509df..33adb15da 100644 --- a/src/database/graph_db.hpp +++ b/src/database/graph_db.hpp @@ -1,22 +1,22 @@ #pragma once +#include + #include "data_structures/concurrent/concurrent_set.hpp" #include "data_structures/concurrent/skiplist.hpp" +#include "database/graph_db_datatypes.hpp" #include "mvcc/version_list.hpp" +#include "storage/edge.hpp" +#include "storage/garbage_collector.hpp" #include "storage/unique_object_store.hpp" +#include "storage/vertex.hpp" #include "transactions/engine.hpp" #include "utils/pass_key.hpp" -// forward declaring Edge and Vertex because they use -// GraphDb::Label etc., and therefore include this header -class Vertex; -class Edge; - // TODO: Maybe split this in another layer between Db and Dbms. Where the new // layer would hold SnapshotEngine and his kind of concept objects. Some // guidelines would be: retain objects which are necessary to implement querys // in Db, the rest can be moved to the new layer. - /** * Main class which represents Database concept in code. * This class is essentially a data structure. It exposes @@ -25,11 +25,6 @@ class Edge; */ class GraphDb { public: - // definitions for what data types are used for a Label, Property, EdgeType - using Label = std::string *; - using EdgeType = std::string *; - using Property = std::string *; - /** * Construct database with a custom name. * @@ -61,8 +56,11 @@ class GraphDb { // main storage for the graph SkipList *> vertices_; SkipList *> edges_; + GarbageCollector gc_vertices_; + GarbageCollector gc_edges_; // unique object stores + // TODO this should be also garbage collected ConcurrentSet labels_; ConcurrentSet edge_types_; ConcurrentSet properties_; diff --git a/src/database/graph_db_accessor.cpp b/src/database/graph_db_accessor.cpp index d3e4e5184..acfc81754 100644 --- a/src/database/graph_db_accessor.cpp +++ b/src/database/graph_db_accessor.cpp @@ -7,7 +7,7 @@ #include "storage/vertex_accessor.hpp" #include "utils/assert.hpp" -GraphDbAccessor::GraphDbAccessor(GraphDb& db) +GraphDbAccessor::GraphDbAccessor(GraphDb &db) : db_(db), transaction_(db.tx_engine.begin()) {} GraphDbAccessor::~GraphDbAccessor() { @@ -16,7 +16,7 @@ GraphDbAccessor::~GraphDbAccessor() { } } -const std::string& GraphDbAccessor::name() const { return db_.name_; } +const std::string &GraphDbAccessor::name() const { return db_.name_; } void GraphDbAccessor::advance_command() { transaction_->engine.advance(transaction_->id); @@ -38,8 +38,8 @@ void GraphDbAccessor::abort() { VertexAccessor GraphDbAccessor::insert_vertex() { // create a vertex - auto vertex_vlist = new mvcc::VersionList(); - Vertex* vertex = vertex_vlist->insert(*transaction_); + Vertex *vertex = nullptr; + auto vertex_vlist = new mvcc::VersionList(*transaction_, vertex); // insert the newly created record into the main storage // TODO make the number of tries configurable @@ -52,7 +52,7 @@ VertexAccessor GraphDbAccessor::insert_vertex() { throw CreationException("Unable to create a Vertex after 5 attempts"); } -bool GraphDbAccessor::remove_vertex(VertexAccessor& vertex_accessor) { +bool GraphDbAccessor::remove_vertex(VertexAccessor &vertex_accessor) { // TODO consider if this works well with MVCC if (vertex_accessor.out_degree() > 0 || vertex_accessor.in_degree() > 0) return false; @@ -61,7 +61,7 @@ bool GraphDbAccessor::remove_vertex(VertexAccessor& vertex_accessor) { return true; } -void GraphDbAccessor::detach_remove_vertex(VertexAccessor& vertex_accessor) { +void GraphDbAccessor::detach_remove_vertex(VertexAccessor &vertex_accessor) { // removing edges via accessors is both safe // and it should remove all the pointers in the relevant // vertices (including this one) @@ -70,13 +70,13 @@ void GraphDbAccessor::detach_remove_vertex(VertexAccessor& vertex_accessor) { for (auto edge_accessor : vertex_accessor.out()) remove_edge(edge_accessor); } -EdgeAccessor GraphDbAccessor::insert_edge(VertexAccessor& from, - VertexAccessor& to, - GraphDb::EdgeType edge_type) { +EdgeAccessor GraphDbAccessor::insert_edge(VertexAccessor &from, + VertexAccessor &to, + GraphDbTypes::EdgeType edge_type) { // create an edge - auto edge_vlist = new mvcc::VersionList(); - Edge* edge = - edge_vlist->insert(*transaction_, *from.vlist_, *to.vlist_, edge_type); + Edge *edge = nullptr; + auto edge_vlist = new mvcc::VersionList( + *transaction_, edge, *from.vlist_, *to.vlist_, edge_type); // set the vertex connections to this edge from.update().out_.emplace_back(edge_vlist); @@ -97,43 +97,45 @@ EdgeAccessor GraphDbAccessor::insert_edge(VertexAccessor& from, * Removes the given edge pointer from a vector of pointers. * Does NOT maintain edge pointer ordering (for efficiency). */ -void swap_out_edge(std::vector*>& edges, - mvcc::VersionList* edge) { +void swap_out_edge(std::vector *> &edges, + mvcc::VersionList *edge) { auto found = std::find(edges.begin(), edges.end(), edge); debug_assert(found != edges.end(), "Edge doesn't exist."); std::swap(*found, edges.back()); edges.pop_back(); } -void GraphDbAccessor::remove_edge(EdgeAccessor& edge_accessor) { +void GraphDbAccessor::remove_edge(EdgeAccessor &edge_accessor) { swap_out_edge(edge_accessor.from().update().out_, edge_accessor.vlist_); swap_out_edge(edge_accessor.to().update().in_, edge_accessor.vlist_); edge_accessor.vlist_->remove(&edge_accessor.update(), *transaction_); } -GraphDb::Label GraphDbAccessor::label(const std::string& label_name) { +GraphDbTypes::Label GraphDbAccessor::label(const std::string &label_name) { return &(*db_.labels_.access().insert(label_name).first); } -std::string& GraphDbAccessor::label_name(const GraphDb::Label label) const { +std::string &GraphDbAccessor::label_name( + const GraphDbTypes::Label label) const { return *label; } -GraphDb::EdgeType GraphDbAccessor::edge_type( - const std::string& edge_type_name) { +GraphDbTypes::EdgeType GraphDbAccessor::edge_type( + const std::string &edge_type_name) { return &(*db_.edge_types_.access().insert(edge_type_name).first); } -std::string& GraphDbAccessor::edge_type_name( - const GraphDb::EdgeType edge_type) const { +std::string &GraphDbAccessor::edge_type_name( + const GraphDbTypes::EdgeType edge_type) const { return *edge_type; } -GraphDb::Property GraphDbAccessor::property(const std::string& property_name) { +GraphDbTypes::Property GraphDbAccessor::property( + const std::string &property_name) { return &(*db_.properties_.access().insert(property_name).first); } -std::string& GraphDbAccessor::property_name( - const GraphDb::Property property) const { +std::string &GraphDbAccessor::property_name( + const GraphDbTypes::Property property) const { return *property; } diff --git a/src/database/graph_db_accessor.hpp b/src/database/graph_db_accessor.hpp index 2d7f836e8..983782e99 100644 --- a/src/database/graph_db_accessor.hpp +++ b/src/database/graph_db_accessor.hpp @@ -97,7 +97,7 @@ class GraphDbAccessor { * @return An accessor to the edge. */ EdgeAccessor insert_edge(VertexAccessor& from, VertexAccessor& to, - GraphDb::EdgeType type); + GraphDbTypes::EdgeType type); /** * Removes an edge from the graph. @@ -126,7 +126,7 @@ class GraphDbAccessor { * Obtains the Label for the label's name. * @return See above. */ - GraphDb::Label label(const std::string& label_name); + GraphDbTypes::Label label(const std::string& label_name); /** * Obtains the label name (a string) for the given label. @@ -134,13 +134,13 @@ class GraphDbAccessor { * @param label a Label. * @return See above. */ - std::string& label_name(const GraphDb::Label label) const; + std::string& label_name(const GraphDbTypes::Label label) const; /** * Obtains the EdgeType for it's name. * @return See above. */ - GraphDb::EdgeType edge_type(const std::string& edge_type_name); + GraphDbTypes::EdgeType edge_type(const std::string& edge_type_name); /** * Obtains the edge type name (a string) for the given edge type. @@ -148,13 +148,13 @@ class GraphDbAccessor { * @param edge_type an EdgeType. * @return See above. */ - std::string& edge_type_name(const GraphDb::EdgeType edge_type) const; + std::string& edge_type_name(const GraphDbTypes::EdgeType edge_type) const; /** * Obtains the Property for it's name. * @return See above. */ - GraphDb::Property property(const std::string& property_name); + GraphDbTypes::Property property(const std::string& property_name); /** * Obtains the property name (a string) for the given property. @@ -162,7 +162,7 @@ class GraphDbAccessor { * @param property a Property. * @return See above. */ - std::string& property_name(const GraphDb::Property property) const; + std::string& property_name(const GraphDbTypes::Property property) const; /** * Advances transaction's command id by 1. diff --git a/src/database/graph_db_datatypes.hpp b/src/database/graph_db_datatypes.hpp new file mode 100644 index 000000000..a1b78cd31 --- /dev/null +++ b/src/database/graph_db_datatypes.hpp @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace GraphDbTypes { +// definitions for what data types are used for a Label, Property, EdgeType +using Label = std::string *; +using EdgeType = std::string *; +using Property = std::string *; +}; diff --git a/src/logging/logger.hpp b/src/logging/logger.hpp index 068b9465b..980d1291f 100644 --- a/src/logging/logger.hpp +++ b/src/logging/logger.hpp @@ -13,15 +13,15 @@ class Logger { Message(Timestamp timestamp, std::string location, std::string message) : timestamp(timestamp), location(location), message(message) {} - const Timestamp& when() const override { return timestamp; } + const Timestamp &when() const override { return timestamp; } - const std::string& where() const override { return location; } + const std::string &where() const override { return location; } unsigned level() const override { return Level::level; } - const std::string& level_str() const override { return Level::text; } + const std::string &level_str() const override { return Level::text; } - const std::string& text() const override { return message; } + const std::string &text() const override { return message; } private: Timestamp timestamp; @@ -32,10 +32,10 @@ class Logger { public: Logger() = default; - Logger(Log* log, const std::string& name) : log(log), name(name) {} + Logger(Log *log, const std::string &name) : log(log), name(name) {} template - void emit(Args&&... args) { + void emit(Args &&... args) { debug_assert(log != nullptr, "Log object has to be defined."); auto message = std::make_unique>( @@ -44,8 +44,14 @@ class Logger { log->emit(std::move(message)); } + /** + *@brief Return if the logger is initialized. + *@return true if initialized, false otherwise. + */ + bool Initialized() { return log != nullptr; } + template - void trace(Args&&... args) { + void trace(Args &&... args) { #ifndef NDEBUG #ifndef LOG_NO_TRACE emit(std::forward(args)...); @@ -54,7 +60,7 @@ class Logger { } template - void debug(Args&&... args) { + void debug(Args &&... args) { #ifndef NDEBUG #ifndef LOG_NO_DEBUG emit(std::forward(args)...); @@ -63,27 +69,27 @@ class Logger { } template - void info(Args&&... args) { + void info(Args &&... args) { #ifndef LOG_NO_INFO emit(std::forward(args)...); #endif } template - void warn(Args&&... args) { + void warn(Args &&... args) { #ifndef LOG_NO_WARN emit(std::forward(args)...); #endif } template - void error(Args&&... args) { + void error(Args &&... args) { #ifndef LOG_NO_ERROR emit(std::forward(args)...); #endif } private: - Log* log; + Log *log; std::string name; }; diff --git a/src/mvcc/version_list.hpp b/src/mvcc/version_list.hpp index 526ffa464..6f358f870 100644 --- a/src/mvcc/version_list.hpp +++ b/src/mvcc/version_list.hpp @@ -12,16 +12,24 @@ namespace mvcc { template class VersionList { - // TODO what is this Accessor? Dead code? - friend class Accessor; - public: using uptr = std::unique_ptr>; using item_t = T; - VersionList() = default; + /* @brief Constructor that is used to insert one item into VersionList. + @param t - transaction + @param T item - item which will point to new and only entry in + version_list. + @param args - args forwarded to constructor of item T. + */ + template + VersionList(tx::Transaction &t, T *&item, Args &&... args) { + item = insert(t, std::forward(args)...); + } + VersionList() = delete; VersionList(const VersionList &) = delete; + VersionList &operator=(const VersionList &) = delete; /* @brief Move constructs the version list * Note: use only at the beginning of the "other's" lifecycle since this @@ -48,59 +56,58 @@ class VersionList { return stream; } - auto gc_lock_acquire() { return std::unique_lock(lock); } - - // Frees all records which are deleted by transaction older than given id. - // EXPECTS THAT THERE IS NO ACTIVE TRANSACTION WITH ID LESS THAN GIVEN ID. - // EXPECTS THAT THERE WON'T BE SIMULATAIUS CALLS FROM DIFFERENT THREADS OF - // THIS METHOD. - // True if this whole version list isn't needed any more. There is still - // possibilty that someone is reading it at this moment but he cant change - // it or get anything from it. - // TODO: Validate this method - bool gc_deleted(const Id &id) { - auto r = head.load(std::memory_order_seq_cst); - T *bef = nullptr; + /** + * This method is NOT thread-safe. This should never be called with a + * transaction id newer than the oldest active transaction id. + * Garbage collect (delete) all records which are no longer visible for any + * transaction with an id greater or equal to id. + * @param id - transaction id from which to start garbage collection + * @return true - If version list is empty after garbage collection. + */ + bool GcDeleted(const Id &id) { + auto newest_deleted_record = head.load(std::memory_order_seq_cst); + T *oldest_not_deleted_record = nullptr; // nullptr // | // [v1] ... // | - // [v2] <------+ + // [v2] <------+ newest_deleted_record // | | - // [v3] <------+ + // [v3] <------+ oldest_not_deleted_record // | | Jump backwards until you find a first old deleted - // [VerList] ----+ version, or you reach the end of the list + // [VerList] ----+ record, or you reach the end of the list // - while (r != nullptr && !r->is_deleted_before(id)) { - bef = r; - r = r->next(std::memory_order_seq_cst); + while (newest_deleted_record != nullptr && + !newest_deleted_record->is_deleted_before(id)) { + oldest_not_deleted_record = newest_deleted_record; + newest_deleted_record = + newest_deleted_record->next(std::memory_order_seq_cst); } - if (bef == nullptr) { - // if r==nullptr he is needed and it is expecting insert. - // if r!=nullptr vertex has been explicitly deleted. It can't be - // updated because for update, visible record is needed and at this - // point whe know that there is no visible record for any - // transaction. Also it cant be inserted because head isn't nullptr. - // Remove also requires visible record. Find wont return any record - // because none is visible. - return r != nullptr; - } else { - if (r != nullptr) { - // Bef is possible visible to some transaction but r is not and - // the implementation of this version list guarantees that - // record r and older records aren't accessed. - bef->next(nullptr, std::memory_order_seq_cst); - delete r; // THIS IS ISSUE IF MULTIPLE THREADS TRY TO DO THIS - } - - return false; + if (oldest_not_deleted_record == nullptr) { + // This can happen only if the head already points to a deleted record or + // the version list is empty. This means that the version_list is ready + // for complete destruction. + if (newest_deleted_record != nullptr) delete newest_deleted_record; + head.store(nullptr, std::memory_order_seq_cst); + return true; } + // oldest_not_deleted_record might be visible to some transaction but + // newest_deleted_record is not. + oldest_not_deleted_record->next( + nullptr, std::memory_order_seq_cst); // No transaction will look + // further than this record and + // that's why it's safe to set + // next to nullptr. + // Call destructor which will clean everything older than this record since + // they are called recursively. + if (newest_deleted_record != nullptr) + delete newest_deleted_record; // THIS IS ISSUE IF MULTIPLE THREADS TRY TO + // DO THIS + return false; } - void vacuum() {} - T *find(const tx::Transaction &t) const { auto r = head.load(std::memory_order_seq_cst); @@ -120,25 +127,6 @@ class VersionList { return r; } - /** - * @Args forwarded to the constructor of T - */ - template - T *insert(tx::Transaction &t, Args &&... args) { - debug_assert(head == nullptr, "Head is not nullptr on creation."); - - // create a first version of the record - // TODO replace 'new' with something better - auto v1 = new T(std::forward(args)...); - - // mark the record as created by the transaction t - v1->mark_created(t); - - head.store(v1, std::memory_order_seq_cst); - - return v1; - } - T *update(tx::Transaction &t) { debug_assert(head != nullptr, "Head is nullptr on update."); auto record = find(t); @@ -202,6 +190,27 @@ class VersionList { throw SerializationError(); } + /** + * This is private because this should only be called from the constructor. + * Otherwise head might be nullptr while version_list exists and it could + * interfere with GC. + * @tparam Args forwarded to the constructor of T + */ + template + T *insert(tx::Transaction &t, Args &&... args) { + debug_assert(head == nullptr, "Head is not nullptr on creation."); + + // create a first version of the record + // TODO replace 'new' with something better + auto v1 = new T(std::forward(args)...); + + // mark the record as created by the transaction t + v1->mark_created(t); + + head.store(v1, std::memory_order_seq_cst); + return v1; + } + std::atomic head{nullptr}; RecordLock lock; }; diff --git a/src/query/frontend/ast/ast.hpp b/src/query/frontend/ast/ast.hpp index fe46aa9ec..660f50995 100644 --- a/src/query/frontend/ast/ast.hpp +++ b/src/query/frontend/ast/ast.hpp @@ -347,16 +347,18 @@ class PropertyLookup : public Expression { } Expression *expression_ = nullptr; - GraphDb::Property property_ = nullptr; + GraphDbTypes::Property property_ = nullptr; // TODO potential problem: property lookups are allowed on both map literals // and records, but map literals have strings as keys and records have - // GraphDb::Property + // GraphDbTypes::Property // - // possible solution: store both string and GraphDb::Property here and choose + // possible solution: store both string and GraphDbTypes::Property here and + // choose // between the two depending on Expression result protected: - PropertyLookup(int uid, Expression *expression, GraphDb::Property property) + PropertyLookup(int uid, Expression *expression, + GraphDbTypes::Property property) : Expression(uid), expression_(expression), property_(property) {} }; @@ -402,9 +404,9 @@ class NodeAtom : public PatternAtom { visitor.PostVisit(*this); } - std::vector labels_; + std::vector labels_; // TODO: change to unordered_map - std::map properties_; + std::map properties_; protected: using PatternAtom::PatternAtom; @@ -426,9 +428,9 @@ class EdgeAtom : public PatternAtom { } Direction direction_ = Direction::BOTH; - std::vector edge_types_; + std::vector edge_types_; // TODO: change to unordered_map - std::map properties_; + std::map properties_; protected: using PatternAtom::PatternAtom; @@ -620,12 +622,12 @@ class SetLabels : public Clause { visitor.PostVisit(*this); } Identifier *identifier_ = nullptr; - std::vector labels_; + std::vector labels_; protected: SetLabels(int uid) : Clause(uid) {} SetLabels(int uid, Identifier *identifier, - const std::vector &labels) + const std::vector &labels) : Clause(uid), identifier_(identifier), labels_(labels) {} }; diff --git a/src/query/frontend/ast/cypher_main_visitor.cpp b/src/query/frontend/ast/cypher_main_visitor.cpp index cb5e63e00..e3071f702 100644 --- a/src/query/frontend/ast/cypher_main_visitor.cpp +++ b/src/query/frontend/ast/cypher_main_visitor.cpp @@ -148,19 +148,20 @@ antlrcpp::Any CypherMainVisitor::visitNodePattern( } if (ctx->nodeLabels()) { node->labels_ = - ctx->nodeLabels()->accept(this).as>(); + ctx->nodeLabels()->accept(this).as>(); } if (ctx->properties()) { - node->properties_ = ctx->properties() - ->accept(this) - .as>(); + node->properties_ = + ctx->properties() + ->accept(this) + .as>(); } return node; } antlrcpp::Any CypherMainVisitor::visitNodeLabels( CypherParser::NodeLabelsContext *ctx) { - std::vector labels; + std::vector labels; for (auto *node_label : ctx->nodeLabel()) { labels.push_back(ctx_.db_accessor_.label(node_label->accept(this))); } @@ -182,7 +183,7 @@ antlrcpp::Any CypherMainVisitor::visitProperties( antlrcpp::Any CypherMainVisitor::visitMapLiteral( CypherParser::MapLiteralContext *ctx) { - std::map map; + std::map map; for (int i = 0; i < (int)ctx->propertyKeyName().size(); ++i) { map[ctx->propertyKeyName()[i]->accept(this)] = ctx->expression()[i]->accept(this); @@ -267,13 +268,14 @@ antlrcpp::Any CypherMainVisitor::visitRelationshipPattern( edge->edge_types_ = ctx->relationshipDetail() ->relationshipTypes() ->accept(this) - .as>(); + .as>(); } if (ctx->relationshipDetail()->properties()) { - edge->properties_ = ctx->relationshipDetail() - ->properties() - ->accept(this) - .as>(); + edge->properties_ = + ctx->relationshipDetail() + ->properties() + ->accept(this) + .as>(); } if (ctx->relationshipDetail()->rangeLiteral()) { // TODO: implement other clauses. @@ -311,7 +313,7 @@ antlrcpp::Any CypherMainVisitor::visitRelationshipDetail( antlrcpp::Any CypherMainVisitor::visitRelationshipTypes( CypherParser::RelationshipTypesContext *ctx) { - std::vector types; + std::vector types; for (auto *edge_type : ctx->relTypeName()) { types.push_back(ctx_.db_accessor_.edge_type(edge_type->accept(this))); } @@ -738,7 +740,7 @@ antlrcpp::Any CypherMainVisitor::visitSetItem( set_labels->identifier_ = storage_.Create( ctx->variable()->accept(this).as()); set_labels->labels_ = - ctx->nodeLabels()->accept(this).as>(); + ctx->nodeLabels()->accept(this).as>(); return static_cast(set_labels); } diff --git a/src/query/frontend/ast/cypher_main_visitor.hpp b/src/query/frontend/ast/cypher_main_visitor.hpp index deba4486f..4742d074f 100644 --- a/src/query/frontend/ast/cypher_main_visitor.hpp +++ b/src/query/frontend/ast/cypher_main_visitor.hpp @@ -181,22 +181,22 @@ class CypherMainVisitor : public antlropencypher::CypherBaseVisitor { CypherParser::NodePatternContext *ctx) override; /** - * @return vector + * @return vector */ antlrcpp::Any visitNodeLabels(CypherParser::NodeLabelsContext *ctx) override; /** - * @return unordered_map + * @return unordered_map */ antlrcpp::Any visitProperties(CypherParser::PropertiesContext *ctx) override; /** - * @return unordered_map + * @return unordered_map */ antlrcpp::Any visitMapLiteral(CypherParser::MapLiteralContext *ctx) override; /** - * @return GraphDb::Property + * @return GraphDbTypes::Property */ antlrcpp::Any visitPropertyKeyName( CypherParser::PropertyKeyNameContext *ctx) override; @@ -244,7 +244,7 @@ class CypherMainVisitor : public antlropencypher::CypherBaseVisitor { CypherParser::RelationshipDetailContext *ctx) override; /** - * @return vector + * @return vector */ antlrcpp::Any visitRelationshipTypes( CypherParser::RelationshipTypesContext *ctx) override; diff --git a/src/query/frontend/logical/operator.hpp b/src/query/frontend/logical/operator.hpp index ebcde4f72..be93b4b1d 100644 --- a/src/query/frontend/logical/operator.hpp +++ b/src/query/frontend/logical/operator.hpp @@ -6,6 +6,7 @@ #include #include "database/graph_db_accessor.hpp" +#include "database/graph_db_datatypes.hpp" #include "query/frontend/ast/ast.hpp" #include "query/frontend/interpret/interpret.hpp" #include "query/frontend/semantic/symbol_table.hpp" @@ -593,7 +594,7 @@ class EdgeFilter : public LogicalOperator { SymbolTable &symbol_table) { // edge type filtering - logical OR const auto &types = self_.edge_atom_->edge_types_; - GraphDb::EdgeType type = edge.edge_type(); + GraphDbTypes::EdgeType type = edge.edge_type(); if (!std::any_of(types.begin(), types.end(), [type](auto t) { return t == type; })) return false; @@ -999,7 +1000,7 @@ class SetLabels : public LogicalOperator { public: SetLabels(const std::shared_ptr input, const Symbol input_symbol, - const std::vector &labels) + const std::vector &labels) : input_(input), input_symbol_(input_symbol), labels_(labels) {} void Accept(LogicalOperatorVisitor &visitor) override { @@ -1019,8 +1020,7 @@ class SetLabels : public LogicalOperator { TypedValue vertex_value = frame[self_.input_symbol_]; VertexAccessor vertex = vertex_value.Value(); - for (auto label : self_.labels_) - vertex.add_label(label); + for (auto label : self_.labels_) vertex.add_label(label); return true; } @@ -1038,7 +1038,7 @@ class SetLabels : public LogicalOperator { private: std::shared_ptr input_; const Symbol input_symbol_; - std::vector labels_; + std::vector labels_; }; } // namespace plan diff --git a/src/storage/edge.hpp b/src/storage/edge.hpp index e6a3a816e..f150c9fdc 100644 --- a/src/storage/edge.hpp +++ b/src/storage/edge.hpp @@ -1,6 +1,6 @@ #pragma once -#include "database/graph_db.hpp" +#include "database/graph_db_datatypes.hpp" #include "mvcc/record.hpp" #include "mvcc/version_list.hpp" #include "storage/property_value_store.hpp" @@ -10,12 +10,12 @@ class Vertex; class Edge : public mvcc::Record { public: - Edge(mvcc::VersionList& from, mvcc::VersionList& to, - GraphDb::EdgeType edge_type) + Edge(mvcc::VersionList &from, mvcc::VersionList &to, + GraphDbTypes::EdgeType edge_type) : from_(from), to_(to), edge_type_(edge_type) {} - mvcc::VersionList& from_; - mvcc::VersionList& to_; - GraphDb::EdgeType edge_type_; - PropertyValueStore properties_; + mvcc::VersionList &from_; + mvcc::VersionList &to_; + GraphDbTypes::EdgeType edge_type_; + PropertyValueStore properties_; }; diff --git a/src/storage/edge_accessor.cpp b/src/storage/edge_accessor.cpp index 1fbec3038..ae2e8dec4 100644 --- a/src/storage/edge_accessor.cpp +++ b/src/storage/edge_accessor.cpp @@ -1,11 +1,11 @@ #include "storage/edge_accessor.hpp" #include "storage/vertex_accessor.hpp" -void EdgeAccessor::set_edge_type(GraphDb::EdgeType edge_type) { +void EdgeAccessor::set_edge_type(GraphDbTypes::EdgeType edge_type) { update().edge_type_ = edge_type; } -GraphDb::EdgeType EdgeAccessor::edge_type() const { return view().edge_type_; } +GraphDbTypes::EdgeType EdgeAccessor::edge_type() const { return view().edge_type_; } VertexAccessor EdgeAccessor::from() const { return VertexAccessor(view().from_, db_accessor()); diff --git a/src/storage/edge_accessor.hpp b/src/storage/edge_accessor.hpp index ef033edae..2fa954aa9 100644 --- a/src/storage/edge_accessor.hpp +++ b/src/storage/edge_accessor.hpp @@ -25,13 +25,13 @@ class EdgeAccessor : public RecordAccessor { * Sets a new edge type. * @param edge_type The new type. */ - void set_edge_type(GraphDb::EdgeType edge_type); + void set_edge_type(GraphDbTypes::EdgeType edge_type); /** * Returns the edge type. * @return */ - GraphDb::EdgeType edge_type() const; + GraphDbTypes::EdgeType edge_type() const; /** * Returns an accessor to the originating Vertex of this edge. diff --git a/src/storage/garbage_collector.hpp b/src/storage/garbage_collector.hpp new file mode 100644 index 000000000..ed7420b2e --- /dev/null +++ b/src/storage/garbage_collector.hpp @@ -0,0 +1,82 @@ +#pragma once + +#include +#include +#include + +#include "config/config.hpp" +#include "data_structures/concurrent/skiplist.hpp" +#include "logging/loggable.hpp" +#include "mvcc/id.hpp" +#include "mvcc/version_list.hpp" +#include "transactions/engine.hpp" + +/** + @template T type of underlying record in mvcc + */ + +template +class GarbageCollector : public Loggable { + public: + GarbageCollector(SkipList *> *skiplist, + tx::Engine *engine) + : Loggable("MvccGc"), skiplist_(skiplist), engine_(engine) { + permanent_assert(skiplist != nullptr, "Skiplist can't be nullptr."); + permanent_assert(engine != nullptr, "Engine can't be nullptr."); + }; + + ~GarbageCollector() { + destruction_.store(true); + condition_variable_.notify_one(); + if (run_thread_.joinable()) run_thread_.join(); + } + + /** + *@brief - Runs garbage collector. Starts a new thread which garbage collects + *in the background. + *@param pause - How long to sleep between successive garbage collector + *thread. If this parameter is -1 the GC will not run. + */ + void Run(const std::chrono::seconds &pause) { + if (pause == std::chrono::seconds(-1)) return; + // Invoke new thread which will do the GC work. + run_thread_ = std::thread([this, pause]() { + for (;;) { + // If the whole class is being destructed we should end this thread. + if (this->destruction_.load(std::memory_order_seq_cst)) break; + + auto accessor = this->skiplist_->access(); + uint64_t count = 0; + // Acquire id of either the oldest active transaction, or the id of a + // transaction that will be assigned next. We should make sure that we + // get count before we ask for active transactions since some + // transaction could possibly increase the count while we ask for + // oldest_active transaction. + const auto next_id = engine_->count() + 1; + const auto id = this->engine_->oldest_active().get_or(next_id); + if (logger.Initialized()) + logger.trace("Gc started cleaning everything deleted before {}", id); + for (auto x : accessor) { + // If the mvcc is empty, i.e. there is nothing else to be read from it + // we can delete it. + if (x->GcDeleted(id)) count += accessor.remove(x); + } + if (logger.Initialized()) logger.trace("Destroyed: {}", count); + + std::unique_lock lk(mutex_); + condition_variable_.wait_for(lk, std::chrono::seconds(pause), [&] { + return this->destruction_ == true; + }); + lk.unlock(); + } + }); + } + + private: + SkipList *> *skiplist_{nullptr}; // Not owned. + tx::Engine *engine_{nullptr}; // Not owned. + std::thread run_thread_; + std::atomic destruction_; + std::mutex mutex_; + std::condition_variable condition_variable_; +}; diff --git a/src/storage/record_accessor.cpp b/src/storage/record_accessor.cpp index aef54bffc..9c5a7735c 100644 --- a/src/storage/record_accessor.cpp +++ b/src/storage/record_accessor.cpp @@ -22,12 +22,12 @@ RecordAccessor::RecordAccessor(mvcc::VersionList &vlist, template const PropertyValue &RecordAccessor::PropsAt( - GraphDb::Property key) const { + GraphDbTypes::Property key) const { return view().properties_.at(key); } template -size_t RecordAccessor::PropsErase(GraphDb::Property key) { +size_t RecordAccessor::PropsErase(GraphDbTypes::Property key) { return update().properties_.erase(key); } @@ -37,14 +37,15 @@ void RecordAccessor::PropsClear() { } template -const PropertyValueStore +const PropertyValueStore &RecordAccessor::Properties() const { return view().properties_; } template void RecordAccessor::PropertiesAccept( - std::function + std::function handler, std::function finish) const { view().properties_.Accept(handler, finish); diff --git a/src/storage/record_accessor.hpp b/src/storage/record_accessor.hpp index 5894ad364..044b8a202 100644 --- a/src/storage/record_accessor.hpp +++ b/src/storage/record_accessor.hpp @@ -62,7 +62,7 @@ class RecordAccessor { * @param key * @return */ - const PropertyValue& PropsAt(GraphDb::Property key) const; + const PropertyValue& PropsAt(GraphDbTypes::Property key) const; /** * Sets a value on the record for the given property. @@ -72,7 +72,7 @@ class RecordAccessor { * @param value The value to set. */ template - void PropsSet(GraphDb::Property key, TValue value) { + void PropsSet(GraphDbTypes::Property key, TValue value) { update().properties_.set(key, value); } @@ -82,7 +82,7 @@ class RecordAccessor { * @param key * @return */ - size_t PropsErase(GraphDb::Property key); + size_t PropsErase(GraphDbTypes::Property key); /** * Removes all the properties from this record. @@ -93,9 +93,9 @@ class RecordAccessor { * Returns the properties of this record. * @return */ - const PropertyValueStore& Properties() const; + const PropertyValueStore& Properties() const; - void PropertiesAccept(std::function handler, std::function finish = {}) const; diff --git a/src/storage/vertex.hpp b/src/storage/vertex.hpp index f8a39cc39..8c186eff1 100644 --- a/src/storage/vertex.hpp +++ b/src/storage/vertex.hpp @@ -2,7 +2,7 @@ #include -#include "database/graph_db.hpp" +#include "database/graph_db_datatypes.hpp" #include "mvcc/record.hpp" #include "mvcc/version_list.hpp" #include "storage/property_value_store.hpp" @@ -12,8 +12,8 @@ class Edge; class Vertex : public mvcc::Record { public: - std::vector*> out_; - std::vector*> in_; - std::vector labels_; - PropertyValueStore properties_; + std::vector *> out_; + std::vector *> in_; + std::vector labels_; + PropertyValueStore properties_; }; diff --git a/src/storage/vertex_accessor.cpp b/src/storage/vertex_accessor.cpp index e6074e9d9..123f81393 100644 --- a/src/storage/vertex_accessor.cpp +++ b/src/storage/vertex_accessor.cpp @@ -8,7 +8,7 @@ size_t VertexAccessor::out_degree() const { return view().out_.size(); } size_t VertexAccessor::in_degree() const { return view().in_.size(); } -bool VertexAccessor::add_label(GraphDb::Label label) { +bool VertexAccessor::add_label(GraphDbTypes::Label label) { auto &labels_view = view().labels_; auto found = std::find(labels_view.begin(), labels_view.end(), label); if (found != labels_view.end()) return false; @@ -18,7 +18,7 @@ bool VertexAccessor::add_label(GraphDb::Label label) { return true; } -size_t VertexAccessor::remove_label(GraphDb::Label label) { +size_t VertexAccessor::remove_label(GraphDbTypes::Label label) { auto &labels = update().labels_; auto found = std::find(labels.begin(), labels.end(), label); if (found == labels.end()) return 0; @@ -28,11 +28,11 @@ size_t VertexAccessor::remove_label(GraphDb::Label label) { return 1; } -bool VertexAccessor::has_label(GraphDb::Label label) const { +bool VertexAccessor::has_label(GraphDbTypes::Label label) const { auto &labels = this->view().labels_; return std::find(labels.begin(), labels.end(), label) != labels.end(); } -const std::vector &VertexAccessor::labels() const { +const std::vector &VertexAccessor::labels() const { return this->view().labels_; } diff --git a/src/storage/vertex_accessor.hpp b/src/storage/vertex_accessor.hpp index edd83f392..287135a3a 100644 --- a/src/storage/vertex_accessor.hpp +++ b/src/storage/vertex_accessor.hpp @@ -41,27 +41,27 @@ class VertexAccessor : public RecordAccessor { * @param label A label. * @return If or not a new Label was set on this Vertex. */ - bool add_label(GraphDb::Label label); + bool add_label(GraphDbTypes::Label label); /** * Removes a label from the Vertex. * @param label The label to remove. * @return The number of removed labels (can be 0 or 1). */ - size_t remove_label(GraphDb::Label label); + size_t remove_label(GraphDbTypes::Label label); /** * Indicates if the Vertex has the given label. * @param label A label. * @return */ - bool has_label(GraphDb::Label label) const; + bool has_label(GraphDbTypes::Label label) const; /** * Returns all the Labels of the Vertex. * @return */ - const std::vector& labels() const; + const std::vector& labels() const; /** * Returns EdgeAccessors for all incoming edges. diff --git a/src/utils/random_graph_generator.hpp b/src/utils/random_graph_generator.hpp index 56da0d228..b83328604 100644 --- a/src/utils/random_graph_generator.hpp +++ b/src/utils/random_graph_generator.hpp @@ -11,6 +11,7 @@ #include #include "database/graph_db_accessor.hpp" +#include "database/graph_db_datatypes.hpp" #include "storage/property_value.hpp" #include "storage/vertex_accessor.hpp" #include "utils/assert.hpp" @@ -44,7 +45,7 @@ class RandomGraphGenerator { void AddVertices(uint count, std::vector label_names) { permanent_assert(!did_commit_, "Already committed"); - std::vector labels; + std::vector labels; for (const auto &label_name : label_names) labels.push_back(dba_.label(label_name)); diff --git a/tests/integration/stream/print_record_stream.hpp b/tests/integration/stream/print_record_stream.hpp index 878732eee..525eef3cd 100644 --- a/tests/integration/stream/print_record_stream.hpp +++ b/tests/integration/stream/print_record_stream.hpp @@ -10,7 +10,7 @@ #include "storage/vertex_accessor.hpp" void write_properties(std::ostream &os, const GraphDbAccessor &access, - const PropertyValueStore &properties) { + const PropertyValueStore &properties) { if (properties.size() > 0) { os << "{"; for (auto x : properties) { @@ -22,7 +22,7 @@ void write_properties(std::ostream &os, const GraphDbAccessor &access, std::ostream &operator<<(std::ostream &os, const VertexAccessor &vertex) { if (vertex.labels().size() > 0) { - for (GraphDb::Label label : vertex.labels()) { + for (GraphDbTypes::Label label : vertex.labels()) { os << vertex.db_accessor().property_name(label) << ", "; } os << "\n"; diff --git a/tests/unit/cypher_main_visitor.cpp b/tests/unit/cypher_main_visitor.cpp index de58284fe..635041e96 100644 --- a/tests/unit/cypher_main_visitor.cpp +++ b/tests/unit/cypher_main_visitor.cpp @@ -380,7 +380,7 @@ TEST(CypherMainVisitorTest, NodePattern) { ast_generator.db_accessor_->label("label1"), ast_generator.db_accessor_->label("label2"), ast_generator.db_accessor_->label("label3"))); - std::unordered_map properties; + std::unordered_map properties; for (auto x : node->properties_) { auto *literal = dynamic_cast(x.second); ASSERT_TRUE(literal); @@ -457,7 +457,7 @@ TEST(CypherMainVisitorTest, RelationshipPatternDetails) { edge->edge_types_, UnorderedElementsAre(ast_generator.db_accessor_->edge_type("type1"), ast_generator.db_accessor_->edge_type("type2"))); - std::unordered_map properties; + std::unordered_map properties; for (auto x : edge->properties_) { auto *literal = dynamic_cast(x.second); ASSERT_TRUE(literal); diff --git a/tests/unit/graph_db_accessor.cpp b/tests/unit/graph_db_accessor.cpp index badbdb414..aa63e1f7b 100644 --- a/tests/unit/graph_db_accessor.cpp +++ b/tests/unit/graph_db_accessor.cpp @@ -263,7 +263,7 @@ TEST(GraphDbAccessorTest, Labels) { Dbms dbms; auto dba1 = dbms.active(); - GraphDb::Label label_friend = dba1->label("friend"); + GraphDbTypes::Label label_friend = dba1->label("friend"); EXPECT_EQ(label_friend, dba1->label("friend")); EXPECT_NE(label_friend, dba1->label("friend2")); EXPECT_EQ(dba1->label_name(label_friend), "friend"); @@ -277,7 +277,7 @@ TEST(GraphDbAccessorTest, EdgeTypes) { Dbms dbms; auto dba1 = dbms.active(); - GraphDb::EdgeType edge_type = dba1->edge_type("likes"); + GraphDbTypes::EdgeType edge_type = dba1->edge_type("likes"); EXPECT_EQ(edge_type, dba1->edge_type("likes")); EXPECT_NE(edge_type, dba1->edge_type("hates")); EXPECT_EQ(dba1->edge_type_name(edge_type), "likes"); @@ -291,7 +291,7 @@ TEST(GraphDbAccessorTest, Properties) { Dbms dbms; auto dba1 = dbms.active(); - GraphDb::EdgeType prop = dba1->property("name"); + GraphDbTypes::EdgeType prop = dba1->property("name"); EXPECT_EQ(prop, dba1->property("name")); EXPECT_NE(prop, dba1->property("surname")); EXPECT_EQ(dba1->property_name(prop), "name"); diff --git a/tests/unit/interpreter.cpp b/tests/unit/interpreter.cpp index d1fc63bf4..3f3cb3caf 100644 --- a/tests/unit/interpreter.cpp +++ b/tests/unit/interpreter.cpp @@ -185,8 +185,8 @@ TEST(Interpreter, NodeFilterLabelsAndProperties) { auto dba = dbms.active(); // add a few nodes to the database - GraphDb::Label label = dba->label("Label"); - GraphDb::Property property = dba->property("Property"); + GraphDbTypes::Label label = dba->label("Label"); + GraphDbTypes::Property property = dba->property("Property"); auto v1 = dba->insert_vertex(); auto v2 = dba->insert_vertex(); auto v3 = dba->insert_vertex(); @@ -231,9 +231,9 @@ TEST(Interpreter, NodeFilterMultipleLabels) { auto dba = dbms.active(); // add a few nodes to the database - GraphDb::Label label1 = dba->label("label1"); - GraphDb::Label label2 = dba->label("label2"); - GraphDb::Label label3 = dba->label("label3"); + GraphDbTypes::Label label1 = dba->label("label1"); + GraphDbTypes::Label label2 = dba->label("label2"); + GraphDbTypes::Label label3 = dba->label("label3"); // the test will look for nodes that have label1 and label2 dba->insert_vertex(); // NOT accepted dba->insert_vertex().add_label(label1); // NOT accepted @@ -278,8 +278,8 @@ TEST(Interpreter, CreateNodeWithAttributes) { Dbms dbms; auto dba = dbms.active(); - GraphDb::Label label = dba->label("Person"); - GraphDb::Property property = dba->label("age"); + GraphDbTypes::Label label = dba->label("Person"); + GraphDbTypes::Property property = dba->label("age"); AstTreeStorage storage; SymbolTable symbol_table; @@ -312,8 +312,8 @@ TEST(Interpreter, CreateReturn) { Dbms dbms; auto dba = dbms.active(); - GraphDb::Label label = dba->label("Person"); - GraphDb::Property property = dba->label("age"); + GraphDbTypes::Label label = dba->label("Person"); + GraphDbTypes::Property property = dba->label("age"); AstTreeStorage storage; SymbolTable symbol_table; @@ -354,10 +354,10 @@ TEST(Interpreter, CreateExpand) { Dbms dbms; auto dba = dbms.active(); - GraphDb::Label label_node_1 = dba->label("Node1"); - GraphDb::Label label_node_2 = dba->label("Node2"); - GraphDb::Property property = dba->label("prop"); - GraphDb::EdgeType edge_type = dba->label("edge_type"); + GraphDbTypes::Label label_node_1 = dba->label("Node1"); + GraphDbTypes::Label label_node_2 = dba->label("Node2"); + GraphDbTypes::Property property = dba->label("prop"); + GraphDbTypes::EdgeType edge_type = dba->label("edge_type"); SymbolTable symbol_table; AstTreeStorage storage; @@ -404,7 +404,7 @@ TEST(Interpreter, CreateExpand) { for (VertexAccessor vertex : dba->vertices()) { EXPECT_EQ(vertex.labels().size(), 1); - GraphDb::Label label = vertex.labels()[0]; + GraphDbTypes::Label label = vertex.labels()[0]; if (label == label_node_1) { // node created by first op EXPECT_EQ(vertex.PropsAt(property).Value(), 1); @@ -460,10 +460,10 @@ TEST(Interpreter, MatchCreateExpand) { dba->insert_vertex(); dba->advance_command(); - // GraphDb::Label label_node_1 = dba->label("Node1"); - // GraphDb::Label label_node_2 = dba->label("Node2"); - // GraphDb::Property property = dba->label("prop"); - GraphDb::EdgeType edge_type = dba->label("edge_type"); + // GraphDbTypes::Label label_node_1 = dba->label("Node1"); + // GraphDbTypes::Label label_node_2 = dba->label("Node2"); + // GraphDbTypes::Property property = dba->label("prop"); + GraphDbTypes::EdgeType edge_type = dba->label("edge_type"); SymbolTable symbol_table; AstTreeStorage storage; @@ -507,11 +507,11 @@ TEST(Interpreter, Expand) { // make a V-graph (v3)<-[r2]-(v1)-[r1]->(v2) auto v1 = dba->insert_vertex(); - v1.add_label((GraphDb::Label)1); + v1.add_label((GraphDbTypes::Label)1); auto v2 = dba->insert_vertex(); - v2.add_label((GraphDb::Label)2); + v2.add_label((GraphDbTypes::Label)2); auto v3 = dba->insert_vertex(); - v3.add_label((GraphDb::Label)3); + v3.add_label((GraphDbTypes::Label)3); auto edge_type = dba->edge_type("Edge"); dba->insert_edge(v1, v2, edge_type); dba->insert_edge(v1, v3, edge_type); @@ -585,11 +585,11 @@ TEST(Interpreter, ExpandEdgeCycle) { // make a V-graph (v3)<-[r2]-(v1)-[r1]->(v2) auto v1 = dba->insert_vertex(); - v1.add_label((GraphDb::Label)1); + v1.add_label((GraphDbTypes::Label)1); auto v2 = dba->insert_vertex(); - v2.add_label((GraphDb::Label)2); + v2.add_label((GraphDbTypes::Label)2); auto v3 = dba->insert_vertex(); - v3.add_label((GraphDb::Label)3); + v3.add_label((GraphDbTypes::Label)3); auto edge_type = dba->edge_type("Edge"); dba->insert_edge(v1, v2, edge_type); dba->insert_edge(v1, v3, edge_type); @@ -631,12 +631,12 @@ TEST(Interpreter, EdgeFilter) { // where only one edge will qualify // and there are all combinations of // (edge_type yes|no) * (property yes|absent|no) - std::vector edge_types; + std::vector edge_types; for (int j = 0; j < 2; ++j) edge_types.push_back(dba->edge_type("et" + std::to_string(j))); std::vector vertices; for (int i = 0; i < 7; ++i) vertices.push_back(dba->insert_vertex()); - GraphDb::Property prop = dba->property("prop"); + GraphDbTypes::Property prop = dba->property("prop"); std::vector edges; for (int i = 0; i < 6; ++i) { edges.push_back( @@ -835,7 +835,7 @@ TEST(Interpreter, Filter) { auto dba = dbms.active(); // add a 6 nodes with property 'prop', 2 have true as value - GraphDb::Property property = dba->property("Property"); + GraphDbTypes::Property property = dba->property("Property"); for (int i = 0; i < 6; ++i) dba->insert_vertex().PropsSet(property, i % 3 == 0); dba->insert_vertex(); // prop not set, gives NULL @@ -995,7 +995,7 @@ TEST(Interpreter, SetLabels) { auto n = MakeScanAll(storage, symbol_table, "n"); auto label_set = std::make_shared( - n.op_, n.sym_, std::vector{label2, label3}); + n.op_, n.sym_, std::vector{label2, label3}); EXPECT_EQ(2, PullAll(label_set, *dba, symbol_table)); for (VertexAccessor vertex : dba->vertices()) { diff --git a/tests/unit/mvcc.cpp b/tests/unit/mvcc.cpp index 7cf1157b9..9148b05a6 100644 --- a/tests/unit/mvcc.cpp +++ b/tests/unit/mvcc.cpp @@ -10,9 +10,9 @@ class Prop : public mvcc::Record {}; TEST(MVCC, Case1Test3) { tx::Engine engine; - mvcc::VersionList version_list; auto t1 = engine.begin(); - version_list.insert(*t1); + Prop *prop; + mvcc::VersionList version_list(*t1, prop); t1->commit(); auto t2 = engine.begin(); @@ -28,14 +28,18 @@ TEST(MVCC, Case1Test3) { TEST(MVCC, InSnapshot) { tx::Engine engine; - mvcc::VersionList version_list; + Prop *prop; auto t1 = engine.begin(); - auto v1 = version_list.insert(*t1); - version_list.update(*t1); // expire old record and create new - auto t2 = engine.begin(); // t1 is in snapshot of t2 + mvcc::VersionList version_list(*t1, prop); t1->commit(); - EXPECT_THROW(version_list.update(v1, *t2), SerializationError); + auto t2 = engine.begin(); + version_list.update(*t2); + auto t3 = engine.begin(); // t2 is in snapshot of t3 + auto v = version_list.find(*t3); + t2->commit(); + + EXPECT_THROW(version_list.update(v, *t3), SerializationError); } int main(int argc, char **argv) { diff --git a/tests/unit/mvcc_gc.cpp b/tests/unit/mvcc_gc.cpp new file mode 100644 index 000000000..6e433968d --- /dev/null +++ b/tests/unit/mvcc_gc.cpp @@ -0,0 +1,122 @@ +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +#include +#include + +#include "config/config.hpp" +#include "data_structures/concurrent/skiplist.hpp" +#include "logging/logger.hpp" +#include "logging/streams/stdout.hpp" +#include "mvcc/record.hpp" +#include "mvcc/version_list.hpp" +#include "storage/garbage_collector.hpp" +#include "storage/vertex.hpp" +#include "transactions/engine.hpp" + +/** + * Class which takes an atomic variable to count number of destructor calls (to + * test if GC is actually deleting records). + */ +class Prop : public mvcc::Record { + public: + Prop(std::atomic &count) : count_(count) {} + ~Prop() { ++count_; } + + private: + std::atomic &count_; +}; + +/** + * Test will the mvcc gc delete records inside the version list because they + * are not longer visible. + */ +TEST(VersionList, GcDeleted) { + const int UPDATES = 10; + tx::Engine engine; + std::vector ids; + auto t1 = engine.begin(); + std::atomic count{0}; + Prop *prop; + mvcc::VersionList version_list(*t1, prop, count); + ids.push_back(t1->id); + t1->commit(); + + for (int i = 0; i < UPDATES; ++i) { + auto t2 = engine.begin(); + ids.push_back(t2->id); + prop = version_list.update(prop, *t2); + t2->commit(); + } + + EXPECT_EQ(version_list.GcDeleted(ids[0]), false); + EXPECT_EQ(count, 0); + EXPECT_EQ(version_list.GcDeleted(ids.back() + 1), false); + EXPECT_EQ(count, UPDATES); + + auto tl = engine.begin(); + version_list.remove(*tl); + EXPECT_EQ(version_list.GcDeleted(tl->id + 1), true); + EXPECT_EQ(count, UPDATES + 1); + tl->commit(); +} + +/** + * Test integration of garbage collector with MVCC GC. Delete mvcc's which are + * empty (not visible from any future transaction) from the skiplist. + */ +TEST(GarbageCollector, WaitAndClean) { + SkipList *> skiplist; + tx::Engine engine; + GarbageCollector gc(&skiplist, &engine); + gc.Run(std::chrono::seconds(1)); + + auto t1 = engine.begin(); + Prop *prop; + std::atomic count; + auto vl = new mvcc::VersionList(*t1, prop, count); + + auto access = skiplist.access(); + access.insert(vl); + t1->commit(); + auto t2 = engine.begin(); + EXPECT_EQ(vl->remove(*t2), true); + t2->commit(); + + std::this_thread::sleep_for(std::chrono::seconds(3)); + EXPECT_EQ(access.size(), (size_t)0); +} + +/** + * Same as above, but, the GarbageCollector will never be run because the time + * between garbage collections is set to -1. + */ +TEST(GarbageCollector, WaitAndDontClean) { + SkipList *> skiplist; + tx::Engine engine; + GarbageCollector gc(&skiplist, &engine); + gc.Run(std::chrono::seconds(-1)); // Never run GC. This test is identical to + // the top one except GC is never run. + + auto t1 = engine.begin(); + Prop *prop; + std::atomic count; + auto vl = new mvcc::VersionList(*t1, prop, count); + + auto access = skiplist.access(); + access.insert(vl); + t1->commit(); + auto t2 = engine.begin(); + EXPECT_EQ(vl->remove(*t2), true); + t2->commit(); + + std::this_thread::sleep_for(std::chrono::seconds(3)); + EXPECT_EQ(access.size(), (size_t)1); +} + +int main(int argc, char **argv) { + ::logging::init_async(); + ::logging::log->pipe(std::make_unique()); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/tests/unit/query_common.hpp b/tests/unit/query_common.hpp index cdb63cc7d..329385683 100644 --- a/tests/unit/query_common.hpp +++ b/tests/unit/query_common.hpp @@ -1,3 +1,5 @@ +#include "database/graph_db_datatypes.hpp" + namespace query { namespace test_common { @@ -8,7 +10,7 @@ namespace test_common { /// Name is used to create the Identifier which is used for property lookup. /// auto GetPropertyLookup(AstTreeStorage &storage, const std::string &name, - GraphDb::Property property) { + GraphDbTypes::Property property) { return storage.Create(storage.Create(name), property); } @@ -19,7 +21,7 @@ auto GetPropertyLookup(AstTreeStorage &storage, const std::string &name, /// Name is used to create the Identifier which is assigned to the edge. /// auto GetEdge(AstTreeStorage &storage, const std::string &name, - GraphDb::EdgeType edge_type = nullptr, + GraphDbTypes::EdgeType edge_type = nullptr, EdgeAtom::Direction dir = EdgeAtom::Direction::BOTH) { auto edge = storage.Create(storage.Create(name), dir); if (edge_type) edge->edge_types_.emplace_back(edge_type); @@ -40,7 +42,7 @@ auto GetEdge(AstTreeStorage &storage, const std::string &name, /// Name is used to create the Identifier which is assigned to the node. /// auto GetNode(AstTreeStorage &storage, const std::string &name, - GraphDb::Label label = nullptr) { + GraphDbTypes::Label label = nullptr) { auto node = storage.Create(storage.Create(name)); if (label) node->labels_.emplace_back(label); return node; @@ -156,7 +158,7 @@ auto GetSet(AstTreeStorage &storage, const std::string &name, Expression *expr, /// Create a set labels clause for given identifier name and labels. /// auto GetSet(AstTreeStorage &storage, const std::string &name, - std::vector labels) { + std::vector labels) { return storage.Create(storage.Create(name), labels); } diff --git a/tests/unit/record_edge_vertex_accessor.cpp b/tests/unit/record_edge_vertex_accessor.cpp index c54c5b6ba..ee3bdb6e5 100644 --- a/tests/unit/record_edge_vertex_accessor.cpp +++ b/tests/unit/record_edge_vertex_accessor.cpp @@ -85,8 +85,8 @@ TEST(RecordAccessor, VertexLabels) { EXPECT_EQ(v1.labels().size(), 0); - GraphDb::Label l1 = dba->label("label1"); - GraphDb::Label l2 = dba->label("label2"); + GraphDbTypes::Label l1 = dba->label("label1"); + GraphDbTypes::Label l2 = dba->label("label2"); // adding labels EXPECT_FALSE(v1.has_label(l1)); @@ -106,7 +106,7 @@ TEST(RecordAccessor, VertexLabels) { EXPECT_EQ(labels.size(), 2); // removing labels - GraphDb::Label l3 = dba->label("label3"); + GraphDbTypes::Label l3 = dba->label("label3"); EXPECT_EQ(v1.remove_label(l3), 0); EXPECT_EQ(labels.size(), 2); @@ -124,8 +124,8 @@ TEST(RecordAccessor, EdgeType) { auto v1 = dba->insert_vertex(); auto v2 = dba->insert_vertex(); - GraphDb::EdgeType likes = dba->edge_type("likes"); - GraphDb::EdgeType hates = dba->edge_type("hates"); + GraphDbTypes::EdgeType likes = dba->edge_type("likes"); + GraphDbTypes::EdgeType hates = dba->edge_type("hates"); auto edge = dba->insert_edge(v1, v2, likes); EXPECT_EQ(edge.edge_type(), likes);