diff --git a/CMakeLists.txt b/CMakeLists.txt index 40dc465c8..0813af66a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -429,8 +429,14 @@ set(memgraph_src_files ${src_dir}/storage/model/properties/array.cpp ${src_dir}/storage/model/properties/properties.cpp ${src_dir}/storage/model/properties/property_family.cpp + ${src_dir}/storage/indexes/index_base.cpp + ${src_dir}/storage/indexes/index_record.cpp + ${src_dir}/storage/indexes/index_update.cpp + ${src_dir}/storage/indexes/index_holder.cpp + ${src_dir}/storage/indexes/impl/unique_ordered_index.cpp ${src_dir}/storage/indexes/impl/nonunique_unordered_index.cpp ${src_dir}/storage/locking/record_lock.cpp + ${src_dir}/storage/garbage/garbage.cpp ${src_dir}/storage/vertex_accessor.cpp ${src_dir}/transactions/transaction.cpp ${src_dir}/template_engine/engine.cpp @@ -442,6 +448,7 @@ set(memgraph_src_files ${src_dir}/logging/log.cpp ${src_dir}/io/network/tls.cpp ${src_dir}/database/db.cpp + ${src_dir}/database/db_transaction.cpp ${src_dir}/database/db_accessor.cpp ${src_dir}/storage/edge_accessor.cpp ${src_dir}/storage/record_accessor.cpp diff --git a/include/communication/bolt/v1/serialization/bolt_serializer.hpp b/include/communication/bolt/v1/serialization/bolt_serializer.hpp index d05609645..4e01b7948 100644 --- a/include/communication/bolt/v1/serialization/bolt_serializer.hpp +++ b/include/communication/bolt/v1/serialization/bolt_serializer.hpp @@ -33,7 +33,7 @@ public: * } * */ - void write(const Vertex::Accessor &vertex) + void write(const VertexAccessor &vertex) { // write signatures for the node struct and node data type encoder.write_struct_header(3); @@ -72,7 +72,7 @@ public: * } * */ - void write(const Edge::Accessor &edge) + void write(const EdgeAccessor &edge) { // write signatures for the edge struct and edge data type encoder.write_struct_header(5); diff --git a/include/communication/bolt/v1/serialization/record_stream.hpp b/include/communication/bolt/v1/serialization/record_stream.hpp index aa4ab438f..39c5534ea 100644 --- a/include/communication/bolt/v1/serialization/record_stream.hpp +++ b/include/communication/bolt/v1/serialization/record_stream.hpp @@ -78,8 +78,8 @@ public: } // -- BOLT SPECIFIC METHODS ----------------------------------------------- - void write(const Vertex::Accessor &vertex) { serializer.write(vertex); } - void write(const Edge::Accessor &edge) { serializer.write(edge); } + void write(const VertexAccessor &vertex) { serializer.write(vertex); } + void write(const EdgeAccessor &edge) { serializer.write(edge); } void write(const Property &prop) { serializer.write(prop); } void write(const Bool& prop) { serializer.write(prop); } diff --git a/include/data_structures/concurrent/concurrent_list.hpp b/include/data_structures/concurrent/concurrent_list.hpp index 76e547eda..04e221220 100644 --- a/include/data_structures/concurrent/concurrent_list.hpp +++ b/include/data_structures/concurrent/concurrent_list.hpp @@ -2,6 +2,7 @@ #include #include +#include #include "utils/crtp.hpp" template diff --git a/include/data_structures/concurrent/concurrent_map.hpp b/include/data_structures/concurrent/concurrent_map.hpp index 4e751a109..ae24568f1 100644 --- a/include/data_structures/concurrent/concurrent_map.hpp +++ b/include/data_structures/concurrent/concurrent_map.hpp @@ -8,53 +8,66 @@ using std::pair; template class ConcurrentMap { - typedef Item item_t; - typedef SkipList list; - typedef typename SkipList::Iterator list_it; - typedef typename SkipList::ConstIterator list_it_con; - - public: - ConcurrentMap() {} - - class Accessor : public AccessorBase - { - friend class ConcurrentMap; - - using AccessorBase::AccessorBase; - -private: - using AccessorBase::accessor; + typedef Item item_t; + typedef SkipList list; + typedef typename SkipList::Iterator list_it; + typedef typename SkipList::ConstIterator list_it_con; public: - std::pair insert(const K &key, const T &data) + ConcurrentMap() {} + + class Accessor : public AccessorBase { - return accessor.insert(item_t(key, data)); - } + friend class ConcurrentMap; - std::pair insert(const K &key, T &&data) - { - return accessor.insert(item_t(key, std::forward(data))); - } + using AccessorBase::AccessorBase; - std::pair insert(K &&key, T &&data) - { - return accessor.insert( - item_t(std::forward(key), std::forward(data))); - } + private: + using AccessorBase::accessor; - list_it_con find(const K &key) const { return accessor.find(key); } + public: + std::pair insert(const K &key, const T &data) + { + return accessor.insert(item_t(key, data)); + } - list_it find(const K &key) { return accessor.find(key); } + std::pair insert(const K &key, T &&data) + { + return accessor.insert(item_t(key, std::forward(data))); + } - bool contains(const K &key) const { return this->find(key) != this->end(); } + std::pair insert(K &&key, T &&data) + { + return accessor.insert( + item_t(std::forward(key), std::forward(data))); + } - bool remove(const K &key) { return accessor.remove(key); } - }; + list_it_con find(const K &key) const { return accessor.find(key); } - Accessor access() { return Accessor(&skiplist); } + list_it find(const K &key) { return accessor.find(key); } - const Accessor access() const { return Accessor(&skiplist); } + list_it_con find_or_larger(const T &item) const + { + return accessor.find_or_larger(item); + } - private: - list skiplist; + list_it find_or_larger(const T &item) + { + return accessor.find_or_larger(item); + } + + bool contains(const K &key) const + { + return this->find(key) != this->end(); + } + + bool remove(const K &key) { return accessor.remove(key); } + }; + + Accessor access() { return Accessor(&skiplist); } + + const Accessor access() const { return Accessor(&skiplist); } + +private: + list skiplist; }; diff --git a/include/data_structures/concurrent/concurrent_multimap.hpp b/include/data_structures/concurrent/concurrent_multimap.hpp index cddce9914..1ae53ce75 100644 --- a/include/data_structures/concurrent/concurrent_multimap.hpp +++ b/include/data_structures/concurrent/concurrent_multimap.hpp @@ -53,6 +53,16 @@ public: list_it find(const K &key) { return accessor.find(key); } + list_it_con find_or_larger(const T &item) const + { + return accessor.find_or_larger(item); + } + + list_it find_or_larger(const T &item) + { + return accessor.find_or_larger(item); + } + bool contains(const K &key) const { return this->find(key) != this->end(); diff --git a/include/data_structures/concurrent/concurrent_multiset.hpp b/include/data_structures/concurrent/concurrent_multiset.hpp index eae01c2d0..858620b1c 100644 --- a/include/data_structures/concurrent/concurrent_multiset.hpp +++ b/include/data_structures/concurrent/concurrent_multiset.hpp @@ -5,46 +5,59 @@ template class ConcurrentMultiSet { - typedef SkipList list; - typedef typename SkipList::Iterator list_it; - typedef typename SkipList::ConstIterator list_it_con; - - public: - ConcurrentMultiSet() {} - - class Accessor : public AccessorBase - { - friend class ConcurrentMultiSet; - - using AccessorBase::AccessorBase; - -private: - using AccessorBase::accessor; + typedef SkipList list; + typedef typename SkipList::Iterator list_it; + typedef typename SkipList::ConstIterator list_it_con; public: - list_it insert(const T &item) { return accessor.insert_non_unique(item); } + ConcurrentMultiSet() {} - list_it insert(T &&item) + class Accessor : public AccessorBase { - return accessor.insert_non_unique(std::forward(item)); - } + friend class ConcurrentMultiSet; - list_it_con find(const T &item) const { return accessor.find(item); } + using AccessorBase::AccessorBase; - list_it find(const T &item) { return accessor.find(item); } + private: + using AccessorBase::accessor; - bool contains(const T &item) const - { - return this->find(item) != this->end(); - } + public: + list_it insert(const T &item) + { + return accessor.insert_non_unique(item); + } - bool remove(const T &item) { return accessor.remove(item); } - }; + list_it insert(T &&item) + { + return accessor.insert_non_unique(std::forward(item)); + } - Accessor access() { return Accessor(&skiplist); } + list_it_con find(const T &item) const { return accessor.find(item); } - const Accessor access() const { return Accessor(&skiplist); } + list_it find(const T &item) { return accessor.find(item); } - private: - list skiplist; + list_it_con find_or_larger(const T &item) const + { + return accessor.find_or_larger(item); + } + + list_it find_or_larger(const T &item) + { + return accessor.find_or_larger(item); + } + + bool contains(const T &item) const + { + return this->find(item) != this->end(); + } + + bool remove(const T &item) { return accessor.remove(item); } + }; + + Accessor access() { return Accessor(&skiplist); } + + const Accessor access() const { return Accessor(&skiplist); } + +private: + list skiplist; }; diff --git a/include/data_structures/concurrent/concurrent_set.hpp b/include/data_structures/concurrent/concurrent_set.hpp index 036463ad2..e0d749bee 100644 --- a/include/data_structures/concurrent/concurrent_set.hpp +++ b/include/data_structures/concurrent/concurrent_set.hpp @@ -37,6 +37,24 @@ public: list_it find(const T &item) { return accessor.find(item); } + template + list_it_con find_or_larger(const K &item) const + { + return accessor.find_or_larger(item); + } + + template + list_it find_or_larger(const K &item) + { + return accessor.find_or_larger(item); + } + + template + list_it_con cfind_or_larger(const K &item) + { + return accessor.template find_or_larger(item); + } + bool contains(const T &item) const { return this->find(item) != this->end(); diff --git a/include/data_structures/concurrent/skiplist.hpp b/include/data_structures/concurrent/skiplist.hpp index bb7834a60..227c6559e 100644 --- a/include/data_structures/concurrent/skiplist.hpp +++ b/include/data_structures/concurrent/skiplist.hpp @@ -556,6 +556,19 @@ public: return skiplist->find(item); } + template + ConstIterator find_or_larger(const K &item) const + { + return static_cast(*skiplist).find_or_larger( + item); + } + + template + It find_or_larger(const K &item) + { + return skiplist->find_or_larger(item); + } + template bool contains(const K &item) const { @@ -637,6 +650,18 @@ private: template It find_node(const K &item) + { + auto it = find_or_larger(item); + if (it.node == nullptr || item == *it) { + return std::move(it); + } else { + return It(); + } + } + + // Returns iterator on searched element or the first larger element. + template + It find_or_larger(const K &item) { Node *node, *pred = header; int h = static_cast(pred->height) - 1; @@ -647,7 +672,7 @@ private: } // if we overshoot at every layer, item doesn't exist - if (h < 0) return It(); + if (h < 0) return It(node); // the item is farther to the right, continue going right as long // as the key is greater than the current node's key diff --git a/include/database/db.hpp b/include/database/db.hpp index 028ed518e..f28a1189d 100644 --- a/include/database/db.hpp +++ b/include/database/db.hpp @@ -1,5 +1,9 @@ #pragma once +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +#include "storage/garbage/garbage.hpp" #include "storage/graph.hpp" #include "transactions/engine.hpp" @@ -14,9 +18,24 @@ public: Graph graph; tx::Engine tx_engine; + Garbage garbage; std::string &name(); + // INDEXES + + // TG - type group + // I - type of function I:const tx::Transaction& -> + // std::unique_ptr> + // G - type of collection (verrtex/edge) + template + bool create_index_on_vertex_property_family(const char *name, G &coll, + I &create_index); + + // Removes index IndexHolder. True if there was index to remove. + template + bool remove_index(IndexHolder &ih); + private: std::string name_; }; diff --git a/include/database/db_accessor.hpp b/include/database/db_accessor.hpp index 6f6bb9fb6..3f9e8fa75 100644 --- a/include/database/db_accessor.hpp +++ b/include/database/db_accessor.hpp @@ -1,9 +1,8 @@ #pragma once #include "database/db_transaction.hpp" -#include "storage/vertex_accessor.hpp" +#include "storage/model/properties/property_family.hpp" #include "utils/border.hpp" -// #include "utils/iterator/iterator.hpp" #include "utils/option.hpp" namespace tx @@ -11,23 +10,29 @@ namespace tx class Transaction; } +class Label; +class EdgeType; + +using EdgePropertyFamily = PropertyFamily; +using VertexPropertyFamily = PropertyFamily; + /* * DbAccessor * -Guarantees that access to Vertex and Edge is possible only through -* Vertex::Accessor and Edge::Accessor. +* VertexAccessor and EdgeAccessor. * -Guarantees that changing Vertex and Edge is possible only using -* Vertex::Accessor returned by vertex_insert() method and -* Edge::Accessor returned by edge_insert() method. +* VertexAccessor returned by vertex_insert() method and +* EdgeAccessor returned by edge_insert() method. * -Offers CRUD for Vertex and Edge except iterating over all edges. * -* Vertex::Accessor +* VertexAccessor * By default Vertex::accessor is empty. Caller has to call fill() method * to fetch valid data and check it's return value. fill() method returns * true if there is valid data for current transaction false otherwise. * Only exception to this rule is vertex_insert() method in DbAccessor -* which returns by default filled Vertex::Accessor. +* which returns by default filled VertexAccessor. * -* Edge::Accessor +* EdgeAccessor * By default Edge::accessor is empty. Caller has to call fill() method * to * fetch valid data and check it's return value. fill() method returns @@ -35,7 +40,7 @@ class Transaction; * if there is valid data for current transaction false otherwise. * Only exception to this rule is edge_insert() method in DbAccessor * which -* returns by default filled Edge::Accessor. +* returns by default filled EdgeAccessor. */ class DbAccessor { @@ -43,22 +48,24 @@ class DbAccessor public: DbAccessor(Db &db); + DbAccessor(Db &db, tx::Transaction &t); + //*******************VERTEX METHODS auto vertex_access(); - Option vertex_find(const Id &id); + Option vertex_find(const Id &id); - // Creates new Vertex and returns filled Vertex::Accessor. - Vertex::Accessor vertex_insert(); + // Creates new Vertex and returns filled VertexAccessor. + VertexAccessor vertex_insert(); // ******************* EDGE METHODS - Option edge_find(const Id &id); + Option edge_find(const Id &id); - // Creates new Edge and returns filled Edge::Accessor. - Edge::Accessor edge_insert(Vertex::Accessor const &from, - Vertex::Accessor const &to); + // Creates new Edge and returns filled EdgeAccessor. + EdgeAccessor edge_insert(VertexAccessor const &from, + VertexAccessor const &to); // ******************* LABEL METHODS @@ -74,25 +81,37 @@ public: // ******************** PROPERTY METHODS - PropertyFamily &vertex_property_family_get(const std::string &name); + VertexPropertyFamily &vertex_property_family_get(const std::string &name); - PropertyFamily &edge_property_family_get(const std::string &name); + EdgePropertyFamily &edge_property_family_get(const std::string &name); // ******************** PROPERTY HELPER METHODS - PropertyFamily::PropertyType::PropertyFamilyKey + VertexPropertyFamily::PropertyType::PropertyFamilyKey vertex_property_key(const std::string &name, Type type); - PropertyFamily::PropertyType::PropertyFamilyKey + EdgePropertyFamily::PropertyType::PropertyFamilyKey edge_property_key(const std::string &name, Type type); + template + VertexPropertyFamily::PropertyType::PropertyTypeKey + vertex_property_key(const std::string &name); + + template + EdgePropertyFamily::PropertyType::PropertyTypeKey + edge_property_key(const std::string &name); + // ******************** TRANSACTION METHODS - void commit(); + // True if commit was successful, or false if transaction was aborted. + bool commit(); void abort(); private: + // TODO: make this friend generic for all indexes. template friend class NonUniqueUnorderedIndex; + template + friend class UniqueOrderedIndex; DbTransaction db_transaction; }; diff --git a/include/database/db_transaction.hpp b/include/database/db_transaction.hpp index a36dd1d18..07972557e 100644 --- a/include/database/db_transaction.hpp +++ b/include/database/db_transaction.hpp @@ -1,10 +1,13 @@ #pragma once +#include "storage/indexes/index_update.hpp" #include "transactions/transaction.hpp" class Db; class DbAccessor; +using index_updates_t = std::vector; + // Inner structures local to transaction can hold ref to this structure and use // its methods. // Also serves as a barrier for calling methods defined public but meant for @@ -21,6 +24,21 @@ public: // This should provide cleaner hierarchy of operations on database. // For example cleaner. + // Updates indexes of Vertex/Edges in index_updates. True if indexes are + // updated successfully. False means that transaction failed. + bool update_indexes(); + + // Will update indexes for given element TG::record_t. Actual update happens + // with call update_indexes + template + void to_update_index(typename TG::vlist_t *vlist, + typename TG::record_t *record) + { + index_updates.push_back(make_index_update(vlist, record)); + } + + index_updates_t index_updates; + tx::Transaction &trans; Db &db; diff --git a/include/mvcc/record.hpp b/include/mvcc/record.hpp index 2731c2b56..bc2e0ece2 100644 --- a/include/mvcc/record.hpp +++ b/include/mvcc/record.hpp @@ -3,14 +3,14 @@ #include #include -#include "transactions/transaction.hpp" #include "transactions/commit_log.hpp" #include "transactions/engine.hpp" +#include "transactions/transaction.hpp" -#include "mvcc/id.hpp" #include "mvcc/cre_exp.hpp" -#include "mvcc/version.hpp" #include "mvcc/hints.hpp" +#include "mvcc/id.hpp" +#include "mvcc/version.hpp" #include "storage/locking/record_lock.hpp" // the mvcc implementation used here is very much like postgresql's @@ -41,7 +41,7 @@ public: RecordLock lock; // check if this record is visible to the transaction t - bool visible(const tx::Transaction& t) + bool visible(const tx::Transaction &t) { // TODO check if the record was created by a transaction that has been // aborted. one might implement this by checking the hints in mvcc @@ -51,87 +51,95 @@ public: // if you think they're not, you're wrong, and you should think about it // again. i know, it happened to me. - return ((tx.cre() == t.id && // inserted by the current transaction - cmd.cre() <= t.cid && // before this command, and - (tx.exp() == Id(0) || // the row has not been deleted, or - (tx.exp() == t.id && // it was deleted by the current - // transaction - cmd.exp() >= t.cid))) // but not before this command, - || // or - (cre_committed(tx.cre(), t) && // the record was inserted by a - // committed transaction, and - (tx.exp() == Id(0) || // the record has not been deleted, or - (tx.exp() == t.id && // the row is being deleted by this - // transaction - cmd.exp() >= t.cid) || // but it's not deleted "yet", or - (tx.exp() != t.id && // the row was deleted by another - // transaction - !exp_committed(tx.exp(), t) // that has not been committed - )))); + return ((tx.cre() == t.id && // inserted by the current transaction + cmd.cre() <= t.cid && // before this command, and + (tx.exp() == Id(0) || // the row has not been deleted, or + (tx.exp() == t.id && // it was deleted by the current + // transaction + cmd.exp() >= t.cid))) // but not before this command, + || // or + (cre_committed(tx.cre(), t) && // the record was inserted by a + // committed transaction, and + (tx.exp() == Id(0) || // the record has not been deleted, or + (tx.exp() == t.id && // the row is being deleted by this + // transaction + cmd.exp() >= t.cid) || // but it's not deleted "yet", or + (tx.exp() != t.id && // the row was deleted by another + // transaction + !exp_committed(tx.exp(), t) // that has not been committed + )))); } - void mark_created(const tx::Transaction& t) + void mark_created(const tx::Transaction &t) { tx.cre(t.id); cmd.cre(t.cid); } - void mark_deleted(const tx::Transaction& t) + void mark_deleted(const tx::Transaction &t) { tx.exp(t.id); cmd.exp(t.cid); } - bool exp_committed(const Id& id, const tx::Transaction& t) + bool exp_committed(const Id &id, const tx::Transaction &t) { return committed(hints.exp, id, t); } - bool exp_committed(const tx::Transaction& t) + bool exp_committed(const tx::Transaction &t) { return committed(hints.exp, tx.exp(), t); } - bool cre_committed(const Id& id, const tx::Transaction& t) + bool cre_committed(const Id &id, const tx::Transaction &t) { return committed(hints.cre, id, t); } - bool cre_committed(const tx::Transaction& t) + bool cre_committed(const tx::Transaction &t) { return committed(hints.cre, tx.cre(), t); } + // TODO: Test this + // True if this record is visible for write. + bool is_visible_write(const tx::Transaction &t) + { + return (tx.cre() == t.id && // inserted by the current transaction + cmd.cre() <= t.cid && // before this command, and + (tx.exp() == Id(0) || // the row has not been deleted, or + (tx.exp() == t.id && // it was deleted by the current + // transaction + cmd.exp() >= t.cid))); // but not before this command, + } + protected: template - bool committed(U& hints, const Id& id, const tx::Transaction& t) + bool committed(U &hints, const Id &id, const tx::Transaction &t) { // you certainly can't see the transaction with id greater than yours // as that means it started after this transaction and if it committed, // it committed after this transaction had started. - if(id > t.id) - return false; + if (id > t.id) return false; + + // The creating transaction is still in progress (examine snapshot) + if (t.is_active(id)) return false; auto hint_bits = hints.load(); + // TODO: Validate if this position is valid for next if. // if hints are set, return if xid is committed - if(!hint_bits.is_unknown()) - return hint_bits.is_committed(); + if (!hint_bits.is_unknown()) return hint_bits.is_committed(); // if hints are not set: - // - the creating transaction is still in progress (examine snapshot) - if(t.snapshot.is_active(id)) - return false; - // - you are the first one to check since it ended, consult commit log auto info = t.engine.clog.fetch_info(id); - if(info.is_committed()) - return hints.set_committed(), true; + if (info.is_committed()) return hints.set_committed(), true; assert(info.is_aborted()); return hints.set_aborted(), false; } }; - } diff --git a/include/mvcc/version_list.hpp b/include/mvcc/version_list.hpp index e6126201f..29161fe27 100644 --- a/include/mvcc/version_list.hpp +++ b/include/mvcc/version_list.hpp @@ -103,13 +103,6 @@ public: T *update(T *record, tx::Transaction &t) { assert(record != nullptr); - // TODO: VALIDATE NEXT IF BLOCK - if (record->tx.cre() == t.id) { - // THEN ONLY THIS TRANSACTION CAN SEE THIS DATA WHICH MENS THAT IT - // CAN CHANGE IT. - return record; - } - lock_and_validate(record, t); auto updated = new T(); @@ -149,7 +142,6 @@ private: void lock_and_validate(T *record, tx::Transaction &t) { assert(record != nullptr); - assert(record == find(t)); // take a lock on this node t.take_lock(lock); @@ -167,9 +159,3 @@ private: RecordLock lock; }; } - -class Vertex; -class Edge; - -using VertexRecord = mvcc::VersionList; -// using EdgeRecord = mvcc::VersionList; diff --git a/include/query_engine/hardcode/queries.hpp b/include/query_engine/hardcode/queries.hpp index 8513b82de..bab402ce0 100644 --- a/include/query_engine/hardcode/queries.hpp +++ b/include/query_engine/hardcode/queries.hpp @@ -6,10 +6,12 @@ #include "query_engine/query_stripper.hpp" #include "query_engine/util.hpp" #include "storage/indexes/impl/nonunique_unordered_index.cpp" +// #include "storage/model/properties/properties.cpp" #include "storage/model/properties/property.hpp" #include "storage/model/properties/property_family.hpp" #include "utils/command_line/arguments.hpp" #include "utils/iterator/iterator.hpp" +// #include "utils/utils.cpp" auto load_queries(Db &db) { diff --git a/include/query_engine/query_result.hpp b/include/query_engine/query_result.hpp index 992df23c0..6ed096daf 100644 --- a/include/query_engine/query_result.hpp +++ b/include/query_engine/query_result.hpp @@ -10,10 +10,11 @@ #include "storage/model/properties/properties.hpp" +template struct ResultList { using sptr = std::shared_ptr; - using data_t = std::vector; + using data_t = std::vector *>; ResultList() = default; ResultList(ResultList &other) = delete; @@ -22,13 +23,15 @@ struct ResultList explicit operator bool() const { return data.size() > 0; } - std::vector data; + std::vector *> data; }; +template struct QueryResult { using sptr = std::shared_ptr; - using data_t = std::unordered_map; + using data_t = + std::unordered_map::sptr>; QueryResult() = default; QueryResult(QueryResult &other) = delete; diff --git a/include/query_engine/util.hpp b/include/query_engine/util.hpp index 40b5b5a57..0d853b258 100644 --- a/include/query_engine/util.hpp +++ b/include/query_engine/util.hpp @@ -11,7 +11,8 @@ using std::cout; using std::endl; -void print_props(const Properties &properties); +template +void print_props(const Properties &properties); #ifdef NDEBUG #define PRINT_PROPS(_) @@ -19,7 +20,8 @@ void print_props(const Properties &properties); #define PRINT_PROPS(_PROPS_) print_props(_PROPS_); #endif -void cout_properties(const Properties &properties); +template +void cout_properties(const Properties &properties); void cout_property(const std::string &key, const Property &property); diff --git a/include/storage/edge.hpp b/include/storage/edge.hpp index 55a3ea182..4e87c4d7d 100644 --- a/include/storage/edge.hpp +++ b/include/storage/edge.hpp @@ -2,7 +2,7 @@ #include "mvcc/record.hpp" #include "storage/model/edge_model.hpp" -#include "storage/model/properties/traversers/jsonwriter.hpp" +// #include "storage/model/properties/traversers/jsonwriter.hpp" class Edge : public mvcc::Record { @@ -10,14 +10,14 @@ public: class Accessor; Edge() = default; - Edge(const EdgeModel& data) : data(data) {} - Edge(EdgeModel&& data) : data(std::move(data)) {} + Edge(const EdgeModel &data) : data(data) {} + Edge(EdgeModel &&data) : data(std::move(data)) {} - Edge(const Edge&) = delete; - Edge(Edge&&) = delete; + Edge(const Edge &) = delete; + Edge(Edge &&) = delete; - Edge& operator=(const Edge&) = delete; - Edge& operator=(Edge&&) = delete; + Edge &operator=(const Edge &) = delete; + Edge &operator=(Edge &&) = delete; EdgeModel data; }; diff --git a/include/storage/edge_accessor.hpp b/include/storage/edge_accessor.hpp index f62328eae..72a4323d0 100644 --- a/include/storage/edge_accessor.hpp +++ b/include/storage/edge_accessor.hpp @@ -7,18 +7,23 @@ #include "utils/assert.hpp" #include "utils/reference_wrapper.hpp" +class EdgeType; +using edge_type_ref_t = ReferenceWrapper; + class Edges; -class Edge::Accessor : public RecordAccessor +class EdgeAccessor : public RecordAccessor { public: using RecordAccessor::RecordAccessor; + typedef Edge record_t; + typedef EdgeRecord record_list_t; void edge_type(edge_type_ref_t edge_type); edge_type_ref_t edge_type() const; - Vertex::Accessor from() const; + VertexAccessor from() const; - Vertex::Accessor to() const; + VertexAccessor to() const; }; diff --git a/include/storage/edge_record.hpp b/include/storage/edge_record.hpp index aa1471abe..50ce0a42d 100644 --- a/include/storage/edge_record.hpp +++ b/include/storage/edge_record.hpp @@ -3,6 +3,8 @@ #include "mvcc/version_list.hpp" #include "storage/edge.hpp" +class VertexRecord; + class EdgeRecord : public mvcc::VersionList { public: diff --git a/include/storage/edge_type/edge_type.hpp b/include/storage/edge_type/edge_type.hpp index 2dd64262c..94367aea1 100644 --- a/include/storage/edge_type/edge_type.hpp +++ b/include/storage/edge_type/edge_type.hpp @@ -3,18 +3,30 @@ #include #include +#include "storage/edge.hpp" +#include "storage/edge_accessor.hpp" +#include "storage/indexes/impl/nonunique_unordered_index.hpp" +#include "storage/type_group_edge.hpp" #include "utils/char_str.hpp" #include "utils/reference_wrapper.hpp" #include "utils/total_ordering.hpp" +using EdgeTypeIndexRecord = IndexRecord; + class EdgeType : public TotalOrdering { public: - EdgeType(); + using type_index_t = NonUniqueUnorderedIndex; + + EdgeType() = delete; + EdgeType(const std::string &id); EdgeType(const char *id); EdgeType(std::string &&id); + EdgeType(const EdgeType &) = delete; + EdgeType(EdgeType &&other) = default; + friend bool operator<(const EdgeType &lhs, const EdgeType &rhs); friend bool operator==(const EdgeType &lhs, const EdgeType &rhs); @@ -25,6 +37,8 @@ public: CharStr char_str() { return CharStr(&id[0]); } + std::unique_ptr index; + private: std::string id; }; diff --git a/include/storage/edges.hpp b/include/storage/edges.hpp index 080d39836..664f6d545 100644 --- a/include/storage/edges.hpp +++ b/include/storage/edges.hpp @@ -1,25 +1,30 @@ #pragma once #include + #include "data_structures/concurrent/concurrent_map.hpp" -#include "mvcc/version_list.hpp" -#include "storage/common.hpp" -#include "storage/edge_accessor.hpp" -#include "storage/model/properties/property_family.hpp" +#include "utils/counters/atomic_counter.hpp" #include "utils/option.hpp" +#include "storage/edge_record.hpp" +#include "storage/model/properties/property_family.hpp" + +class EdgeAccessor; +class DbTransaction; + +using EdgePropertyFamily = PropertyFamily; + class Edges { - using prop_familys_t = ConcurrentMap; + using prop_familys_t = ConcurrentMap; public: - Option find(DbTransaction &t, const Id &id); + Option find(DbTransaction &t, const Id &id); - // Creates new Edge and returns filled Edge::Accessor. - Edge::Accessor insert(DbTransaction &t, VertexRecord *from, - VertexRecord *to); + // Creates new Edge and returns filled EdgeAccessor. + EdgeAccessor insert(DbTransaction &t, VertexRecord *from, VertexRecord *to); - PropertyFamily &property_family_find_or_create(const std::string &name); + EdgePropertyFamily &property_family_find_or_create(const std::string &name); private: ConcurrentMap edges; diff --git a/include/storage/garbage/delete_sensitive.hpp b/include/storage/garbage/delete_sensitive.hpp new file mode 100644 index 000000000..973bde9d2 --- /dev/null +++ b/include/storage/garbage/delete_sensitive.hpp @@ -0,0 +1,9 @@ +#pragma once + +// Base class for all classes which need to be safely disposed. Main usage is +// for garbage class operations. +class DeleteSensitive +{ +public: + virtual ~DeleteSensitive() {} +}; diff --git a/include/storage/garbage/garbage.hpp b/include/storage/garbage/garbage.hpp new file mode 100644 index 000000000..4712fe780 --- /dev/null +++ b/include/storage/garbage/garbage.hpp @@ -0,0 +1,28 @@ +#pragma once + +#include "data_structures/concurrent/concurrent_list.hpp" +#include "mvcc/id.hpp" +#include "storage/garbage/delete_sensitive.hpp" +#include "transactions/snapshot.hpp" + +namespace tx +{ +class Engine; +} + +// Collection of delete sensitive data which need to be safely deleted. That +// meens that all transactions that may have pointer to it must finish before +// the sensitive data can be safely destroyed. +class Garbage +{ +public: + void dispose(tx::Snapshot &&snapshot, DeleteSensitive *data); + + // Cleaner thread shoul call this method every some time. Removes data which + // is + // safe to be deleted. + void clean(tx::Engine &engine); + +private: + List, DeleteSensitive *>> gar; +}; diff --git a/include/storage/indexes/impl/nonunique_unordered_index.hpp b/include/storage/indexes/impl/nonunique_unordered_index.hpp index 01bebb1c5..77c76fc75 100644 --- a/include/storage/indexes/impl/nonunique_unordered_index.hpp +++ b/include/storage/indexes/impl/nonunique_unordered_index.hpp @@ -1,26 +1,29 @@ #pragma once #include "storage/indexes/index_base.hpp" -#include "storage/indexes/index_record.hpp" +// #include "storage/indexes/index_record.hpp" #include "data_structures/concurrent/concurrent_list.hpp" -template -class NonUniqueUnorderedIndex : public IndexBase +template +class NonUniqueUnorderedIndex : public IndexBase { public: - typedef T value_type; - typedef K key_type; + // typedef T value_type; + // typedef K key_type; + // Created with the database NonUniqueUnorderedIndex(); + NonUniqueUnorderedIndex(tx::Transaction const &t); + // Insert's value. // nonunique => always succeds. - bool insert(IndexRecord &&value) final; + bool insert(IndexRecord &&value) final; // Returns iterator which returns valid records in range. // ordered==None => doesn't guarantee any order of submitting records. - std::unique_ptr> + std::unique_ptr> for_range(DbAccessor &t, Border from = Border(), Border to = Border()) final; @@ -34,5 +37,5 @@ public: void clean(DbTransaction &) final; private: - List> list; + List> list; }; diff --git a/include/storage/indexes/impl/unique_ordered_index.hpp b/include/storage/indexes/impl/unique_ordered_index.hpp new file mode 100644 index 000000000..69daeecf7 --- /dev/null +++ b/include/storage/indexes/impl/unique_ordered_index.hpp @@ -0,0 +1,40 @@ +#pragma once + +#include "storage/indexes/index_base.hpp" + +#include "data_structures/concurrent/concurrent_set.hpp" + +template +class UniqueOrderedIndex : public IndexBase +{ +public: + // typedef T value_type; + // typedef K key_type; + + // Created with the database + UniqueOrderedIndex(Order order); + + UniqueOrderedIndex(Order order, tx::Transaction const &t); + + // Insert's value. + // nonunique => always succeds. + bool insert(IndexRecord &&value) final; + + // Returns iterator which returns valid records in range. + // ordered==None => doesn't guarantee any order of submitting records. + std::unique_ptr> + for_range(DbAccessor &t, Border from = Border(), + Border to = Border()) final; + + // Same as for_range just whith known returned iterator. + auto for_range_exact(DbAccessor &t, Border from = Border(), + Border to = Border()); + + // Removes for all transactions obsolete Records. + // Cleaner has to call this method when he decideds that it is time for + // cleaning. + void clean(DbTransaction &) final; + +private: + ConcurrentSet> set; +}; diff --git a/include/storage/indexes/index_base.hpp b/include/storage/indexes/index_base.hpp index dd5e01529..06c00046f 100644 --- a/include/storage/indexes/index_base.hpp +++ b/include/storage/indexes/index_base.hpp @@ -1,41 +1,45 @@ #pragma once -// #include "storage/indexes/index_record.hpp" +#include #include #include +#include "mvcc/id.hpp" + +// #include "storage/indexes/index_record.hpp" +#include "storage/garbage/delete_sensitive.hpp" #include "utils/border.hpp" #include "utils/iterator/iterator_base.hpp" +#include "utils/order.hpp" + +template +class IndexRecord; class DbTransaction; class DbAccessor; - -template -class IndexRecord; - -// Defines ordering of data -enum Order +namespace tx { - None = 0, - Ascending = 1, - Descending = 2, -}; +class Transaction; +} // Interface for all indexes. -// T type of record. +// TG type group // K type of key on which records are ordered -template -class IndexBase +template +class IndexBase : public DeleteSensitive { public: - typedef T value_type; - typedef K key_type; + // typedef T value_type; + // typedef K key_type; - IndexBase(bool unique, Order order) : unique(unique), order(order) {} + // Created with the database + IndexBase(bool unique, Order order); + + IndexBase(bool unique, Order order, const tx::Transaction &t); // Insert's value. // unique => returns false if there is already valid equal value. // nonunique => always succeds. - virtual bool insert(IndexRecord &&value) = 0; + virtual bool insert(IndexRecord &&value) = 0; // Returns iterator which returns valid filled records in range. // order==noe => doesn't guarantee any order of returned records. @@ -44,7 +48,7 @@ public: // order==Descending => guarantees order of returned records will be from // largest to smallest. // Range must be from<=to - virtual std::unique_ptr> + virtual std::unique_ptr> for_range(DbAccessor &, Border from = Border(), Border to = Border()) = 0; @@ -53,8 +57,23 @@ public: // cleaning. virtual void clean(DbTransaction &) = 0; + // Activates index for readers. + void activate(); + + // True if index is ready for reading. + bool can_read(); + + // True if transaction is obliged to insert T into index. + bool is_obliged_to_insert(const tx::Transaction &t); + // Are the records unique const bool unique; // Ordering of the records. const Order order; + +private: + // Id of transaction which created this index. + const Id created; + // Active state + std::atomic_bool active = {false}; }; diff --git a/include/storage/indexes/index_holder.hpp b/include/storage/indexes/index_holder.hpp new file mode 100644 index 000000000..e8dac0386 --- /dev/null +++ b/include/storage/indexes/index_holder.hpp @@ -0,0 +1,49 @@ +#pragma once + +#include + +#include "storage/indexes/index_base.hpp" +#include "utils/option.hpp" +#include "utils/option_ptr.hpp" + +namespace tx +{ +class Transaction; +} + +// Holds onde index which can be changed. +// TG - type group +// K - key of index_records +template +class IndexHolder +{ + +public: + IndexHolder() = default; + + IndexHolder(IndexHolder const &) = delete; + + IndexHolder(IndexHolder &&) = default; + + // Sets index for this property family. Treturns false if index is already + // present. + bool set_index(std::unique_ptr> inx); + + // Returns index for read only if it is present and it's valid for read. + OptionPtr> get_read() const; + + // Returns index for write only if it's present and transaction is + // responsibly for updating it. + OptionPtr> get_write(const tx::Transaction &t) const; + + // Removes index if it is given index. Caller is now responsable of + // disposing index in a safe way. + Option>> + remove_index(IndexBase *index); + + // Caller is now responsable of disposing index in a safe way. + Option>> remove_index(); + +private: + std::atomic *> index = {nullptr}; +}; diff --git a/include/storage/indexes/index_record.hpp b/include/storage/indexes/index_record.hpp index 70f05af71..8381f8292 100644 --- a/include/storage/indexes/index_record.hpp +++ b/include/storage/indexes/index_record.hpp @@ -1,37 +1,32 @@ #pragma once -#include "database/db_transaction.hpp" -#include "mvcc/version_list.hpp" +#include "utils/border.hpp" #include "utils/total_ordering.hpp" -// class DbTransaction; -// namespace tx -// { -// class Transaction; -// } +namespace tx +{ +class Transaction; +} +class DbTransaction; -// T type of record. +// TG type group // K key on which record is ordered. -template -class IndexRecord : public TotalOrdering> +template +class IndexRecord : public TotalOrdering>, + public TotalOrdering, IndexRecord> { public: - using vlist_t = mvcc::VersionList; - IndexRecord() = default; - IndexRecord(K key, T *record, vlist_t *vlist) - : key(std::move(key)), record(record), vlist(vlist) - { - assert(record != nullptr); - assert(vlist != nullptr); - } + IndexRecord(K key, typename TG::record_t *record, + typename TG::vlist_t *vlist); friend bool operator<(const IndexRecord &lhs, const IndexRecord &rhs) { - return lhs.key < rhs.key || - (lhs.key == rhs.key && lhs.vlist == rhs.vlist && - lhs.record < rhs.record); + return (lhs.key < rhs.key || + (lhs.key == rhs.key && lhs.vlist == rhs.vlist && + lhs.record < rhs.record)) ^ + lhs.descending; } friend bool operator==(const IndexRecord &lhs, const IndexRecord &rhs) @@ -40,28 +35,29 @@ public: (lhs.vlist != rhs.vlist || lhs.record == rhs.record); } - bool empty() const { return record == nullptr; } - - bool is_valid(tx::Transaction &t) const + friend bool operator<(const Border &lhs, const IndexRecord &rhs) { - assert(!empty()); - return record == vlist->find(t); + return lhs < rhs.key; } - const auto access(DbTransaction &db) const + friend bool operator==(const Border &lhs, const IndexRecord &rhs) { - return T::Accessor::create(record, vlist, db); + return lhs == rhs.key; } + // Will change ordering of record to descending. + void set_descending(); + + bool empty() const; + + bool is_valid(tx::Transaction &t) const; + + const auto access(DbTransaction &db) const; + const K key; private: - T *const record{nullptr}; - vlist_t *const vlist{nullptr}; + bool descending = false; + typename TG::record_t *const record{nullptr}; + typename TG::vlist_t *const vlist{nullptr}; }; - -template -using VertexIndexRecord = IndexRecord; - -template -using EdgeIndexRecord = IndexRecord; diff --git a/include/storage/indexes/index_update.hpp b/include/storage/indexes/index_update.hpp new file mode 100644 index 000000000..e19650960 --- /dev/null +++ b/include/storage/indexes/index_update.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include "storage/indexes/index_record.hpp" +#include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" + +struct IndexUpdateEdge +{ + EdgeRecord *vlist; + Edge *record; +}; + +struct IndexUpdateVertex +{ + VertexRecord *vlist; + Vertex *record; +}; + +struct IndexUpdate +{ + enum + { + EDGE, + VERTEX + } tag; + + union + { + IndexUpdateEdge e; + IndexUpdateVertex v; + }; +}; + +template +IndexUpdate make_index_update(V *vlist, T *record); + +template <> +IndexUpdate make_index_update(EdgeRecord *vlist, Edge *record); + +template <> +IndexUpdate make_index_update(VertexRecord *vlist, Vertex *record); diff --git a/include/storage/label/label.hpp b/include/storage/label/label.hpp index 441ff5ab2..12dc59f1b 100644 --- a/include/storage/label/label.hpp +++ b/include/storage/label/label.hpp @@ -9,13 +9,16 @@ #include "utils/char_str.hpp" #include "utils/reference_wrapper.hpp" #include "utils/total_ordering.hpp" +// #include "storage/type_group_edge.hpp" +#include "storage/type_group_vertex.hpp" -using LabelIndexRecord = VertexIndexRecord; +using LabelIndexRecord = IndexRecord; class Label : public TotalOrdering