From b94cae12d1e3fbaf9254ea52a5ee218ecb3e9126 Mon Sep 17 00:00:00 2001 From: Marko Budiselic Date: Tue, 5 Jul 2016 04:01:22 +0100 Subject: [PATCH] some new header + source files (vertices, vertex_accessor, properties), first version of Index works, skiplist + concurrent_set + concurrent_map (still has to be finished) --- CMakeLists.txt | 24 +- include/storage/locking/record_lock.hpp | 20 + include/storage/model/properties/bool.hpp | 27 + include/storage/model/properties/handler.hpp | 41 + include/storage/model/properties/null.hpp | 31 + .../storage/model/properties/properties.hpp | 33 + .../storage/model/properties/property.hpp | 29 +- include/storage/model/properties/string.hpp | 29 + include/storage/vertex_accessor.hpp | 25 + include/storage/vertices.hpp | 26 + include/template_engine/engine.hpp | 18 + {src => include}/transactions/transaction.hpp | 27 +- .../concurrent/concurrent_map.hpp | 9 + .../concurrent/concurrent_set.hpp | 8 + .../{skiplist => concurrent}/skiplist.hpp | 182 ++--- src/data_structures/skiplist/skiplist_gc.hpp | 3 + src/data_structures/skiplist/skiplist_map.hpp | 709 ++++++++++++++++++ src/data_structures/skiplist/skiplistset.hpp | 6 +- src/query_engine/main_queries.cpp | 185 ++--- src/storage/common.hpp | 4 +- src/storage/edges.hpp | 3 +- src/storage/indexes/index.hpp | 55 +- src/storage/indexes/index_record.hpp | 46 ++ .../indexes/index_record_collection.hpp | 38 + src/storage/indexes/{ => keys}/index_key.hpp | 6 - src/storage/label_store.hpp | 4 +- src/storage/locking/record_lock.cpp | 27 + src/storage/locking/record_lock.hpp | 42 -- src/storage/model/label.hpp | 4 + src/storage/model/label_collection.hpp | 34 +- src/storage/model/properties/all.hpp | 27 - src/storage/model/properties/bool.cpp | 46 ++ src/storage/model/properties/bool.hpp | 57 -- src/storage/model/properties/null.cpp | 30 + src/storage/model/properties/null.hpp | 51 -- src/storage/model/properties/number.hpp | 14 +- src/storage/model/properties/properties.cpp | 54 ++ src/storage/model/properties/properties.hpp | 69 -- src/storage/model/properties/property.cpp | 13 + src/storage/model/properties/string.cpp | 36 + src/storage/model/properties/string.hpp | 48 -- .../properties/traversers/consolewriter.hpp | 45 +- .../properties/traversers/jsonwriter.hpp | 71 +- src/storage/model/property_model.hpp | 2 +- src/storage/vertex_accessor.cpp | 38 + src/storage/vertex_accessor.hpp | 37 - src/storage/vertices.cpp | 50 ++ src/storage/vertices.hpp | 54 -- src/template_engine/engine.cpp | 21 + src/template_engine/engine.hpp | 31 - src/transactions/commit_log.hpp | 61 +- src/transactions/engine.hpp | 14 +- src/transactions/transaction.cpp | 20 + src/transactions/transaction_store.hpp | 2 +- src/utils/reference_wrapper.hpp | 40 + tests/CMakeLists.txt | 14 +- .../unit/{skiplist.cpp => concurrent_map.cpp} | 4 +- tests/unit/concurrent_set.cpp | 58 ++ tests/unit/{db_index.cpp => db_index.cpp.old} | 0 tests/unit/template_engine.cpp | 13 + 60 files changed, 1864 insertions(+), 851 deletions(-) create mode 100644 include/storage/locking/record_lock.hpp create mode 100644 include/storage/model/properties/bool.hpp create mode 100644 include/storage/model/properties/handler.hpp create mode 100644 include/storage/model/properties/null.hpp create mode 100644 include/storage/model/properties/properties.hpp rename {src => include}/storage/model/properties/property.hpp (76%) create mode 100644 include/storage/model/properties/string.hpp create mode 100644 include/storage/vertex_accessor.hpp create mode 100644 include/storage/vertices.hpp create mode 100644 include/template_engine/engine.hpp rename {src => include}/transactions/transaction.hpp (52%) create mode 100644 src/data_structures/concurrent/concurrent_map.hpp create mode 100644 src/data_structures/concurrent/concurrent_set.hpp rename src/data_structures/{skiplist => concurrent}/skiplist.hpp (78%) create mode 100644 src/data_structures/skiplist/skiplist_map.hpp create mode 100644 src/storage/indexes/index_record.hpp create mode 100644 src/storage/indexes/index_record_collection.hpp rename src/storage/indexes/{ => keys}/index_key.hpp (79%) create mode 100644 src/storage/locking/record_lock.cpp delete mode 100644 src/storage/locking/record_lock.hpp delete mode 100644 src/storage/model/properties/all.hpp create mode 100644 src/storage/model/properties/bool.cpp delete mode 100644 src/storage/model/properties/bool.hpp create mode 100644 src/storage/model/properties/null.cpp delete mode 100644 src/storage/model/properties/null.hpp create mode 100644 src/storage/model/properties/properties.cpp delete mode 100644 src/storage/model/properties/properties.hpp create mode 100644 src/storage/model/properties/property.cpp create mode 100644 src/storage/model/properties/string.cpp delete mode 100644 src/storage/model/properties/string.hpp create mode 100644 src/storage/vertex_accessor.cpp delete mode 100644 src/storage/vertex_accessor.hpp create mode 100644 src/storage/vertices.cpp delete mode 100644 src/storage/vertices.hpp create mode 100644 src/template_engine/engine.cpp delete mode 100644 src/template_engine/engine.hpp create mode 100644 src/transactions/transaction.cpp create mode 100644 src/utils/reference_wrapper.hpp rename tests/unit/{skiplist.cpp => concurrent_map.cpp} (94%) create mode 100644 tests/unit/concurrent_set.cpp rename tests/unit/{db_index.cpp => db_index.cpp.old} (100%) create mode 100644 tests/unit/template_engine.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 506dfe91b..1f32bb0e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,6 +142,7 @@ if(NDEBUG) endif() # includes +include_directories(${CMAKE_SOURCE_DIR}/include) include_directories(${src_dir}) include_directories(${build_include_dir}) include_directories(${fmt_source_dir}) @@ -178,12 +179,25 @@ EXECUTE_PROCESS( # target_link_libraries(query_engine cypher_lib) # target_link_libraries(query_engine ${fmt_static_lib}) -# query hasher executable -add_executable(query_hasher src/query_engine/main_query_hasher.cpp) -target_link_libraries(query_hasher ${fmt_static_lib}) +# # query hasher executable +# add_executable(query_hasher src/query_engine/main_query_hasher.cpp) +# target_link_libraries(query_hasher ${fmt_static_lib}) -# # hard coded implementation of queries -add_executable(queries src/query_engine/main_queries.cpp) +# hard coded implementation of queries +add_executable( + queries + src/query_engine/main_queries.cpp + src/storage/vertices.cpp + src/storage/model/properties/property.cpp + src/storage/model/properties/null.cpp + src/storage/model/properties/bool.cpp + src/storage/model/properties/string.cpp + src/storage/model/properties/properties.cpp + src/storage/locking/record_lock.cpp + src/storage/vertices.cpp + src/storage/vertex_accessor.cpp + src/transactions/transaction.cpp +) target_link_libraries(queries ${fmt_static_lib}) # tests diff --git a/include/storage/locking/record_lock.hpp b/include/storage/locking/record_lock.hpp new file mode 100644 index 000000000..840697244 --- /dev/null +++ b/include/storage/locking/record_lock.hpp @@ -0,0 +1,20 @@ +#pragma once + +#include "threading/sync/futex.hpp" +#include "storage/locking/lock_status.hpp" +#include "mvcc/id.hpp" + +class RecordLock +{ + static constexpr struct timespec timeout {20, 0}; + static constexpr Id INVALID = Id(); + +public: + LockStatus lock(const Id& id); + void lock(); + void unlock(); + +private: + Futex mutex; + Id owner; +}; diff --git a/include/storage/model/properties/bool.hpp b/include/storage/model/properties/bool.hpp new file mode 100644 index 000000000..c551ee3e6 --- /dev/null +++ b/include/storage/model/properties/bool.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include "storage/model/properties/property.hpp" + +class Bool : public Property +{ +public: + static constexpr Flags type = Flags::Bool; + + Bool(bool value); + Bool(const Bool& other) = default; + + bool value() const; + + explicit operator bool() const; + + bool operator==(const Property& other) const override; + + bool operator==(const Bool& other) const; + + bool operator==(bool v) const; + + std::ostream& print(std::ostream& stream) const override; + + friend std::ostream& operator<<(std::ostream& stream, const Bool& prop); +}; + diff --git a/include/storage/model/properties/handler.hpp b/include/storage/model/properties/handler.hpp new file mode 100644 index 000000000..9940f1216 --- /dev/null +++ b/include/storage/model/properties/handler.hpp @@ -0,0 +1,41 @@ +#pragma once + +#include "storage/model/properties/property.hpp" + +#include "storage/model/properties/bool.hpp" +#include "storage/model/properties/double.hpp" +#include "storage/model/properties/float.hpp" +#include "storage/model/properties/int32.hpp" +#include "storage/model/properties/int64.hpp" +#include "storage/model/properties/string.hpp" + +template +void accept(Property &property, Handler &h) +{ + switch (property.flags) { + + case Property::Flags::True: + return h.handle(static_cast(property)); + + case Property::Flags::False: + return h.handle(static_cast(property)); + + case Property::Flags::String: + return h.handle(static_cast(property)); + + case Property::Flags::Int32: + return h.handle(static_cast(property)); + + case Property::Flags::Int64: + return h.handle(static_cast(property)); + + case Property::Flags::Float: + return h.handle(static_cast(property)); + + case Property::Flags::Double: + return h.handle(static_cast(property)); + + default: + return; + } +} diff --git a/include/storage/model/properties/null.hpp b/include/storage/model/properties/null.hpp new file mode 100644 index 000000000..9a450f5ae --- /dev/null +++ b/include/storage/model/properties/null.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include "storage/model/properties/property.hpp" + +class Null : public Property +{ +public: + friend class Property; + + static constexpr Flags type = Flags::Null; + + Null(const Null&) = delete; + Null(Null&&) = delete; + + Null operator=(const Null&) = delete; + + bool operator==(const Property& other) const override; + + bool operator==(const Null&) const; + + explicit operator bool(); + + friend std::ostream& operator<<(std::ostream& stream, const Null&); + + std::ostream& print(std::ostream& stream) const override; + +private: + // the constructor for null is private, it can be constructed only as a + // value inside the Property class, Property::Null + Null(); +}; diff --git a/include/storage/model/properties/properties.hpp b/include/storage/model/properties/properties.hpp new file mode 100644 index 000000000..029265bd3 --- /dev/null +++ b/include/storage/model/properties/properties.hpp @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include "storage/model/properties/property.hpp" + +class Properties +{ +public: + using sptr = std::shared_ptr; + + const Property& at(const std::string& key) const; + + template + void set(const std::string& key, Args&&... args); + + void set(const std::string& key, Property::sptr value); + + void clear(const std::string& key); + + template + void accept(Handler& handler) const + { + for(auto& kv : props) + handler.handle(kv.first, *kv.second); + + handler.finish(); + } + +private: + using props_t = std::map; + props_t props; +}; diff --git a/src/storage/model/properties/property.hpp b/include/storage/model/properties/property.hpp similarity index 76% rename from src/storage/model/properties/property.hpp rename to include/storage/model/properties/property.hpp index bb2a76ae8..28b8423ee 100644 --- a/src/storage/model/properties/property.hpp +++ b/include/storage/model/properties/property.hpp @@ -61,14 +61,11 @@ public: static const Null Null; - Property(Flags flags) : flags(flags) {} + Property(Flags flags); virtual bool operator==(const Property& other) const = 0; - bool operator!=(const Property& other) const - { - return !operator==(other); - } + bool operator!=(const Property& other) const; template bool is() const @@ -92,13 +89,23 @@ public: virtual std::ostream& print(std::ostream& stream) const = 0; - friend std::ostream& operator<<(std::ostream& stream, const Property& prop) - { - return prop.print(stream); - } + friend std::ostream& operator<<(std::ostream& stream, const Property& prop); - template - void accept(Handler& handler); +// template +// void accept(Handler& h) +// { +// switch(flags) +// { +// case Flags::True: return h.handle(static_cast(*this)); +// case Flags::False: return h.handle(static_cast(*this)); +// case Flags::String: return h.handle(static_cast(*this)); +// case Flags::Int32: return h.handle(static_cast(*this)); +// case Flags::Int64: return h.handle(static_cast(*this)); +// case Flags::Float: return h.handle(static_cast(*this)); +// case Flags::Double: return h.handle(static_cast(*this)); +// default: return; +// } +// } const Flags flags; }; diff --git a/include/storage/model/properties/string.hpp b/include/storage/model/properties/string.hpp new file mode 100644 index 000000000..6230c9d98 --- /dev/null +++ b/include/storage/model/properties/string.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include "storage/model/properties/property.hpp" + +class String : public Property +{ +public: + static constexpr Flags type = Flags::String; + + String(const String&) = default; + String(String&&) = default; + + String(const std::string& value); + String(std::string&& value); + + operator const std::string&() const; + + bool operator==(const Property& other) const override; + + bool operator==(const String& other) const; + + bool operator==(const std::string& other) const; + + friend std::ostream& operator<<(std::ostream& stream, const String& prop); + + std::ostream& print(std::ostream& stream) const override; + + std::string value; +}; diff --git a/include/storage/vertex_accessor.hpp b/include/storage/vertex_accessor.hpp new file mode 100644 index 000000000..6c02f2595 --- /dev/null +++ b/include/storage/vertex_accessor.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include "storage/record_accessor.hpp" +#include "storage/vertex.hpp" + +class Vertices; + +class Vertex::Accessor + : public RecordAccessor +{ +public: + using RecordAccessor::RecordAccessor; + + size_t out_degree() const; + + size_t in_degree() const; + + size_t degree() const; + + void add_label(const Label &label); + + bool has_label(const Label &label) const; + + const std::set& labels() const; +}; diff --git a/include/storage/vertices.hpp b/include/storage/vertices.hpp new file mode 100644 index 000000000..03af92d1f --- /dev/null +++ b/include/storage/vertices.hpp @@ -0,0 +1,26 @@ +#pragma once + +#include "data_structures/concurrent/concurrent_map.hpp" +#include "storage/common.hpp" +#include "storage/indexes/index.hpp" +#include "storage/indexes/index_record_collection.hpp" +#include "storage/vertex_accessor.hpp" + +class Vertices +{ +public: + const Vertex::Accessor find(tx::Transaction &t, const Id &id); + + Vertex::Accessor insert(tx::Transaction &t); + + void update_label_index(const Label &label, + VertexIndexRecord &&index_record); + + VertexIndexRecordCollection& find_label_index(const Label& label); + +private: + Index label_index; + + ConcurrentMap vertices; + AtomicCounter counter; +}; diff --git a/include/template_engine/engine.hpp b/include/template_engine/engine.hpp new file mode 100644 index 000000000..1df56f802 --- /dev/null +++ b/include/template_engine/engine.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +namespace template_engine +{ + +using std::string; +using data = std::unordered_map; + +class TemplateEngine +{ +public: + string render(const string& form, const data& partials); +}; + +} diff --git a/src/transactions/transaction.hpp b/include/transactions/transaction.hpp similarity index 52% rename from src/transactions/transaction.hpp rename to include/transactions/transaction.hpp index dde533062..f287282df 100644 --- a/src/transactions/transaction.hpp +++ b/include/transactions/transaction.hpp @@ -1,13 +1,13 @@ #pragma once -#include #include +#include #include #include "mvcc/id.hpp" -#include "snapshot.hpp" -#include "lock_store.hpp" #include "storage/locking/record_lock.hpp" +#include "transactions/lock_store.hpp" +#include "transactions/snapshot.hpp" namespace tx { @@ -19,35 +19,24 @@ class Transaction friend class Engine; public: - Transaction(const Id& id, const Snapshot& snapshot, Engine& engine) - : id(id), cid(1), snapshot(snapshot), engine(engine) {} - - Transaction(const Transaction&) = delete; - Transaction(Transaction&&) = delete; + Transaction(const Id &id, const Snapshot &snapshot, Engine &engine); + Transaction(const Transaction &) = delete; + Transaction(Transaction &&) = delete; // index of this transaction const Id id; - // index of the current command in the current transaction; uint8_t cid; - // a snapshot of currently active transactions const Snapshot snapshot; - void take_lock(RecordLock& lock) - { - locks.take(&lock, id); - } - - // convenience methods which call the corresponging methods in the - // transaction engine + void take_lock(RecordLock &lock); void commit(); void abort(); - Engine& engine; + Engine &engine; private: LockStore locks; }; - } diff --git a/src/data_structures/concurrent/concurrent_map.hpp b/src/data_structures/concurrent/concurrent_map.hpp new file mode 100644 index 000000000..a9e04a495 --- /dev/null +++ b/src/data_structures/concurrent/concurrent_map.hpp @@ -0,0 +1,9 @@ +#pragma once + +#include "data_structures/skiplist/skiplist_map.hpp" + +template +class ConcurrentMap : public SkipListMap +{ + // TODO implement with SkipList +}; diff --git a/src/data_structures/concurrent/concurrent_set.hpp b/src/data_structures/concurrent/concurrent_set.hpp new file mode 100644 index 000000000..35993f328 --- /dev/null +++ b/src/data_structures/concurrent/concurrent_set.hpp @@ -0,0 +1,8 @@ +#pragma once + +#include "data_structures/concurrent/skiplist.hpp" + +template +class ConcurrentSet : public SkipList +{ +}; diff --git a/src/data_structures/skiplist/skiplist.hpp b/src/data_structures/concurrent/skiplist.hpp similarity index 78% rename from src/data_structures/skiplist/skiplist.hpp rename to src/data_structures/concurrent/skiplist.hpp index 26b0341c3..abad9828d 100644 --- a/src/data_structures/skiplist/skiplist.hpp +++ b/src/data_structures/concurrent/skiplist.hpp @@ -4,12 +4,13 @@ #include #include +#include "utils/random/fast_binomial.hpp" +#include "utils/placeholder.hpp" + #include "threading/sync/lockable.hpp" #include "threading/sync/spinlock.hpp" -#include "utils/random/fast_binomial.hpp" -#include "utils/placeholder.hpp" -#include "skiplist_gc.hpp" +#include "data_structures/skiplist/skiplist_gc.hpp" /* @brief Concurrent lock-based skiplist with fine grained locking * @@ -45,40 +46,40 @@ * * The implementation has an interface which closely resembles the functions * with arguments and returned types frequently used by the STL. - * + * * Example usage: - * Skiplist skiplist; + * Skiplist skiplist; * * { * auto accessor = skiplist.access(); * - * // inserts into the skiplist and returns + * // inserts item into the skiplist and returns * // pair. iterator points to the newly created * // node and the boolean member evaluates to true denoting that the * // insertion was successful - * accessor.insert_unique(key1, value1); + * accessor.insert(item1); * - * // nothing gets inserted because key1 already exist in the skiplist + * // nothing gets inserted because item1 already exist in the skiplist * // returned iterator points to the existing element and the return * // boolean evaluates to false denoting the failed insertion - * accessor.insert_unique(key1, value1); + * accessor.insert(item1); * - * // returns an iterator to the element pair - * auto it = accessor.find(key1); + * // returns an iterator to the element item1 + * auto it = accessor.find(item1); * * // returns an empty iterator. it == accessor.end() - * auto it = accessor.find(key2); + * auto it = accessor.find(item2); * - * // iterate over all key value pairs + * // iterate over all items * for(auto it = accessor.begin(); it != accessor.end(); ++it) - * cout << it->first << " " << it->second; + * cout << *it << endl; * * // range based for loops also work * for(auto& e : accessor) - * cout << e.first << " " << e.second; + * cout << e << endl; * - * accessor.remove(key1); // returns true - * accessor.remove(key1); // returns false because key1 doesn't exist + * accessor.remove(item1); // returns true + * accessor.remove(item1); // returns false because key1 doesn't exist * } * * // accessor out of scope, garbage collection might occur @@ -86,14 +87,13 @@ * For detailed operations available, please refer to the Accessor class * inside the public section of the SkipList class. * - * @tparam K Type to use as the key - * @tparam T Type to use as the value + * @tparam T Type to use as the item * @tparam H Maximum node height. Determines the effective number of nodes * the skiplist can hold in order to preserve it's log2 properties * @tparam lock_t Lock type used when locking is needed during the creation * and deletion of nodes. */ -template +template class SkipList : private Lockable { public: @@ -101,8 +101,6 @@ public: // with p(k) = (1/2)^k for all k from the interval static thread_local FastBinomial rnd; - using value_type = std::pair; - /* @brief Wrapper class for flags used in the implementation * * MARKED flag is used to logically delete a node. @@ -119,22 +117,22 @@ public: bool is_marked() const { - return flags.load(std::memory_order_acquire) & MARKED; + return flags.load() & MARKED; } void set_marked() { - flags.fetch_or(MARKED, std::memory_order_release); + flags.fetch_or(MARKED); } bool is_fully_linked() const { - return flags.load(std::memory_order_acquire) & FULLY_LINKED; + return flags.load() & FULLY_LINKED; } void set_fully_linked() { - flags.fetch_or(FULLY_LINKED, std::memory_order_release); + flags.fetch_or(FULLY_LINKED); } private: @@ -149,27 +147,12 @@ public: const uint8_t height; Flags flags; - const K& key() const - { - return kv_pair().first; - } - T& value() - { - return kv_pair().second; - } - - const T& value() const - { - return kv_pair().second; - } - - value_type& kv_pair() { return data.get(); } - const value_type& kv_pair() const + const T& value() const { return data.get(); } @@ -181,23 +164,18 @@ public: return new (allocate(height)) Node(height); } - static Node* create(const K& key, const T& item, uint8_t height) + static Node* create(const T& item, uint8_t height) { - return create({key, item}, height); + return create(item, height); } - static Node* create(const K& key, T&& item, uint8_t height) - { - return create({key, std::forward(item)}, height); - } - - static Node* create(value_type&& data, uint8_t height) + static Node* create(T&& item, uint8_t height) { auto node = allocate(height); // we have raw memory and we need to construct an object // of type Node on it - return new (node) Node(std::forward(data), height); + return new (node) Node(std::forward(item), height); } static void destroy(Node* node) @@ -208,12 +186,12 @@ public: Node* forward(size_t level) const { - return tower[level].load(std::memory_order_acquire); + return tower[level].load(); } void forward(size_t level, Node* next) { - tower[level].store(next, std::memory_order_release); + tower[level].store(next); } private: @@ -226,9 +204,9 @@ public: new (&tower[i]) std::atomic {nullptr}; } - Node(value_type&& data, uint8_t height) : Node(height) + Node(T&& data, uint8_t height) : Node(height) { - this->data.set(std::forward(data)); + this->data.set(std::forward(data)); } ~Node() @@ -252,8 +230,7 @@ public: return node; } - Placeholder data; - + Placeholder data; // this creates an array of the size zero. we can't put any sensible // value here since we don't know what size it will be untill the @@ -276,22 +253,22 @@ public: IteratorBase() = default; IteratorBase(const IteratorBase&) = default; - value_type& operator*() + T& operator*() { assert(node != nullptr); - return node->kv_pair(); + return node->value(); } - value_type* operator->() + T* operator->() { assert(node != nullptr); - return &node->kv_pair(); + return &node->value(); } - operator value_type&() + operator T&() { assert(node != nullptr); - return node->kv_pair(); + return node->value(); } It& operator++() @@ -326,19 +303,19 @@ public: ConstIterator() = default; ConstIterator(const ConstIterator&) = default; - const value_type& operator*() + const T& operator*() { return IteratorBase::operator*(); } - const value_type* operator->() + const T* operator->() { return IteratorBase::operator->(); } - operator const value_type&() + operator const T&() { - return IteratorBase::operator value_type&(); + return IteratorBase::operator T&(); } }; @@ -413,29 +390,34 @@ public: return skiplist->cend(); } - std::pair insert_unique(const K& key, const T& item) + std::pair insert(const T& item) { - return skiplist->insert({key, item}, preds, succs); + return skiplist->insert(item, preds, succs); } - std::pair insert_unique(const K& key, T&& item) + std::pair insert(T&& item) { - return skiplist->insert({key, std::forward(item)}, preds, succs); + return skiplist->insert(std::forward(item), preds, succs); } - ConstIterator find(const K& key) const + ConstIterator find(const T& item) const { - return static_cast(*skiplist).find(key); + return static_cast(*skiplist).find(item); } - Iterator find(const K& key) + Iterator find(const T& item) { - return skiplist->find(key); + return skiplist->find(item); } - bool remove(const K& key) + bool contains(const T& item) const { - return skiplist->remove(key, preds, succs); + return this->find(item) != this->end(); + } + + bool remove(const T& item) + { + return skiplist->remove(item, preds, succs); } size_t size() const @@ -496,28 +478,28 @@ private: return count.load(); } - bool greater(const K& key, const Node* const node) + bool greater(const T& item, const Node* const node) { - return node && key > node->key(); + return node && item > node->value(); } - bool less(const K& key, const Node* const node) + bool less(const T& item, const Node* const node) { - return (node == nullptr) || key < node->key(); + return (node == nullptr) || item < node->value(); } - ConstIterator find(const K& key) const + ConstIterator find(const T& item) const { - return const_cast(this)->find_node(key); + return const_cast(this)->find_node(item); } - Iterator find(const K& key) + Iterator find(const T& item) { - return find_node(key); + return find_node(item); } template - It find_node(const K& key) + It find_node(const T& item) { Node* node, *pred = header; int h = static_cast(pred->height) - 1; @@ -525,7 +507,7 @@ private: while(true) { // try to descend down first the next key on this layer overshoots - for(; h >= 0 && less(key, node = pred->forward(h)); --h) {} + for(; h >= 0 && less(item, node = pred->forward(h)); --h) {} // if we overshoot at every layer, item doesn't exist if(h < 0) @@ -533,16 +515,16 @@ private: // the item is farther to the right, continue going right as long // as the key is greater than the current node's key - while(greater(key, node)) + while(greater(item, node)) pred = node, node = node->forward(h); // check if we have a hit. if not, we need to descend down again - if(!less(key, node) && !node->flags.is_marked()) + if(!less(item, node) && !node->flags.is_marked()) return It(node); } } - int find_path(Node* from, int start, const K& key, + int find_path(Node* from, int start, const T& item, Node* preds[], Node* succs[]) { int level_found = -1; @@ -552,10 +534,10 @@ private: { Node* node = pred->forward(level); - while(greater(key, node)) + while(greater(item, node)) pred = node, node = pred->forward(level); - if(level_found == -1 && !less(key, node)) + if(level_found == -1 && !less(item, node)) level_found = level; preds[level] = pred; @@ -589,11 +571,12 @@ private: } std::pair - insert(value_type&& data, Node* preds[], Node* succs[]) + insert(T&& data, Node* preds[], Node* succs[]) { while(true) { - auto level = find_path(header, H - 1, data.first, preds, succs); + // TODO: before here was data.first + auto level = find_path(header, H - 1, data, preds, succs); if(level != -1) { @@ -618,7 +601,7 @@ private: continue; // you have the locks, create a new node - auto new_node = Node::create(std::forward(data), height); + auto new_node = Node::create(std::forward(data), height); // link the predecessors and successors, e.g. // @@ -646,7 +629,7 @@ private: && !node->flags.is_marked(); } - bool remove(const K& key, Node* preds[], Node* succs[]) + bool remove(const T& item, Node* preds[], Node* succs[]) { Node* node = nullptr; guard_t node_guard; @@ -655,7 +638,7 @@ private: while(true) { - auto level = find_path(header, H - 1, key, preds, succs); + auto level = find_path(header, H - 1, item, preds, succs); if(!marked && (level == -1 || !ok_delete(succs[level], level))) return false; @@ -695,6 +678,5 @@ private: SkiplistGC gc; }; -template -thread_local FastBinomial SkipList::rnd; - +template +thread_local FastBinomial SkipList::rnd; diff --git a/src/data_structures/skiplist/skiplist_gc.hpp b/src/data_structures/skiplist/skiplist_gc.hpp index 86e597325..c2ac53fd0 100644 --- a/src/data_structures/skiplist/skiplist_gc.hpp +++ b/src/data_structures/skiplist/skiplist_gc.hpp @@ -1,5 +1,8 @@ #pragma once +// TODO: remove from here and from the project +#include + #include "memory/freelist.hpp" #include "memory/lazy_gc.hpp" #include "threading/sync/spinlock.hpp" diff --git a/src/data_structures/skiplist/skiplist_map.hpp b/src/data_structures/skiplist/skiplist_map.hpp new file mode 100644 index 000000000..c67a8f652 --- /dev/null +++ b/src/data_structures/skiplist/skiplist_map.hpp @@ -0,0 +1,709 @@ +#pragma once + +#include +#include +#include + +#include "threading/sync/lockable.hpp" +#include "threading/sync/spinlock.hpp" + +#include "utils/random/fast_binomial.hpp" +#include "utils/placeholder.hpp" +#include "skiplist_gc.hpp" + +/* @brief Concurrent lock-based skiplist with fine grained locking + * + * From Wikipedia: + * "A skip list is a data structure that allows fast search within an + * ordered sequence of elements. Fast search is made possible by + * maintaining a linked hierarchy of subsequences, each skipping over + * fewer elements. Searching starts in the sparsest subsequence until + * two consecutive elements have been found, one smaller and one + * larger than or equal to the element searched for." + * + * [_]---------------->[+]----------->[_] + * [_]->[+]----------->[+]------>[+]->[_] + * [_]->[+]------>[+]->[+]------>[+]->[_] + * [_]->[+]->[+]->[+]->[+]->[+]->[+]->[_] + * head 1 2 4 5 8 9 nil + * + * The logarithmic properties are maintained by randomizing the height for + * every new node using the binomial distribution + * p(k) = (1/2)^k for k in [1...H]. + * + * The implementation is based on the work described in the paper + * "A Provably Correct Scalable Concurrent Skip List" + * URL: https://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/OPODIS2006-BA.pdf + * + * The proposed implementation is in Java so the authors don't worry about + * garbage collection, but obviously we have to. This implementation uses + * lazy garbage collection. When all clients stop using the skiplist, we can + * be sure that all logically removed nodes are not visible to anyone so + * we can safely remove them. The idea of counting active clients implies + * the use of a intermediary structure (called Accessor) when accessing the + * skiplist. + * + * The implementation has an interface which closely resembles the functions + * with arguments and returned types frequently used by the STL. + * + * Example usage: + * Skiplist skiplist; + * + * { + * auto accessor = skiplist.access(); + * + * // inserts into the skiplist and returns + * // pair. iterator points to the newly created + * // node and the boolean member evaluates to true denoting that the + * // insertion was successful + * accessor.insert_unique(key1, value1); + * + * // nothing gets inserted because key1 already exist in the skiplist + * // returned iterator points to the existing element and the return + * // boolean evaluates to false denoting the failed insertion + * accessor.insert_unique(key1, value1); + * + * // returns an iterator to the element pair + * auto it = accessor.find(key1); + * + * // returns an empty iterator. it == accessor.end() + * auto it = accessor.find(key2); + * + * // iterate over all key value pairs + * for(auto it = accessor.begin(); it != accessor.end(); ++it) + * cout << it->first << " " << it->second; + * + * // range based for loops also work + * for(auto& e : accessor) + * cout << e.first << " " << e.second; + * + * accessor.remove(key1); // returns true + * accessor.remove(key1); // returns false because key1 doesn't exist + * } + * + * // accessor out of scope, garbage collection might occur + * + * For detailed operations available, please refer to the Accessor class + * inside the public section of the SkipList class. + * + * @tparam K Type to use as the key + * @tparam T Type to use as the value + * @tparam H Maximum node height. Determines the effective number of nodes + * the skiplist can hold in order to preserve it's log2 properties + * @tparam lock_t Lock type used when locking is needed during the creation + * and deletion of nodes. + */ +template +class SkipListMap : private Lockable +{ +public: + // computes the height for the new node from the interval [1...H] + // with p(k) = (1/2)^k for all k from the interval + static thread_local FastBinomial rnd; + + using value_type = std::pair; + + /* @brief Wrapper class for flags used in the implementation + * + * MARKED flag is used to logically delete a node. + * FULLY_LINKED is used to mark the node as fully inserted, i.e. linked + * at all layers in the skiplist up to the node height + */ + struct Flags + { + enum node_flags : uint8_t + { + MARKED = 0x01, + FULLY_LINKED = 0x10, + }; + + bool is_marked() const + { + return flags.load(std::memory_order_acquire) & MARKED; + } + + void set_marked() + { + flags.fetch_or(MARKED, std::memory_order_release); + } + + bool is_fully_linked() const + { + return flags.load(std::memory_order_acquire) & FULLY_LINKED; + } + + void set_fully_linked() + { + flags.fetch_or(FULLY_LINKED, std::memory_order_release); + } + + private: + std::atomic flags {0}; + }; + + class Node : Lockable + { + public: + friend class SkipListMap; + + const uint8_t height; + Flags flags; + + const K& key() const + { + return kv_pair().first; + } + + T& value() + { + return kv_pair().second; + } + + const T& value() const + { + return kv_pair().second; + } + + value_type& kv_pair() + { + return data.get(); + } + + const value_type& kv_pair() const + { + return data.get(); + } + + static Node* sentinel(uint8_t height) + { + // we have raw memory and we need to construct an object + // of type Node on it + return new (allocate(height)) Node(height); + } + + static Node* create(const K& key, const T& item, uint8_t height) + { + return create({key, item}, height); + } + + static Node* create(const K& key, T&& item, uint8_t height) + { + return create({key, std::forward(item)}, height); + } + + static Node* create(value_type&& data, uint8_t height) + { + auto node = allocate(height); + + // we have raw memory and we need to construct an object + // of type Node on it + return new (node) Node(std::forward(data), height); + } + + static void destroy(Node* node) + { + node->~Node(); + free(node); + } + + Node* forward(size_t level) const + { + return tower[level].load(std::memory_order_acquire); + } + + void forward(size_t level, Node* next) + { + tower[level].store(next, std::memory_order_release); + } + + private: + Node(uint8_t height) : height(height) + { + // here we assume, that the memory for N towers (N = height) has + // been allocated right after the Node structure so we need to + // initialize that memory + for(auto i = 0; i < height; ++i) + new (&tower[i]) std::atomic {nullptr}; + } + + Node(value_type&& data, uint8_t height) : Node(height) + { + this->data.set(std::forward(data)); + } + + ~Node() + { + for(auto i = 0; i < height; ++i) + tower[i].~atomic(); + } + + static Node* allocate(uint8_t height) + { + // [ Node ][Node*][Node*][Node*]...[Node*] + // | | | | | + // | 0 1 2 height-1 + // |----------------||-----------------------------| + // space for Node space for tower pointers + // structure right after the Node + // structure + auto size = sizeof(Node) + height * sizeof(std::atomic); + auto node = static_cast(std::malloc(size)); + + return node; + } + + Placeholder data; + + // this creates an array of the size zero. we can't put any sensible + // value here since we don't know what size it will be untill the + // node is allocated. we could make it a Node** but then we would + // have two memory allocations, one for node and one for the forward + // list. this way we avoid expensive malloc/free calls and also cache + // thrashing when following a pointer on the heap + std::atomic tower[0]; + }; + +public: + template + class IteratorBase : public Crtp + { + protected: + IteratorBase(Node* node) : node(node) {} + + Node* node {nullptr}; + public: + IteratorBase() = default; + IteratorBase(const IteratorBase&) = default; + + value_type& operator*() + { + assert(node != nullptr); + return node->kv_pair(); + } + + value_type* operator->() + { + assert(node != nullptr); + return &node->kv_pair(); + } + + operator value_type&() + { + assert(node != nullptr); + return node->kv_pair(); + } + + It& operator++() + { + assert(node != nullptr); + node = node->forward(0); + return this->derived(); + } + + It& operator++(int) + { + return operator++(); + } + + friend bool operator==(const It& a, const It& b) + { + return a.node == b.node; + } + + friend bool operator!=(const It& a, const It& b) + { + return !(a == b); + } + }; + + class ConstIterator : public IteratorBase + { + friend class SkipListMap; + ConstIterator(Node* node) : IteratorBase(node) {} + + public: + ConstIterator() = default; + ConstIterator(const ConstIterator&) = default; + + const value_type& operator*() + { + return IteratorBase::operator*(); + } + + const value_type* operator->() + { + return IteratorBase::operator->(); + } + + operator const value_type&() + { + return IteratorBase::operator value_type&(); + } + }; + + class Iterator : public IteratorBase + { + friend class SkipListMap; + Iterator(Node* node) : IteratorBase(node) {} + + public: + Iterator() = default; + Iterator(const Iterator&) = default; + }; + + SkipListMap() : header(Node::sentinel(H)) {} + + friend class Accessor; + + class Accessor + { + friend class SkipListMap; + + Accessor(SkipListMap* skiplist) : skiplist(skiplist) + { + assert(skiplist != nullptr); + + skiplist->gc.add_ref(); + } + + public: + Accessor(const Accessor&) = delete; + + Accessor(Accessor&& other) : skiplist(other.skiplist) + { + other.skiplist = nullptr; + } + + ~Accessor() + { + if(skiplist == nullptr) + return; + + skiplist->gc.release_ref(); + } + + Iterator begin() + { + return skiplist->begin(); + } + + ConstIterator begin() const + { + return skiplist->cbegin(); + } + + ConstIterator cbegin() const + { + return skiplist->cbegin(); + } + + Iterator end() + { + return skiplist->end(); + } + + ConstIterator end() const + { + return skiplist->cend(); + } + + ConstIterator cend() const + { + return skiplist->cend(); + } + + std::pair insert_unique(const K& key, const T& item) + { + return skiplist->insert({key, item}, preds, succs); + } + + std::pair insert_unique(const K& key, T&& item) + { + return skiplist->insert({key, std::forward(item)}, preds, succs); + } + + ConstIterator find(const K& key) const + { + return static_cast(*skiplist).find(key); + } + + Iterator find(const K& key) + { + return skiplist->find(key); + } + + bool contains(const K& key) const + { + return skiplist->contains(key); + } + + bool remove(const K& key) + { + return skiplist->remove(key, preds, succs); + } + + size_t size() const + { + return skiplist->size(); + } + + private: + SkipListMap* skiplist; + Node* preds[H], *succs[H]; + }; + + Accessor access() + { + return Accessor(this); + } + + const Accessor access() const + { + return Accessor(this); + } + +private: + using guard_t = std::unique_lock; + + Iterator begin() + { + return Iterator(header->forward(0)); + } + + ConstIterator begin() const + { + return ConstIterator(header->forward(0)); + } + + ConstIterator cbegin() const + { + return ConstIterator(header->forward(0)); + } + + Iterator end() + { + return Iterator(); + } + + ConstIterator end() const + { + return ConstIterator(); + } + + ConstIterator cend() const + { + return ConstIterator(); + } + + size_t size() const + { + return count.load(); + } + + bool greater(const K& key, const Node* const node) + { + return node && key > node->key(); + } + + bool less(const K& key, const Node* const node) + { + return (node == nullptr) || key < node->key(); + } + + ConstIterator find(const K& key) const + { + return const_cast(this)->find_node(key); + } + + Iterator find(const K& key) + { + return find_node(key); + } + + bool contains(const K& key) + { + return this->find(key) != this->end(); + } + + template + It find_node(const K& key) + { + Node* node, *pred = header; + int h = static_cast(pred->height) - 1; + + while(true) + { + // try to descend down first the next key on this layer overshoots + for(; h >= 0 && less(key, node = pred->forward(h)); --h) {} + + // if we overshoot at every layer, item doesn't exist + if(h < 0) + return It(); + + // the item is farther to the right, continue going right as long + // as the key is greater than the current node's key + while(greater(key, node)) + pred = node, node = node->forward(h); + + // check if we have a hit. if not, we need to descend down again + if(!less(key, node) && !node->flags.is_marked()) + return It(node); + } + } + + int find_path(Node* from, int start, const K& key, + Node* preds[], Node* succs[]) + { + int level_found = -1; + Node* pred = from; + + for(int level = start; level >= 0; --level) + { + Node* node = pred->forward(level); + + while(greater(key, node)) + pred = node, node = pred->forward(level); + + if(level_found == -1 && !less(key, node)) + level_found = level; + + preds[level] = pred; + succs[level] = node; + } + + return level_found; + } + + template + bool lock_nodes(uint8_t height, guard_t guards[], + Node* preds[], Node* succs[]) + { + Node *prepred, *pred, *succ = nullptr; + bool valid = true; + + for(int level = 0; valid && level < height; ++level) + { + pred = preds[level], succ = succs[level]; + + if(pred != prepred) + guards[level] = pred->acquire_unique(), prepred = pred; + + valid = !pred->flags.is_marked() && pred->forward(level) == succ; + + if(ADDING) + valid = valid && (succ == nullptr || !succ->flags.is_marked()); + } + + return valid; + } + + std::pair + insert(value_type&& data, Node* preds[], Node* succs[]) + { + while(true) + { + auto level = find_path(header, H - 1, data.first, preds, succs); + + if(level != -1) + { + auto found = succs[level]; + + if(found->flags.is_marked()) + continue; + + while(!found->flags.is_fully_linked()) + usleep(250); + + return {Iterator {succs[level]}, false}; + } + + auto height = rnd(); + guard_t guards[H]; + + // try to acquire the locks for predecessors up to the height of + // the new node. release the locks and try again if someone else + // has the locks + if(!lock_nodes(height, guards, preds, succs)) + continue; + + // you have the locks, create a new node + auto new_node = Node::create(std::forward(data), height); + + // link the predecessors and successors, e.g. + // + // 4 HEAD ... P ------------------------> S ... NULL + // 3 HEAD ... ... P -----> NEW ---------> S ... NULL + // 2 HEAD ... ... P -----> NEW -----> S ... ... NULL + // 1 HEAD ... ... ... P -> NEW -> S ... ... ... NULL + for(uint8_t level = 0; level < height; ++level) + { + new_node->forward(level, succs[level]); + preds[level]->forward(level, new_node); + } + + new_node->flags.set_fully_linked(); + count.fetch_add(1); + + return {Iterator {new_node}, true}; + } + } + + bool ok_delete(Node* node, int level) + { + return node->flags.is_fully_linked() + && node->height - 1 == level + && !node->flags.is_marked(); + } + + bool remove(const K& key, Node* preds[], Node* succs[]) + { + Node* node = nullptr; + guard_t node_guard; + bool marked = false; + int height = 0; + + while(true) + { + auto level = find_path(header, H - 1, key, preds, succs); + + if(!marked && (level == -1 || !ok_delete(succs[level], level))) + return false; + + if(!marked) + { + node = succs[level]; + height = node->height; + node_guard = node->acquire_unique(); + + if(node->flags.is_marked()) + return false; + + node->flags.set_marked(); + marked = true; + } + + guard_t guards[H]; + + if(!lock_nodes(height, guards, preds, succs)) + continue; + + for(int level = height - 1; level >= 0; --level) + preds[level]->forward(level, node->forward(level)); + + // TODO: review and test + gc.collect(node); + + count.fetch_sub(1); + return true; + } + } + + // number of elements + std::atomic count {0}; + Node* header; + SkiplistGC gc; +}; + +template +thread_local FastBinomial SkipListMap::rnd; + diff --git a/src/data_structures/skiplist/skiplistset.hpp b/src/data_structures/skiplist/skiplistset.hpp index bf77607d5..4baf95a58 100644 --- a/src/data_structures/skiplist/skiplistset.hpp +++ b/src/data_structures/skiplist/skiplistset.hpp @@ -2,7 +2,7 @@ #include -#include "skiplist.hpp" +#include "data_structures/skiplist/skiplist_map.hpp" #include "utils/total_ordering.hpp" template @@ -46,7 +46,7 @@ class SkipListSet const T* item {nullptr}; }; - using skiplist_t = SkipList; + using skiplist_t = SkipListMap; using iter_t = typename skiplist_t::Iterator; using const_iter_t = typename skiplist_t::ConstIterator; @@ -230,5 +230,5 @@ public: } private: - SkipList data; + SkipListMap data; }; diff --git a/src/query_engine/main_queries.cpp b/src/query_engine/main_queries.cpp index 1c70a1ad0..0ca3152fb 100644 --- a/src/query_engine/main_queries.cpp +++ b/src/query_engine/main_queries.cpp @@ -1,116 +1,123 @@ #include +#include "cypher/common.hpp" #include "database/db.hpp" #include "query_stripper.hpp" #include "storage/model/properties/property.hpp" -#include "utils/command_line/arguments.hpp" -#include "cypher/common.hpp" -#include "utils/time/timer.hpp" #include "storage/model/properties/traversers/consolewriter.hpp" +#include "utils/command_line/arguments.hpp" +#include "utils/time/timer.hpp" using namespace std; // -- // DESCRIPTION: create account -// FULL: CREATE (n:ACCOUNT {id: 50, name: "Nikola", country: "Croatia", created_at: 14634563}) RETURN n +// FULL: CREATE (n:ACCOUNT {id: 50, name: "Nikola", country: "Croatia", +// created_at: 14634563}) RETURN n // STRIPPED: CREATE(n:ACCOUNT{id:0,name:1,country:2,created_at:3})RETURNn // HASH: 10597108978382323595 // STATUS: DONE // -- // DESCRIPTION: create personnel -// FULL: CREATE (n:PERSONNEL {id: 23, role: "CTO", created_at: 1235345}) RETURN n +// FULL: CREATE (n:PERSONNEL {id: 23, role: "CTO", created_at: 1235345}) +// RETURN n // STRIPPED: CREATE(n:PERSONNEL{id:0,role:1,created_at:2})RETURNn -// HASH: 4037885257628527960 +// HASH: 4037885257628527960 // STATUS: TODO // -- // DESCRIPTION: create edge between ACCOUNT node and PERSONNEL node (IS type) -// FULL: MATCH (a:ACCOUNT {id:50}), (p:PERSONNEL {id: 23}) CREATE (a)-[:IS]->(p) +// FULL: MATCH (a:ACCOUNT {id:50}), (p:PERSONNEL {id: 23}) CREATE +// (a)-[:IS]->(p) // STRIPPED: MATCH(a:ACCOUNT{id:0}),(p:PERSONNEL{id:1})CREATE(a)-[:IS]->(p) -// HASH: 16888190822925675190 +// HASH: 16888190822925675190 // STATUS: TODO // -- // DESCRIPTION: find ACCOUNT node, PERSONNEL node and edge between them -// FULL: MATCH (a:ACCOUNT {id:50})-[r:IS]->(p:PERSONNEL {id: 23}) RETURN a,r,p +// FULL: MATCH (a:ACCOUNT {id:50})-[r:IS]->(p:PERSONNEL {id: 23}) RETURN +// a,r,p // STRIPPED: MATCH(a:ACCOUNT{id:0})-[r:IS]->(p:PERSONNEL{id:1})RETURNa,r,p -// HASH: 9672752533852902744 +// HASH: 9672752533852902744 // STATUS: TODO // -- // DESCRIPTION: create OPPORTUNITY -// FULL: -// STRIPPED: -// HASH: +// FULL: +// STRIPPED: +// HASH: // STATUS: TODO // -- // DESCRIPTION: create PERSONNEL-[:CREATED]->OPPORTUNITY -// FULL: -// STRIPPED: -// HASH: +// FULL: +// STRIPPED: +// HASH: // STATUS: TODO // -- // DESCRIPTION: create COMPANY -// FULL: -// STRIPPED: -// HASH: +// FULL: +// STRIPPED: +// HASH: // STATUS: TODO // -- // DESCRIPTION: create OPPORTUNITY-[:MATCH]->COMPANY -// FULL: -// STRIPPED: -// HASH: +// FULL: +// STRIPPED: +// HASH: // STATUS: TODO // -- // DESCRIPTION: create an edge between two nodes that are found by the ID -// FULL: MATCH (a {id:0}), (p {id: 1}) CREATE (a)-[r:IS]->(p) RETURN r +// FULL: MATCH (a {id:0}), (p {id: 1}) CREATE (a)-[r:IS]->(p) RETURN r // STRIPPED: MATCH(a{id:0}),(p{id:1})CREATE(a)-[r:IS]->(p)RETURNr -// HASH: 7939106225150551899 +// HASH: 7939106225150551899 // STATUS: DONE // -- // DESCRIPTION: fine node by the ID // FULL: MATCH (n {id: 0}) RETURN n // STRIPPED: MATCH(n{id:0})RETURNn -// HASH: 11198568396549106428 +// HASH: 11198568396549106428 // STATUS: DONE // -- // DESCRIPTION: find edge by the ID // FULL: MATCH ()-[r]-() WHERE ID(r)=0 RETURN r // STRIPPED: MATCH()-[r]-()WHEREID(r)=0RETURNr -// HASH: 8320600413058284114 +// HASH: 8320600413058284114 // STATUS: DONE // -- // DESCRIPTION: update node that is found by the ID -// FULL: MATCH (n: {id: 0}) SET n.name = "TEST100" RETURN n -// STRIPPED: MATCH(n:{id:0})SETn.name=1RETURNn -// HASH: 6813335159006269041 +// FULL: MATCH (n: {id: 0}) SET n.name = "TEST100" RETURN n +// STRIPPED: MATCH(n:{id:0})SETn.name=1RETURNn +// HASH: 6813335159006269041 // STATUS: DONE // -- // DESCRIPTION: find shortest path between two nodes specified by ID -// FULL: -// STRIPPED: -// HASH: +// FULL: +// STRIPPED: +// HASH: +// STATUS: TODO +// -- +// DESCRIPTION: find all nodes by label name +// FULL: MATCH (n:LABEL) RETURN n +// STRIPPPED: MATCH(n:LABEL)RETURNn +// HASH: 4857652843629217005 // STATUS: TODO // -- // TODO: Labels and Types have to be extracted from a query during the // query compile time -void cout_properties(const Properties& properties) +void cout_properties(const Properties &properties) { ConsoleWriter writer; properties.accept(writer); } -int main(int argc, char** argv) +int main(int argc, char **argv) { Db db; - std::map> queries; + std::map> queries; - auto create_labeled_and_named_node = [&db](const properties_t& args) - { - auto& t = db.tx_engine.begin(); + auto create_labeled_and_named_node = [&db](const properties_t &args) { + auto &t = db.tx_engine.begin(); auto vertex_accessor = db.graph.vertices.insert(t); - vertex_accessor.property( - "name", args[0] - ); - auto label = db.graph.label_store.find_or_create("LABEL"); + vertex_accessor.property("name", args[0]); + auto &label = db.graph.label_store.find_or_create("LABEL"); vertex_accessor.add_label(label); cout_properties(vertex_accessor.properties()); t.commit(); @@ -118,25 +125,16 @@ int main(int argc, char** argv) return true; }; - auto create_account = [&db](const properties_t& args) - { - auto& t = db.tx_engine.begin(); + auto create_account = [&db](const properties_t &args) { + auto &t = db.tx_engine.begin(); auto vertex_accessor = db.graph.vertices.insert(t); // this can be a loop - vertex_accessor.property( - "id", args[0] - ); - vertex_accessor.property( - "name", args[1] - ); - vertex_accessor.property( - "country", args[2] - ); - vertex_accessor.property( - "created_at", args[3] - ); + vertex_accessor.property("id", args[0]); + vertex_accessor.property("name", args[1]); + vertex_accessor.property("country", args[2]); + vertex_accessor.property("created_at", args[3]); // here is the problem because LABEL can't be stripped out - auto label = db.graph.label_store.find_or_create("ACCOUNT"); + auto &label = db.graph.label_store.find_or_create("ACCOUNT"); vertex_accessor.add_label(label); cout_properties(vertex_accessor.properties()); t.commit(); @@ -144,35 +142,37 @@ int main(int argc, char** argv) return true; }; - auto find_node_by_internal_id = [&db](const properties_t& args) - { - auto& t = db.tx_engine.begin(); - auto id = static_cast(*args[0]); + auto find_node_by_internal_id = [&db](const properties_t &args) { + auto &t = db.tx_engine.begin(); + auto id = static_cast(*args[0]); auto vertex_accessor = db.graph.vertices.find(t, Id(id.value)); if (!vertex_accessor) { cout << "vertex doesn't exist" << endl; t.commit(); - return false; + return false; } cout_properties(vertex_accessor.properties()); + + cout << "LABELS:" << endl; + for (auto label_ref : vertex_accessor.labels()) { + cout << label_ref.get() << endl; + } + t.commit(); return true; }; - auto create_edge = [&db](const properties_t& args) - { - auto& t = db.tx_engine.begin(); + auto create_edge = [&db](const properties_t &args) { + auto &t = db.tx_engine.begin(); auto v1 = db.graph.vertices.find(t, args[0]->as().value); - if (!v1) - return t.commit(), false; + if (!v1) return t.commit(), false; auto v2 = db.graph.vertices.find(t, args[1]->as().value); - if (!v2) - return t.commit(), false; + if (!v2) return t.commit(), false; auto e = db.graph.edges.insert(t); @@ -192,12 +192,10 @@ int main(int argc, char** argv) return true; }; - auto find_edge_by_internal_id = [&db](const properties_t& args) - { - auto& t = db.tx_engine.begin(); + auto find_edge_by_internal_id = [&db](const properties_t &args) { + auto &t = db.tx_engine.begin(); auto e = db.graph.edges.find(t, args[0]->as().value); - if (!e) - return t.commit(), false; + if (!e) return t.commit(), false; // print edge type and properties cout << "EDGE_TYPE: " << e.edge_type() << endl; @@ -215,14 +213,12 @@ int main(int argc, char** argv) return true; }; - auto update_node = [&db](const properties_t& args) - { - auto& t = db.tx_engine.begin(); + auto update_node = [&db](const properties_t &args) { + auto &t = db.tx_engine.begin(); auto v = db.graph.vertices.find(t, args[0]->as().value); - if (!v) - return t.commit(), false; - + if (!v) return t.commit(), false; + v.property("name", args[1]); cout_properties(v.properties()); @@ -231,18 +227,33 @@ int main(int argc, char** argv) return true; }; + auto find_by_label = [&db](const properties_t &args) + { + auto &t = db.tx_engine.begin(); + auto &label = db.graph.label_store.find_or_create("LABEL"); + auto &index_record_collection = + db.graph.vertices.find_label_index(label); + auto accessor = index_record_collection.access(); + cout << "VERTICES" << endl; + for (auto& v : accessor) { + cout << v.record->data.props.at("name").as().value << endl; + } + return true; + }; + queries[10597108978382323595u] = create_account; - queries[5397556489557792025u] = create_labeled_and_named_node; - queries[7939106225150551899u] = create_edge; + queries[5397556489557792025u] = create_labeled_and_named_node; + queries[7939106225150551899u] = create_edge; queries[11198568396549106428u] = find_node_by_internal_id; - queries[8320600413058284114u] = find_edge_by_internal_id; - queries[6813335159006269041u] = update_node; + queries[8320600413058284114u] = find_edge_by_internal_id; + queries[6813335159006269041u] = update_node; + queries[4857652843629217005u] = find_by_label; // auto arguments = all_arguments(argc, argv); // auto input_query = extract_query(arguments); auto stripper = make_query_stripper(TK_INT, TK_FLOAT, TK_STR, TK_BOOL); // auto stripped = stripper.strip(input_query); - // + // // auto time = timer([&stripped, &queries]() { // for (int i = 0; i < 1000000; ++i) { // queries[stripped.hash](stripped.arguments); @@ -264,7 +275,7 @@ int main(int argc, char** argv) cout << "unsupported query" << endl; continue; } - + auto result = queries[stripped.hash](stripped.arguments); cout << "RETURN: " << result << endl; diff --git a/src/storage/common.hpp b/src/storage/common.hpp index 4601ec474..8e29ab91e 100644 --- a/src/storage/common.hpp +++ b/src/storage/common.hpp @@ -1,6 +1,6 @@ #pragma once -#include "transactions/transaction.hpp" +#include "data_structures/concurrent/concurrent_map.hpp" #include "mvcc/version_list.hpp" -#include "data_structures/skiplist/skiplist.hpp" +#include "transactions/transaction.hpp" #include "utils/counters/atomic_counter.hpp" diff --git a/src/storage/edges.hpp b/src/storage/edges.hpp index 21c8ec081..23871419d 100644 --- a/src/storage/edges.hpp +++ b/src/storage/edges.hpp @@ -2,6 +2,7 @@ #include "common.hpp" #include "edge_accessor.hpp" +#include "data_structures/concurrent/concurrent_map.hpp" class Edges { @@ -46,6 +47,6 @@ public: } private: - SkipList edges; + ConcurrentMap edges; AtomicCounter counter; }; diff --git a/src/storage/indexes/index.hpp b/src/storage/indexes/index.hpp index acb038552..9dd98dba7 100644 --- a/src/storage/indexes/index.hpp +++ b/src/storage/indexes/index.hpp @@ -1,48 +1,45 @@ #pragma once -#include "data_structures/skiplist/skiplist.hpp" -#include "keys/unique_key.hpp" +#include -#include "storage/cursor.hpp" +#include "data_structures/concurrent/concurrent_map.hpp" +#include "storage/indexes/index_record.hpp" +#include "storage/indexes/index_record_collection.hpp" +#include "storage/model/label.hpp" template class Index { public: - using skiplist_t = SkipList; - using iterator_t = typename skiplist_t::Iterator; - using accessor_t = typename skiplist_t::Accessor; - using K = typename Key::key_t; - using cursor_t = Cursor; + using container_t = ConcurrentMap; - // cursor_t insert(const K& key, Item* item, tx::Transaction& t) - auto insert(const K& key, Item* item) + Index() : index(std::make_unique()) {} + + auto update(const Label &label, VertexIndexRecord &&index_record) { - // Item has to be some kind of container for the real data like Vertex, - // the container has to be transactionally aware - // in other words index or something that wraps index has to be - // transactionally aware + auto accessor = index->access(); + auto label_ref = label_ref_t(label); - auto accessor = skiplist.access(); - auto result = accessor.insert_unique(key, item); + // create Index Record Collection if it doesn't exist + if (!accessor.contains(label_ref)) { + accessor.insert_unique(label_ref, + std::move(VertexIndexRecordCollection())); + } - // TODO: handle existing insert - - return result; + // add Vertex Index Record to the Record Collection + auto &record_collection = (*accessor.find(label_ref)).second; + record_collection.add(std::forward(index_record)); } - auto remove(const K& key) + VertexIndexRecordCollection& find(const Label& label) { - auto accessor = skiplist.access(); - return accessor.remove(key); + // TODO: accessor should be outside? + // bacause otherwise GC could delete record that has just be returned + auto label_ref = label_ref_t(label); + auto accessor = index->access(); + return (*accessor.find(label_ref)).second; } - auto find(const K& key) - { - auto accessor = skiplist.access(); - return accessor.find(key); - } - private: - skiplist_t skiplist; + std::unique_ptr index; }; diff --git a/src/storage/indexes/index_record.hpp b/src/storage/indexes/index_record.hpp new file mode 100644 index 000000000..b7e63efaa --- /dev/null +++ b/src/storage/indexes/index_record.hpp @@ -0,0 +1,46 @@ +#pragma once + +#include "mvcc/version_list.hpp" +#include "utils/total_ordering.hpp" + +template +class IndexRecord : TotalOrdering> +{ +public: + using vlist_t = mvcc::VersionList; + + IndexRecord() = default; + + IndexRecord(T *record, vlist_t *vlist) : record(record), vlist(vlist) + { + assert(record != nullptr); + assert(vlist != nullptr); + } + + friend bool operator<(const IndexRecord& lhs, const IndexRecord& rhs) + { + return lhs.record < rhs.record; + } + + friend bool operator==(const IndexRecord& lhs, const IndexRecord& rhs) + { + return lhs.record == rhs.record; + } + + bool empty() const { return record == nullptr; } + + // const typename T::Accessor get() + // { + // // TODO: if somebody wants to read T content + // // const T::Accessor has to be returned from here + // // the problem is that here we don't have pointer to store + // // TODO: figure it out + // } + +// private: + T *const record{nullptr}; + vlist_t *const vlist{nullptr}; +}; + +using VertexIndexRecord = IndexRecord; +using EdgeIndexRecord = IndexRecord; diff --git a/src/storage/indexes/index_record_collection.hpp b/src/storage/indexes/index_record_collection.hpp new file mode 100644 index 000000000..0d06e2a91 --- /dev/null +++ b/src/storage/indexes/index_record_collection.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include + +#include "data_structures/concurrent/concurrent_set.hpp" +#include "storage/indexes/index_record.hpp" + +template +class IndexRecordCollection +{ +public: + using index_record_t = IndexRecord; + using index_record_collection_t = ConcurrentSet; + + IndexRecordCollection() + : records(std::make_unique()) + { + } + + void add(index_record_t &&record) + { + auto accessor = records->access(); + accessor.insert(std::forward(record)); + } + + auto access() + { + return records->access(); + } + + // TODO: iterator and proxy + +private: + std::unique_ptr records; +}; + +using VertexIndexRecordCollection = IndexRecordCollection; +using EdgeIndexRecordCollection = IndexRecordCollection; diff --git a/src/storage/indexes/index_key.hpp b/src/storage/indexes/keys/index_key.hpp similarity index 79% rename from src/storage/indexes/index_key.hpp rename to src/storage/indexes/keys/index_key.hpp index 68beba984..b2555a782 100644 --- a/src/storage/indexes/index_key.hpp +++ b/src/storage/indexes/keys/index_key.hpp @@ -2,7 +2,6 @@ #include "utils/total_ordering.hpp" -#include "ordering.hpp" #include "storage/model/properties/properties.hpp" #include "storage/model/properties/property.hpp" @@ -10,15 +9,10 @@ template class UniqueIndexKey { public: - namespace - -private: - Property& key; }; template class IndexKey { public: - }; diff --git a/src/storage/label_store.hpp b/src/storage/label_store.hpp index 13de699bb..cc81056f4 100644 --- a/src/storage/label_store.hpp +++ b/src/storage/label_store.hpp @@ -3,7 +3,7 @@ #include #include "model/label.hpp" -#include "data_structures/skiplist/skiplistset.hpp" +#include "data_structures/concurrent/concurrent_set.hpp" class LabelStore { @@ -25,5 +25,5 @@ public: // return { Label, is_found } private: - SkipListSet