Merge branch 'dev' into mgcore_T159_inotify

include/data_structures/kdtree/build.hpp

@@ -0,0 +1,67 @@
+#pragma once
+
+#include <vector>
+#include <algorithm>
+#include <functional>
+
+#include "math.hpp"
+#include "kdnode.hpp"
+
+namespace kd {
+
+template <class T, class U>
+using Nodes = std::vector<KdNode<T, U>*>;
+
+template <class T, class U>
+KdNode<T, U>* build(Nodes<T, U>& nodes, byte axis = 0) 
+{
+    // if there are no elements left, we've completed building of this branch
+    if(nodes.empty())
+        return nullptr;
+
+    // comparison function to use for sorting the elements
+    auto fsort = [axis](KdNode<T, U>* a, KdNode<T, U>* b) -> bool
+        { return kd::math::axial_distance(a->coord, b->coord, axis) < 0; };
+
+    size_t median = nodes.size() / 2;
+
+    // partial sort nodes vector to compute median and ensure that elements
+    // less than median are positioned before the median so we can slice it
+    // nicely
+
+    // internal implementation is O(n) worst case
+    // tl;dr http://en.wikipedia.org/wiki/Introselect
+    std::nth_element(nodes.begin(), nodes.begin() + median, nodes.end(), fsort);
+
+    // set axis for the node
+    auto node = nodes.at(median);
+    node->axis = axis;
+    
+    // slice the vector into two halves
+    auto left = Nodes<T, U>(nodes.begin(), nodes.begin() + median);
+    auto right = Nodes<T, U>(nodes.begin() + median + 1, nodes.end());
+    
+    // recursively build left and right branches
+    node->left = build(left, axis ^ 1);
+    node->right = build(right, axis ^ 1);
+
+    return node;
+}
+
+template <class T, class U, class It>
+KdNode<T, U>* build(It first, It last)
+{
+    Nodes<T, U> kdnodes;
+
+    std::transform(first, last, std::back_inserter(kdnodes),
+        [&](const std::pair<Point<T>, U>& element) {
+            auto key = element.first;
+            auto data = element.second;
+             return new KdNode<T, U>(key, data);
+        });
+
+    // build the tree from the kdnodes and return the root node
+    return build(kdnodes);
+}
+
+}

include/data_structures/kdtree/kdnode.hpp

@@ -0,0 +1,44 @@
+#pragma once
+
+#include <memory>
+
+#include "point.hpp"
+
+namespace kd {
+
+template <class T, class U>
+class KdNode
+{
+public:
+    KdNode(const U& data)
+        : axis(0), coord(Point<T>(0, 0)), left(nullptr), right(nullptr), data(data) { }
+
+    KdNode(const Point<T>& coord, const U& data)
+        : axis(0), coord(coord), left(nullptr), right(nullptr), data(data) { }
+
+    KdNode(unsigned char axis, const Point<T>& coord, const U& data)
+        : axis(axis), coord(coord), left(nullptr), right(nullptr), data(data) { }
+
+    KdNode(unsigned char axis, const Point<T>& coord, KdNode<T, U>* left, KdNode<T, U>* right, const U& data)
+        : axis(axis), coord(coord), left(left), right(right), data(data) { }
+
+    ~KdNode();
+
+    unsigned char axis;
+
+    Point<T> coord;
+
+    KdNode<T, U>* left;
+    KdNode<T, U>* right;
+
+    U data;
+};
+
+template <class T, class U>
+KdNode<T, U>::~KdNode()
+{
+    delete left;
+    delete right;
+}
+
+}

include/data_structures/kdtree/kdtree.hpp

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <vector>
+
+#include "build.hpp"
+#include "nns.hpp"
+
+namespace kd
+{
+
+template <class T, class U>
+class KdTree
+{
+public:
+    KdTree() {}
+
+    template <class It>
+    KdTree(It first, It last);
+
+    const U& lookup(const Point<T>& pk) const;
+
+protected:
+    std::unique_ptr<KdNode<float, U>> root;
+};
+
+template <class T, class U>
+const U& KdTree<T, U>::lookup(const Point<T>& pk) const
+{
+    // do a nearest neighbour search on the tree
+    return kd::nns(pk, root.get())->data;
+}
+
+template <class T, class U>
+template <class It>
+KdTree<T, U>::KdTree(It first, It last)
+{
+    root.reset(kd::build<T, U, It>(first, last));
+}
+
+}

include/data_structures/kdtree/math.hpp

@@ -0,0 +1,40 @@
+#pragma once
+
+#include <limits>
+#include <cmath>
+
+#include "point.hpp"
+
+namespace kd {
+namespace math {
+
+using byte = unsigned char;
+
+// returns the squared distance between two points
+template<class T>
+T distance_sq(const Point<T>& a, const Point<T>& b)
+{
+    auto dx = a.longitude - b.longitude;
+    auto dy = a.latitude - b.latitude;
+    return dx * dx + dy * dy;
+}
+
+// returns the distance between two points
+template<class T>
+T distance(const Point<T>& a, const Point<T>& b)
+{
+    return std::sqrt(distance_sq(a, b));
+}
+
+// returns the distance between two points looking at a specific axis
+// \param axis 0 if abscissa else 1 if ordinate
+template <class T>
+T axial_distance(const Point<T>& a, const Point<T>& b, byte axis)
+{
+    return axis == 0 ?
+        a.longitude - b.longitude:
+        a.latitude  - b.latitude;
+}
+
+}
+}

include/data_structures/kdtree/nns.hpp

@@ -0,0 +1,101 @@
+#pragma once
+
+#include "math.hpp"
+#include "point.hpp"
+#include "kdnode.hpp"
+
+namespace kd {
+
+// helper class for calculating the nearest neighbour in a kdtree 
+template <class T, class U>
+struct Result
+{
+    Result()
+        : node(nullptr), distance_sq(std::numeric_limits<T>::infinity()) {}
+
+    Result(const KdNode<T, U>* node, T distance_sq)
+        : node(node), distance_sq(distance_sq) {}
+
+    const KdNode<T, U>* node;
+    T distance_sq;
+};
+
+
+// a recursive implementation for the kdtree nearest neighbour search
+// \param p    the point for which we search for the nearest neighbour
+// \param node the root of the subtree during recursive descent
+// \param best the place to save the best result so far
+template <class T, class U>
+void nns(const Point<T>& p, const KdNode<T, U>* const node, Result<T, U>& best)
+{
+    if(node == nullptr)
+        return;
+
+    T d = math::distance_sq(p, node->coord);
+
+    // keep record of the closest point C found so far
+    if(d < best.distance_sq)
+    {
+        best.node = node;
+        best.distance_sq = d;
+    }
+
+    // where to traverse next?
+    // what to prune?
+
+    //            |
+    //  possible  |
+    //  prune     *
+    //  area      | - - - - -* P
+    //            |
+    //
+    //            |----------|
+    //                 dx
+    //
+    
+    //        possible prune
+    // RIGHT    area
+    //               
+    //  --------*------   ---
+    //          |          |
+    // LEFT                |
+    //          |          |  dy
+    //                     |
+    //          |          |
+    //          * p       ---
+
+    T axd = math::axial_distance(p, node->coord, node->axis);
+
+    // traverse the subtree in order that
+    // maximizes the probability for pruning
+    auto near = axd > 0 ? node->right : node->left;
+    auto far  = axd > 0 ? node->left  : node->right; 
+
+    // try near first
+    nns(p, near, best);
+
+    // prune subtrees once their bounding boxes say
+    // that they can't contain any point closer than C
+    if(axd * axd >= best.distance_sq)
+        return;
+
+    // try other subtree
+    nns(p, far, best);
+}
+
+// an implementation for the kdtree nearest neighbour search
+// \param  p    the point for which we search for the nearest neighbour
+// \param  root the root of the tree
+// \return the nearest neighbour for the point p
+template <class T, class U>
+const KdNode<T, U>* nns(const Point<T>& p, const KdNode<T, U>* root)
+{
+    Result<T, U> best;
+
+    // begin recursive search
+    nns(p, root, best);
+
+    return best.node;
+}
+
+}

include/data_structures/kdtree/point.hpp

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <ostream>
+
+namespace kd {
+
+template <class T>
+class Point
+{
+public:
+    Point(T latitude, T longitude)
+        : latitude(latitude), longitude(longitude) {}
+
+    // latitude
+    //    y
+    //    ^
+    //    |
+    //    0---> x longitude
+
+    T latitude;
+    T longitude;
+
+    /// nice stream formatting with the standard << operator
+    friend std::ostream& operator<< (std::ostream& stream, const Point& p) {
+        return stream << "(lat: " << p.latitude
+                      << ", lng: " << p.longitude << ')';
+    }
+};
+
+}

include/data_structures/linked_list.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <list>
+
+#include "threading/sync/lockable.hpp"
+#include "threading/sync/spinlock.hpp"
+
+template <typename value_type, typename lock_type = SpinLock>
+class LinkedList : public Lockable<lock_type>
+{
+public:
+    std::size_t size() const
+    {
+        auto guard = this->acquire_unique();
+        return data.size();
+    }
+
+    void push_front(const value_type &value)
+    {
+        auto guard = this->acquire_unique();
+        data.push_front(value);
+    }
+
+    void push_front(value_type &&value)
+    {
+        auto guard = this->acquire_unique();
+        data.push_front(std::forward<value_type>(value));
+    }
+
+    void pop_front()
+    {
+        auto guard = this->acquire_unique();
+        data.pop_front();
+    }
+
+    // value_type& as return value
+    // would not be concurrent
+    value_type front()
+    {
+        auto guard = this->acquire_unique();
+        return data.front();
+    }
+
+private:
+    std::list<value_type> data;
+};

include/data_structures/list/lockfree_list.hpp

@@ -0,0 +1,243 @@
+#pragma once
+
+#include <atomic>
+#include <unistd.h>
+
+#include "threading/sync/lockable.hpp"
+#include "memory/hp.hpp"
+
+namespace lockfree
+{
+
+template <class T, size_t sleep_time = 250>
+class List : Lockable<SpinLock>
+{
+public:
+    List() = default;
+
+    List(List&) = delete;
+    List(List&&) = delete;
+
+    void operator=(List&) = delete;
+
+    class read_iterator
+    {
+    public:
+        // constructor
+        read_iterator(T* curr) : 
+            curr(curr),
+            hazard_ref(std::move(memory::HP::get().insert(curr))) {}
+
+        // no copy constructor
+        read_iterator(read_iterator& other) = delete;
+
+        // move constructor
+        read_iterator(read_iterator&& other) :
+            curr(other.curr),
+            hazard_ref(std::move(other.hazard_ref)) {}
+  
+        T& operator*() { return *curr; }
+        T* operator->() { return curr; }
+
+        operator T*() { return curr; }
+
+        read_iterator& operator++()
+        {
+            auto& hp = memory::HP::get();
+            hazard_ref = std::move(hp.insert(curr->next.load()));
+
+            curr = curr->next.load();
+            return *this;
+        }
+  
+        read_iterator& operator++(int)
+        {
+            return operator++();
+        }
+
+        bool has_next()
+        {
+            if (curr->next == nullptr)
+                return false;
+            return true;
+        }
+  
+    private:
+        T* curr;
+        memory::HP::reference hazard_ref;
+    };
+  
+    class read_write_iterator
+    {
+        friend class List<T, sleep_time>;
+
+    public:
+        read_write_iterator(T* prev, T* curr) :
+            prev(prev),
+            curr(curr),
+            hazard_ref(std::move(memory::HP::get().insert(curr))) {}
+      
+         // no copy constructor
+        read_write_iterator(read_write_iterator& other) = delete;
+
+        // move constructor
+        read_write_iterator(read_write_iterator&& other) :
+            prev(other.prev),
+            curr(other.curr),
+            hazard_ref(std::move(other.hazard_ref)) {}
+ 
+        T& operator*() { return *curr; }
+        T* operator->() { return curr; }
+
+        operator T*() { return curr; }
+
+        read_write_iterator& operator++()
+        {
+            auto& hp = memory::HP::get();
+            hazard_ref = std::move(hp.insert(curr->next.load()));
+
+            prev = curr;
+            curr = curr->next.load();
+            return *this;
+        }
+  
+        read_write_iterator& operator++(int)
+        {
+            return operator++();
+        }
+  
+    private:
+        T* prev;
+        T* curr;
+        memory::HP::reference hazard_ref;
+    };
+
+    read_iterator begin()
+    {
+        return read_iterator(head.load());
+    }
+
+    read_write_iterator rw_begin()
+    {
+        return read_write_iterator(nullptr, head.load());
+    }
+
+    void push_front(T* node)
+    {
+        // we want to push an item to front of a list like this
+        // HEAD --> [1] --> [2] --> [3] --> ...
+        
+        // read the value of head atomically and set the node's next pointer
+        // to point to the same location as head
+
+        // HEAD --------> [1] --> [2] --> [3] --> ...
+        //                 |
+        //                 |
+        //      NODE ------+
+
+        T* h = node->next = head.load();
+
+        // atomically do: if the value of node->next is equal to current value
+        // of head, make the head to point to the node.
+        // if this fails (another thread has just made progress), update the
+        // value of node->next to the current value of head and retry again
+        // until you succeed
+
+        // HEAD ----|CAS|----------> [1] --> [2] --> [3] --> ...
+        //  |         |               |
+        //  |         v               |
+        //  +-------|CAS|---> NODE ---+
+
+        while(!head.compare_exchange_weak(h, node))
+        {
+            node->next.store(h);
+            usleep(sleep_time);
+        }
+
+        // the final state of the list after compare-and-swap looks like this
+
+        // HEAD          [1] --> [2] --> [3] --> ...
+        //  |             |
+        //  |             |
+        //  +---> NODE ---+
+    }
+
+    bool remove(read_write_iterator& it)
+    {
+        // acquire an exclusive guard.
+        // we only care about push_front and iterator performance so we can
+        // we only care about push_front and iterator performance so we can
+        // tradeoff some remove speed for better reads and inserts. remove is
+        // used exclusively by the GC thread(s) so it can be slower
+        auto guard = acquire_unique();
+
+        // even though concurrent removes are synchronized, we need to worry
+        // about concurrent reads (solved by using atomics) and concurrent 
+        // inserts to head (VERY dangerous, suffers from ABA problem, solved
+        // by simply not deleting the head node until it gets pushed further
+        // down the list)
+
+        // check if we're deleting the head node. we can't do that because of
+        // the ABA problem so just return false for now. the logic behind this
+        // is that this node will move further down the list next time the
+        // garbage collector traverses this list and therefore it will become
+        // deletable
+        if(it.prev == nullptr) {
+            std::cout << "prev null" << std::endl;
+            return false;
+        }
+
+        //  HEAD --> ... --> [i] --> [i + 1] --> [i + 2] --> ...
+        //                  
+        //                  prev       curr        next
+
+        auto prev = it.prev;
+        auto curr = it.curr;
+        auto next = curr->next.load(std::memory_order_acquire);
+
+        // effectively remove the curr node from the list
+
+        //                    +---------------------+
+        //                    |                     |
+        //                    |                     v
+        //  HEAD --> ... --> [i]     [i + 1] --> [i + 2] --> ...
+        //                  
+        //                  prev       curr        next
+
+        prev->next.store(next, std::memory_order_release);
+
+        // curr is now removed from the list so no iterators will be able
+        // to reach it at this point, but we still need to check the hazard
+        // pointers and wait until everyone who currently holds a reference to
+        // it has stopped using it before we can physically delete it
+       
+        // TODO: test more appropriate 
+        auto& hp = memory::HP::get();
+
+        while(hp.find(reinterpret_cast<uintptr_t>(curr)))
+            sleep(sleep_time);
+        
+        delete curr;
+        
+        return true;
+    }
+
+private:
+    std::atomic<T*> head { nullptr };
+};
+
+template <class T, size_t sleep_time>
+bool operator==(typename List<T, sleep_time>::read_iterator& a,
+                typename List<T, sleep_time>::read_iterator& b)
+{
+    return a->curr == b->curr;
+}
+
+template <class T, size_t sleep_time>
+bool operator!=(typename List<T, sleep_time>::read_iterator& a,
+                typename List<T, sleep_time>::read_iterator& b)
+{
+    return !operator==(a, b);
+}
+
+}

include/data_structures/queue/bounded_spsc_queue.hpp

@@ -0,0 +1,87 @@
+#pragma once
+
+#include <atomic>
+#include <memory>
+
+namespace lockfree
+{
+
+template <class T, size_t N>
+class BoundedSpscQueue
+{
+public:
+    static constexpr size_t size = N;
+
+    BoundedSpscQueue() = default;
+
+    BoundedSpscQueue(const BoundedSpscQueue&) = delete;
+    BoundedSpscQueue(BoundedSpscQueue&&) = delete;
+
+    BoundedSpscQueue& operator=(const BoundedSpscQueue&) = delete;
+
+    bool push(const T& item)
+    {
+        // load the current tail
+        // [] [] [1] [2] [3] [4] [5] [$] []
+        //        H                   T
+        auto t = tail.load(std::memory_order_relaxed);
+
+        // what will next tail be after we push
+        // [] [] [1] [2] [3] [4] [5] [$] [ ]
+        //        H                   T   T' 
+        auto next = increment(t);
+
+        // check if queue is full and do nothing if it is
+        // [3] [4] [5] [6] [7] [8] [$] [ 1 ] [2]
+        //                          T   T'H
+        if(next == head.load(std::memory_order_acquire))
+            return false;
+
+        // insert the item into the empty spot
+        // [] [] [1] [2] [3] [4] [5] [ ] []
+        //        H               T   T' 
+        items[t] = item;
+
+        // release the tail to the consumer (serialization point)
+        // [] [] [1] [2] [3] [4] [5] [ $ ] []
+        //        H                   T T' 
+        tail.store(next, std::memory_order_release);
+
+        return true;
+    }
+
+    bool pop(T& item)
+    {
+        // [] [] [1] [2] [3] [4] [5] [$] []
+        //        H                   T
+        auto h = head.load(std::memory_order_relaxed);
+
+        // [] [] [] [] [ $ ] [] [] [] []
+        //              H T        
+        if(h == tail.load(std::memory_order_acquire))
+            return false;
+
+        // move an item from the queue
+        item = std::move(items[h]);
+
+        // serialization point wrt producer
+        // [] [] [] [2] [3] [4] [5] [$] []
+        //           H               T
+        head.store(increment(h), std::memory_order_release);
+
+        return true;
+    }
+
+private:
+    static constexpr size_t capacity = N + 1;
+
+    std::array<T, capacity> items;
+    std::atomic<size_t> head {0}, tail {0};
+
+    size_t increment(size_t idx) const
+    {
+        return (idx + 1) % capacity;
+    }
+};
+
+}

include/data_structures/queue/mpsc_queue.hpp

@@ -0,0 +1,149 @@
+#pragma once
+
+#include <atomic>
+#include <memory>
+
+namespace lockfree
+{
+
+/** @brief Multiple-Producer Single-Consumer Queue
+ *  A wait-free (*) multiple-producer single-consumer queue.
+ *  
+ *  features:
+ *    - wait-free
+ *    - fast producers (only one atomic XCHG and and one atomic store with
+ *      release semantics)
+ *    - extremely fast consumer (only atomic loads with acquire semantics on
+ *      the fast path and atomic loads + atomic XCHG on the slow path)
+ *    - no need for order reversion -> pop() is always O(1)
+ *    - ABA free
+ *
+ *  great for using in loggers, garbage collectors etc.
+ *
+ *  (*) there is a small window of inconsistency from the lock free design
+ *      see the url below for details
+ *  URL: http://www.1024cores.net/home/lock-free-algorithms/queues/intrusive-mpsc-node-based-queue
+ *
+ *  mine is not intrusive for better modularity, but with slightly worse
+ *  performance because it needs to do two memory allocations instead of
+ *  one
+ *
+ *  @tparam T Type of the items to store in the queue
+ */
+template <class T>
+class MpscQueue
+{
+    struct Node
+    {
+        Node(Node* next, std::unique_ptr<T>&& item)
+            : next(next), item(std::forward<std::unique_ptr<T>>(item)) {}
+
+        std::atomic<Node*> next;
+        std::unique_ptr<T> item;
+    };
+
+public:
+    MpscQueue()
+    {
+        auto stub = new Node(nullptr, nullptr);
+        head.store(stub);
+        tail = stub;
+    }
+
+    ~MpscQueue()
+    {
+        // purge all elements from the queue
+        while(pop()) {}
+
+        // we are left with a stub, delete that
+        delete tail;
+    }
+
+    MpscQueue(MpscQueue&) = delete;
+    MpscQueue(MpscQueue&&) = delete;
+
+    /** @brief Pushes an item into the queue.
+     *
+     *  Pushes an item into the front of the queue.
+     *
+     *  @param item std::unique_ptr<T> An item to push into the queue
+     *  @return void
+     */
+    void push(std::unique_ptr<T>&& item)
+    {
+        push(new Node(nullptr, std::forward<std::unique_ptr<T>>(item)));
+    }
+
+    /** @brief Pops a node from the queue.
+     *
+     *  Pops and returns a node from the back of the queue.
+     *
+     *  @return std::unique_ptr<T> A pointer to the node popped from the
+     *      queue, nullptr if nothing was popped
+     */
+    std::unique_ptr<T> pop()
+    {
+        auto tail = this->tail;
+        
+        // serialization point wrt producers
+        auto next = tail->next.load(std::memory_order_acquire);
+
+        if(next)
+        {
+            // remove the last stub from the queue
+            // make [2] the next stub and return it's data
+            //
+            // H --> [n] <- ... <- [2] <--+--[STUB]   +-- T
+            //                            |           |
+            //                            +-----------+
+            this->tail = next;
+
+            // delete the stub node
+            // H --> [n] <- ... <- [STUB] <-- T
+            delete tail;
+
+            return std::move(next->item);
+        }
+
+        return nullptr;
+    }
+
+private:
+    std::atomic<Node*> head;
+    Node* tail;
+
+    /** @brief Pushes a new node into the queue.
+     *
+     *  Pushes a new node containing the item into the front of the queue.
+     *
+     *  @param node Node* A pointer to node you want to push into the queue
+     *  @return void
+     */
+    void push(Node* node)
+    {
+        // initial state
+        // H --> [3] <- [2] <- [STUB] <-- T
+        
+        // serialization point wrt producers, acquire-release
+        auto old = head.exchange(node, std::memory_order_acq_rel);
+
+        // after exchange
+        // H --> [4]    [3] <- [2] <- [STUB] <-- T
+        
+        // this is the window of inconsistency, if the producer is blocked
+        // here, the consumer is also blocked. but this window is extremely
+        // small, it's followed by a store operation which is a
+        // serialization point wrt consumer
+
+        // old holds a pointer to node [3] and we need to link the [3] to a
+        // newly created node [4] using release semantics
+        
+        // serialization point wrt consumer, release
+        old->next.store(node, std::memory_order_release);
+
+        // finally, we have a queue like this
+        // H --> [4] <- [3] <- [2] <- [1] <-- T
+    }
+};
+
+}

include/data_structures/queue/slqueue.hpp

@@ -0,0 +1,63 @@
+#pragma once
+
+#include <queue>
+
+#include "threading/sync/lockable.hpp"
+#include "threading/sync/spinlock.hpp"
+
+template <class T>
+class SlQueue : Lockable<SpinLock>
+{
+public:
+
+    template <class... Args>
+    void emplace(Args&&... args)
+    {
+        auto guard = acquire_unique();
+        queue.emplace(args...);
+    }
+
+    void push(const T& item)
+    {
+        auto guard = acquire_unique();
+        queue.push(item);
+    }
+
+    T front()
+    {
+        auto guard = acquire_unique();
+        return queue.front();
+    }
+
+    void pop()
+    {
+        auto guard = acquire_unique();
+        queue.pop();
+    }
+
+    bool pop(T& item)
+    {
+        auto guard = acquire_unique();
+        if(queue.empty())
+            return false;
+
+        item = std::move(queue.front());
+        queue.pop();
+        return true;
+    }
+
+    bool empty()
+    {
+        auto guard = acquire_unique();
+        return queue.empty();
+    }
+
+    size_t size()
+    {
+        auto guard = acquire_unique();
+        return queue.size();
+    }
+
+private:
+    std::queue<T> queue;
+};

include/data_structures/slrbtree.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <map>
+
+#include "threading/sync/spinlock.hpp"
+
+template <class K, class T>
+class SlRbTree : Lockable<SpinLock>
+{
+public:
+
+private:
+    std::map<K, T> tree;
+};

include/data_structures/slstack.hpp

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <stack>
+
+#include "threading/sync/spinlock.hpp"
+#include "threading/sync/lockable.hpp"
+
+template <class T>
+class SpinLockStack : Lockable<SpinLock>
+{
+public:
+
+    T pop()
+    {
+        auto guard = acquire();
+
+        T elem = stack.top();
+        stack.pop();
+
+        return elem;
+    }
+
+    void push(const T& elem)
+    {
+        auto guard = acquire();
+
+        stack.push(elem);
+    }
+
+private:
+    std::stack<T> stack;
+};

include/data_structures/stack/array_stack.hpp

@@ -0,0 +1,8 @@
+#pragma once
+
+template <class T>
+class ArrayStack
+{
+
+private:
+};

include/data_structures/static_array.hpp

@@ -0,0 +1,65 @@
+#pragma once
+
+#include "utils/assert.hpp"
+
+// data structure namespace short ds
+// TODO: document strategy related to namespace naming
+// (namespace names should be short but eazy to memorize)
+namespace ds
+{
+
+// static array is data structure which size (capacity) can be known at compile
+// time
+// this data structure isn't concurrent
+template <typename T, size_t N>
+class static_array
+{
+public:
+    // default constructor
+    static_array() {}
+
+    // explicit constructor which populates the data array with
+    // initial values, array structure after initialization
+    // is N * [initial_value]
+    explicit static_array(const T &initial_value)
+    {
+        for (size_t i = 0; i < size(); ++i) {
+            data[i] = initial_value;
+        }
+    }
+
+    // returns array size
+    size_t size() const { return N; }
+
+    // returns element reference on specific index
+    T &operator[](size_t index)
+    {
+        runtime_assert(index < N, "Index " << index << " must be less than "
+                                           << N);
+        return data[index];
+    }
+
+    // returns const element reference on specific index
+    const T &operator[](size_t index) const
+    {
+        runtime_assert(index < N, "Index " << index << " must be less than "
+                                           << N);
+        return data[index];
+    }
+
+    // returns begin iterator
+    T *begin() { return &data[0]; }
+
+    // returns const begin iterator
+    const T *begin() const { return &data[0]; }
+
+    // returns end iterator
+    T *end() { return &data[N]; }
+
+    // returns const end iterator
+    const T *end() const { return &data[N]; }
+
+private:
+    T data[N];
+};
+}

include/data_structures/union_find/union_find.hpp

@@ -0,0 +1,84 @@
+#pragma once
+
+#include <memory>
+
+template <class uintXX_t = uint32_t,
+          class allocator = std::allocator<uintXX_t>>
+class UnionFind
+{
+public:
+    UnionFind(uintXX_t n) : N(n), n(n)
+    {
+        count = alloc.allocate(n);
+        parent = alloc.allocate(n);
+
+        for(auto i = 0; i < n; ++i)
+            count[i] = 1, parent[i] = i;
+    }
+
+    ~UnionFind()
+    {
+        alloc.deallocate(count, N);
+        alloc.deallocate(parent, N);
+    }
+
+    // this is O(lg* n)
+    void connect(uintXX_t p, uintXX_t q)
+    {
+        auto rp = root(p);
+        auto rq = root(q);
+
+        // if roots are equal, we don't have to do anything
+        if(rp == rq)
+            return;
+
+        // merge the smaller subtree to the root of the larger subtree
+        if(count[rp] < count[rq])
+            parent[rp] = rq, count[rp] += count[rp];
+        else
+            parent[rq] = rp, count[rp] += count[rq];
+
+        // update the number of groups
+        n--;
+    }
+
+    // O(lg* n)
+    bool find(uintXX_t p, uintXX_t q)
+    {
+        return root(p) == root(q);
+    }
+
+    // O(lg* n)
+    uintXX_t root(uintXX_t p)
+    {
+        auto r = p;
+        auto newp = p;
+
+        // find the node connected to itself, that's the root
+        while(parent[r] != r)
+            r = parent[r];
+
+        // do some path compression to enable faster searches
+        while(p != r)
+            newp = parent[p], parent[p] = r, p = newp;
+
+        return r;
+    }
+
+    uintXX_t size() const
+    {
+        return n;
+    }
+
+private:
+    allocator alloc;
+
+    const uintXX_t N;
+    uintXX_t n;
+
+    // array of subtree counts
+    uintXX_t* count;
+
+    // array of tree indices
+    uintXX_t* parent;
+};

include/memory/deferred_recycler.hpp

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "recycler.hpp"
+
+template <class T, class Allocator>
+class DeferredRecycler : Recycler<T, Allocator>
+{
+public:
+    using Recycler<T, Allocator>::acquire;
+
+    void recycle(T* item)
+    {
+        auto guard = this->acquire_unique();
+        dirty.push_back(item);
+    }
+
+    void clean()
+    {
+        auto guard = this->acquire_unique();
+
+        for(auto item : dirty)
+            this->recycle_or_delete(item);
+
+        dirty.clear();
+    }
+
+private:
+    std::queue<T*> dirty;
+};

include/memory/hp.hpp

@@ -0,0 +1,148 @@
+#pragma once
+
+#include <atomic>
+#include <cassert>
+#include <unistd.h>
+#include <iostream>
+
+namespace memory
+{
+
+constexpr const size_t HP_SIZE = 128;
+
+class HP
+{
+public:
+    // this object can't be copied or moved
+    HP(HP&) = delete;
+    HP(HP&&) = delete;
+
+    // grabs a singleton instance
+    static HP& get()
+    {
+        static HP hp;
+        return hp;
+    }
+
+    class reference
+    {
+        friend class HP;
+    public:
+        reference(reference&) = delete;
+
+        // this type shouldn't be copyable to avoid calling its destructor
+        // multiple times, but should be movable
+        reference(reference&& other)
+        {
+            this->idx = other.idx;
+
+            // set the index to a negative number to indicate that this
+            // index has been moved and that you should not free its
+            // hazard pointer
+            other.idx = -1;
+        }
+        // hazard pointer is cleared once reference goes out of scope
+        ~reference()
+        {
+            // TODO: remove
+            // std::cout << "reference destructor called: ";
+            // std::cout << this->idx;
+            // std::cout << std::endl;
+
+            // check if this reference was moved during its lifetime
+            if(idx < 0)
+                return;
+
+            auto& hp = HP::get();
+            hp.clear(*this);
+        }
+
+        reference& operator=(reference&& other)
+        {
+            return *this;
+        }
+
+    private:
+        reference(int64_t idx) : idx(idx) {}
+        int64_t idx;
+    };
+
+    friend class reference;
+
+    template <class T>
+    reference insert(T* ptr)
+    {
+        auto p = reinterpret_cast<uintptr_t>(ptr);
+
+        while(true)
+        {
+            // try to find a free spot in the hazard pointer list
+            for(size_t i = 0; i < HP_SIZE; ++i)
+            {
+                auto hazard = ptr_list[i].load();
+
+                // if this spot isn't free, continue searching
+                if(hazard != 0)
+                    continue;
+
+                // try to take this spot, if we fail, then another thread has
+                // just taken it. continue searching for a new one
+                if(!ptr_list[i].compare_exchange_strong(hazard, p))
+                    continue;
+
+                // found a free spot! return a reference to this spot so it
+                // can be cleared later
+                return reference(i);
+            }
+
+            // we didn't find any free spots, sleep for a while and try again
+            // from the beginning, some other thread might have freed a spot
+            // while we were traversing the lsit.
+            usleep(250);
+        }
+    }
+
+    bool find(uintptr_t hptr)
+    {
+        for (size_t i = 0; i < HP_SIZE; ++i) {
+            auto& hptr_i = ptr_list[i];
+
+            if (hptr_i != hptr)
+                continue;
+
+            if (hptr_i.load() == 1)
+                return true;
+
+            if (hptr_i.load() == 0)
+                return false;
+        }
+
+        return false;
+    }
+
+    friend std::ostream& operator<<(std::ostream& os, const HP& hp)
+    {
+        os << "Hazard pointers: ";
+        for (size_t i = 0; i < HP_SIZE; ++i) {
+            auto& hptr_i = hp.ptr_list[i];
+            os << hptr_i.load() << " ";
+        }
+        return os << std::endl;
+    }
+
+private:
+    HP()
+    {
+        for(size_t i = 0; i < HP_SIZE; ++i)
+            ptr_list[i].store(0);
+    }
+
+    void clear(reference& ref)
+    {
+        ptr_list[ref.idx].store(0);
+    }
+
+    std::atomic<uintptr_t> ptr_list[HP_SIZE];
+};
+
+}

include/memory/literals.hpp

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <cstdint>
+
+namespace memory
+{
+namespace literals
+{
+
+constexpr unsigned long long operator"" _GB(unsigned long long gb)
+{
+        return 1024 * 1024 * 1024 * gb;
+}
+
+constexpr unsigned long long operator"" _MB(unsigned long long mb)
+{
+        return 1024 * 1024 * mb;
+}
+
+constexpr unsigned long long operator"" _kB(unsigned long long kb)
+{
+        return 1024 * kb;
+}
+
+}
+}

include/memory/memory.hpp

@@ -0,0 +1,55 @@
+#pragma once
+
+#include <atomic>
+#include <mutex>
+
+#include "storage/model/record.hpp"
+#include "storage/model/vertex.hpp"
+#include "storage/model/edge.hpp"
+
+// TODO implement the memory engine using the allocator style allocation to
+// make this class non-dependent on the memory allocation strategy
+
+// TODO implement real recycling of vertices and edges to improve performance
+class MemoryEngine
+{
+public:
+    
+    template <class T,
+              typename... Args>
+    T* create(Args&&... args)
+    {
+        return new T(std::forward<Args>(args)...);
+    }
+
+    template<class T>
+    T* allocate()
+    {
+        return static_cast<T*>(malloc(sizeof(T)));
+    }
+
+    template <class T>
+    void recycle(Record<T>* record)
+    {
+        recycle(&record->derived());
+    }
+
+    void recycle(Vertex* v)
+    {
+        delete v;
+    }
+
+    void recycle(Edge* e)
+    {
+        delete e;
+    }
+
+private:
+    
+    std::unique_lock<SpinLock> acquire()
+    {
+        return std::unique_lock<SpinLock>(lock);
+    }
+
+    SpinLock lock;
+};

include/memory/recycler.hpp

@@ -0,0 +1,46 @@
+#pragma once
+
+#include <memory>
+#include <queue>
+
+#include "threading/sync/lockable.hpp"
+#include "threading/sync/spinlock.hpp"
+
+template <class T, class Allocator=std::allocator<T>>
+class Recycler : public Lockable<SpinLock>
+{
+    static constexpr size_t default_max_reserved = 100;
+
+public:
+    Recycler() = default;
+    Recycler(size_t max_reserved) : max_reserved(max_reserved) {}
+
+    template <class... Args>
+    T* acquire(Args&&... args)
+    {
+        auto guard = acquire_unique();
+        return fetch_or_create(std::forward<Args>(args)...);
+    }
+
+    void release(T* item)
+    {
+        auto guard = acquire_unique();
+        return recycle_or_delete(item);
+    }
+
+protected:
+    Allocator alloc;
+    size_t max_reserved {default_max_reserved};
+    std::queue<T*> items;
+
+    template <class... Args>
+    T* fetch_or_create(Args&&... args)
+    {
+        return new T(std::forward<Args>(args)...); // todo refactor :D
+    }
+
+    void recycle_or_delete(T* item)
+    {
+        delete item; // todo refactor :D
+    }
+};