tursom/memgraph
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Hardcoded query infrastructure - first concrete version - USEFUL FOR: POCs & pilots

Browse Source 
Summary: Hardcoded query infrastructure - first concrete version - USEFUL FOR: POCs & pilots

Test Plan: manual + jenkins

Reviewers: sale, florijan

Reviewed By: florijan

Subscribers: pullbot, buda

Differential Revision: https://phabricator.memgraph.io/D45
This commit is contained in:
Marko Budiselic
2017-02-14 09:37:32 +01:00
parent 6d981c9cd0
commit 0fcda94162
85 changed files with 1456 additions and 885 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
include/config/config.hpp
View File

@@ -22,8 +22,8 @@ public:
};
// -- all possible Memgraph's keys --
constexpr const char *COMPILE_CPU_PATH = "compile_cpu_path";
constexpr const char *TEMPLATE_CPU_CPP_PATH = "template_cpu_cpp_path";
constexpr const char *COMPILE_PATH = "compile_path";
constexpr const char *TEMPLATE_CPP_PATH = "template_cpp_path";
constexpr const char *SNAPSHOTS_PATH = "snapshots_path";
constexpr const char *CLEANING_CYCLE_SEC = "cleaning_cycle_sec";
constexpr const char *SNAPSHOT_CYCLE_SEC = "snapshot_cycle_sec";

3
include/data_structures/concurrent/concurrent_map.hpp
View File

@@ -85,7 +85,8 @@ public:
Accessor access() { return Accessor(&skiplist); }
const Accessor access() const { return Accessor(&skiplist); }
// TODO:
// const Accessor access() const { return Accessor(&skiplist); }
private:
list skiplist;

9
include/data_structures/concurrent/skiplist_gc.hpp
View File

@@ -6,13 +6,13 @@
#include "memory/freelist.hpp"
#include "memory/lazy_gc.hpp"
#include "threading/sync/spinlock.hpp"
#include "logging/default.hpp"
#include "logging/loggable.hpp"
template <class T, class lock_t = SpinLock>
class SkiplistGC : public LazyGC<SkiplistGC<T, lock_t>, lock_t>
class SkiplistGC : public LazyGC<SkiplistGC<T, lock_t>, lock_t>, public Loggable
{
public:
SkiplistGC() : logger(logging::log->logger("SkiplistGC")) {}
SkiplistGC() : Loggable("SkiplistGC") {}
// release_ref method should be called by a thread
// when the thread finish it job over object
@@ -53,9 +53,6 @@ public:
void collect(T *node) { freelist.add(node); }
protected:
Logger logger;
private:
FreeList<T> freelist;
};

93
include/dc/dynamic_lib.hpp
View File

@@ -1,93 +0,0 @@
#pragma once
#include <iostream>
#include <string>
#include <stdexcept>
#include <dlfcn.h>
#include <atomic>
using std::cout;
using std::endl;
template<typename T>
class DynamicLib
{
private:
// IMPORTANT: all dynamic libraries must have produce and destruct methods!
const std::string produce_name = "produce";
const std::string destruct_name = "destruct";
using produce_t = typename T::produce;
using destruct_t = typename T::destruct;
public:
produce_t produce_method;
destruct_t destruct_method;
DynamicLib(const std::string& lib_path) :
lib_path(lib_path),
lib_object(nullptr)
{
load();
}
typename T::lib_object* instance()
{
// TODO singleton, concurrency
if (lib_object == nullptr) {
lib_object = this->produce_method();
}
return lib_object;
}
void load()
{
load_lib();
load_produce_func();
load_destruct_func();
}
~DynamicLib()
{
if (lib_object != nullptr) {
destruct_method(lib_object);
}
}
private:
std::string lib_path;
void *dynamic_lib;
typename T::lib_object *lib_object;
void load_lib()
{
dynamic_lib = dlopen(lib_path.c_str(), RTLD_LAZY);
if (!dynamic_lib) {
throw std::runtime_error(dlerror());
}
dlerror();
}
void load_produce_func()
{
produce_method = (produce_t) dlsym(
dynamic_lib,
produce_name.c_str()
);
const char* dlsym_error = dlerror();
if (dlsym_error) {
throw std::runtime_error(dlsym_error);
}
}
void load_destruct_func()
{
destruct_method = (destruct_t) dlsym(
dynamic_lib,
destruct_name.c_str()
);
const char *dlsym_error = dlerror();
if (dlsym_error) {
throw std::runtime_error(dlsym_error);
}
}
};

16
include/logging/loggable.hpp
View File

@@ -2,16 +2,28 @@
#include "logging/default.hpp"
/**
* @class Loggable
*
* @brief Base class that could be used in all classed which need a logging
* functionality.
*/
class Loggable
{
public:
Loggable(std::string &&name)
: logger(logging::log->logger(std::forward<std::string>(name)))
/**
* Sets logger name.
*/
Loggable(const std::string &name)
: logger(logging::log->logger(name))
{
}
virtual ~Loggable() {}
protected:
/**
* Logger instance that can be used only from derived classes.
*/
Logger logger;
};

4
include/query/backend/cpp_old/cypher.hpp
View File

@@ -15,7 +15,7 @@ public:
CypherBackend() : logger(logging::log->logger("CypherBackend"))
{
// load template file
std::string template_path = CONFIG(config::TEMPLATE_CPU_CPP_PATH);
std::string template_path = CONFIG(config::TEMPLATE_CPP_PATH);
template_text = utils::read_text(fs::path(template_path));
}
@@ -38,7 +38,7 @@ public:
// save the code
std::string generated = template_engine::render(
template_text.str(), {{"class_name", "CodeCPU"},
template_text.str(), {{"class_name", "CPUPlan"},
{"stripped_hash", std::to_string(stripped_hash)},
{"query", query},
{"stream", type_name<Stream>().to_string()},

21
include/query/dynamic_lib.hpp
View File

@@ -1,21 +0,0 @@
#pragma once
#include "query/i_plan_cpu.hpp"
#include "dc/dynamic_lib.hpp"
namespace
{
template <typename Stream>
class MemgraphDynamicLib
{
public:
using produce = produce_t<Stream>;
using destruct = destruct_t<Stream>;
using lib_object = IPlanCPU<Stream>;
};
template <typename Stream>
using CodeLib = DynamicLib<MemgraphDynamicLib<Stream>>;
}

235
include/query/engine.hpp
View File

@@ -1,68 +1,223 @@
#pragma once
#include <experimental/filesystem>
#include "database/db.hpp"
#include "logging/default.hpp"
#include "query/exception/query_engine.hpp"
#include "query/plan/program.hpp"
#include "query/plan/program_loader.hpp"
#include "query/plan/program_executor.hpp"
/*
* Current arhitecture:
* query -> code_loader -> query_stripper -> [code_generator]
* -> [code_compiler] -> code_executor
*/
namespace fs = std::experimental::filesystem;
// query engine has to be aware of the Stream because Stream
// is passed to the dynamic shared library
template <typename Stream>
class QueryEngine
{
public:
QueryEngine() : logger(logging::log->logger("QueryEngine")) {}
#include "database/db.hpp"
#include "logging/loggable.hpp"
#include "query/exception/query_engine.hpp"
#include "query/plan_compiler.hpp"
#include "query/plan_generator.hpp"
#include "query/plan_interface.hpp"
#include "query/preprocessor.hpp"
#include "utils/dynamic_lib.hpp"
#include "data_structures/concurrent/concurrent_map.hpp"
auto execute(const std::string &query, Db &db, Stream &stream)
// TODO: replace with openCypher and Antlr
#include "query/frontend/cypher.hpp"
// TODO: depricated
#include "query/backend/cpp_old/cypher.hpp"
/**
* Responsible for query execution. Depends on Db and compiler.
*
* Current Query Engine arhitecture:
* query -> query_stripper -> [plan_generator] -> [plan_compiler] -> execution
*
* @tparam Stream the query engine has to be aware of the Stream because Stream
* is passed to the dynamic shared library because that is the way how
* the results should be returned (more optimal then just return
* the whole result set)
*/
template <typename Stream>
class QueryEngine : public Loggable
{
private:
using QueryPlanLib = DynamicLib<QueryPlanTrait<Stream>>;
using HashT = QueryPreprocessor::HashT;
public:
QueryEngine() : Loggable("QueryEngine") {}
/**
* Reloads query plan (plan_path contains compiled query plan).
* This methdo just calculates stripped query and offloads everything else
* to the LoadCpp method.
*
* @param query a query for which the plan will be loaded
* @param plan_path a custom made cpp query plan
*
* @return void
*/
auto ReloadCustom(const std::string &query, const fs::path &plan_path)
{
try {
auto program = program_loader.load(query);
auto result = program_executor.execute(program, db, stream);
if (UNLIKELY(!result)) {
auto preprocessed = preprocessor.preprocess(query);
Unload(query);
LoadCpp(plan_path, preprocessed.hash);
}
/**
* Executes query on the database with the stream. If a query plan is cached
* (based on query hash) it will be used.
*
* @param query a query that is going to be executed
* @param db database againt the query is going to be executed
* @param stream the resuts will be send to the stream
*
* @return query execution status:
* false if query wasn't executed successfully
* true if query execution was successfull
*/
auto Run(const std::string &query, Db &db, Stream &stream)
{
try
{
auto preprocessed = preprocessor.preprocess(query);
auto plan = LoadCypher(preprocessed);
auto result = plan->run(db, preprocessed.arguments, stream);
if (UNLIKELY(!result))
{
// info because it might be something like deadlock in which
// case one thread is stopped and user has try again
logger.info(
"Unable to execute query (executor returned false)");
"Unable to execute query (execution returned false)");
}
return result;
} catch (CypherLexicalError &e) {
}
catch (CypherLexicalError &e)
{
logger.error("CypherLexicalError: {}", std::string(e.what()));
throw e;
} catch (QueryEngineException &e) {
}
catch (QueryEngineException &e)
{
logger.error("QueryEngineException: {}", std::string(e.what()));
throw e;
} catch (std::exception &e) {
throw e;
}
catch (std::exception &e)
{
throw BasicException(e.what());
}
catch (...)
{
throw BasicException("unknown query engine exception");
}
}
// preload functionality
auto load(const uint64_t hash, const fs::path& path)
/**
* Unloads query plan and release the resources (should be automatically).
*
* @param query a query for which the query plan will be unloaded.
*
* return bool is the plan unloaded
*/
auto Unload(const std::string &query)
{
program_loader.load(hash, path);
return query_plans.access().remove(preprocessor.preprocess(query).hash);
}
auto load(const std::string& query)
/**
* Checks is a plan for the query loaded.
*
* @param query for which a plan existance will be checked
*
* return bool
*/
auto Loaded(const std::string &query)
{
program_loader.load(query);
auto plans_accessor = query_plans.access();
return plans_accessor.find(preprocessor.preprocess(query).hash) !=
plans_accessor.end();
}
protected:
Logger logger;
/**
* The number of loaded query plans.
*
* @return size_t the number of loaded query plans
*/
auto Size() { // TODO: const once whan ConcurrentMap::Accessor becomes const
return query_plans.access().size();
}
// return query_plans.access().size(); }
private:
ProgramExecutor<Stream> program_executor;
ProgramLoader<Stream> program_loader;
/**
* Loads query plan eather from hardcoded folder or from the file that is
* generated in this method.
*
* @param stripped a stripped query
*
* @return runnable query plan
*/
auto LoadCypher(const StrippedQuery<HashT> &stripped)
{
auto plans_accessor = query_plans.access();
// code is already compiled and loaded, just return runnable
// instance
auto query_plan_it = plans_accessor.find(stripped.hash);
if (query_plan_it != plans_accessor.end())
return query_plan_it->second->instance();
// find hardcoded query plan if exists
auto hardcoded_path =
fs::path(CONFIG(config::COMPILE_PATH) + "hardcode/" +
std::to_string(stripped.hash) + ".cpp");
if (fs::exists(hardcoded_path))
return LoadCpp(hardcoded_path, stripped.hash);
// generate query plan
auto generated_path =
fs::path(CONFIG(config::COMPILE_PATH) + std::to_string(stripped.hash) + ".cpp");
plan_generator.generate_plan(stripped.query, stripped.hash,
generated_path);
return LoadCpp(generated_path, stripped.hash);
}
/**
* Load cpp query plan from a file. Or if plan is already cached from the
* cache.
*
* @param path_cpp a path to query plan
* @param hash query hash
*
* @return runnable query plan
*/
auto LoadCpp(const fs::path &path_cpp, const QueryPreprocessor::HashT hash)
{
auto plans_accessor = query_plans.access();
// code is already compiled and loaded, just return runnable
// instance
auto query_plan_it = plans_accessor.find(hash);
if (query_plan_it != plans_accessor.end())
return query_plan_it->second->instance();
// generate dynamic lib path
// The timestamp has been added here because dlopen
// uses path and returns the same handler for the same path
// and that is a problem because at this point we want brand new
// dynamic lib. That is the tmp solution. The right solution would be
// to deal with this problem in DynamicLib
auto path_so = CONFIG(config::COMPILE_PATH) + std::to_string(hash) +
"_" + (std::string)Timestamp::now() + ".so";
plan_compiler.compile(path_cpp, path_so);
auto query_plan = std::make_unique<QueryPlanLib>(path_so);
// TODO: underlying object has to be live during query execution
// fix that when Antler will be introduced into the database
auto query_plan_instance = query_plan->instance(); // because of move
plans_accessor.insert(hash, std::move(query_plan));
// return an instance of runnable code (PlanInterface)
return query_plan_instance;
}
QueryPreprocessor preprocessor;
PlanGenerator<cypher::Frontend, CypherBackend<Stream>> plan_generator;
PlanCompiler plan_compiler;
ConcurrentMap<QueryPreprocessor::HashT, std::unique_ptr<QueryPlanLib>>
query_plans;
};

20
include/query/i_plan_cpu.hpp
View File

@@ -1,20 +0,0 @@
#pragma once
#include "communication/communication.hpp"
#include "database/db.hpp"
#include "database/db_accessor.hpp"
#include "query/strip/stripped.hpp"
template <typename Stream>
class IPlanCPU
{
public:
virtual bool run(Db &db, plan_args_t &args, Stream &stream) = 0;
virtual ~IPlanCPU() {}
};
template <typename Stream>
using produce_t = IPlanCPU<Stream> *(*)();
template <typename Stream>
using destruct_t = void (*)(IPlanCPU<Stream> *);

0
include/query/ir/tree/create.hpp
View File

0
include/query/ir/tree/delete.hpp
View File

0
include/query/ir/tree/edge.hpp
View File

27
include/query/ir/tree/node.hpp
View File

@@ -1,27 +0,0 @@
#pragma once
#include <memory>
#include <vector>
#include "query/backend/backend.hpp"
namespace ir
{
class Node
{
public:
virtual ~Node() {}
virtual void accept(Backend* visitor)
{
for (auto &child : childs)
{
visitor->process(child.get());
}
}
std::vector<std::unique_ptr<Node>> childs;
Node *parent;
};
}

5
include/query/ir/tree/query.hpp
View File

@@ -1,5 +0,0 @@
#pragma once
class Query : public Node
{
};

0
include/query/ir/tree/read.hpp
View File

0
include/query/ir/tree/tree.hpp
View File

0
include/query/ir/tree/update.hpp
View File

0
include/query/ir/tree/vertex.hpp
View File

26
include/query/plan/program.hpp
View File

@@ -1,26 +0,0 @@
#pragma once
#include "query/i_plan_cpu.hpp"
#include "query/strip/stripped.hpp"
/*
* Query Program Contains:
* * Query Plan
* * Query Arguments (Stripped)
*/
template <typename Stream>
struct QueryProgram
{
using plan_t = IPlanCPU<Stream>;
QueryProgram(plan_t *plan, QueryStripped &&stripped)
: plan(plan), stripped(std::forward<QueryStripped>(stripped))
{
}
QueryProgram(QueryProgram &other) = delete;
QueryProgram(QueryProgram &&other) = default;
plan_t *plan;
QueryStripped stripped;
};

30
include/query/plan/program_executor.hpp
View File

@@ -1,30 +0,0 @@
#pragma once
#include <string>
#include "database/db.hpp"
#include "query/exception/query_engine.hpp"
#include "query/exception/plan_execution.hpp"
#include "query/plan/program.hpp"
#include "query/util.hpp"
// preparations before execution
// execution
// postprocess the results
template <typename Stream>
class ProgramExecutor
{
public:
// QueryProgram has to know about the Stream
// Stream has to be passed in this function for every execution
auto execute(QueryProgram<Stream> &program, Db &db, Stream &stream)
{
try {
return program.plan->run(db, program.stripped.arguments, stream);
// TODO: catch more exceptions
} catch (...) {
throw PlanExecutionException("");
}
}
};

114
include/query/plan/program_loader.hpp
View File

@@ -1,114 +0,0 @@
#pragma once
#include <memory>
#include <string>
#include <unordered_map>
#include <experimental/filesystem>
#include "config/config.hpp"
#include "logging/default.hpp"
#include "query/backend/cpp_old/cypher.hpp"
#include "query/dynamic_lib.hpp"
#include "query/frontend/cypher.hpp"
#include "query/plan/compiler.hpp"
#include "query/plan/generator.hpp"
#include "query/plan/program.hpp"
#include "query/preprocesor.hpp"
#include "utils/file.hpp"
#include "utils/hashing/fnv.hpp"
namespace fs = std::experimental::filesystem;
template <typename Stream>
class ProgramLoader
{
public:
using code_lib_t = CodeLib<Stream>;
using sptr_code_lib = std::shared_ptr<code_lib_t>;
using query_program_t = QueryProgram<Stream>;
ProgramLoader() : logger(logging::log->logger("PlanLoader")) {}
// TODO: decouple load(query) method
auto load(const uint64_t hash, const fs::path &path)
{
// TODO: get lib path (that same folder as path folder or from config)
// TODO: compile
// TODO: dispose the old lib
// TODO: store the compiled lib
}
auto load(const std::string &query)
{
auto preprocessed = preprocessor.preprocess(query);
logger.debug("stripped_query = {}", preprocessed.query);
logger.debug("query_hash = {}", std::to_string(preprocessed.hash));
auto code_lib_iter = code_libs.find(preprocessed.hash);
// code is already compiled and loaded, just return runnable
// instance
if (code_lib_iter != code_libs.end()) {
auto code = code_lib_iter->second->instance();
return query_program_t(code, std::move(preprocessed));
}
auto base_path = CONFIG(config::COMPILE_CPU_PATH);
auto hash_string = std::to_string(preprocessed.hash);
auto path_cpp = base_path + hash_string + ".cpp";
auto hard_code_cpp = base_path + "hardcode/" + hash_string + ".cpp";
auto stripped_space = preprocessor.strip_space(query);
// cpp files in the hardcode folder have bigger priority then
// other cpp files
if (!utils::fexists(hard_code_cpp)) {
plan_generator.generate_plan(stripped_space.query,
preprocessed.hash, path_cpp);
}
// compile the code
auto path_so = base_path + hash_string + ".so";
// hardcoded queries are compiled to the same folder as generated
// queries (all .so files are in the same folder)
if (utils::fexists(hard_code_cpp)) {
plan_compiler.compile(hard_code_cpp, path_so);
} else {
plan_compiler.compile(path_cpp, path_so);
}
// loads dynamic lib and store it
auto code_lib = load_code_lib(path_so);
code_libs.insert({{preprocessed.hash, code_lib}});
// return an instance of runnable code (ICodeCPU)
return query_program_t(code_lib->instance(), std::move(preprocessed));
}
protected:
Logger logger;
private:
// TODO ifdef MEMGRAPH64 problem, how to use this kind
// of ifdef functions?
// uint64_t depends on fnv function
// TODO: faster datastructure
std::unordered_map<uint64_t, sptr_code_lib> code_libs;
QueryPreprocessor preprocessor;
// TODO: compile time switch between frontends and backends
using frontend_t = cypher::Frontend;
using backend_t = CypherBackend<Stream>;
PlanGenerator<frontend_t, backend_t> plan_generator;
PlanCompiler plan_compiler;
sptr_code_lib load_code_lib(const std::string &path)
{
sptr_code_lib code_lib = std::make_shared<CodeLib<Stream>>(path);
code_lib->load();
return code_lib;
}
};

38
include/query/plan/compiler.hpp → include/query/plan_compiler.hpp
View File

@@ -5,16 +5,28 @@
#include "logging/default.hpp"
#include "query/exception/plan_compilation.hpp"
#include "utils/string/join.hpp"
#include "logging/loggable.hpp"
// TODO:
// * all libraries have to be compiled in the server compile time
// * compile command has to be generated
class PlanCompiler
/**
* Compiles code into shared object (.so)
*/
class PlanCompiler : public Loggable
{
public:
PlanCompiler() : logger(logging::log->logger("PlanCompiler")) {}
PlanCompiler() : Loggable("PlanCompiler") {}
/**
* Compiles in_file into out_file (.cpp -> .so)
*
* @param in_file C++ file that can be compiled into dynamic lib
* @param out_file dynamic lib (on linux .so)
*
* @return void
*/
void compile(const std::string &in_file, const std::string &out_file)
{
std::string flags;
@@ -39,17 +51,22 @@ public:
flags += " -DLOG_NO_ERROR";
#endif
// TODO: load from config (generate compile command)
// generate compile command
auto compile_command = utils::prints(
"clang++" + flags,
// "-std=c++1y -O2 -DNDEBUG", // compile flags
"-std=c++1y", // compile flags // TODO: load from config file
// "-std=c++1y -O2 -DNDEBUG",
"-std=c++1y", // compile flags
in_file, // input file
"-o", out_file, // ouput file
"-I./include", // include paths (TODO: parameter)
"-I./include", // include paths
"-I../include",
"-I../libs/fmt", // TODO: load from config
"-I../../libs/fmt", "-L./ -L../",
"-I../../include",
"-I../../../include",
"-I../libs/fmt",
"-I../../libs/fmt",
"-I../../../libs/fmt",
"-L./ -L../ -L../../",
"-lmemgraph_pic",
"-shared -fPIC" // shared library flags
);
@@ -59,8 +76,10 @@ public:
// synchronous call
auto compile_status = system(compile_command.c_str());
logger.debug("compile status {}", compile_status);
// if compilation has failed throw exception
if (compile_status == -1) {
if (compile_status != 0) {
logger.debug("FAIL: Query Code Compilation: {} -> {}", in_file,
out_file);
throw PlanCompilationException(
@@ -71,7 +90,4 @@ public:
logger.debug("SUCCESS: Query Code Compilation: {} -> {}", in_file,
out_file);
}
protected:
Logger logger;
};

0
include/query/plan/generator.hpp → include/query/plan_generator.hpp
View File

50
include/query/plan_interface.hpp Normal file
View File

@@ -0,0 +1,50 @@
#pragma once
#include "communication/communication.hpp"
#include "database/db.hpp"
#include "database/db_accessor.hpp"
#include "query/stripped.hpp"
/**
* @class PlanInterface
*
* @brief Pure Abstract class that is interface to query plans.
*
* @tparam Stream stream for results writing.
*/
template <typename Stream>
class PlanInterface
{
public:
/**
* In derived classes this method has to be overriden in order to implement
* concrete execution plan.
*
* @param db active database instance
* @param args plan argument (vector of literal values from the query)
* @param stream stream for results writing
*
* @return bool status after execution (success OR fail)
*/
virtual bool run(Db &db, const PlanArgsT &args, Stream &stream) = 0;
/**
* Virtual destructor in base class.
*/
virtual ~PlanInterface() {}
};
/**
* Defines type of underlying extern C functions and library object class name.
* (ObjectPrototype).
*
* @tparam underlying object depends on Stream template parameter because
* it will send the results throughout a custom Stream object.
*/
template <typename Stream>
struct QueryPlanTrait
{
using ObjectPrototype = PlanInterface<Stream>;
using ProducePrototype = PlanInterface<Stream> *(*)();
using DestructPrototype = void (*)(PlanInterface<Stream> *);
};

36
include/query/preprocesor.hpp
View File

@@ -1,36 +0,0 @@
#pragma once
#include "query/strip/stripper.hpp"
/*
* Query preprocessing contains:
* * query stripping
*
* The preprocessing results are:
* * stripped query |
* * stripped arguments |-> QueryStripped
* * stripped query hash |
*/
class QueryPreprocessor
{
public:
QueryPreprocessor()
: stripper(make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL))
{
}
auto strip_space(const std::string& query)
{
return stripper.strip_space(query);
}
auto preprocess(const std::string &query) { return stripper.strip(query); }
private:
// In C++17 the ints should be unnecessary?
// as far as I understand in C++17 class template parameters
// can be deduced just like function template parameters
// TODO: once C++ 17 will be well suported by comilers
// refactor this piece of code
QueryStripper<int, int, int, int> stripper;
};

59
include/query/preprocessor.hpp Normal file
View File

@@ -0,0 +1,59 @@
#pragma once
#include "logging/loggable.hpp"
#include "query/stripper.hpp"
/*
* Query preprocessing contains:
* * query stripping
*
* This class is here because conceptually process of query preprocessing
* might have more than one step + in current version of C++ standard
* isn't trivial to instantiate QueryStripper because of template arguments +
* it depends on underlying lexical analyser.
*
* The preprocessing results are:
* * stripped query |
* * stripped arguments |-> StrippedQuery
* * stripped query hash |
*/
class QueryPreprocessor : public Loggable
{
private:
// In C++17 the ints will be removed.
// as far as I understand in C++17 class template parameters
// can be deduced just like function template parameters
// TODO: once C++ 17 will be well suported by comilers
// refactor this piece of code because it is hard to maintain.
using QueryStripperT = QueryStripper<int, int, int, int>;
public:
using HashT = QueryStripperT::HashT;
QueryPreprocessor() : Loggable("QueryPreprocessor"),
stripper(make_query_stripper(TK_LONG, TK_FLOAT, TK_STR, TK_BOOL))
{
}
/**
* Preprocess the query:
* * strip parameters
* * calculate query hash
*
* @param query that is going to be stripped
*
* @return QueryStripped object
*/
auto preprocess(const std::string &query)
{
auto preprocessed = stripper.strip(query);
logger.info("stripped_query = {}", preprocessed.query);
logger.info("query_hash = {}", preprocessed.hash);
return stripper.strip(query);
}
private:
QueryStripperT stripper;
};

32
include/query/strip/stripped.hpp
View File

@@ -1,32 +0,0 @@
#pragma once
#include <vector>
#include "storage/model/properties/property.hpp"
/*
* Query Plan Arguments Type
*/
using plan_args_t = std::vector<Property>;
/*
* QueryStripped contains:
* * stripped query
* * plan arguments stripped from query
* * hash of stripped query
*/
struct QueryStripped
{
QueryStripped(const std::string &&query, plan_args_t &&arguments,
uint64_t hash)
: query(std::forward<const std::string>(query)),
arguments(std::forward<plan_args_t>(arguments)), hash(hash)
{
}
QueryStripped(QueryStripped &other) = delete;
QueryStripped(QueryStripped &&other) = default;
std::string query;
plan_args_t arguments;
uint64_t hash;
};

55
include/query/stripped.hpp Normal file
View File

@@ -0,0 +1,55 @@
#pragma once
#include "storage/model/properties/property.hpp"
/*
* Query Plan Arguments Type
*/
using PlanArgsT = std::vector<Property>;
/*
* StrippedQuery contains:
* * stripped query
* * plan arguments stripped from query
* * hash of stripped query
*
* @tparam THash a type of query hash
*/
template <typename THash>
struct StrippedQuery
{
StrippedQuery(const std::string &&query, PlanArgsT &&arguments, THash hash)
: query(std::forward<const std::string>(query)),
arguments(std::forward<PlanArgsT>(arguments)), hash(hash)
{
}
/**
* Copy constructor is deleted because we don't want to make unecessary
* copies of this object (copying of string and vector could be expensive)
*/
StrippedQuery(const StrippedQuery &other) = delete;
StrippedQuery &operator=(const StrippedQuery &other) = delete;
/**
* Move is allowed operation because it is not expensive and we can
* move the object after it was created.
*/
StrippedQuery(StrippedQuery &&other) = default;
StrippedQuery &operator=(StrippedQuery &&other) = default;
/**
* Stripped query
*/
const std::string query;
/**
* Stripped arguments
*/
const PlanArgsT arguments;
/**
* Hash based on stripped query.
*/
const THash hash;
};

40
include/query/strip/stripper.hpp → include/query/stripper.hpp
View File

@@ -1,5 +1,6 @@
#pragma once
#include <vector>
#include <iostream>
#include <string>
#include <tuple>
@@ -9,23 +10,42 @@
#include "cypher/cypher.h"
#include "logging/loggable.hpp"
#include "query/language/cypher/tokenizer/cypher_lexer.hpp"
#include "query/strip/stripped.hpp"
#include "storage/model/properties/all.hpp"
#include "utils/hashing/fnv.hpp"
#include "utils/string/transform.hpp"
#include "utils/variadic/variadic.hpp"
#include "query/stripped.hpp"
// TODO: all todos will be resolved once Antler will be integrated
// TODO: Maybe std::move(v) is faster, but it must be cheked for validity.
template <class T, class V>
void store_query_param(plan_args_t &arguments, V &&v)
void store_query_param(PlanArgsT &arguments, V &&v)
{
arguments.emplace_back(Property(T(std::move(v)), T::type));
}
// TODO: hash function should be a template parameter
// TODO: move StrippedQuery into this file, maybe into QueryStripper object
/**
* @class QueryStripper
*
* @brief QueryStripper is responsible for the query stripping
* (taking literal values from the query) and for hash calculation based
* on the stripped query. The whole task is done at once and StrippedQuery
* object is returned as a result.
*
* @tparam Ts type of token ids from underlying lexical analyser. The
* lexical analyser is needed because we have to know what is and
* what isn't a literal value.
*/
template <typename... Ts>
class QueryStripper : public Loggable
{
public:
using HashT = uint64_t;
QueryStripper(Ts &&... strip_types)
: Loggable("QueryStripper"),
strip_types(std::make_tuple(std::forward<Ts>(strip_types)...)),
@@ -41,9 +61,7 @@ public:
{
}
auto strip_space(const std::string &query) { return strip(query, " "); }
auto strip(const std::string &query, const std::string &separator = "")
auto strip(const std::string &query, const std::string &separator = " ")
{
// -------------------------------------------------------------------
// TODO: write speed tests and then optimize, because this
@@ -60,7 +78,7 @@ public:
constexpr auto size = std::tuple_size<decltype(strip_types)>::value;
int counter = 0;
plan_args_t stripped_arguments;
PlanArgsT stripped_arguments;
std::string stripped_query;
stripped_query.reserve(query.size());
@@ -104,6 +122,9 @@ public:
// should be the same
// TODO: probably we shoud do the lowercase before
// or during the tokenization (SPEED TESTS)
// TODO: stripped shouldn't be responsible for the process
// of lowercasing -> reorganize this in the process of
// Antlr integration
if (token.id == TK_OR || token.id == TK_AND ||
token.id == TK_NOT || token.id == TK_WITH ||
token.id == TK_SET || token.id == TK_CREATE ||
@@ -120,10 +141,9 @@ public:
}
}
// TODO: hash function should be a template parameter
auto hash = fnv(stripped_query);
return QueryStripped(std::move(stripped_query),
std::move(stripped_arguments), hash);
return StrippedQuery<HashT>(std::move(stripped_query),
std::move(stripped_arguments), hash);
}
private:
@@ -134,7 +154,7 @@ private:
bool _or(Value &&value, Tuple &&tuple, std::index_sequence<index...>)
{
return utils::or_vargs(std::forward<Value>(value),
std::get<index>(std::forward<Tuple>(tuple))...);
std::get<index>(std::forward<Tuple>(tuple))...);
}
};

9
include/utils/command_line/arguments.hpp
View File

@@ -45,16 +45,17 @@ class ProgramArguments {
bool is_valid() {
for (const auto &arg : required_arguments_)
for (const auto &a : arguments_)
if (!arguments_.count(arg)) return false;
return true;
}
protected:
ProgramArguments() {}
~ProgramArguments() {}
ProgramArguments() {
}
public:
~ProgramArguments() {
}
class Argument {
private:
std::string arg_;

122
include/utils/dynamic_lib.hpp Normal file
View File

@@ -0,0 +1,122 @@
#pragma once
#include <dlfcn.h>
#include <experimental/filesystem>
#include <stdexcept>
#include <string>
namespace fs = std::experimental::filesystem;
#include "logging/loggable.hpp"
/**
* DynamicLib is a wrapper aroung dynamic object returned by dlopen.
*
* Dynamic object must have extern C functions which names should be
* "produce" and "destruct" (that's by convention).
*
* The functions prototypes can be defined with template parameter
* type trait (T).
*
* DynamicLib isn't implemented for concurrent access.
*
* @tparam T type trait which defines the prototypes of extern C functions
* of undelying dynamic object.
*/
template <typename T>
class DynamicLib : public Loggable
{
public:
/**
* Initializes dynamic library (loads lib, produce and
* destruct functions)
*
* @param lib_path file system path to dynamic library
*/
DynamicLib(const fs::path &lib_path)
: Loggable("DynamicLib"), lib_path(lib_path), lib_object(nullptr)
{
// load dynamic lib
dynamic_lib = dlopen(lib_path.c_str(), RTLD_NOW);
if (!dynamic_lib) throw std::runtime_error(dlerror());
dlerror(); /* Clear any existing error */
logger.trace("dynamic lib at ADDRESS({}) was opened", dynamic_lib);
// load produce method
this->produce_method =
(typename T::ProducePrototype)dlsym(dynamic_lib, "produce");
const char *dlsym_produce_error = dlerror();
if (dlsym_produce_error) throw std::runtime_error(dlsym_produce_error);
// load destruct method
this->destruct_method =
(typename T::DestructPrototype)dlsym(dynamic_lib, "destruct");
const char *dlsym_destruct_error = dlerror();
if (dlsym_destruct_error)
throw std::runtime_error(dlsym_destruct_error);
}
// becuase we are dealing with pointers
// and conceptualy there is no need to copy the instance of this class
// the copy constructor and copy assignment operator are deleted
// the same applies to move methods
DynamicLib(const DynamicLib &other) = delete;
DynamicLib &operator=(const DynamicLib &other) = delete;
DynamicLib(DynamicLib &&other) = delete;
DynamicLib &operator=(DynamicLib &&other) = delete;
/**
* Singleton method. Returns the same instance of underlying class
* for every call. The instance of underlying class is returned by
* extern C produce function.
*
* @return T the instance of lib class
*/
typename T::ObjectPrototype *instance()
{
if (lib_object == nullptr) lib_object = this->produce_method();
return lib_object;
}
/**
* Clean underlying object and close the lib
*/
~DynamicLib()
{
// first destroy underlying object
if (lib_object != nullptr)
{
logger.trace("shared object at ADDRESS({}) will be destroyed",
(void *)lib_object);
this->destruct_method(lib_object);
}
// then destroy dynamic lib
logger.trace("unloading lib {}", lib_path.c_str());
if (dynamic_lib != nullptr)
{
logger.trace("closing dynamic lib ADDRESS({})",
(void *)dynamic_lib);
// // IMPORTANT: weird problem the program SEGFAULT on dlclose
// // TODO: FIX somehow
// // maybe something similar is:
// //
// http://stackoverflow.com/questions/6450828/segmentation-fault-when-using-dlclose-on-android-platform
// // for now it is not crucial so I've created a task for that
// // ! 0 is success
// int closing_status = dlclose(dynamic_lib);
// if (closing_status != 0)
// throw std::runtime_error("dynamic lib closing error");
}
else
{
logger.trace("unload lib was called but lib ptr is null");
}
}
private:
std::string lib_path;
void *dynamic_lib;
typename T::ObjectPrototype *lib_object;
typename T::ProducePrototype produce_method;
typename T::DestructPrototype destruct_method;
};

58
include/utils/file.hpp
View File

@@ -2,19 +2,59 @@
#include <fstream>
// TODO: remove experimental from here once that becomes possible (C++17
// standard)
#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;
namespace utils
{
inline bool fexists(const char *filename)
/**
* Loads all file paths in the specified directory. Optionally
* the paths are filtered by extension.
*
* NOTE: the call isn't recursive
*
* @param directory a path to directory that will be scanned in order to find
* all paths
* @param extension paths will be filtered by this extension
*
* @return std::vector of paths founded in the directory
*/
inline auto LoadFilePaths(const fs::path &directory,
const std::string &extension = "")
{
std::ifstream ifile(filename);
return (bool)ifile;
}
// result container
std::vector<fs::path> file_paths;
inline bool fexists(const std::string& filename)
{
std::ifstream ifile(filename.c_str());
return (bool)ifile;
}
for (auto &directory_entry : fs::recursive_directory_iterator(directory))
{
auto path = directory_entry.path().string();
// skip directories
if (!fs::is_regular_file(directory_entry)) continue;
// if extension isn't defined then put all file paths from the directory
// to the result set
if (!extension.empty()) {
// skip paths that don't have appropriate extension
auto file_extension = path.substr(path.find_last_of(".") + 1);
if (file_extension != extension) continue;
}
file_paths.emplace_back(path);
// skip paths that don't have appropriate extension
auto file_extension = path.substr(path.find_last_of(".") + 1);
if (file_extension != extension) continue;
// path has the right extension and can be placed in the result
// container
file_paths.emplace_back(path);
}
return file_paths;
}
}

46
include/utils/fswatcher.hpp
View File

@@ -54,19 +54,38 @@ constexpr uint64_t IN_HEADER_SIZE = sizeof(struct inotify_event);
* a reasonable size and doesn't have to be configurable before compile or run
* time.
*/
constexpr uint64_t IN_BUFF_LEN = 10 * (IN_HEADER_SIZE + NAME_MAX + 1);
constexpr uint64_t IN_BUFF_SLOT_LEN = IN_HEADER_SIZE + NAME_MAX + 1;
constexpr uint64_t IN_BUFF_LEN = 10 * IN_BUFF_SLOT_LEN;
/**
* File System Event Type - abstraction for underlying event types
*/
enum class FSEventType : os_mask_t
{
Created = IN_CREATE,
Modified = IN_MODIFY,
Deleted = IN_DELETE,
All = Created | Modified | Deleted
Accessed = IN_ACCESS,
MetadataChanged = IN_ATTRIB,
CloseWrite = IN_CLOSE_WRITE,
CloseNowrite = IN_CLOSE_NOWRITE,
Created = IN_CREATE,
Deleted = IN_DELETE,
DeletedSelf = IN_DELETE_SELF,
Modified = IN_MODIFY,
Renamed = IN_MOVE_SELF,
MovedFrom = IN_MOVED_FROM,
MovedTo = IN_MOVED_TO,
Close = IN_CLOSE,
Opened = IN_OPEN,
Ignored = IN_IGNORED,
All = Accessed | MetadataChanged | CloseWrite | CloseNowrite | Created |
Deleted | DeletedSelf | Modified | Renamed | MovedFrom | MovedTo |
Close | Opened | Ignored
};
inline FSEventType operator|(FSEventType lhs, FSEventType rhs)
{
return (FSEventType)(underlying_cast(lhs) | underlying_cast(rhs));
}
/**
* @struct FSEventBase
*
@@ -156,6 +175,8 @@ public:
FSWatcher(ms check_interval = ms(100))
: Loggable("FSWatcher"), check_interval_(check_interval)
{
logger.debug("Inotify header length: {}", IN_HEADER_SIZE);
logger.debug("Inotify buffer length: {}", IN_BUFF_LEN);
inotify_fd_ = inotify_init();
if (inotify_fd_ == -1)
throw FSWatcherException("Unable to initialize inotify\n");
@@ -286,6 +307,8 @@ public:
*/
void start()
{
if (is_running_.load()) return;
is_running_.store(true);
// run separate thread
@@ -313,9 +336,20 @@ public:
auto in_event = reinterpret_cast<in_event_t *>(p);
auto in_event_length = IN_HEADER_SIZE + in_event->len;
// sometimes inotify system returns event->len that is
// longer then the length of the buffer
// TODO: figure out why (it is not easy)
if (((p - buffer_) + in_event_length) > IN_BUFF_LEN) break;
// here should be an assertion
// runtime_assert(in_event_length <= IN_BUFF_SLOT_LEN,
// "Inotify event length cannot be bigger
// than "
// "Inotify slot length");
// skip if in_event is undefined OR is equal to IN_IGNORED
if ((in_event->len == 0 && in_event->mask == 0) ||
in_event->mask == IN_IGNORED)
in_event->mask == IN_IGNORED ||
in_event_length == IN_HEADER_SIZE) // skip empty paths
{
p += in_event_length;
continue;

5
include/utils/string/file.hpp
View File

@@ -1,8 +1,6 @@
#pragma once
#include <cerrno>
// TODO: remove experimental from here once that becomes possible
#include <experimental/filesystem>
#include <fstream>
#include <ostream>
#include <stdexcept>
@@ -11,7 +9,8 @@
#include <fmt/format.h>
// TODO: remove experimental from here once it becomes possible
// TODO: remove experimental from here once that becomes possible
#include <experimental/filesystem>
namespace fs = std::experimental::filesystem;
namespace utils

25
include/utils/string/trim.hpp Normal file
View File

@@ -0,0 +1,25 @@
#pragma once
#include <string>
namespace utils
{
/**
* Removes whitespace characters from the start and from the end of a string.
*
* @param str string that is going to be trimmed
*
* @return trimmed string
*/
std::string trim(const std::string &str)
{
size_t first = str.find_first_not_of(' ');
if (std::string::npos == first)
{
return str;
}
size_t last = str.find_last_not_of(' ');
return str.substr(first, (last - first + 1));
}
}

13
include/web/client.hpp
View File

@@ -1,13 +0,0 @@
#pragma once
#include <string>
namespace web
{
class Client
{
void post(const std::string& url, const std::string& body);
};
}

21
include/web/logger.hpp
View File

@@ -1,21 +0,0 @@
#pragma once
#include <string>
namespace web
{
class Logger
{
public:
Logger() { /* create connection */ }
// TODO: singleton
void up_ping();
void send(const std::string&);
void down_ping();
};
}