Compare commits

..

4 Commits

Author SHA1 Message Date
antoniofilipovic
01e1c7f4d4 remove unneccessary code comment 2024-02-07 11:51:31 +01:00
antoniofilipovic
8a50abfb69 add code comment 2024-02-07 11:49:07 +01:00
antoniofilipovic
6e37b46eee fix issues on memory limit 2024-02-07 11:47:58 +01:00
antoniofilipovic
5385c741ad fix bug 2024-02-07 11:23:27 +01:00
6 changed files with 12 additions and 58 deletions

1
libs/.gitignore vendored
View File

@@ -7,4 +7,3 @@
!pulsar.patch
!antlr4.10.1.patch
!rocksdb8.1.1.patch
!jemalloc-oversize_threshold.patch

View File

@@ -1,39 +0,0 @@
diff --git a/src/ctl.c b/src/ctl.c
index 48afaa61..1e545710 100644
--- a/src/ctl.c
+++ b/src/ctl.c
@@ -2376,11 +2376,11 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx);
MIB_UNSIGNED(arena_ind, 1);
- if (arena_ind < narenas_total_get()) {
+ if (arena_ind <= narenas_total_get()) {
extent_hooks_t *old_extent_hooks;
arena = arena_get(tsd_tsdn(tsd), arena_ind, false);
if (arena == NULL) {
- if (arena_ind >= narenas_auto) {
+ if (arena_ind >= narenas_auto+1) {
ret = EFAULT;
goto label_return;
}
@@ -2388,6 +2388,20 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen,
(extent_hooks_t *)&extent_hooks_default;
READ(old_extent_hooks, extent_hooks_t *);
if (newp != NULL) {
+ if (arena_ind == narenas_auto) {
+ // init some number for huge arena
+ arena_init_huge();
+ //init arena really
+ arena = arena_choose_huge(tsd);
+ extent_hooks_t *new_extent_hooks
+ JEMALLOC_CC_SILENCE_INIT(NULL);
+ WRITE(new_extent_hooks, extent_hooks_t *);
+ old_extent_hooks = extent_hooks_set(tsd, arena,
+ new_extent_hooks);
+ READ(old_extent_hooks, extent_hooks_t *);
+ ret = 0;
+ goto label_return;
+ }
/* Initialize a new arena as a side effect. */
extent_hooks_t *new_extent_hooks
JEMALLOC_CC_SILENCE_INIT(NULL);

View File

@@ -265,7 +265,7 @@ repo_clone_try_double "${primary_urls[jemalloc]}" "${secondary_urls[jemalloc]}"
# this is hack for cmake in libs to set path, and for FindJemalloc to use Jemalloc_INCLUDE_DIR
pushd jemalloc
patch -p1 < ../jemalloc-oversize_threshold.patch
./autogen.sh
MALLOC_CONF="retain:false,percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:5000,dirty_decay_ms:5000" \
./configure \

View File

@@ -60,10 +60,12 @@ void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t
unsigned arena_ind) {
// This needs to be before, to throw exception in case of too big alloc
if (*commit) [[likely]] {
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
// This needs to happen before global alloc, because if thread tracker throws
// we will be left with incorrect state of global alloc
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
}
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
}
auto *ptr = old_hooks->alloc(extent_hooks, new_addr, size, alignment, zero, commit, arena_ind);
@@ -224,20 +226,6 @@ void SetHooks() {
LOG_FATAL("Error setting custom hooks for jemalloc arena {}", i);
}
}
auto last_arena = n_arenas;
// For oversize arena, no need to read hooks, it will be intialized
// with arena, just set our custom hooks
std::string func_name = "arena." + std::to_string(last_arena) + ".extent_hooks";
if (err) {
LOG_FATAL("Error setting jemalloc hooks for jemalloc arena {}", last_arena);
}
err = mallctl(func_name.c_str(), nullptr, nullptr, &new_hooks, sizeof(new_hooks));
if (err) {
LOG_FATAL("Error setting custom hooks for jemalloc oversize threshold arena {}", last_arena);
}
#endif
}

View File

@@ -4172,6 +4172,10 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
utils::Downcast<ConstraintQuery>(parsed_query.query) != nullptr ||
upper_case_query.find(kSchemaAssert) != std::string::npos;
SetupDatabaseTransaction(could_commit, unique);
query_execution_ptr->get()->transaction_id_ = *current_db_.db_transactional_accessor_->GetTransactionId();
}
if (in_explicit_transaction_) {
query_execution_ptr->get()->transaction_id_ = *current_db_.db_transactional_accessor_->GetTransactionId();
}
#ifdef MG_ENTERPRISE

View File

@@ -312,6 +312,8 @@ class Interpreter final {
std::map<std::string, TypedValue> summary;
std::vector<Notification> notifications;
// Used for tracking of query and procedure memory limit
uint64_t transaction_id_;
static auto Create(std::variant<utils::MonotonicBufferResource, utils::PoolResource> memory_resource,
std::optional<PreparedQuery> prepared_query = std::nullopt) -> std::unique_ptr<QueryExecution> {
@@ -427,8 +429,8 @@ std::map<std::string, TypedValue> Interpreter::Pull(TStream *result_stream, std:
// If the query finished executing, we have received a value which tells
// us what to do after.
if (maybe_res) {
if (current_transaction_) {
memgraph::memory::TryStopTrackingOnTransaction(*current_transaction_);
if (query_execution->transaction_id_) {
memgraph::memory::TryStopTrackingOnTransaction(query_execution->transaction_id_);
}
// Save its summary
maybe_summary.emplace(std::move(query_execution->summary));