Compare commits
51 Commits
v2.12.0
...
agg-prop-l
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c5af23ee23 | ||
|
|
7fc9b89634 | ||
|
|
ed5bdb841f | ||
|
|
bb2a7b8f21 | ||
|
|
6680f99e4d | ||
|
|
72d47fc3bf | ||
|
|
3cf13e2deb | ||
|
|
7f5a55f1b2 | ||
|
|
70db2fca56 | ||
|
|
8b9e1fa08b | ||
|
|
e4f94c15c6 | ||
|
|
0ce6dcf194 | ||
|
|
1d90b60f56 | ||
|
|
108d83cc8c | ||
|
|
eff857447a | ||
|
|
d03fafcef6 | ||
|
|
c31a7f9648 | ||
|
|
6053a91ef8 | ||
|
|
645568a75b | ||
|
|
d3f4c35362 | ||
|
|
c037cddb0e | ||
|
|
ced08fd7bc | ||
|
|
9cc060c4b0 | ||
|
|
e671a0737e | ||
|
|
11be3972c4 | ||
|
|
fdab42a023 | ||
|
|
e5b2c19ea2 | ||
|
|
e907817854 | ||
|
|
0756cd6898 | ||
|
|
38ad5e2146 | ||
|
|
3c413a7e50 | ||
|
|
17915578f8 | ||
|
|
4e9a036881 | ||
|
|
c8fe9ee7d9 | ||
|
|
e4afddf518 | ||
|
|
4d5ea03dfa | ||
|
|
ece4b0dba8 | ||
|
|
66487a6dce | ||
|
|
f4b97fc03d | ||
|
|
58648d1a70 | ||
|
|
1ab7f6ac78 | ||
|
|
dbc6054689 | ||
|
|
16b8c7b27c | ||
|
|
48631d1e37 | ||
|
|
93e6d058d2 | ||
|
|
3e9f25b8e4 | ||
|
|
c94201621a | ||
|
|
fdbc390d53 | ||
|
|
5e6c5618f5 | ||
|
|
4aacd45640 | ||
|
|
157b36162b |
@@ -1,4 +1,4 @@
|
||||
name: Package All
|
||||
name: Package memgraph
|
||||
|
||||
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
|
||||
|
||||
@@ -6,18 +6,56 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
memgraph_version:
|
||||
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
|
||||
description: "Memgraph version to upload as. Leave this field empty if you don't want to upload binaries to S3. Format: 'X.Y.Z'"
|
||||
required: false
|
||||
build_type:
|
||||
type: choice
|
||||
description: "Memgraph Build type. Default value is Release."
|
||||
description: "Memgraph Build type. Default value is Release"
|
||||
default: 'Release'
|
||||
options:
|
||||
- Release
|
||||
- RelWithDebInfo
|
||||
target_os:
|
||||
type: choice
|
||||
description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS. Default is Ubuntu 22.04"
|
||||
default: 'ubuntu-22_04'
|
||||
options:
|
||||
- all
|
||||
- amzn-2
|
||||
- centos-7
|
||||
- centos-9
|
||||
- debian-10
|
||||
- debian-11
|
||||
- debian-11-arm
|
||||
- debian-11-platform
|
||||
- docker
|
||||
- fedora-36
|
||||
- ubuntu-18_04
|
||||
- ubuntu-20_04
|
||||
- ubuntu-22_04
|
||||
- ubuntu-22_04-arm
|
||||
|
||||
jobs:
|
||||
amzn-2:
|
||||
if: ${{ github.event.inputs.target_os == 'amzn-2' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: amzn-2
|
||||
path: build/output/amzn-2/memgraph*.rpm
|
||||
|
||||
centos-7:
|
||||
if: ${{ github.event.inputs.target_os == 'centos-7' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -35,6 +73,7 @@ jobs:
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
|
||||
centos-9:
|
||||
if: ${{ github.event.inputs.target_os == 'centos-9' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -52,6 +91,7 @@ jobs:
|
||||
path: build/output/centos-9/memgraph*.rpm
|
||||
|
||||
debian-10:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -69,6 +109,7 @@ jobs:
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
|
||||
debian-11:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -85,7 +126,44 @@ jobs:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
debian-11-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
if: ${{ github.event.inputs.target_os == 'debian-11-platform' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
docker:
|
||||
if: ${{ github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -104,75 +182,8 @@ jobs:
|
||||
name: docker
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
|
||||
ubuntu-1804:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-18.04
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-20.04
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
ubuntu-2204:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
|
||||
fedora-36:
|
||||
if: ${{ github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -189,7 +200,8 @@ jobs:
|
||||
name: fedora-36
|
||||
path: build/output/fedora-36/memgraph*.rpm
|
||||
|
||||
amzn-2:
|
||||
ubuntu-18_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
@@ -199,16 +211,17 @@ jobs:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
|
||||
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: amzn-2
|
||||
path: build/output/amzn-2/memgraph*.rpm
|
||||
name: ubuntu-18.04
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
debian-11-arm:
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
ubuntu-20_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
@@ -216,14 +229,33 @@ jobs:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
|
||||
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
name: ubuntu-20.04
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
ubuntu-2204-arm:
|
||||
ubuntu-22_04:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, X64]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- name: "Set up repository"
|
||||
uses: actions/checkout@v3
|
||||
with:
|
||||
fetch-depth: 0 # Required because of release/get_version.py
|
||||
- name: "Build package"
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
ubuntu-22_04-arm:
|
||||
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' || github.event.inputs.target_os == 'all' }}
|
||||
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
|
||||
timeout-minutes: 120
|
||||
steps:
|
||||
@@ -243,7 +275,7 @@ jobs:
|
||||
upload-to-s3:
|
||||
# only run upload if we specified version. Allows for runs without upload
|
||||
if: "${{ github.event.inputs.memgraph_version != '' }}"
|
||||
needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm]
|
||||
needs: [amzn-2, centos-7, centos-9, debian-10, debian-11, debian-11-arm, debian-11-platform, docker, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04, ubuntu-22_04-arm]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
22
.sonarcloud.properties
Normal file
22
.sonarcloud.properties
Normal file
@@ -0,0 +1,22 @@
|
||||
# Path to sources
|
||||
sonar.sources = .
|
||||
# sonar.exclusions=
|
||||
sonar.inclusions=src,include,query_modules
|
||||
|
||||
# Path to tests
|
||||
sonar.tests = tests/
|
||||
# sonar.test.exclusions=
|
||||
# sonar.test.inclusions=
|
||||
|
||||
# Source encoding
|
||||
# sonar.sourceEncoding=
|
||||
|
||||
# Exclusions for copy-paste detection
|
||||
# sonar.cpd.exclusions=
|
||||
|
||||
# Python version (for python projects only)
|
||||
# sonar.python.version=
|
||||
|
||||
# C++ standard version (for C++ projects only)
|
||||
# If not specified, it defaults to the latest supported standard
|
||||
# sonar.cfamily.reportingCppStandardOverride=c++98|c++11|c++14|c++17|c++20
|
||||
@@ -42,10 +42,6 @@ endif()
|
||||
|
||||
project(memgraph LANGUAGES C CXX)
|
||||
|
||||
# NOTE: once in a while this needs to be toggled to check headers are
|
||||
# correct and PCH isn't masking any include issues
|
||||
set(CMAKE_DISABLE_PRECOMPILE_HEADERS OFF)
|
||||
|
||||
#TODO: upgrade to cmake 3.24 + CheckIPOSupported
|
||||
#cmake_policy(SET CMP0138 NEW)
|
||||
#include(CheckIPOSupported)
|
||||
@@ -296,7 +292,7 @@ if (MG_ENTERPRISE)
|
||||
add_definitions(-DMG_ENTERPRISE)
|
||||
endif()
|
||||
|
||||
set(ENABLE_JEMALLOC ON)
|
||||
option(ENABLE_JEMALLOC "Use jemalloc" ON)
|
||||
|
||||
if (ASAN)
|
||||
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")
|
||||
|
||||
@@ -236,6 +236,54 @@ inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke<mgp_type *>(mgp
|
||||
|
||||
// mgp_graph
|
||||
|
||||
inline bool create_label_index(mgp_graph *graph, const char *label) {
|
||||
return MgInvoke<int>(mgp_create_label_index, graph, label);
|
||||
}
|
||||
|
||||
inline bool drop_label_index(mgp_graph *graph, const char *label) {
|
||||
return MgInvoke<int>(mgp_drop_label_index, graph, label);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_label_indices(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_label_indices, graph, memory);
|
||||
}
|
||||
|
||||
inline bool create_label_property_index(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_create_label_property_index, graph, label, property);
|
||||
}
|
||||
|
||||
inline bool drop_label_property_index(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_drop_label_property_index, graph, label, property);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_label_property_indices, graph, memory);
|
||||
}
|
||||
|
||||
inline bool create_existence_constraint(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_create_existence_constraint, graph, label, property);
|
||||
}
|
||||
|
||||
inline bool drop_existence_constraint(mgp_graph *graph, const char *label, const char *property) {
|
||||
return MgInvoke<int>(mgp_drop_existence_constraint, graph, label, property);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_existence_constraints, graph, memory);
|
||||
}
|
||||
|
||||
inline bool create_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) {
|
||||
return MgInvoke<int>(mgp_create_unique_constraint, memgraph_graph, label, properties);
|
||||
}
|
||||
|
||||
inline bool drop_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) {
|
||||
return MgInvoke<int>(mgp_drop_unique_constraint, memgraph_graph, label, properties);
|
||||
}
|
||||
|
||||
inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory) {
|
||||
return MgInvoke<mgp_list *>(mgp_list_all_unique_constraints, graph, memory);
|
||||
}
|
||||
|
||||
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
|
||||
|
||||
inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) {
|
||||
|
||||
@@ -876,6 +876,65 @@ enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *m
|
||||
enum mgp_error mgp_graph_get_vertex_by_id(struct mgp_graph *g, struct mgp_vertex_id id, struct mgp_memory *memory,
|
||||
struct mgp_vertex **result);
|
||||
|
||||
/// Creates label index for given label.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if label index already exists, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_label_index(struct mgp_graph *graph, const char *label, int *result);
|
||||
|
||||
/// Drop label index.
|
||||
enum mgp_error mgp_drop_label_index(struct mgp_graph *graph, const char *label, int *result);
|
||||
|
||||
/// List all label indices.
|
||||
enum mgp_error mgp_list_all_label_indices(struct mgp_graph *graph, struct mgp_memory *memory, struct mgp_list **result);
|
||||
|
||||
/// Creates label-property index for given label and propery.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if label property index already exists, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_label_property_index(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// Drops label-property index for given label and propery.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if dropping label property index failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_drop_label_property_index(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// List all label+property indices.
|
||||
enum mgp_error mgp_list_all_label_property_indices(struct mgp_graph *graph, struct mgp_memory *memory,
|
||||
struct mgp_list **result);
|
||||
|
||||
/// Creates existence constraint for given label and property.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if creating existence constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_existence_constraint(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// Drops existence constraint for given label and property.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if dropping existence constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_drop_existence_constraint(struct mgp_graph *graph, const char *label, const char *property,
|
||||
int *result);
|
||||
|
||||
/// List all existence constraints.
|
||||
enum mgp_error mgp_list_all_existence_constraints(struct mgp_graph *graph, struct mgp_memory *memory,
|
||||
struct mgp_list **result);
|
||||
|
||||
/// Creates unique constraint for given label and properties.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if creating unique constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_create_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties,
|
||||
int *result);
|
||||
|
||||
/// Drops unique constraint for given label and properties.
|
||||
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
|
||||
/// if dropping unique constraint failed, result will be 0, otherwise 1.
|
||||
enum mgp_error mgp_drop_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties,
|
||||
int *result);
|
||||
|
||||
/// List all unique constraints
|
||||
enum mgp_error mgp_list_all_unique_constraints(struct mgp_graph *graph, struct mgp_memory *memory,
|
||||
struct mgp_list **result);
|
||||
|
||||
/// Result is non-zero if the graph can be modified.
|
||||
/// If a graph is immutable, then vertices cannot be created or deleted, and all of the returned vertices will be
|
||||
/// immutable also. The same applies for edges.
|
||||
|
||||
374
include/mgp.hpp
374
include/mgp.hpp
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,7 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
|
||||
add_library(example_c SHARED example.c)
|
||||
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(example_c PRIVATE -Wall)
|
||||
target_link_libraries(example_c PRIVATE -static-libgcc -static-libstdc++)
|
||||
# Strip C example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET example_c POST_BUILD
|
||||
@@ -28,6 +29,7 @@ install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
|
||||
add_library(example_cpp SHARED example.cpp)
|
||||
target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(example_cpp PRIVATE -Wall)
|
||||
target_link_libraries(example_cpp PRIVATE -static-libgcc -static-libstdc++)
|
||||
# Strip C++ example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET example_cpp POST_BUILD
|
||||
@@ -43,6 +45,7 @@ install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
|
||||
add_library(schema SHARED schema.cpp)
|
||||
target_include_directories(schema PRIVATE ${CMAKE_SOURCE_DIR}/include)
|
||||
target_compile_options(schema PRIVATE -Wall)
|
||||
target_link_libraries(schema PRIVATE -static-libgcc -static-libstdc++)
|
||||
# Strip C++ example in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET schema POST_BUILD
|
||||
|
||||
@@ -10,18 +10,33 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <mgp.hpp>
|
||||
#include "utils/string.hpp"
|
||||
|
||||
#include <optional>
|
||||
|
||||
namespace Schema {
|
||||
|
||||
/*NodeTypeProperties and RelTypeProperties constants*/
|
||||
constexpr std::string_view kStatusKept = "Kept";
|
||||
constexpr std::string_view kStatusCreated = "Created";
|
||||
constexpr std::string_view kStatusDropped = "Dropped";
|
||||
constexpr std::string_view kReturnNodeType = "nodeType";
|
||||
constexpr std::string_view kProcedureNodeType = "node_type_properties";
|
||||
constexpr std::string_view kProcedureRelType = "rel_type_properties";
|
||||
constexpr std::string_view kProcedureAssert = "assert";
|
||||
constexpr std::string_view kReturnLabels = "nodeLabels";
|
||||
constexpr std::string_view kReturnRelType = "relType";
|
||||
constexpr std::string_view kReturnPropertyName = "propertyName";
|
||||
constexpr std::string_view kReturnPropertyType = "propertyTypes";
|
||||
constexpr std::string_view kReturnMandatory = "mandatory";
|
||||
constexpr std::string_view kReturnLabel = "label";
|
||||
constexpr std::string_view kReturnKey = "key";
|
||||
constexpr std::string_view kReturnKeys = "keys";
|
||||
constexpr std::string_view kReturnUnique = "unique";
|
||||
constexpr std::string_view kReturnAction = "action";
|
||||
constexpr std::string_view kParameterIndices = "indices";
|
||||
constexpr std::string_view kParameterUniqueConstraints = "unique_constraints";
|
||||
constexpr std::string_view kParameterExistenceConstraints = "existence_constraints";
|
||||
constexpr std::string_view kParameterDropExisting = "drop_existing";
|
||||
|
||||
std::string TypeOf(const mgp::Type &type);
|
||||
|
||||
@@ -35,6 +50,7 @@ void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, con
|
||||
|
||||
void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
void Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
|
||||
} // namespace Schema
|
||||
|
||||
/*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this
|
||||
@@ -92,21 +108,21 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t
|
||||
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
|
||||
}
|
||||
|
||||
void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result,
|
||||
mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
;
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
const mgp::Graph graph = mgp::Graph(memgraph_graph);
|
||||
for (auto node : graph.Nodes()) {
|
||||
std::string type = "";
|
||||
std::string type;
|
||||
mgp::List labels = mgp::List();
|
||||
for (auto label : node.Labels()) {
|
||||
labels.AppendExtend(mgp::Value(label));
|
||||
type += ":`" + std::string(label) + "`";
|
||||
}
|
||||
|
||||
if (node.Properties().size() == 0) {
|
||||
if (node.Properties().empty()) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
|
||||
continue;
|
||||
@@ -126,16 +142,15 @@ void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_r
|
||||
}
|
||||
}
|
||||
|
||||
void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
;
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
try {
|
||||
const mgp::Graph graph = mgp::Graph(memgraph_graph);
|
||||
|
||||
for (auto rel : graph.Relationships()) {
|
||||
std::string type = ":`" + std::string(rel.Type()) + "`";
|
||||
if (rel.Properties().size() == 0) {
|
||||
if (rel.Properties().empty()) {
|
||||
auto record = record_factory.NewRecord();
|
||||
ProcessPropertiesRel<std::string>(record, type, "", "", false);
|
||||
continue;
|
||||
@@ -155,29 +170,435 @@ void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_re
|
||||
}
|
||||
}
|
||||
|
||||
void InsertRecordForLabelIndex(const auto &record_factory, const std::string_view label,
|
||||
const std::string_view status) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
|
||||
record.Insert(std::string(Schema::kReturnKey).c_str(), "");
|
||||
record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List());
|
||||
record.Insert(std::string(Schema::kReturnUnique).c_str(), false);
|
||||
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
|
||||
}
|
||||
|
||||
void InsertRecordForUniqueConstraint(const auto &record_factory, const std::string_view label,
|
||||
const mgp::List &properties, const std::string_view status) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
|
||||
record.Insert(std::string(Schema::kReturnKey).c_str(), properties.ToString());
|
||||
record.Insert(std::string(Schema::kReturnKeys).c_str(), properties);
|
||||
record.Insert(std::string(Schema::kReturnUnique).c_str(), true);
|
||||
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
|
||||
}
|
||||
|
||||
void InsertRecordForLabelPropertyIndexAndExistenceConstraint(const auto &record_factory, const std::string_view label,
|
||||
const std::string_view property,
|
||||
const std::string_view status) {
|
||||
auto record = record_factory.NewRecord();
|
||||
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
|
||||
record.Insert(std::string(Schema::kReturnKey).c_str(), property);
|
||||
record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List({mgp::Value(property)}));
|
||||
record.Insert(std::string(Schema::kReturnUnique).c_str(), false);
|
||||
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
|
||||
}
|
||||
|
||||
void ProcessCreatingLabelIndex(const std::string_view label, const std::set<std::string_view> &existing_label_indices,
|
||||
mgp_graph *memgraph_graph, const auto &record_factory) {
|
||||
if (existing_label_indices.contains(label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusKept);
|
||||
} else if (mgp::CreateLabelIndex(memgraph_graph, label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TFunc>
|
||||
void ProcessCreatingLabelPropertyIndexAndExistenceConstraint(const std::string_view label,
|
||||
const std::string_view property,
|
||||
const std::set<std::string_view> &existing_collection,
|
||||
const TFunc &func_creation, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory) {
|
||||
const auto label_property_search_key = std::string(label) + ":" + std::string(property);
|
||||
if (existing_collection.contains(label_property_search_key)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusKept);
|
||||
} else if (func_creation(memgraph_graph, label, property)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusCreated);
|
||||
}
|
||||
}
|
||||
|
||||
/// We collect properties for which index was created.
|
||||
using AssertedIndices = std::set<std::string, std::less<>>;
|
||||
AssertedIndices CreateIndicesForLabel(const std::string_view label, const mgp::Value &properties_val,
|
||||
mgp_graph *memgraph_graph, const auto &record_factory,
|
||||
const std::set<std::string_view> &existing_label_indices,
|
||||
const std::set<std::string_view> &existing_label_property_indices) {
|
||||
AssertedIndices asserted_indices;
|
||||
if (!properties_val.IsList()) {
|
||||
return {};
|
||||
}
|
||||
if (const auto properties = properties_val.ValueList();
|
||||
properties.Empty() && mgp::CreateLabelIndex(memgraph_graph, label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated);
|
||||
asserted_indices.emplace("");
|
||||
} else {
|
||||
std::for_each(properties.begin(), properties.end(),
|
||||
[&label, &existing_label_indices, &existing_label_property_indices, &memgraph_graph, &record_factory,
|
||||
&asserted_indices](const mgp::Value &property) {
|
||||
if (!property.IsString()) {
|
||||
return;
|
||||
}
|
||||
const auto property_str = property.ValueString();
|
||||
if (property_str.empty()) {
|
||||
ProcessCreatingLabelIndex(label, existing_label_indices, memgraph_graph, record_factory);
|
||||
asserted_indices.emplace("");
|
||||
} else {
|
||||
ProcessCreatingLabelPropertyIndexAndExistenceConstraint(
|
||||
label, property_str, existing_label_property_indices, mgp::CreateLabelPropertyIndex,
|
||||
memgraph_graph, record_factory);
|
||||
asserted_indices.emplace(property_str);
|
||||
}
|
||||
});
|
||||
}
|
||||
return asserted_indices;
|
||||
}
|
||||
|
||||
void ProcessIndices(const mgp::Map &indices_map, mgp_graph *memgraph_graph, const auto &record_factory,
|
||||
bool drop_existing) {
|
||||
auto mgp_existing_label_indices = mgp::ListAllLabelIndices(memgraph_graph);
|
||||
auto mgp_existing_label_property_indices = mgp::ListAllLabelPropertyIndices(memgraph_graph);
|
||||
|
||||
std::set<std::string_view> existing_label_indices;
|
||||
std::transform(mgp_existing_label_indices.begin(), mgp_existing_label_indices.end(),
|
||||
std::inserter(existing_label_indices, existing_label_indices.begin()),
|
||||
[](const mgp::Value &index) { return index.ValueString(); });
|
||||
|
||||
std::set<std::string_view> existing_label_property_indices;
|
||||
std::transform(mgp_existing_label_property_indices.begin(), mgp_existing_label_property_indices.end(),
|
||||
std::inserter(existing_label_property_indices, existing_label_property_indices.begin()),
|
||||
[](const mgp::Value &index) { return index.ValueString(); });
|
||||
|
||||
std::set<std::string> asserted_label_indices;
|
||||
std::set<std::string> asserted_label_property_indices;
|
||||
|
||||
auto merge_label_property = [](const std::string &label, const std::string &property) {
|
||||
return label + ":" + property;
|
||||
};
|
||||
|
||||
for (const auto &index : indices_map) {
|
||||
const std::string_view label = index.key;
|
||||
const mgp::Value &properties_val = index.value;
|
||||
|
||||
AssertedIndices asserted_indices_new = CreateIndicesForLabel(
|
||||
label, properties_val, memgraph_graph, record_factory, existing_label_indices, existing_label_property_indices);
|
||||
|
||||
if (!drop_existing) {
|
||||
continue;
|
||||
}
|
||||
std::ranges::for_each(asserted_indices_new, [&asserted_label_indices, &asserted_label_property_indices, label,
|
||||
&merge_label_property](const std::string &property) {
|
||||
if (property.empty()) {
|
||||
asserted_label_indices.emplace(label);
|
||||
} else {
|
||||
asserted_label_property_indices.emplace(merge_label_property(std::string(label), property));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (!drop_existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::set<std::string_view> label_indices_to_drop;
|
||||
std::ranges::set_difference(existing_label_indices, asserted_label_indices,
|
||||
std::inserter(label_indices_to_drop, label_indices_to_drop.begin()));
|
||||
|
||||
std::ranges::for_each(label_indices_to_drop, [memgraph_graph, &record_factory](const std::string_view label) {
|
||||
if (mgp::DropLabelIndex(memgraph_graph, label)) {
|
||||
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
|
||||
std::set<std::string_view> label_property_indices_to_drop;
|
||||
std::ranges::set_difference(existing_label_property_indices, asserted_label_property_indices,
|
||||
std::inserter(label_property_indices_to_drop, label_property_indices_to_drop.begin()));
|
||||
|
||||
auto decouple_label_property = [](const std::string_view label_property) {
|
||||
const auto label_size = label_property.find(':');
|
||||
const auto label = std::string(label_property.substr(0, label_size));
|
||||
const auto property = std::string(label_property.substr(label_size + 1));
|
||||
return std::make_pair(label, property);
|
||||
};
|
||||
|
||||
std::ranges::for_each(label_property_indices_to_drop, [memgraph_graph, &record_factory, decouple_label_property](
|
||||
const std::string_view label_property) {
|
||||
const auto [label, property] = decouple_label_property(label_property);
|
||||
if (mgp::DropLabelPropertyIndex(memgraph_graph, label, property)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
using ExistenceConstraintsStorage = std::set<std::string_view>;
|
||||
|
||||
ExistenceConstraintsStorage CreateExistenceConstraintsForLabel(
|
||||
const std::string_view label, const mgp::Value &properties_val, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory, const std::set<std::string_view> &existing_existence_constraints) {
|
||||
ExistenceConstraintsStorage asserted_existence_constraints;
|
||||
if (!properties_val.IsList()) {
|
||||
return asserted_existence_constraints;
|
||||
}
|
||||
|
||||
auto validate_property = [](const mgp::Value &property) -> bool {
|
||||
return property.IsString() && !property.ValueString().empty();
|
||||
};
|
||||
|
||||
const auto &properties = properties_val.ValueList();
|
||||
std::for_each(properties.begin(), properties.end(),
|
||||
[&label, &existing_existence_constraints, &asserted_existence_constraints, &memgraph_graph,
|
||||
&record_factory, &validate_property](const mgp::Value &property) {
|
||||
if (!validate_property(property)) {
|
||||
return;
|
||||
}
|
||||
const std::string_view property_str = property.ValueString();
|
||||
asserted_existence_constraints.emplace(property_str);
|
||||
ProcessCreatingLabelPropertyIndexAndExistenceConstraint(
|
||||
label, property_str, existing_existence_constraints, mgp::CreateExistenceConstraint,
|
||||
memgraph_graph, record_factory);
|
||||
});
|
||||
return asserted_existence_constraints;
|
||||
}
|
||||
|
||||
void ProcessExistenceConstraints(const mgp::Map &existence_constraints_map, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory, bool drop_existing) {
|
||||
auto mgp_existing_existence_constraints = mgp::ListAllExistenceConstraints(memgraph_graph);
|
||||
std::set<std::string_view> existing_existence_constraints;
|
||||
std::transform(mgp_existing_existence_constraints.begin(), mgp_existing_existence_constraints.end(),
|
||||
std::inserter(existing_existence_constraints, existing_existence_constraints.begin()),
|
||||
[](const mgp::Value &constraint) { return constraint.ValueString(); });
|
||||
|
||||
auto merge_label_property = [](const std::string_view label, const std::string_view property) {
|
||||
auto str = std::string(label) + ":";
|
||||
str += property;
|
||||
return str;
|
||||
};
|
||||
|
||||
ExistenceConstraintsStorage asserted_existence_constraints;
|
||||
|
||||
for (const auto &existing_constraint : existence_constraints_map) {
|
||||
const std::string_view label = existing_constraint.key;
|
||||
const mgp::Value &properties_val = existing_constraint.value;
|
||||
auto asserted_existence_constraints_new = CreateExistenceConstraintsForLabel(
|
||||
label, properties_val, memgraph_graph, record_factory, existing_existence_constraints);
|
||||
if (!drop_existing) {
|
||||
continue;
|
||||
}
|
||||
|
||||
std::ranges::for_each(asserted_existence_constraints_new, [&asserted_existence_constraints, &merge_label_property,
|
||||
label](const std::string_view property) {
|
||||
asserted_existence_constraints.emplace(merge_label_property(label, property));
|
||||
});
|
||||
}
|
||||
|
||||
if (!drop_existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::set<std::string_view> existence_constraints_to_drop;
|
||||
std::ranges::set_difference(existing_existence_constraints, asserted_existence_constraints,
|
||||
std::inserter(existence_constraints_to_drop, existence_constraints_to_drop.begin()));
|
||||
|
||||
auto decouple_label_property = [](const std::string_view label_property) {
|
||||
const auto label_size = label_property.find(':');
|
||||
const auto label = std::string(label_property.substr(0, label_size));
|
||||
const auto property = std::string(label_property.substr(label_size + 1));
|
||||
return std::make_pair(label, property);
|
||||
};
|
||||
|
||||
std::ranges::for_each(existence_constraints_to_drop, [&](const std::string_view label_property) {
|
||||
const auto [label, property] = decouple_label_property(label_property);
|
||||
if (mgp::DropExistenceConstraint(memgraph_graph, label, property)) {
|
||||
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
using AssertedUniqueConstraintsStorage = std::set<std::set<std::string_view>>;
|
||||
AssertedUniqueConstraintsStorage CreateUniqueConstraintsForLabel(
|
||||
const std::string_view label, const mgp::Value &unique_props_nested,
|
||||
const std::map<std::string_view, AssertedUniqueConstraintsStorage> &existing_unique_constraints,
|
||||
mgp_graph *memgraph_graph, const auto &record_factory) {
|
||||
AssertedUniqueConstraintsStorage asserted_unique_constraints;
|
||||
if (!unique_props_nested.IsList()) {
|
||||
return asserted_unique_constraints;
|
||||
}
|
||||
|
||||
auto validate_unique_constraint_props = [](const mgp::Value &properties) -> bool {
|
||||
if (!properties.IsList()) {
|
||||
return false;
|
||||
}
|
||||
const auto &properties_list = properties.ValueList();
|
||||
if (properties_list.Empty()) {
|
||||
return false;
|
||||
}
|
||||
return std::all_of(properties_list.begin(), properties_list.end(), [](const mgp::Value &property) {
|
||||
return property.IsString() && !property.ValueString().empty();
|
||||
});
|
||||
};
|
||||
|
||||
auto unique_constraint_exists =
|
||||
[](const std::string_view label, const std::set<std::string_view> &properties,
|
||||
const std::map<std::string_view, AssertedUniqueConstraintsStorage> &existing_unique_constraints) -> bool {
|
||||
auto iter = existing_unique_constraints.find(label);
|
||||
if (iter == existing_unique_constraints.end()) {
|
||||
return false;
|
||||
}
|
||||
return iter->second.find(properties) != iter->second.end();
|
||||
};
|
||||
|
||||
for (const auto unique_props_nested_list = unique_props_nested.ValueList();
|
||||
const auto &properties : unique_props_nested_list) {
|
||||
if (!validate_unique_constraint_props(properties)) {
|
||||
continue;
|
||||
}
|
||||
const auto properties_list = properties.ValueList();
|
||||
std::set<std::string_view> properties_coll;
|
||||
std::transform(properties_list.begin(), properties_list.end(),
|
||||
std::inserter(properties_coll, properties_coll.begin()),
|
||||
[](const mgp::Value &property) { return property.ValueString(); });
|
||||
|
||||
if (unique_constraint_exists(label, properties_coll, existing_unique_constraints)) {
|
||||
InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusKept);
|
||||
} else if (mgp::CreateUniqueConstraint(memgraph_graph, label, properties.ptr())) {
|
||||
InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusCreated);
|
||||
}
|
||||
asserted_unique_constraints.emplace(std::move(properties_coll));
|
||||
}
|
||||
return asserted_unique_constraints;
|
||||
}
|
||||
|
||||
void ProcessUniqueConstraints(const mgp::Map &unique_constraints_map, mgp_graph *memgraph_graph,
|
||||
const auto &record_factory, bool drop_existing) {
|
||||
auto mgp_existing_unique_constraints = mgp::ListAllUniqueConstraints(memgraph_graph);
|
||||
// label-unique_constraints pair
|
||||
std::map<std::string_view, AssertedUniqueConstraintsStorage> existing_unique_constraints;
|
||||
for (const auto &constraint : mgp_existing_unique_constraints) {
|
||||
auto constraint_list = constraint.ValueList();
|
||||
std::set<std::string_view> properties;
|
||||
for (int i = 1; i < constraint_list.Size(); i++) {
|
||||
properties.emplace(constraint_list[i].ValueString());
|
||||
}
|
||||
const std::string_view label = constraint_list[0].ValueString();
|
||||
auto [it, inserted] = existing_unique_constraints.try_emplace(label, AssertedUniqueConstraintsStorage{properties});
|
||||
if (!inserted) {
|
||||
it->second.emplace(std::move(properties));
|
||||
}
|
||||
}
|
||||
|
||||
std::map<std::string_view, AssertedUniqueConstraintsStorage> asserted_unique_constraints;
|
||||
|
||||
for (const auto &[label, unique_props_nested] : unique_constraints_map) {
|
||||
auto asserted_unique_constraints_new = CreateUniqueConstraintsForLabel(
|
||||
label, unique_props_nested, existing_unique_constraints, memgraph_graph, record_factory);
|
||||
if (drop_existing) {
|
||||
asserted_unique_constraints.emplace(label, std::move(asserted_unique_constraints_new));
|
||||
}
|
||||
}
|
||||
|
||||
if (!drop_existing) {
|
||||
return;
|
||||
}
|
||||
|
||||
std::vector<std::pair<std::string_view, std::set<std::string_view>>> unique_constraints_to_drop;
|
||||
|
||||
// Check for each label for we found existing constraint in the DB whether it was asserted.
|
||||
// If no unique constraint was found with label, we can drop all unique constraints for this label. (if branch)
|
||||
// If some unique constraint was found with label, we can drop only those unique constraints that were not asserted.
|
||||
// (else branch.)
|
||||
std::ranges::for_each(existing_unique_constraints, [&asserted_unique_constraints, &unique_constraints_to_drop](
|
||||
const auto &existing_label_unique_constraints) {
|
||||
const auto &label = existing_label_unique_constraints.first;
|
||||
const auto &existing_unique_constraints_for_label = existing_label_unique_constraints.second;
|
||||
const auto &asserted_unique_constraints_for_label = asserted_unique_constraints.find(label);
|
||||
if (asserted_unique_constraints_for_label == asserted_unique_constraints.end()) {
|
||||
std::ranges::for_each(
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.begin()),
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.end()),
|
||||
[&unique_constraints_to_drop, &label](std::set<std::string_view> existing_unique_constraint_for_label) {
|
||||
unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label));
|
||||
});
|
||||
} else {
|
||||
const auto &asserted_unique_constraints_for_label_coll = asserted_unique_constraints_for_label->second;
|
||||
std::ranges::for_each(
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.begin()),
|
||||
std::make_move_iterator(existing_unique_constraints_for_label.end()),
|
||||
[&unique_constraints_to_drop, &label, &asserted_unique_constraints_for_label_coll](
|
||||
std::set<std::string_view> existing_unique_constraint_for_label) {
|
||||
if (!asserted_unique_constraints_for_label_coll.contains(existing_unique_constraint_for_label)) {
|
||||
unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label));
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
std::ranges::for_each(
|
||||
unique_constraints_to_drop, [memgraph_graph, &record_factory](const auto &label_unique_constraint) {
|
||||
const auto &[label, unique_constraint] = label_unique_constraint;
|
||||
|
||||
auto unique_constraint_list = mgp::List();
|
||||
std::ranges::for_each(unique_constraint, [&unique_constraint_list](const std::string_view &property) {
|
||||
unique_constraint_list.AppendExtend(mgp::Value(property));
|
||||
});
|
||||
|
||||
if (mgp::DropUniqueConstraint(memgraph_graph, label, mgp::Value(unique_constraint_list).ptr())) {
|
||||
InsertRecordForUniqueConstraint(record_factory, label, unique_constraint_list, Schema::kStatusDropped);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
const auto record_factory = mgp::RecordFactory(result);
|
||||
auto arguments = mgp::List(args);
|
||||
auto indices_map = arguments[0].ValueMap();
|
||||
auto unique_constraints_map = arguments[1].ValueMap();
|
||||
auto existence_constraints_map = arguments[2].ValueMap();
|
||||
auto drop_existing = arguments[3].ValueBool();
|
||||
|
||||
ProcessIndices(indices_map, memgraph_graph, record_factory, drop_existing);
|
||||
ProcessExistenceConstraints(existence_constraints_map, memgraph_graph, record_factory, drop_existing);
|
||||
ProcessUniqueConstraints(unique_constraints_map, memgraph_graph, record_factory, drop_existing);
|
||||
}
|
||||
|
||||
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
|
||||
try {
|
||||
mgp::MemoryDispatcherGuard guard{memory};
|
||||
;
|
||||
|
||||
AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read,
|
||||
{},
|
||||
{mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
|
||||
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
|
||||
AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {},
|
||||
{mgp::Return(Schema::kReturnNodeType, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnLabels, {mgp::Type::List, mgp::Type::String}),
|
||||
mgp::Return(Schema::kReturnPropertyName, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any),
|
||||
mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)},
|
||||
module, memory);
|
||||
|
||||
AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read,
|
||||
{},
|
||||
{mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
|
||||
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
|
||||
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
|
||||
AddProcedure(Schema::RelTypeProperties, Schema::kProcedureRelType, mgp::ProcedureType::Read, {},
|
||||
{mgp::Return(Schema::kReturnRelType, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnPropertyName, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any),
|
||||
mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)},
|
||||
module, memory);
|
||||
|
||||
AddProcedure(
|
||||
Schema::Assert, Schema::kProcedureAssert, mgp::ProcedureType::Read,
|
||||
{
|
||||
mgp::Parameter(Schema::kParameterIndices, {mgp::Type::Map, mgp::Type::Any}),
|
||||
mgp::Parameter(Schema::kParameterUniqueConstraints, {mgp::Type::Map, mgp::Type::Any}),
|
||||
mgp::Parameter(Schema::kParameterExistenceConstraints, {mgp::Type::Map, mgp::Type::Any},
|
||||
mgp::Value(mgp::Map{})),
|
||||
mgp::Parameter(Schema::kParameterDropExisting, mgp::Type::Bool, mgp::Value(true)),
|
||||
},
|
||||
{mgp::Return(Schema::kReturnLabel, mgp::Type::String), mgp::Return(Schema::kReturnKey, mgp::Type::String),
|
||||
mgp::Return(Schema::kReturnKeys, {mgp::Type::List, mgp::Type::String}),
|
||||
mgp::Return(Schema::kReturnUnique, mgp::Type::Bool), mgp::Return(Schema::kReturnAction, mgp::Type::String)},
|
||||
module, memory);
|
||||
} catch (const std::exception &e) {
|
||||
std::cerr << "Error while initializing query module: " << e.what() << std::endl;
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
@@ -104,7 +104,9 @@ def retry(retry_limit, timeout=100):
|
||||
except Exception:
|
||||
time.sleep(timeout)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
return inner_func
|
||||
|
||||
|
||||
@@ -200,19 +202,19 @@ if args.version:
|
||||
try:
|
||||
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
|
||||
if current_branch != "master":
|
||||
branches = get_output("git", "branch")
|
||||
if "master" in branches:
|
||||
branches = get_output("git", "branch", "-r", "--list", "origin/master")
|
||||
if "origin/master" in branches:
|
||||
# If master is present locally, the fetch is allowed to fail
|
||||
# because this script will still be able to compare against the
|
||||
# master branch.
|
||||
try:
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
get_output("git", "fetch", "origin", "master")
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# If master is not present locally, the fetch command has to
|
||||
# succeed because something else will fail otherwise.
|
||||
get_output("git", "fetch", "origin", "master:master")
|
||||
get_output("git", "fetch", "origin", "master")
|
||||
except Exception:
|
||||
print("Fatal error while ensuring local master branch.")
|
||||
sys.exit(1)
|
||||
@@ -232,7 +234,7 @@ for branch in branches:
|
||||
match = branch_regex.match(branch)
|
||||
if match is not None:
|
||||
version = tuple(map(int, match.group(1).split(".")))
|
||||
master_branch_merge = get_output("git", "merge-base", "master", branch)
|
||||
master_branch_merge = get_output("git", "merge-base", "origin/master", branch)
|
||||
versions.append((version, branch, master_branch_merge))
|
||||
versions.sort(reverse=True)
|
||||
|
||||
@@ -243,7 +245,7 @@ current_version = None
|
||||
for version in versions:
|
||||
version_tuple, branch, master_branch_merge = version
|
||||
current_branch_merge = get_output("git", "merge-base", current_hash, branch)
|
||||
master_current_merge = get_output("git", "merge-base", current_hash, "master")
|
||||
master_current_merge = get_output("git", "merge-base", current_hash, "origin/master")
|
||||
# The first check checks whether this commit is a child of `master` and
|
||||
# the version branch was created before us.
|
||||
# The second check checks whether this commit is a child of the version
|
||||
|
||||
@@ -143,7 +143,7 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
|
||||
# Memgraph CSV Import Tool Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
add_executable(mg_import_csv mg_import_csv.cpp)
|
||||
target_link_libraries(mg_import_csv mg-storage-v2)
|
||||
target_link_libraries(mg_import_csv mg-storage-v2 mg-dbms)
|
||||
|
||||
# Strip the executable in release build.
|
||||
if(lower_build_type STREQUAL "release")
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <json/json.hpp>
|
||||
#include <utility>
|
||||
|
||||
#include "storage/v2/temporal.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -87,8 +88,8 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
|
||||
: storage_directory_(storage_directory),
|
||||
Log::Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
|
||||
: storage_directory_(std::move(storage_directory)),
|
||||
buffer_size_(buffer_size),
|
||||
buffer_flush_interval_millis_(buffer_flush_interval_millis),
|
||||
started_(false) {}
|
||||
|
||||
@@ -36,7 +36,7 @@ class Log {
|
||||
};
|
||||
|
||||
public:
|
||||
Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
|
||||
Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
|
||||
|
||||
~Log();
|
||||
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
|
||||
#include <cstdint>
|
||||
#include <regex>
|
||||
#include <utility>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
@@ -560,20 +561,20 @@ Databases Databases::Deserialize(const nlohmann::json &data) {
|
||||
}
|
||||
#endif
|
||||
|
||||
User::User() {}
|
||||
User::User() = default;
|
||||
|
||||
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
|
||||
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
|
||||
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
|
||||
User::User(const std::string &username, std::string password_hash, const Permissions &permissions)
|
||||
: username_(utils::ToLowerCase(username)), password_hash_(std::move(password_hash)), permissions_(permissions) {}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
|
||||
User::User(const std::string &username, std::string password_hash, const Permissions &permissions,
|
||||
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
|
||||
: username_(utils::ToLowerCase(username)),
|
||||
password_hash_(password_hash),
|
||||
password_hash_(std::move(password_hash)),
|
||||
permissions_(permissions),
|
||||
fine_grained_access_handler_(std::move(fine_grained_access_handler)),
|
||||
database_access_(db_access) {}
|
||||
database_access_(std::move(db_access)) {}
|
||||
#endif
|
||||
|
||||
bool User::CheckPassword(const std::string &password) {
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <unordered_map>
|
||||
|
||||
#include <json/json.hpp>
|
||||
#include <utility>
|
||||
#include "dbms/constants.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
@@ -301,8 +302,8 @@ class Databases final {
|
||||
bool Contains(const std::string &db) const;
|
||||
|
||||
bool GetAllowAll() const { return allow_all_; }
|
||||
const std::set<std::string> &GetGrants() const { return grants_dbs_; }
|
||||
const std::set<std::string> &GetDenies() const { return denies_dbs_; }
|
||||
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
|
||||
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
|
||||
const std::string &GetDefault() const;
|
||||
|
||||
nlohmann::json Serialize() const;
|
||||
@@ -310,14 +311,17 @@ class Databases final {
|
||||
static Databases Deserialize(const nlohmann::json &data);
|
||||
|
||||
private:
|
||||
Databases(bool allow_all, std::set<std::string> grant, std::set<std::string> deny,
|
||||
const std::string &default_db = dbms::kDefaultDB)
|
||||
: grants_dbs_(grant), denies_dbs_(deny), allow_all_(allow_all), default_db_(default_db) {}
|
||||
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
|
||||
std::string default_db = dbms::kDefaultDB)
|
||||
: grants_dbs_(std::move(grant)),
|
||||
denies_dbs_(std::move(deny)),
|
||||
allow_all_(allow_all),
|
||||
default_db_(std::move(default_db)) {}
|
||||
|
||||
std::set<std::string> grants_dbs_; //!< set of databases with granted access
|
||||
std::set<std::string> denies_dbs_; //!< set of databases with denied access
|
||||
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
|
||||
std::string default_db_; //!< user's default database
|
||||
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
|
||||
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
|
||||
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
|
||||
std::string default_db_; //!< user's default database
|
||||
};
|
||||
#endif
|
||||
|
||||
@@ -327,9 +331,9 @@ class User final {
|
||||
User();
|
||||
|
||||
explicit User(const std::string &username);
|
||||
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
|
||||
User(const std::string &username, std::string password_hash, const Permissions &permissions);
|
||||
#ifdef MG_ENTERPRISE
|
||||
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
|
||||
User(const std::string &username, std::string password_hash, const Permissions &permissions,
|
||||
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
|
||||
#endif
|
||||
User(const User &) = default;
|
||||
|
||||
@@ -25,4 +25,3 @@ target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils
|
||||
find_package(OpenSSL REQUIRED)
|
||||
target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})
|
||||
target_include_directories(mg-communication SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
target_precompile_headers(mg-communication INTERFACE http/server.hpp <boost/beast/websocket.hpp> bolt/v1/session.hpp)
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "communication/bolt/v1/codes.hpp"
|
||||
@@ -34,8 +35,8 @@ class FailureResponseException : public utils::BasicException {
|
||||
|
||||
explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {}
|
||||
|
||||
FailureResponseException(const std::string &code, const std::string &message)
|
||||
: utils::BasicException{message}, code_{code} {}
|
||||
FailureResponseException(std::string code, const std::string &message)
|
||||
: utils::BasicException{message}, code_{std::move(code)} {}
|
||||
|
||||
const std::string &code() const { return code_; }
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -51,7 +51,7 @@ enum class ChunkState : uint8_t {
|
||||
template <typename TBuffer>
|
||||
class ChunkedDecoderBuffer {
|
||||
public:
|
||||
ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
|
||||
explicit ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
|
||||
|
||||
/**
|
||||
* Reads data from the internal buffer.
|
||||
|
||||
@@ -401,11 +401,11 @@ class Decoder {
|
||||
}
|
||||
auto &labels = dv.ValueList();
|
||||
vertex.labels.reserve(labels.size());
|
||||
for (size_t i = 0; i < labels.size(); ++i) {
|
||||
if (labels[i].type() != Value::Type::String) {
|
||||
for (auto &label : labels) {
|
||||
if (label.type() != Value::Type::String) {
|
||||
return false;
|
||||
}
|
||||
vertex.labels.emplace_back(std::move(labels[i].ValueString()));
|
||||
vertex.labels.emplace_back(std::move(label.ValueString()));
|
||||
}
|
||||
|
||||
// read properties
|
||||
|
||||
@@ -111,12 +111,12 @@ class BaseEncoder {
|
||||
|
||||
void WriteList(const std::vector<Value> &value) {
|
||||
WriteTypeSize(value.size(), MarkerList);
|
||||
for (auto &x : value) WriteValue(x);
|
||||
for (const auto &x : value) WriteValue(x);
|
||||
}
|
||||
|
||||
void WriteMap(const std::map<std::string, Value> &value) {
|
||||
WriteTypeSize(value.size(), MarkerMap);
|
||||
for (auto &x : value) {
|
||||
for (const auto &x : value) {
|
||||
WriteString(x.first);
|
||||
WriteValue(x.second);
|
||||
}
|
||||
@@ -205,11 +205,11 @@ class BaseEncoder {
|
||||
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
|
||||
WriteRAW(utils::UnderlyingCast(Signature::Path));
|
||||
WriteTypeSize(path.vertices.size(), MarkerList);
|
||||
for (auto &v : path.vertices) WriteVertex(v);
|
||||
for (const auto &v : path.vertices) WriteVertex(v);
|
||||
WriteTypeSize(path.edges.size(), MarkerList);
|
||||
for (auto &e : path.edges) WriteEdge(e);
|
||||
for (const auto &e : path.edges) WriteEdge(e);
|
||||
WriteTypeSize(path.indices.size(), MarkerList);
|
||||
for (auto &i : path.indices) WriteInt(i);
|
||||
for (const auto &i : path.indices) WriteInt(i);
|
||||
}
|
||||
|
||||
void WriteDate(const utils::Date &date) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -48,7 +48,7 @@ namespace memgraph::communication::bolt {
|
||||
template <class TOutputStream>
|
||||
class ChunkedEncoderBuffer {
|
||||
public:
|
||||
ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
|
||||
explicit ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
|
||||
|
||||
/**
|
||||
* Writes n values into the buffer. If n is bigger than whole chunk size
|
||||
|
||||
@@ -39,7 +39,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
|
||||
using BaseEncoder<Buffer>::buffer_;
|
||||
|
||||
public:
|
||||
ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
explicit ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
|
||||
using BaseEncoder<Buffer>::UpdateVersion;
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ class Encoder : private BaseEncoder<Buffer> {
|
||||
using BaseEncoder<Buffer>::buffer_;
|
||||
|
||||
public:
|
||||
Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
explicit Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
|
||||
|
||||
using BaseEncoder<Buffer>::UpdateVersion;
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<i
|
||||
return State::Close;
|
||||
}
|
||||
|
||||
if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
|
||||
if (summary.contains("has_more") && summary.at("has_more").ValueBool()) {
|
||||
return State::Result;
|
||||
}
|
||||
|
||||
@@ -148,13 +148,13 @@ State HandlePullDiscardV4(TSession &session, const State state, const Marker mar
|
||||
spdlog::trace("Couldn't read extra field!");
|
||||
}
|
||||
const auto &extra_map = extra.ValueMap();
|
||||
if (extra_map.count("n")) {
|
||||
if (extra_map.contains("n")) {
|
||||
if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
|
||||
n = n_value;
|
||||
}
|
||||
}
|
||||
|
||||
if (extra_map.count("qid")) {
|
||||
if (extra_map.contains("qid")) {
|
||||
if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
|
||||
qid = qid_value;
|
||||
}
|
||||
@@ -209,7 +209,11 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
|
||||
|
||||
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
|
||||
|
||||
#if MG_ENTERPRISE
|
||||
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
|
||||
#else
|
||||
spdlog::debug("[Run] '{}'", query.ValueString());
|
||||
#endif
|
||||
|
||||
// Increment number of queries in the metrics
|
||||
IncrementQueryMetrics(session);
|
||||
@@ -276,7 +280,11 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
|
||||
return HandleFailure(session, e);
|
||||
}
|
||||
|
||||
#if MG_ENTERPRISE
|
||||
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
|
||||
#else
|
||||
spdlog::debug("[Run] '{}'", query.ValueString());
|
||||
#endif
|
||||
|
||||
// Increment number of queries in the metrics
|
||||
IncrementQueryMetrics(session);
|
||||
|
||||
@@ -42,11 +42,11 @@ std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
|
||||
std::string username;
|
||||
std::string password;
|
||||
if (data["scheme"].ValueString() == "basic") {
|
||||
if (!data.count("principal")) { // Special case principal = ""
|
||||
if (!data.contains("principal")) { // Special case principal = ""
|
||||
spdlog::warn("The client didn't supply the principal field! Trying with \"\"...");
|
||||
data["principal"] = "";
|
||||
}
|
||||
if (!data.count("credentials")) { // Special case credentials = ""
|
||||
if (!data.contains("credentials")) { // Special case credentials = ""
|
||||
spdlog::warn("The client didn't supply the credentials field! Trying with \"\"...");
|
||||
data["credentials"] = "";
|
||||
}
|
||||
@@ -118,7 +118,7 @@ std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
|
||||
}
|
||||
|
||||
auto &data = metadata.ValueMap();
|
||||
if (!data.count("user_agent")) {
|
||||
if (!data.contains("user_agent")) {
|
||||
spdlog::warn("The client didn't supply the user agent!");
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -142,7 +142,7 @@ std::optional<Value> GetInitDataV5(TSession &session, const Marker marker) {
|
||||
}
|
||||
|
||||
const auto &data = metadata.ValueMap();
|
||||
if (!data.count("user_agent")) {
|
||||
if (!data.contains("user_agent")) {
|
||||
spdlog::warn("The client didn't supply the user agent!");
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ struct UnboundedEdge {
|
||||
* The decoder writes data into this structure.
|
||||
*/
|
||||
struct Path {
|
||||
Path() {}
|
||||
Path() = default;
|
||||
|
||||
Path(const std::vector<Vertex> &vertices, const std::vector<Edge> &edges) {
|
||||
// Helper function. Looks for the given element in the collection. If found,
|
||||
|
||||
@@ -132,7 +132,7 @@ class Client final {
|
||||
*/
|
||||
class ClientInputStream final {
|
||||
public:
|
||||
ClientInputStream(Client &client);
|
||||
explicit ClientInputStream(Client &client);
|
||||
|
||||
ClientInputStream(const ClientInputStream &) = delete;
|
||||
ClientInputStream(ClientInputStream &&) = delete;
|
||||
@@ -156,7 +156,7 @@ class ClientInputStream final {
|
||||
*/
|
||||
class ClientOutputStream final {
|
||||
public:
|
||||
ClientOutputStream(Client &client);
|
||||
explicit ClientOutputStream(Client &client);
|
||||
|
||||
ClientOutputStream(const ClientOutputStream &) = delete;
|
||||
ClientOutputStream(ClientOutputStream &&) = delete;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -34,7 +34,7 @@ ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) {
|
||||
}
|
||||
|
||||
ClientContext::ClientContext(const std::string &key_file, const std::string &cert_file) : ClientContext(true) {
|
||||
if (key_file != "" && cert_file != "") {
|
||||
if (!key_file.empty() && !cert_file.empty()) {
|
||||
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1,
|
||||
"Couldn't load client certificate from file: {}", cert_file);
|
||||
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1,
|
||||
@@ -124,7 +124,7 @@ ServerContext &ServerContext::operator=(ServerContext &&other) noexcept {
|
||||
return *this;
|
||||
}
|
||||
|
||||
ServerContext::~ServerContext() {}
|
||||
ServerContext::~ServerContext() = default;
|
||||
|
||||
SSL_CTX *ServerContext::context() {
|
||||
MG_ASSERT(ctx_);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
namespace memgraph::communication {
|
||||
|
||||
const std::string SslGetLastError() {
|
||||
std::string SslGetLastError() {
|
||||
char buff[2048];
|
||||
auto err = ERR_get_error();
|
||||
ERR_error_string_n(err, buff, sizeof(buff));
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,6 +18,6 @@ namespace memgraph::communication {
|
||||
/**
|
||||
* This function reads and returns a string describing the last OpenSSL error.
|
||||
*/
|
||||
const std::string SslGetLastError();
|
||||
std::string SslGetLastError();
|
||||
|
||||
} // namespace memgraph::communication
|
||||
|
||||
@@ -38,7 +38,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TRequestHand
|
||||
Listener(Listener &&) = delete;
|
||||
Listener &operator=(const Listener &) = delete;
|
||||
Listener &operator=(Listener &&) = delete;
|
||||
~Listener() {}
|
||||
~Listener() = default;
|
||||
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Listener> Create(Args &&...args) {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
@@ -51,13 +52,13 @@ class Listener final {
|
||||
using SessionHandler = Session<TSession, TSessionContext>;
|
||||
|
||||
public:
|
||||
Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, const std::string &service_name,
|
||||
Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, std::string service_name,
|
||||
size_t workers_count)
|
||||
: data_(data),
|
||||
alive_(false),
|
||||
context_(context),
|
||||
inactivity_timeout_sec_(inactivity_timeout_sec),
|
||||
service_name_(service_name),
|
||||
service_name_(std::move(service_name)),
|
||||
workers_count_(workers_count) {}
|
||||
|
||||
~Listener() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -80,7 +80,7 @@ class ResultStreamFaker {
|
||||
std::transform(header.begin(), header.end(), column_widths.begin(), [](const auto &s) { return s.size(); });
|
||||
|
||||
// convert all the results into strings, and track max column width
|
||||
auto &results_data = results.GetResults();
|
||||
const auto &results_data = results.GetResults();
|
||||
std::vector<std::vector<std::string>> result_strings(results_data.size(),
|
||||
std::vector<std::string>(column_widths.size()));
|
||||
for (int row_ind = 0; row_ind < static_cast<int>(results_data.size()); ++row_ind) {
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <thread>
|
||||
#include <utility>
|
||||
|
||||
#include <openssl/bio.h>
|
||||
#include <openssl/err.h>
|
||||
@@ -51,7 +52,8 @@ using InputStream = Buffer::ReadEnd;
|
||||
*/
|
||||
class OutputStream final {
|
||||
public:
|
||||
OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function) : write_function_(write_function) {}
|
||||
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
|
||||
: write_function_(std::move(write_function)) {}
|
||||
|
||||
OutputStream(const OutputStream &) = delete;
|
||||
OutputStream(OutputStream &&) = delete;
|
||||
|
||||
@@ -47,7 +47,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
|
||||
Listener(Listener &&) = delete;
|
||||
Listener &operator=(const Listener &) = delete;
|
||||
Listener &operator=(Listener &&) = delete;
|
||||
~Listener() {}
|
||||
~Listener() = default;
|
||||
|
||||
template <typename... Args>
|
||||
static std::shared_ptr<Listener> Create(Args &&...args) {
|
||||
|
||||
@@ -76,7 +76,7 @@ using tcp = boost::asio::ip::tcp;
|
||||
class OutputStream final {
|
||||
public:
|
||||
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
|
||||
: write_function_(write_function) {}
|
||||
: write_function_(std::move(write_function)) {}
|
||||
|
||||
OutputStream(const OutputStream &) = delete;
|
||||
OutputStream(OutputStream &&) = delete;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -16,6 +16,7 @@
|
||||
#include <spdlog/sinks/base_sink.h>
|
||||
#include <boost/asio/io_context.hpp>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <utility>
|
||||
|
||||
#include "communication/websocket/listener.hpp"
|
||||
#include "io/network/endpoint.hpp"
|
||||
@@ -45,7 +46,7 @@ class Server final {
|
||||
|
||||
class LoggingSink : public spdlog::sinks::base_sink<std::mutex> {
|
||||
public:
|
||||
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(listener) {}
|
||||
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(std::move(listener)) {}
|
||||
|
||||
private:
|
||||
void sink_it_(const spdlog::details::log_msg &msg) override;
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
|
||||
add_library(mg-dbms STATIC database.cpp)
|
||||
add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp)
|
||||
target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query)
|
||||
|
||||
@@ -15,4 +15,10 @@ namespace memgraph::dbms {
|
||||
|
||||
constexpr static const char *kDefaultDB = "memgraph"; //!< Name of the default database
|
||||
|
||||
#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY
|
||||
constexpr bool allow_mt_repl = true;
|
||||
#else
|
||||
constexpr bool allow_mt_repl = false;
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "dbms/database.hpp"
|
||||
#include "dbms/inmemory/storage_helper.hpp"
|
||||
#include "dbms/replication_handler.hpp"
|
||||
#include "flags/storage_mode.hpp"
|
||||
#include "storage/v2/disk/storage.hpp"
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
@@ -19,14 +21,16 @@ template struct memgraph::utils::Gatekeeper<memgraph::dbms::Database>;
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
Database::Database(const storage::Config &config)
|
||||
Database::Database(storage::Config config, replication::ReplicationState &repl_state)
|
||||
: trigger_store_(config.durability.storage_directory / "triggers"),
|
||||
streams_{config.durability.storage_directory / "streams"} {
|
||||
streams_{config.durability.storage_directory / "streams"},
|
||||
plan_cache_{FLAGS_query_plan_cache_max_size},
|
||||
repl_state_(&repl_state) {
|
||||
if (config.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk ||
|
||||
utils::DirExists(config.disk.main_storage_directory)) {
|
||||
storage_ = std::make_unique<storage::DiskStorage>(config);
|
||||
storage_ = std::make_unique<storage::DiskStorage>(std::move(config));
|
||||
} else {
|
||||
storage_ = std::make_unique<storage::InMemoryStorage>(config, config.storage_mode);
|
||||
storage_ = dbms::CreateInMemoryStorage(std::move(config), repl_state);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,8 @@
|
||||
#include "query/trigger.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "utils/gatekeeper.hpp"
|
||||
#include "utils/lru_cache.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
@@ -46,7 +48,7 @@ class Database {
|
||||
*
|
||||
* @param config storage configuration
|
||||
*/
|
||||
explicit Database(const storage::Config &config);
|
||||
explicit Database(storage::Config config, replication::ReplicationState &repl_state);
|
||||
|
||||
/**
|
||||
* @brief Returns the raw storage pointer.
|
||||
@@ -56,6 +58,7 @@ class Database {
|
||||
* @return storage::Storage*
|
||||
*/
|
||||
storage::Storage *storage() { return storage_.get(); }
|
||||
storage::Storage const *storage() const { return storage_.get(); }
|
||||
|
||||
/**
|
||||
* @brief Storage's Accessor
|
||||
@@ -65,12 +68,12 @@ class Database {
|
||||
*/
|
||||
std::unique_ptr<storage::Storage::Accessor> Access(
|
||||
std::optional<storage::IsolationLevel> override_isolation_level = {}) {
|
||||
return storage_->Access(override_isolation_level);
|
||||
return storage_->Access(override_isolation_level, repl_state_->IsMain());
|
||||
}
|
||||
|
||||
std::unique_ptr<storage::Storage::Accessor> UniqueAccess(
|
||||
std::optional<storage::IsolationLevel> override_isolation_level = {}) {
|
||||
return storage_->UniqueAccess(override_isolation_level);
|
||||
return storage_->UniqueAccess(override_isolation_level, repl_state_->IsMain());
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -145,9 +148,9 @@ class Database {
|
||||
/**
|
||||
* @brief Returns the PlanCache vector raw pointer
|
||||
*
|
||||
* @return utils::SkipList<query::PlanCacheEntry>*
|
||||
* @return utils::Synchronized<utils::LRUCache<uint64_t, std::shared_ptr<PlanWrapper>>, utils::RWSpinLock>
|
||||
*/
|
||||
utils::SkipList<query::PlanCacheEntry> *plan_cache() { return &plan_cache_; }
|
||||
query::PlanCacheLRU *plan_cache() { return &plan_cache_; }
|
||||
|
||||
private:
|
||||
std::unique_ptr<storage::Storage> storage_; //!< Underlying storage
|
||||
@@ -156,7 +159,9 @@ class Database {
|
||||
query::stream::Streams streams_; //!< Streams associated with the storage
|
||||
|
||||
// TODO: Move to a better place
|
||||
utils::SkipList<query::PlanCacheEntry> plan_cache_; //!< Plan cache associated with the storage
|
||||
query::PlanCacheLRU plan_cache_; //!< Plan cache associated with the storage
|
||||
|
||||
const replication::ReplicationState *repl_state_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -51,7 +51,7 @@ class DatabaseHandler : public Handler<Database> {
|
||||
* @param config Storage configuration
|
||||
* @return HandlerT::NewResult
|
||||
*/
|
||||
HandlerT::NewResult New(std::string_view name, storage::Config config) {
|
||||
HandlerT::NewResult New(std::string_view name, storage::Config config, replication::ReplicationState &repl_state) {
|
||||
// Control that no one is using the same data directory
|
||||
if (std::any_of(begin(), end(), [&](auto &elem) {
|
||||
auto db_acc = elem.second.access();
|
||||
@@ -62,7 +62,7 @@ class DatabaseHandler : public Handler<Database> {
|
||||
return NewError::EXISTS;
|
||||
}
|
||||
config.name = name; // Set storage id via config
|
||||
return HandlerT::New(std::piecewise_construct, name, config);
|
||||
return HandlerT::New(std::piecewise_construct, name, config, repl_state);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -93,5 +93,4 @@ class DatabaseHandler : public Handler<Database> {
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
#endif
|
||||
|
||||
75
src/dbms/dbms_handler.cpp
Normal file
75
src/dbms/dbms_handler.cpp
Normal file
@@ -0,0 +1,75 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "dbms/dbms_handler.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
#ifdef MG_ENTERPRISE
|
||||
DbmsHandler::DbmsHandler(
|
||||
storage::Config config,
|
||||
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
|
||||
bool recovery_on_startup, bool delete_on_drop)
|
||||
: default_config_{std::move(config)},
|
||||
delete_on_drop_(delete_on_drop),
|
||||
repl_state_{ReplicationStateRootPath(default_config_)} {
|
||||
// TODO: Decouple storage config from dbms config
|
||||
// TODO: Save individual db configs inside the kvstore and restore from there
|
||||
storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases");
|
||||
const auto &db_dir = default_config_.durability.storage_directory;
|
||||
const auto durability_dir = db_dir / ".durability";
|
||||
utils::EnsureDirOrDie(db_dir);
|
||||
utils::EnsureDirOrDie(durability_dir);
|
||||
durability_ = std::make_unique<kvstore::KVStore>(durability_dir);
|
||||
|
||||
// Generate the default database
|
||||
MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB.");
|
||||
|
||||
// Recover previous databases
|
||||
if (recovery_on_startup) {
|
||||
for (const auto &[name, _] : *durability_) {
|
||||
if (name == kDefaultDB) continue; // Already set
|
||||
spdlog::info("Restoring database {}.", name);
|
||||
MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name);
|
||||
spdlog::info("Database {} restored.", name);
|
||||
}
|
||||
} else { // Clear databases from the durability list and auth
|
||||
auto locked_auth = auth->Lock();
|
||||
for (const auto &[name, _] : *durability_) {
|
||||
if (name == kDefaultDB) continue;
|
||||
locked_auth->DeleteDatabase(name);
|
||||
durability_->Delete(name);
|
||||
}
|
||||
}
|
||||
|
||||
// Startup replication state (if recovered at startup)
|
||||
auto replica = [this](replication::RoleReplicaData const &data) {
|
||||
// Register handlers
|
||||
InMemoryReplicationHandlers::Register(this, *data.server);
|
||||
if (!data.server->Start()) {
|
||||
spdlog::error("Unable to start the replication server.");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
};
|
||||
// Replication frequent check start
|
||||
auto main = [this](replication::RoleMainData &data) {
|
||||
for (auto &client : data.registered_replicas_) {
|
||||
StartReplicaClient(*this, client);
|
||||
}
|
||||
return true;
|
||||
};
|
||||
// Startup proccess for main/replica
|
||||
MG_ASSERT(std::visit(memgraph::utils::Overloaded{replica, main}, repl_state_.ReplicationData()),
|
||||
"Replica recovery failure!");
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
@@ -26,7 +26,11 @@
|
||||
#include "auth/auth.hpp"
|
||||
#include "constants.hpp"
|
||||
#include "dbms/database.hpp"
|
||||
#include "dbms/inmemory/replication_handlers.hpp"
|
||||
#ifdef MG_ENTERPRISE
|
||||
#include "dbms/database_handler.hpp"
|
||||
#endif
|
||||
#include "dbms/replication_client.hpp"
|
||||
#include "global.hpp"
|
||||
#include "query/config.hpp"
|
||||
#include "query/interpreter_context.hpp"
|
||||
@@ -81,58 +85,44 @@ static inline nlohmann::json ToJson(const Statistics &stats) {
|
||||
return res;
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
using DeleteResult = utils::BasicResult<DeleteError>;
|
||||
|
||||
/**
|
||||
* @brief Multi-database session contexts handler.
|
||||
*/
|
||||
class DbmsHandler {
|
||||
public:
|
||||
using LockT = utils::RWLock;
|
||||
#ifdef MG_ENTERPRISE
|
||||
|
||||
using NewResultT = utils::BasicResult<NewError, DatabaseAccess>;
|
||||
using DeleteResult = utils::BasicResult<DeleteError>;
|
||||
|
||||
/**
|
||||
* @brief Initialize the handler.
|
||||
*
|
||||
* @param configs storage and interpreter configurations
|
||||
* @param configs storage configuration
|
||||
* @param auth pointer to the global authenticator
|
||||
* @param recovery_on_startup restore databases (and its content) and authentication data
|
||||
* @param delete_on_drop when dropping delete any associated directories on disk
|
||||
*/
|
||||
DbmsHandler(storage::Config config, auto *auth, bool recovery_on_startup, bool delete_on_drop)
|
||||
: lock_{utils::RWLock::Priority::READ}, default_config_{std::move(config)}, delete_on_drop_(delete_on_drop) {
|
||||
// TODO: Decouple storage config from dbms config
|
||||
// TODO: Save individual db configs inside the kvstore and restore from there
|
||||
storage::UpdatePaths(*default_config_, default_config_->durability.storage_directory / "databases");
|
||||
const auto &db_dir = default_config_->durability.storage_directory;
|
||||
const auto durability_dir = db_dir / ".durability";
|
||||
utils::EnsureDirOrDie(db_dir);
|
||||
utils::EnsureDirOrDie(durability_dir);
|
||||
durability_ = std::make_unique<kvstore::KVStore>(durability_dir);
|
||||
|
||||
// Generate the default database
|
||||
MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB.");
|
||||
|
||||
// Recover previous databases
|
||||
if (recovery_on_startup) {
|
||||
for (const auto &[name, _] : *durability_) {
|
||||
if (name == kDefaultDB) continue; // Already set
|
||||
spdlog::info("Restoring database {}.", name);
|
||||
MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name);
|
||||
spdlog::info("Database {} restored.", name);
|
||||
}
|
||||
} else { // Clear databases from the durability list and auth
|
||||
auto locked_auth = auth->Lock();
|
||||
for (const auto &[name, _] : *durability_) {
|
||||
if (name == kDefaultDB) continue;
|
||||
locked_auth->DeleteDatabase(name);
|
||||
durability_->Delete(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
DbmsHandler(storage::Config config,
|
||||
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
|
||||
bool recovery_on_startup, bool delete_on_drop); // TODO If more arguments are added use a config strut
|
||||
#else
|
||||
/**
|
||||
* @brief Initialize the handler. A single database is supported in community edition.
|
||||
*
|
||||
* @param configs storage configuration
|
||||
*/
|
||||
DbmsHandler(storage::Config config)
|
||||
: repl_state_{ReplicationStateRootPath(config)},
|
||||
db_gatekeeper_{[&] {
|
||||
config.name = kDefaultDB;
|
||||
return std::move(config);
|
||||
}(),
|
||||
repl_state_} {}
|
||||
#endif
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
/**
|
||||
* @brief Create a new Database associated with the "name" database
|
||||
*
|
||||
@@ -151,11 +141,24 @@ class DbmsHandler {
|
||||
* @return DatabaseAccess
|
||||
* @throw UnknownDatabaseException if database not found
|
||||
*/
|
||||
DatabaseAccess Get(std::string_view name) {
|
||||
DatabaseAccess Get(std::string_view name = kDefaultDB) {
|
||||
std::shared_lock<LockT> rd(lock_);
|
||||
return Get_(name);
|
||||
}
|
||||
#else
|
||||
/**
|
||||
* @brief Get the context associated with the default database
|
||||
*
|
||||
* @return DatabaseAccess
|
||||
*/
|
||||
DatabaseAccess Get() {
|
||||
auto acc = db_gatekeeper_.access();
|
||||
MG_ASSERT(acc, "Failed to get default database!");
|
||||
return *acc;
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
/**
|
||||
* @brief Delete database.
|
||||
*
|
||||
@@ -201,6 +204,7 @@ class DbmsHandler {
|
||||
|
||||
return {}; // Success
|
||||
}
|
||||
#endif
|
||||
|
||||
/**
|
||||
* @brief Return all active databases.
|
||||
@@ -208,10 +212,20 @@ class DbmsHandler {
|
||||
* @return std::vector<std::string>
|
||||
*/
|
||||
std::vector<std::string> All() const {
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::shared_lock<LockT> rd(lock_);
|
||||
return db_handler_.All();
|
||||
#else
|
||||
return {db_gatekeeper_.access()->get()->id()};
|
||||
#endif
|
||||
}
|
||||
|
||||
replication::ReplicationState &ReplicationState() { return repl_state_; }
|
||||
replication::ReplicationState const &ReplicationState() const { return repl_state_; }
|
||||
|
||||
bool IsMain() const { return repl_state_.IsMain(); }
|
||||
bool IsReplica() const { return repl_state_.IsReplica(); }
|
||||
|
||||
/**
|
||||
* @brief Return the statistics all databases.
|
||||
*
|
||||
@@ -220,24 +234,30 @@ class DbmsHandler {
|
||||
Statistics Stats() {
|
||||
Statistics stats{};
|
||||
// TODO: Handle overflow?
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::shared_lock<LockT> rd(lock_);
|
||||
for (auto &[_, db_gk] : db_handler_) {
|
||||
#else
|
||||
{
|
||||
auto &db_gk = db_gatekeeper_;
|
||||
#endif
|
||||
auto db_acc_opt = db_gk.access();
|
||||
if (!db_acc_opt) continue;
|
||||
auto &db_acc = *db_acc_opt;
|
||||
const auto &info = db_acc->GetInfo();
|
||||
const auto &storage_info = info.storage_info;
|
||||
stats.num_vertex += storage_info.vertex_count;
|
||||
stats.num_edges += storage_info.edge_count;
|
||||
stats.triggers += info.triggers;
|
||||
stats.streams += info.streams;
|
||||
++stats.num_databases;
|
||||
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
|
||||
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
|
||||
++stats.storage_modes[(int)storage_info.storage_mode];
|
||||
++stats.isolation_levels[(int)storage_info.isolation_level];
|
||||
stats.snapshot_enabled += storage_info.durability_snapshot_enabled;
|
||||
stats.wal_enabled += storage_info.durability_wal_enabled;
|
||||
if (db_acc_opt) {
|
||||
auto &db_acc = *db_acc_opt;
|
||||
const auto &info = db_acc->GetInfo();
|
||||
const auto &storage_info = info.storage_info;
|
||||
stats.num_vertex += storage_info.vertex_count;
|
||||
stats.num_edges += storage_info.edge_count;
|
||||
stats.triggers += info.triggers;
|
||||
stats.streams += info.streams;
|
||||
++stats.num_databases;
|
||||
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
|
||||
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
|
||||
++stats.storage_modes[(int)storage_info.storage_mode];
|
||||
++stats.isolation_levels[(int)storage_info.isolation_level];
|
||||
stats.snapshot_enabled += storage_info.durability_snapshot_enabled;
|
||||
stats.wal_enabled += storage_info.durability_wal_enabled;
|
||||
}
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
@@ -249,13 +269,19 @@ class DbmsHandler {
|
||||
*/
|
||||
std::vector<DatabaseInfo> Info() {
|
||||
std::vector<DatabaseInfo> res;
|
||||
res.reserve(std::distance(db_handler_.cbegin(), db_handler_.cend()));
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::shared_lock<LockT> rd(lock_);
|
||||
res.reserve(std::distance(db_handler_.cbegin(), db_handler_.cend()));
|
||||
for (auto &[_, db_gk] : db_handler_) {
|
||||
#else
|
||||
{
|
||||
auto &db_gk = db_gatekeeper_;
|
||||
#endif
|
||||
auto db_acc_opt = db_gk.access();
|
||||
if (!db_acc_opt) continue;
|
||||
auto &db_acc = *db_acc_opt;
|
||||
res.push_back(db_acc->GetInfo());
|
||||
if (db_acc_opt) {
|
||||
auto &db_acc = *db_acc_opt;
|
||||
res.push_back(db_acc->GetInfo());
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
@@ -267,15 +293,21 @@ class DbmsHandler {
|
||||
* @param ic global InterpreterContext
|
||||
*/
|
||||
void RestoreTriggers(query::InterpreterContext *ic) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::lock_guard<LockT> wr(lock_);
|
||||
for (auto &[_, db_gk] : db_handler_) {
|
||||
#else
|
||||
{
|
||||
auto &db_gk = db_gatekeeper_;
|
||||
#endif
|
||||
auto db_acc_opt = db_gk.access();
|
||||
if (!db_acc_opt) continue;
|
||||
auto &db_acc = *db_acc_opt;
|
||||
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id());
|
||||
auto storage_accessor = db_acc->Access();
|
||||
auto dba = memgraph::query::DbAccessor{storage_accessor.get()};
|
||||
db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker);
|
||||
if (db_acc_opt) {
|
||||
auto &db_acc = *db_acc_opt;
|
||||
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id());
|
||||
auto storage_accessor = db_acc->Access();
|
||||
auto dba = memgraph::query::DbAccessor{storage_accessor.get()};
|
||||
db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -286,17 +318,67 @@ class DbmsHandler {
|
||||
* @param ic global InterpreterContext
|
||||
*/
|
||||
void RestoreStreams(query::InterpreterContext *ic) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::lock_guard<LockT> wr(lock_);
|
||||
for (auto &[_, db_gk] : db_handler_) {
|
||||
#else
|
||||
{
|
||||
auto &db_gk = db_gatekeeper_;
|
||||
#endif
|
||||
auto db_acc = db_gk.access();
|
||||
if (!db_acc) continue;
|
||||
auto *db = db_acc->get();
|
||||
spdlog::debug("Restoring streams for database \"{}\"", db->id());
|
||||
db->streams()->RestoreStreams(*db_acc, ic);
|
||||
if (db_acc) {
|
||||
auto *db = db_acc->get();
|
||||
spdlog::debug("Restoring streams for database \"{}\"", db->id());
|
||||
db->streams()->RestoreStreams(*db_acc, ic);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief todo
|
||||
*
|
||||
* @param f
|
||||
*/
|
||||
void ForEach(auto f) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::shared_lock<LockT> rd(lock_);
|
||||
for (auto &[_, db_gk] : db_handler_) {
|
||||
#else
|
||||
{
|
||||
auto &db_gk = db_gatekeeper_;
|
||||
#endif
|
||||
auto db_acc = db_gk.access();
|
||||
if (db_acc) { // This isn't an error, just a defunct db
|
||||
f(db_acc->get());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @brief todo
|
||||
*
|
||||
* @param f
|
||||
*/
|
||||
void ForOne(auto f) {
|
||||
#ifdef MG_ENTERPRISE
|
||||
std::shared_lock<LockT> rd(lock_);
|
||||
for (auto &[_, db_gk] : db_handler_) {
|
||||
auto db_acc = db_gk.access();
|
||||
if (db_acc) { // This isn't an error, just a defunct db
|
||||
if (f(db_acc->get())) break; // Run until the first successful one
|
||||
}
|
||||
}
|
||||
#else
|
||||
{
|
||||
auto db_acc = db_gatekeeper_.access();
|
||||
MG_ASSERT(db_acc, "Should always have the database");
|
||||
f(db_acc->get());
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
private:
|
||||
#ifdef MG_ENTERPRISE
|
||||
/**
|
||||
* @brief return the storage directory of the associated database
|
||||
*
|
||||
@@ -328,13 +410,9 @@ class DbmsHandler {
|
||||
* @return NewResultT context on success, error on failure
|
||||
*/
|
||||
NewResultT New_(const std::string &name, std::filesystem::path storage_subdir) {
|
||||
if (default_config_) {
|
||||
auto config_copy = *default_config_;
|
||||
storage::UpdatePaths(config_copy, default_config_->durability.storage_directory / storage_subdir);
|
||||
return New_(name, config_copy);
|
||||
}
|
||||
spdlog::info("Trying to generate session context without any configurations.");
|
||||
return NewError::NO_CONFIGS;
|
||||
auto config_copy = default_config_;
|
||||
storage::UpdatePaths(config_copy, default_config_.durability.storage_directory / storage_subdir);
|
||||
return New_(name, config_copy);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -351,7 +429,7 @@ class DbmsHandler {
|
||||
return NewError::DEFUNCT;
|
||||
}
|
||||
|
||||
auto new_db = db_handler_.New(name, storage_config);
|
||||
auto new_db = db_handler_.New(name, storage_config, repl_state_);
|
||||
if (new_db.HasValue()) {
|
||||
// Success
|
||||
if (durability_) durability_->Put(name, "ok"); // TODO: Serialize the configuration?
|
||||
@@ -436,14 +514,17 @@ class DbmsHandler {
|
||||
throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name);
|
||||
}
|
||||
|
||||
// Should storage objects ever be deleted?
|
||||
mutable LockT lock_; //!< protective lock
|
||||
DatabaseHandler db_handler_; //!< multi-tenancy storage handler
|
||||
std::optional<storage::Config> default_config_; //!< Storage configuration used when creating new databases
|
||||
std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation)
|
||||
std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures
|
||||
bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory
|
||||
};
|
||||
mutable LockT lock_{utils::RWLock::Priority::READ}; //!< protective lock
|
||||
storage::Config default_config_; //!< Storage configuration used when creating new databases
|
||||
DatabaseHandler db_handler_; //!< multi-tenancy storage handler
|
||||
std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation)
|
||||
bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory
|
||||
std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures
|
||||
#endif
|
||||
replication::ReplicationState repl_state_; //!< Global replication state
|
||||
#ifndef MG_ENTERPRISE
|
||||
mutable utils::Gatekeeper<Database> db_gatekeeper_; //!< Single databases gatekeeper
|
||||
#endif
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
enum class DeleteError : uint8_t {
|
||||
DEFAULT_DB,
|
||||
USING,
|
||||
@@ -34,11 +35,7 @@ enum class NewError : uint8_t {
|
||||
GENERIC,
|
||||
};
|
||||
|
||||
enum class SetForResult : uint8_t {
|
||||
SUCCESS,
|
||||
ALREADY_SET,
|
||||
FAIL,
|
||||
};
|
||||
#endif
|
||||
|
||||
/**
|
||||
* UnknownSession Exception
|
||||
|
||||
@@ -38,7 +38,7 @@ class Handler {
|
||||
* @brief Empty Handler constructor.
|
||||
*
|
||||
*/
|
||||
Handler() {}
|
||||
Handler() = default;
|
||||
|
||||
/**
|
||||
* @brief Generate a new context and corresponding configuration.
|
||||
@@ -49,7 +49,7 @@ class Handler {
|
||||
* @return NewResult
|
||||
*/
|
||||
template <typename... Args>
|
||||
NewResult New(std::piecewise_construct_t /* marker */, std::string_view name, Args... args) {
|
||||
NewResult New(std::piecewise_construct_t /* marker */, std::string_view name, Args &&...args) {
|
||||
// Make sure the emplace will succeed, since we don't want to create temporary objects that could break something
|
||||
if (!Has(name)) {
|
||||
auto [itr, _] = items_.emplace(std::piecewise_construct, std::forward_as_tuple(name),
|
||||
|
||||
@@ -9,7 +9,11 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "storage/v2/inmemory/replication/replication_server.hpp"
|
||||
#include "dbms/inmemory/replication_handlers.hpp"
|
||||
#include "dbms/constants.hpp"
|
||||
#include "dbms/dbms_handler.hpp"
|
||||
#include "replication/replication_server.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
#include "storage/v2/durability/durability.hpp"
|
||||
#include "storage/v2/durability/snapshot.hpp"
|
||||
#include "storage/v2/durability/version.hpp"
|
||||
@@ -17,9 +21,20 @@
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/inmemory/unique_constraints.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
using memgraph::storage::Delta;
|
||||
using memgraph::storage::EdgeAccessor;
|
||||
using memgraph::storage::EdgeRef;
|
||||
using memgraph::storage::EdgeTypeId;
|
||||
using memgraph::storage::LabelIndexStats;
|
||||
using memgraph::storage::LabelPropertyIndexStats;
|
||||
using memgraph::storage::PropertyId;
|
||||
using memgraph::storage::UniqueConstraints;
|
||||
using memgraph::storage::View;
|
||||
using memgraph::storage::durability::WalDeltaData;
|
||||
|
||||
namespace memgraph::dbms {
|
||||
namespace {
|
||||
std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder *decoder) {
|
||||
std::pair<uint64_t, WalDeltaData> ReadDelta(storage::durability::BaseDecoder *decoder) {
|
||||
try {
|
||||
auto timestamp = ReadWalDeltaHeader(decoder);
|
||||
SPDLOG_INFO(" Timestamp {}", timestamp);
|
||||
@@ -27,78 +42,114 @@ std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder
|
||||
return {timestamp, delta};
|
||||
} catch (const slk::SlkReaderException &) {
|
||||
throw utils::BasicException("Missing data!");
|
||||
} catch (const durability::RecoveryFailure &) {
|
||||
} catch (const storage::durability::RecoveryFailure &) {
|
||||
throw utils::BasicException("Invalid data!");
|
||||
}
|
||||
};
|
||||
|
||||
std::optional<DatabaseAccess> GetDatabaseAccessor(dbms::DbmsHandler *dbms_handler, std::string_view db_name) {
|
||||
try {
|
||||
#ifdef MG_ENTERPRISE
|
||||
auto acc = dbms_handler->Get(db_name);
|
||||
#else
|
||||
if (db_name != dbms::kDefaultDB) {
|
||||
spdlog::warn("Trying to replicate a non-default database on a community replica.");
|
||||
return std::nullopt;
|
||||
}
|
||||
auto acc = dbms_handler->Get();
|
||||
#endif
|
||||
if (!acc) {
|
||||
spdlog::error("Failed to get access to ", db_name);
|
||||
return std::nullopt;
|
||||
}
|
||||
auto *inmem_storage = dynamic_cast<storage::InMemoryStorage *>(acc.get()->storage());
|
||||
if (!inmem_storage || inmem_storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
|
||||
spdlog::error("Database \"{}\" is not IN_MEMORY_TRANSACTIONAL.", db_name);
|
||||
return std::nullopt;
|
||||
}
|
||||
return std::optional{std::move(acc)};
|
||||
} catch (const dbms::UnknownDatabaseException &e) {
|
||||
spdlog::warn("No database \"{}\" on replica!", db_name);
|
||||
return std::nullopt;
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
InMemoryReplicationServer::InMemoryReplicationServer(InMemoryStorage *storage,
|
||||
const memgraph::replication::ReplicationServerConfig &config,
|
||||
memgraph::replication::ReplicationEpoch *repl_epoch)
|
||||
: ReplicationServer{config}, storage_(storage), repl_epoch_{repl_epoch} {
|
||||
rpc_server_.Register<replication::HeartbeatRpc>([this](auto *req_reader, auto *res_builder) {
|
||||
void InMemoryReplicationHandlers::Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server) {
|
||||
server.rpc_server_.Register<storage::replication::HeartbeatRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received HeartbeatRpc");
|
||||
this->HeartbeatHandler(req_reader, res_builder);
|
||||
InMemoryReplicationHandlers::HeartbeatHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
|
||||
rpc_server_.Register<replication::AppendDeltasRpc>([this](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received AppendDeltasRpc");
|
||||
this->AppendDeltasHandler(req_reader, res_builder);
|
||||
});
|
||||
rpc_server_.Register<replication::SnapshotRpc>([this](auto *req_reader, auto *res_builder) {
|
||||
server.rpc_server_.Register<storage::replication::AppendDeltasRpc>(
|
||||
[dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received AppendDeltasRpc");
|
||||
InMemoryReplicationHandlers::AppendDeltasHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
server.rpc_server_.Register<storage::replication::SnapshotRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received SnapshotRpc");
|
||||
this->SnapshotHandler(req_reader, res_builder);
|
||||
InMemoryReplicationHandlers::SnapshotHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
rpc_server_.Register<replication::WalFilesRpc>([this](auto *req_reader, auto *res_builder) {
|
||||
server.rpc_server_.Register<storage::replication::WalFilesRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received WalFilesRpc");
|
||||
this->WalFilesHandler(req_reader, res_builder);
|
||||
InMemoryReplicationHandlers::WalFilesHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
rpc_server_.Register<replication::CurrentWalRpc>([this](auto *req_reader, auto *res_builder) {
|
||||
server.rpc_server_.Register<storage::replication::CurrentWalRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received CurrentWalRpc");
|
||||
this->CurrentWalHandler(req_reader, res_builder);
|
||||
InMemoryReplicationHandlers::CurrentWalHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
rpc_server_.Register<replication::TimestampRpc>([this](auto *req_reader, auto *res_builder) {
|
||||
server.rpc_server_.Register<storage::replication::TimestampRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
|
||||
spdlog::debug("Received TimestampRpc");
|
||||
this->TimestampHandler(req_reader, res_builder);
|
||||
InMemoryReplicationHandlers::TimestampHandler(dbms_handler, req_reader, res_builder);
|
||||
});
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::HeartbeatReq req;
|
||||
void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::HeartbeatReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
replication::HeartbeatRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load(),
|
||||
std::string{repl_epoch_->id()}};
|
||||
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
|
||||
if (!db_acc) return;
|
||||
|
||||
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
|
||||
auto const *storage = db_acc->get()->storage();
|
||||
storage::replication::HeartbeatRes res{storage->id(), true,
|
||||
storage->repl_storage_state_.last_commit_timestamp_.load(),
|
||||
std::string{storage->repl_storage_state_.epoch_.id()}};
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::AppendDeltasReq req;
|
||||
void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::AppendDeltasReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
|
||||
if (!db_acc) return;
|
||||
|
||||
replication::Decoder decoder(req_reader);
|
||||
storage::replication::Decoder decoder(req_reader);
|
||||
|
||||
auto maybe_epoch_id = decoder.ReadString();
|
||||
MG_ASSERT(maybe_epoch_id, "Invalid replication message");
|
||||
|
||||
auto &repl_storage_state = storage_->repl_storage_state_;
|
||||
if (*maybe_epoch_id != repl_epoch_->id()) {
|
||||
auto prev_epoch = repl_epoch_->SetEpoch(*maybe_epoch_id);
|
||||
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
|
||||
auto &repl_storage_state = storage->repl_storage_state_;
|
||||
if (*maybe_epoch_id != storage->repl_storage_state_.epoch_.id()) {
|
||||
auto prev_epoch = storage->repl_storage_state_.epoch_.SetEpoch(*maybe_epoch_id);
|
||||
repl_storage_state.AddEpochToHistoryForce(prev_epoch);
|
||||
}
|
||||
|
||||
if (storage_->wal_file_) {
|
||||
if (req.seq_num > storage_->wal_file_->SequenceNumber() || *maybe_epoch_id != repl_epoch_->id()) {
|
||||
storage_->wal_file_->FinalizeWal();
|
||||
storage_->wal_file_.reset();
|
||||
storage_->wal_seq_num_ = req.seq_num;
|
||||
if (storage->wal_file_) {
|
||||
if (req.seq_num > storage->wal_file_->SequenceNumber() ||
|
||||
*maybe_epoch_id != storage->repl_storage_state_.epoch_.id()) {
|
||||
storage->wal_file_->FinalizeWal();
|
||||
storage->wal_file_.reset();
|
||||
storage->wal_seq_num_ = req.seq_num;
|
||||
spdlog::trace("Finalized WAL file");
|
||||
} else {
|
||||
MG_ASSERT(storage_->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
|
||||
storage_->wal_seq_num_ = req.seq_num + 1;
|
||||
MG_ASSERT(storage->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
|
||||
storage->wal_seq_num_ = req.seq_num + 1;
|
||||
}
|
||||
} else {
|
||||
storage_->wal_seq_num_ = req.seq_num;
|
||||
storage->wal_seq_num_ = req.seq_num;
|
||||
}
|
||||
|
||||
if (req.previous_commit_timestamp != repl_storage_state.last_commit_timestamp_.load()) {
|
||||
@@ -107,144 +158,162 @@ void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk
|
||||
while (!transaction_complete) {
|
||||
SPDLOG_INFO("Skipping delta");
|
||||
const auto [timestamp, delta] = ReadDelta(&decoder);
|
||||
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(
|
||||
delta.type, durability::kVersion); // TODO: Check if we are always using the latest version when replicating
|
||||
transaction_complete = storage::durability::IsWalDeltaDataTypeTransactionEnd(
|
||||
delta.type,
|
||||
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
|
||||
}
|
||||
|
||||
replication::AppendDeltasRes res{false, repl_storage_state.last_commit_timestamp_.load()};
|
||||
storage::replication::AppendDeltasRes res{storage->id(), false, repl_storage_state.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
return;
|
||||
}
|
||||
|
||||
ReadAndApplyDelta(storage_, &decoder,
|
||||
durability::kVersion); // TODO: Check if we are always using the latest version when replicating
|
||||
ReadAndApplyDelta(
|
||||
storage, &decoder,
|
||||
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
|
||||
|
||||
replication::AppendDeltasRes res{true, repl_storage_state.last_commit_timestamp_.load()};
|
||||
storage::replication::AppendDeltasRes res{storage->id(), true, repl_storage_state.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::SnapshotReq req;
|
||||
void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::SnapshotReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
|
||||
if (!db_acc) return;
|
||||
|
||||
replication::Decoder decoder(req_reader);
|
||||
storage::replication::Decoder decoder(req_reader);
|
||||
|
||||
utils::EnsureDirOrDie(storage_->snapshot_directory_);
|
||||
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
|
||||
utils::EnsureDirOrDie(storage->snapshot_directory_);
|
||||
|
||||
const auto maybe_snapshot_path = decoder.ReadFile(storage_->snapshot_directory_);
|
||||
const auto maybe_snapshot_path = decoder.ReadFile(storage->snapshot_directory_);
|
||||
MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!");
|
||||
spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path);
|
||||
|
||||
auto storage_guard = std::unique_lock{storage_->main_lock_};
|
||||
auto storage_guard = std::unique_lock{storage->main_lock_};
|
||||
spdlog::trace("Clearing database since recovering from snapshot.");
|
||||
// Clear the database
|
||||
storage_->vertices_.clear();
|
||||
storage_->edges_.clear();
|
||||
storage->vertices_.clear();
|
||||
storage->edges_.clear();
|
||||
|
||||
storage_->constraints_.existence_constraints_ = std::make_unique<ExistenceConstraints>();
|
||||
storage_->constraints_.unique_constraints_ = std::make_unique<InMemoryUniqueConstraints>();
|
||||
storage_->indices_.label_index_ = std::make_unique<InMemoryLabelIndex>();
|
||||
storage_->indices_.label_property_index_ = std::make_unique<InMemoryLabelPropertyIndex>();
|
||||
storage->constraints_.existence_constraints_ = std::make_unique<storage::ExistenceConstraints>();
|
||||
storage->constraints_.unique_constraints_ = std::make_unique<storage::InMemoryUniqueConstraints>();
|
||||
storage->indices_.label_index_ = std::make_unique<storage::InMemoryLabelIndex>();
|
||||
storage->indices_.label_property_index_ = std::make_unique<storage::InMemoryLabelPropertyIndex>();
|
||||
try {
|
||||
spdlog::debug("Loading snapshot");
|
||||
auto recovered_snapshot = durability::LoadSnapshot(
|
||||
*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_, &storage_->repl_storage_state_.history,
|
||||
storage_->name_id_mapper_.get(), &storage_->edge_count_, storage_->config_);
|
||||
auto recovered_snapshot = storage::durability::LoadSnapshot(
|
||||
*maybe_snapshot_path, &storage->vertices_, &storage->edges_, &storage->repl_storage_state_.history,
|
||||
storage->name_id_mapper_.get(), &storage->edge_count_, storage->config_);
|
||||
spdlog::debug("Snapshot loaded successfully");
|
||||
// If this step is present it should always be the first step of
|
||||
// the recovery so we use the UUID we read from snasphost
|
||||
storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
|
||||
repl_epoch_->SetEpoch(std::move(recovered_snapshot.snapshot_info.epoch_id));
|
||||
storage->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
|
||||
storage->repl_storage_state_.epoch_.SetEpoch(std::move(recovered_snapshot.snapshot_info.epoch_id));
|
||||
const auto &recovery_info = recovered_snapshot.recovery_info;
|
||||
storage_->vertex_id_ = recovery_info.next_vertex_id;
|
||||
storage_->edge_id_ = recovery_info.next_edge_id;
|
||||
storage_->timestamp_ = std::max(storage_->timestamp_, recovery_info.next_timestamp);
|
||||
storage->vertex_id_ = recovery_info.next_vertex_id;
|
||||
storage->edge_id_ = recovery_info.next_edge_id;
|
||||
storage->timestamp_ = std::max(storage->timestamp_, recovery_info.next_timestamp);
|
||||
|
||||
spdlog::trace("Recovering indices and constraints from snapshot.");
|
||||
durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage_->indices_,
|
||||
&storage_->constraints_, &storage_->vertices_);
|
||||
} catch (const durability::RecoveryFailure &e) {
|
||||
storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_,
|
||||
&storage->constraints_, &storage->vertices_,
|
||||
storage->name_id_mapper_.get());
|
||||
} catch (const storage::durability::RecoveryFailure &e) {
|
||||
LOG_FATAL("Couldn't load the snapshot because of: {}", e.what());
|
||||
}
|
||||
storage_guard.unlock();
|
||||
|
||||
replication::SnapshotRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
storage::replication::SnapshotRes res{storage->id(), true,
|
||||
storage->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
|
||||
spdlog::trace("Deleting old snapshot files due to snapshot recovery.");
|
||||
// Delete other durability files
|
||||
auto snapshot_files = durability::GetSnapshotFiles(storage_->snapshot_directory_, storage_->uuid_);
|
||||
auto snapshot_files = storage::durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_);
|
||||
for (const auto &[path, uuid, _] : snapshot_files) {
|
||||
if (path != *maybe_snapshot_path) {
|
||||
spdlog::trace("Deleting snapshot file {}", path);
|
||||
storage_->file_retainer_.DeleteFile(path);
|
||||
storage->file_retainer_.DeleteFile(path);
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::trace("Deleting old WAL files due to snapshot recovery.");
|
||||
auto wal_files = durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_);
|
||||
auto wal_files = storage::durability::GetWalFiles(storage->wal_directory_, storage->uuid_);
|
||||
if (wal_files) {
|
||||
for (const auto &wal_file : *wal_files) {
|
||||
spdlog::trace("Deleting WAL file {}", wal_file.path);
|
||||
storage_->file_retainer_.DeleteFile(wal_file.path);
|
||||
storage->file_retainer_.DeleteFile(wal_file.path);
|
||||
}
|
||||
|
||||
storage_->wal_file_.reset();
|
||||
storage->wal_file_.reset();
|
||||
}
|
||||
spdlog::debug("Replication recovery from snapshot finished!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::WalFilesReq req;
|
||||
void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::WalFilesReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
|
||||
if (!db_acc) return;
|
||||
|
||||
const auto wal_file_number = req.file_number;
|
||||
spdlog::debug("Received WAL files: {}", wal_file_number);
|
||||
|
||||
replication::Decoder decoder(req_reader);
|
||||
storage::replication::Decoder decoder(req_reader);
|
||||
|
||||
utils::EnsureDirOrDie(storage_->wal_directory_);
|
||||
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
|
||||
utils::EnsureDirOrDie(storage->wal_directory_);
|
||||
|
||||
for (auto i = 0; i < wal_file_number; ++i) {
|
||||
LoadWal(storage_, *repl_epoch_, &decoder);
|
||||
LoadWal(storage, &decoder);
|
||||
}
|
||||
|
||||
replication::WalFilesRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
storage::replication::WalFilesRes res{storage->id(), true,
|
||||
storage->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::CurrentWalReq req;
|
||||
void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::CurrentWalReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
|
||||
if (!db_acc) return;
|
||||
|
||||
replication::Decoder decoder(req_reader);
|
||||
storage::replication::Decoder decoder(req_reader);
|
||||
|
||||
utils::EnsureDirOrDie(storage_->wal_directory_);
|
||||
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
|
||||
utils::EnsureDirOrDie(storage->wal_directory_);
|
||||
|
||||
LoadWal(storage_, *repl_epoch_, &decoder);
|
||||
LoadWal(storage, &decoder);
|
||||
|
||||
replication::CurrentWalRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
storage::replication::CurrentWalRes res{storage->id(), true,
|
||||
storage->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
spdlog::debug("Replication recovery from current WAL ended successfully, replica is now up to date!");
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::replication::ReplicationEpoch &epoch,
|
||||
replication::Decoder *decoder) {
|
||||
const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory;
|
||||
void InMemoryReplicationHandlers::LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder) {
|
||||
const auto temp_wal_directory =
|
||||
std::filesystem::temp_directory_path() / "memgraph" / storage::durability::kWalDirectory;
|
||||
utils::EnsureDir(temp_wal_directory);
|
||||
auto maybe_wal_path = decoder->ReadFile(temp_wal_directory);
|
||||
MG_ASSERT(maybe_wal_path, "Failed to load WAL!");
|
||||
spdlog::trace("Received WAL saved to {}", *maybe_wal_path);
|
||||
try {
|
||||
auto wal_info = durability::ReadWalInfo(*maybe_wal_path);
|
||||
auto wal_info = storage::durability::ReadWalInfo(*maybe_wal_path);
|
||||
if (wal_info.seq_num == 0) {
|
||||
storage->uuid_ = wal_info.uuid;
|
||||
}
|
||||
|
||||
if (wal_info.epoch_id != epoch.id()) {
|
||||
auto prev_epoch = epoch.SetEpoch(wal_info.epoch_id);
|
||||
auto &replica_epoch = storage->repl_storage_state_.epoch_;
|
||||
if (wal_info.epoch_id != replica_epoch.id()) {
|
||||
auto prev_epoch = replica_epoch.SetEpoch(wal_info.epoch_id);
|
||||
storage->repl_storage_state_.AddEpochToHistoryForce(prev_epoch);
|
||||
}
|
||||
|
||||
@@ -259,11 +328,12 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::repl
|
||||
storage->wal_seq_num_ = wal_info.seq_num;
|
||||
}
|
||||
spdlog::trace("Loading WAL deltas from {}", *maybe_wal_path);
|
||||
durability::Decoder wal;
|
||||
const auto version = wal.Initialize(*maybe_wal_path, durability::kWalMagic);
|
||||
storage::durability::Decoder wal;
|
||||
const auto version = wal.Initialize(*maybe_wal_path, storage::durability::kWalMagic);
|
||||
spdlog::debug("WAL file {} loaded successfully", *maybe_wal_path);
|
||||
if (!version) throw durability::RecoveryFailure("Couldn't read WAL magic and/or version!");
|
||||
if (!durability::IsVersionSupported(*version)) throw durability::RecoveryFailure("Invalid WAL version!");
|
||||
if (!version) throw storage::durability::RecoveryFailure("Couldn't read WAL magic and/or version!");
|
||||
if (!storage::durability::IsVersionSupported(*version))
|
||||
throw storage::durability::RecoveryFailure("Invalid WAL version!");
|
||||
wal.SetPosition(wal_info.offset_deltas);
|
||||
|
||||
for (size_t i = 0; i < wal_info.num_deltas;) {
|
||||
@@ -271,38 +341,46 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::repl
|
||||
}
|
||||
|
||||
spdlog::debug("Replication from current WAL successful!");
|
||||
} catch (const durability::RecoveryFailure &e) {
|
||||
} catch (const storage::durability::RecoveryFailure &e) {
|
||||
LOG_FATAL("Couldn't recover WAL deltas from {} because of: {}", *maybe_wal_path, e.what());
|
||||
}
|
||||
}
|
||||
|
||||
void InMemoryReplicationServer::TimestampHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
|
||||
replication::TimestampReq req;
|
||||
void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
|
||||
slk::Builder *res_builder) {
|
||||
storage::replication::TimestampReq req;
|
||||
slk::Load(&req, req_reader);
|
||||
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
|
||||
if (!db_acc) return;
|
||||
|
||||
replication::TimestampRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
|
||||
auto const *storage = db_acc->get()->storage();
|
||||
storage::replication::TimestampRes res{storage->id(), true,
|
||||
storage->repl_storage_state_.last_commit_timestamp_.load()};
|
||||
slk::Save(res, res_builder);
|
||||
}
|
||||
|
||||
uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder,
|
||||
const uint64_t version) {
|
||||
uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage *storage,
|
||||
storage::durability::BaseDecoder *decoder,
|
||||
const uint64_t version) {
|
||||
auto edge_acc = storage->edges_.access();
|
||||
auto vertex_acc = storage->vertices_.access();
|
||||
|
||||
constexpr bool kUniqueAccess = true;
|
||||
constexpr bool kSharedAccess = false;
|
||||
|
||||
std::optional<std::pair<uint64_t, InMemoryStorage::ReplicationAccessor>> commit_timestamp_and_accessor;
|
||||
std::optional<std::pair<uint64_t, storage::InMemoryStorage::ReplicationAccessor>> commit_timestamp_and_accessor;
|
||||
auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp,
|
||||
bool unique = !kUniqueAccess) {
|
||||
bool unique = kSharedAccess) {
|
||||
if (!commit_timestamp_and_accessor) {
|
||||
std::unique_ptr<Storage::Accessor> acc = nullptr;
|
||||
std::unique_ptr<storage::Storage::Accessor> acc = nullptr;
|
||||
if (unique) {
|
||||
acc = storage->UniqueAccess(std::nullopt);
|
||||
acc = storage->UniqueAccess(std::nullopt, false /*not main*/);
|
||||
} else {
|
||||
acc = storage->Access(std::nullopt);
|
||||
acc = storage->Access(std::nullopt, false /*not main*/);
|
||||
}
|
||||
auto inmem_acc = std::unique_ptr<InMemoryStorage::InMemoryAccessor>(
|
||||
static_cast<InMemoryStorage::InMemoryAccessor *>(acc.release()));
|
||||
auto inmem_acc = std::unique_ptr<storage::InMemoryStorage::InMemoryAccessor>(
|
||||
static_cast<storage::InMemoryStorage::InMemoryAccessor *>(acc.release()));
|
||||
commit_timestamp_and_accessor.emplace(commit_timestamp, std::move(*inmem_acc));
|
||||
} else if (commit_timestamp_and_accessor->first != commit_timestamp) {
|
||||
throw utils::BasicException("Received more than one transaction!");
|
||||
@@ -319,7 +397,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
max_commit_timestamp = timestamp;
|
||||
}
|
||||
|
||||
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version);
|
||||
transaction_complete = storage::durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version);
|
||||
|
||||
if (timestamp < storage->timestamp_) {
|
||||
continue;
|
||||
@@ -327,13 +405,13 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
|
||||
SPDLOG_INFO(" Delta {}", applied_deltas);
|
||||
switch (delta.type) {
|
||||
case durability::WalDeltaData::Type::VERTEX_CREATE: {
|
||||
case WalDeltaData::Type::VERTEX_CREATE: {
|
||||
spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint());
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
transaction->CreateVertexEx(delta.vertex_create_delete.gid);
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::VERTEX_DELETE: {
|
||||
case WalDeltaData::Type::VERTEX_DELETE: {
|
||||
spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint());
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW);
|
||||
@@ -342,7 +420,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
|
||||
case WalDeltaData::Type::VERTEX_ADD_LABEL: {
|
||||
spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(),
|
||||
delta.vertex_add_remove_label.label);
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
@@ -352,7 +430,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
|
||||
case WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
|
||||
spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(),
|
||||
delta.vertex_add_remove_label.label);
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
@@ -362,7 +440,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
|
||||
case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
|
||||
spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
|
||||
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
@@ -373,7 +451,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::EDGE_CREATE: {
|
||||
case WalDeltaData::Type::EDGE_CREATE: {
|
||||
spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}",
|
||||
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
|
||||
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
|
||||
@@ -388,7 +466,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (edge.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::EDGE_DELETE: {
|
||||
case WalDeltaData::Type::EDGE_DELETE: {
|
||||
spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}",
|
||||
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
|
||||
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
|
||||
@@ -406,7 +484,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
|
||||
case WalDeltaData::Type::EDGE_SET_PROPERTY: {
|
||||
spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
|
||||
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
|
||||
if (!storage->config_.items.properties_on_edges)
|
||||
@@ -469,17 +547,18 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
break;
|
||||
}
|
||||
|
||||
case durability::WalDeltaData::Type::TRANSACTION_END: {
|
||||
case WalDeltaData::Type::TRANSACTION_END: {
|
||||
spdlog::trace(" Transaction end");
|
||||
if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
|
||||
throw utils::BasicException("Invalid commit data!");
|
||||
auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first);
|
||||
auto ret =
|
||||
commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first, false /* not main */);
|
||||
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
commit_timestamp_and_accessor = std::nullopt;
|
||||
break;
|
||||
}
|
||||
|
||||
case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
|
||||
case WalDeltaData::Type::LABEL_INDEX_CREATE: {
|
||||
spdlog::trace(" Create label index on :{}", delta.operation_label.label);
|
||||
// Need to send the timestamp
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
@@ -487,14 +566,14 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
|
||||
case WalDeltaData::Type::LABEL_INDEX_DROP: {
|
||||
spdlog::trace(" Drop label index on :{}", delta.operation_label.label);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
if (transaction->DropIndex(storage->NameToLabel(delta.operation_label.label)).HasError())
|
||||
throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_INDEX_STATS_SET: {
|
||||
case WalDeltaData::Type::LABEL_INDEX_STATS_SET: {
|
||||
spdlog::trace(" Set label index statistics on :{}", delta.operation_label_stats.label);
|
||||
// Need to send the timestamp
|
||||
auto *transaction = get_transaction(timestamp);
|
||||
@@ -506,7 +585,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
transaction->SetIndexStats(label, stats);
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: {
|
||||
case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: {
|
||||
const auto &info = delta.operation_label;
|
||||
spdlog::trace(" Clear label index statistics on :{}", info.label);
|
||||
// Need to send the timestamp
|
||||
@@ -514,7 +593,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
transaction->DeleteLabelIndexStats(storage->NameToLabel(info.label));
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
|
||||
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
|
||||
spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label,
|
||||
delta.operation_label_property.property);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
@@ -525,7 +604,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
|
||||
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
|
||||
spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label,
|
||||
delta.operation_label_property.property);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
@@ -536,7 +615,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: {
|
||||
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: {
|
||||
const auto &info = delta.operation_label_property_stats;
|
||||
spdlog::trace(" Set label-property index statistics on :{}", info.label);
|
||||
// Need to send the timestamp
|
||||
@@ -550,7 +629,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
transaction->SetIndexStats(label, property, stats);
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: {
|
||||
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: {
|
||||
const auto &info = delta.operation_label;
|
||||
spdlog::trace(" Clear label-property index statistics on :{}", info.label);
|
||||
// Need to send the timestamp
|
||||
@@ -558,7 +637,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label));
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
|
||||
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
|
||||
spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
|
||||
delta.operation_label_property.property);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
@@ -568,7 +647,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
|
||||
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
|
||||
spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label,
|
||||
delta.operation_label_property.property);
|
||||
auto *transaction = get_transaction(timestamp, kUniqueAccess);
|
||||
@@ -579,7 +658,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
|
||||
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
|
||||
std::stringstream ss;
|
||||
utils::PrintIterable(ss, delta.operation_label_properties.properties);
|
||||
spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
|
||||
@@ -594,7 +673,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
throw utils::BasicException("Invalid transaction!");
|
||||
break;
|
||||
}
|
||||
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
|
||||
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
|
||||
std::stringstream ss;
|
||||
utils::PrintIterable(ss, delta.operation_label_properties.properties);
|
||||
spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
|
||||
@@ -621,4 +700,4 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
|
||||
return applied_deltas;
|
||||
}
|
||||
|
||||
} // namespace memgraph::storage
|
||||
} // namespace memgraph::dbms
|
||||
49
src/dbms/inmemory/replication_handlers.hpp
Normal file
49
src/dbms/inmemory/replication_handlers.hpp
Normal file
@@ -0,0 +1,49 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "replication/replication_server.hpp"
|
||||
#include "replication/state.hpp"
|
||||
#include "storage/v2/replication/serialization.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
class InMemoryStorage;
|
||||
}
|
||||
namespace memgraph::dbms {
|
||||
|
||||
class DbmsHandler;
|
||||
|
||||
class InMemoryReplicationHandlers {
|
||||
public:
|
||||
static void Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server);
|
||||
|
||||
private:
|
||||
// RPC handlers
|
||||
static void HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
|
||||
|
||||
static void LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder);
|
||||
|
||||
static uint64_t ReadAndApplyDelta(storage::InMemoryStorage *storage, storage::durability::BaseDecoder *decoder,
|
||||
uint64_t version);
|
||||
};
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
61
src/dbms/inmemory/storage_helper.hpp
Normal file
61
src/dbms/inmemory/storage_helper.hpp
Normal file
@@ -0,0 +1,61 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <variant>
|
||||
|
||||
#include "dbms/constants.hpp"
|
||||
#include "dbms/replication_handler.hpp"
|
||||
#include "replication/state.hpp"
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
inline std::unique_ptr<storage::Storage> CreateInMemoryStorage(storage::Config config,
|
||||
::memgraph::replication::ReplicationState &repl_state) {
|
||||
const auto wal_mode = config.durability.snapshot_wal_mode;
|
||||
const auto name = config.name;
|
||||
auto storage = std::make_unique<storage::InMemoryStorage>(std::move(config));
|
||||
|
||||
// Connect replication state and storage
|
||||
storage->CreateSnapshotHandler(
|
||||
[storage = storage.get(), &repl_state]() -> utils::BasicResult<storage::InMemoryStorage::CreateSnapshotError> {
|
||||
if (repl_state.IsReplica()) {
|
||||
return storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica;
|
||||
}
|
||||
return storage->CreateSnapshot();
|
||||
});
|
||||
|
||||
if (allow_mt_repl || name == dbms::kDefaultDB) {
|
||||
// Handle global replication state
|
||||
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
|
||||
// RECOVER REPLICA CONNECTIONS
|
||||
memgraph::dbms::RestoreReplication(repl_state, *storage);
|
||||
} else if (const ::memgraph::replication::RoleMainData *data =
|
||||
std::get_if<::memgraph::replication::RoleMainData>(&repl_state.ReplicationData());
|
||||
data && !data->registered_replicas_.empty()) {
|
||||
spdlog::warn("Multi-tenant replication is currently not supported!");
|
||||
}
|
||||
|
||||
if (wal_mode == storage::Config::Durability::SnapshotWalMode::DISABLED && repl_state.IsMain()) {
|
||||
spdlog::warn(
|
||||
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider "
|
||||
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
|
||||
"without write-ahead logs this instance is not replicating any data.");
|
||||
}
|
||||
|
||||
return std::move(storage);
|
||||
}
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
34
src/dbms/replication_client.cpp
Normal file
34
src/dbms/replication_client.cpp
Normal file
@@ -0,0 +1,34 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "dbms/replication_client.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client) {
|
||||
// No client error, start instance level client
|
||||
auto const &endpoint = client.rpc_client_.Endpoint();
|
||||
spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port);
|
||||
client.StartFrequentCheck([&dbms_handler](std::string_view name) {
|
||||
// Working connection, check if any database has been left behind
|
||||
dbms_handler.ForEach([name](dbms::Database *db) {
|
||||
// Specific database <-> replica client
|
||||
db->storage()->repl_storage_state_.WithClient(name, [&](storage::ReplicationStorageClient *client) {
|
||||
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
|
||||
// Database <-> replica might be behind, check and recover
|
||||
client->TryCheckReplicaStateAsync(db->storage());
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
21
src/dbms/replication_client.hpp
Normal file
21
src/dbms/replication_client.hpp
Normal file
@@ -0,0 +1,21 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "dbms/dbms_handler.hpp"
|
||||
#include "replication/replication_client.hpp"
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client);
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
262
src/dbms/replication_handler.cpp
Normal file
262
src/dbms/replication_handler.cpp
Normal file
@@ -0,0 +1,262 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "dbms/replication_handler.hpp"
|
||||
|
||||
#include "dbms/constants.hpp"
|
||||
#include "dbms/dbms_handler.hpp"
|
||||
#include "dbms/inmemory/replication_handlers.hpp"
|
||||
#include "dbms/inmemory/storage_helper.hpp"
|
||||
#include "dbms/replication_client.hpp"
|
||||
#include "replication/state.hpp"
|
||||
|
||||
using memgraph::replication::ReplicationClientConfig;
|
||||
using memgraph::replication::ReplicationState;
|
||||
using memgraph::replication::RoleMainData;
|
||||
using memgraph::replication::RoleReplicaData;
|
||||
|
||||
namespace memgraph::dbms {
|
||||
|
||||
namespace {
|
||||
|
||||
std::string RegisterReplicaErrorToString(RegisterReplicaError error) {
|
||||
switch (error) {
|
||||
using enum RegisterReplicaError;
|
||||
case NAME_EXISTS:
|
||||
return "NAME_EXISTS";
|
||||
case END_POINT_EXISTS:
|
||||
return "END_POINT_EXISTS";
|
||||
case CONNECTION_FAILED:
|
||||
return "CONNECTION_FAILED";
|
||||
case COULD_NOT_BE_PERSISTED:
|
||||
return "COULD_NOT_BE_PERSISTED";
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
ReplicationHandler::ReplicationHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {}
|
||||
|
||||
bool ReplicationHandler::SetReplicationRoleMain() {
|
||||
auto const main_handler = [](RoleMainData const &) {
|
||||
// If we are already MAIN, we don't want to change anything
|
||||
return false;
|
||||
};
|
||||
auto const replica_handler = [this](RoleReplicaData const &) {
|
||||
// STEP 1) bring down all REPLICA servers
|
||||
dbms_handler_.ForEach([](Database *db) {
|
||||
auto *storage = db->storage();
|
||||
// Remember old epoch + storage timestamp association
|
||||
storage->PrepareForNewEpoch();
|
||||
});
|
||||
|
||||
// STEP 2) Change to MAIN
|
||||
// TODO: restore replication servers if false?
|
||||
if (!dbms_handler_.ReplicationState().SetReplicationRoleMain()) {
|
||||
// TODO: Handle recovery on failure???
|
||||
return false;
|
||||
}
|
||||
|
||||
// STEP 3) We are now MAIN, update storage local epoch
|
||||
const auto &epoch =
|
||||
std::get<RoleMainData>(std::as_const(dbms_handler_.ReplicationState()).ReplicationData()).epoch_;
|
||||
dbms_handler_.ForEach([&](Database *db) {
|
||||
auto *storage = db->storage();
|
||||
storage->repl_storage_state_.epoch_ = epoch;
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
// TODO: under lock
|
||||
return std::visit(utils::Overloaded{main_handler, replica_handler},
|
||||
dbms_handler_.ReplicationState().ReplicationData());
|
||||
}
|
||||
|
||||
bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) {
|
||||
// We don't want to restart the server if we're already a REPLICA
|
||||
if (dbms_handler_.ReplicationState().IsReplica()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are
|
||||
// deleting the replica.
|
||||
// Remove database specific clients
|
||||
dbms_handler_.ForEach([&](Database *db) {
|
||||
auto *storage = db->storage();
|
||||
storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); });
|
||||
});
|
||||
// Remove instance level clients
|
||||
std::get<RoleMainData>(dbms_handler_.ReplicationState().ReplicationData()).registered_replicas_.clear();
|
||||
|
||||
// Creates the server
|
||||
dbms_handler_.ReplicationState().SetReplicationRoleReplica(config);
|
||||
|
||||
// Start
|
||||
const auto success =
|
||||
std::visit(utils::Overloaded{[](RoleMainData const &) {
|
||||
// ASSERT
|
||||
return false;
|
||||
},
|
||||
[this](RoleReplicaData const &data) {
|
||||
// Register handlers
|
||||
InMemoryReplicationHandlers::Register(&dbms_handler_, *data.server);
|
||||
if (!data.server->Start()) {
|
||||
spdlog::error("Unable to start the replication server.");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}},
|
||||
dbms_handler_.ReplicationState().ReplicationData());
|
||||
// TODO Handle error (restore to main?)
|
||||
return success;
|
||||
}
|
||||
|
||||
auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> memgraph::utils::BasicResult<RegisterReplicaError> {
|
||||
MG_ASSERT(dbms_handler_.ReplicationState().IsMain(), "Only main instance can register a replica!");
|
||||
|
||||
auto instance_client = dbms_handler_.ReplicationState().RegisterReplica(config);
|
||||
if (instance_client.HasError()) switch (instance_client.GetError()) {
|
||||
case memgraph::replication::RegisterReplicaError::NOT_MAIN:
|
||||
MG_ASSERT(false, "Only main instance can register a replica!");
|
||||
return {};
|
||||
case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
|
||||
return memgraph::dbms::RegisterReplicaError::NAME_EXISTS;
|
||||
case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS:
|
||||
return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS;
|
||||
case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
|
||||
return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
|
||||
case memgraph::replication::RegisterReplicaError::SUCCESS:
|
||||
break;
|
||||
}
|
||||
|
||||
if (!allow_mt_repl && dbms_handler_.All().size() > 1) {
|
||||
spdlog::warn("Multi-tenant replication is currently not supported!");
|
||||
}
|
||||
|
||||
bool all_clients_good = true;
|
||||
|
||||
// Add database specific clients (NOTE Currently all databases are connected to each replica)
|
||||
dbms_handler_.ForEach([&](Database *db) {
|
||||
auto *storage = db->storage();
|
||||
if (!allow_mt_repl && storage->id() != kDefaultDB) {
|
||||
return;
|
||||
}
|
||||
// TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
|
||||
if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
|
||||
|
||||
all_clients_good &=
|
||||
storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) {
|
||||
auto client = std::make_unique<storage::ReplicationStorageClient>(*instance_client.GetValue());
|
||||
client->Start(storage);
|
||||
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
|
||||
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
|
||||
// Failed to start due to branching of MAIN and REPLICA
|
||||
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
|
||||
return false;
|
||||
}
|
||||
storage_clients.push_back(std::move(client));
|
||||
return true;
|
||||
});
|
||||
});
|
||||
|
||||
// NOTE Currently if any databases fails, we revert back
|
||||
if (!all_clients_good) {
|
||||
spdlog::error("Failed to register all databases to the REPLICA \"{}\"", config.name);
|
||||
UnregisterReplica(config.name);
|
||||
return RegisterReplicaError::CONNECTION_FAILED;
|
||||
}
|
||||
|
||||
// No client error, start instance level client
|
||||
StartReplicaClient(dbms_handler_, *instance_client.GetValue());
|
||||
return {};
|
||||
}
|
||||
|
||||
auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterReplicaResult {
|
||||
auto const replica_handler = [](RoleReplicaData const &) -> UnregisterReplicaResult {
|
||||
return UnregisterReplicaResult::NOT_MAIN;
|
||||
};
|
||||
auto const main_handler = [this, name](RoleMainData &mainData) -> UnregisterReplicaResult {
|
||||
if (!dbms_handler_.ReplicationState().TryPersistUnregisterReplica(name)) {
|
||||
return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED;
|
||||
}
|
||||
// Remove database specific clients
|
||||
dbms_handler_.ForEach([name](Database *db) {
|
||||
db->storage()->repl_storage_state_.replication_clients_.WithLock([&name](auto &clients) {
|
||||
std::erase_if(clients, [name](const auto &client) { return client->Name() == name; });
|
||||
});
|
||||
});
|
||||
// Remove instance level clients
|
||||
auto const n_unregistered =
|
||||
std::erase_if(mainData.registered_replicas_, [name](auto const &client) { return client.name_ == name; });
|
||||
return n_unregistered != 0 ? UnregisterReplicaResult::SUCCESS : UnregisterReplicaResult::CAN_NOT_UNREGISTER;
|
||||
};
|
||||
|
||||
return std::visit(utils::Overloaded{main_handler, replica_handler},
|
||||
dbms_handler_.ReplicationState().ReplicationData());
|
||||
}
|
||||
|
||||
auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole {
|
||||
return dbms_handler_.ReplicationState().GetRole();
|
||||
}
|
||||
|
||||
bool ReplicationHandler::IsMain() const { return dbms_handler_.ReplicationState().IsMain(); }
|
||||
|
||||
bool ReplicationHandler::IsReplica() const { return dbms_handler_.ReplicationState().IsReplica(); }
|
||||
|
||||
// Per storage
|
||||
// NOTE Storage will connect to all replicas. Future work might change this
|
||||
void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage) {
|
||||
spdlog::info("Restoring replication role.");
|
||||
|
||||
/// MAIN
|
||||
auto const recover_main = [&storage](RoleMainData &mainData) {
|
||||
// Each individual client has already been restored and started. Here we just go through each database and start its
|
||||
// client
|
||||
for (auto &instance_client : mainData.registered_replicas_) {
|
||||
spdlog::info("Replica {} restoration started for {}.", instance_client.name_, storage.id());
|
||||
|
||||
const auto &ret = storage.repl_storage_state_.replication_clients_.WithLock(
|
||||
[&](auto &storage_clients) -> utils::BasicResult<RegisterReplicaError> {
|
||||
auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client);
|
||||
client->Start(&storage);
|
||||
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
|
||||
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
|
||||
// Failed to start due to branching of MAIN and REPLICA
|
||||
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
|
||||
spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.",
|
||||
instance_client.name_);
|
||||
}
|
||||
storage_clients.push_back(std::move(client));
|
||||
return {};
|
||||
});
|
||||
|
||||
if (ret.HasError()) {
|
||||
MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError());
|
||||
LOG_FATAL("Failure when restoring replica {}: {}.", instance_client.name_,
|
||||
RegisterReplicaErrorToString(ret.GetError()));
|
||||
}
|
||||
spdlog::info("Replica {} restored for {}.", instance_client.name_, storage.id());
|
||||
}
|
||||
spdlog::info("Replication role restored to MAIN.");
|
||||
};
|
||||
|
||||
/// REPLICA
|
||||
auto const recover_replica = [](RoleReplicaData const &data) { /*nothing to do*/ };
|
||||
|
||||
std::visit(
|
||||
utils::Overloaded{
|
||||
recover_main,
|
||||
recover_replica,
|
||||
},
|
||||
repl_state.ReplicationData());
|
||||
}
|
||||
} // namespace memgraph::dbms
|
||||
@@ -12,6 +12,7 @@
|
||||
#pragma once
|
||||
|
||||
#include "replication/role.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
// BEGIN fwd declares
|
||||
@@ -20,14 +21,10 @@ struct ReplicationState;
|
||||
struct ReplicationServerConfig;
|
||||
struct ReplicationClientConfig;
|
||||
} // namespace memgraph::replication
|
||||
namespace memgraph::storage {
|
||||
class Storage;
|
||||
}
|
||||
// END fwd declares
|
||||
|
||||
namespace memgraph::storage {
|
||||
namespace memgraph::dbms {
|
||||
class DbmsHandler;
|
||||
|
||||
enum class RegistrationMode : std::uint8_t { MUST_BE_INSTANTLY_VALID, RESTORE };
|
||||
enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
|
||||
enum class UnregisterReplicaResult : uint8_t {
|
||||
NOT_MAIN,
|
||||
@@ -39,8 +36,7 @@ enum class UnregisterReplicaResult : uint8_t {
|
||||
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
|
||||
/// TODO: extend to do multiple storages
|
||||
struct ReplicationHandler {
|
||||
ReplicationHandler(memgraph::replication::ReplicationState &replState, Storage &storage)
|
||||
: repl_state_(replState), storage_(storage) {}
|
||||
explicit ReplicationHandler(DbmsHandler &dbms_handler);
|
||||
|
||||
// as REPLICA, become MAIN
|
||||
bool SetReplicationRoleMain();
|
||||
@@ -49,23 +45,23 @@ struct ReplicationHandler {
|
||||
bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config);
|
||||
|
||||
// as MAIN, define and connect to REPLICAs
|
||||
auto RegisterReplica(RegistrationMode registration_mode, const memgraph::replication::ReplicationClientConfig &config)
|
||||
auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
|
||||
-> utils::BasicResult<RegisterReplicaError>;
|
||||
|
||||
// as MAIN, remove a REPLICA connection
|
||||
auto UnregisterReplica(std::string_view name) -> UnregisterReplicaResult;
|
||||
|
||||
// Generic restoration
|
||||
// TODO: decouple storage restoration from epoch restoration
|
||||
void RestoreReplication();
|
||||
|
||||
// Helper pass-through (TODO: remove)
|
||||
auto GetRole() const -> memgraph::replication::ReplicationRole;
|
||||
bool IsMain() const;
|
||||
bool IsReplica() const;
|
||||
|
||||
private:
|
||||
memgraph::replication::ReplicationState &repl_state_;
|
||||
Storage &storage_;
|
||||
DbmsHandler &dbms_handler_;
|
||||
};
|
||||
} // namespace memgraph::storage
|
||||
|
||||
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
|
||||
/// TODO: extend to do multiple storages
|
||||
void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage);
|
||||
|
||||
} // namespace memgraph::dbms
|
||||
@@ -65,7 +65,7 @@ DEFINE_bool(allow_load_csv, true, "Controls whether LOAD CSV clause is allowed i
|
||||
// Storage flags.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).",
|
||||
FLAG_IN_RANGE(1, 24 * 3600));
|
||||
FLAG_IN_RANGE(1, 24UL * 3600));
|
||||
// NOTE: The `storage_properties_on_edges` flag must be the same here and in
|
||||
// `mg_import_csv`. If you change it, make sure to change it there as well.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
|
||||
@@ -8,4 +8,3 @@ target_sources(mg-glue PRIVATE auth.cpp
|
||||
MonitoringServerT.cpp
|
||||
run_id.cpp)
|
||||
target_link_libraries(mg-glue mg-query mg-auth mg-audit mg-flags)
|
||||
target_precompile_headers(mg-glue INTERFACE auth_checker.hpp auth_handler.hpp)
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include "gflags/gflags.h"
|
||||
|
||||
#include "audit/log.hpp"
|
||||
@@ -272,7 +273,7 @@ void SessionHL::Configure(const std::map<std::string, memgraph::communication::b
|
||||
#endif
|
||||
}
|
||||
SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context,
|
||||
const memgraph::communication::v2::ServerEndpoint &endpoint,
|
||||
memgraph::communication::v2::ServerEndpoint endpoint,
|
||||
memgraph::communication::v2::InputStream *input_stream,
|
||||
memgraph::communication::v2::OutputStream *output_stream,
|
||||
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth
|
||||
@@ -289,7 +290,7 @@ SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context,
|
||||
audit_log_(audit_log),
|
||||
#endif
|
||||
auth_(auth),
|
||||
endpoint_(endpoint),
|
||||
endpoint_(std::move(endpoint)),
|
||||
implicit_db_(dbms::kDefaultDB) {
|
||||
// Metrics update
|
||||
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions);
|
||||
|
||||
@@ -23,7 +23,7 @@ class SessionHL final : public memgraph::communication::bolt::Session<memgraph::
|
||||
memgraph::communication::v2::OutputStream> {
|
||||
public:
|
||||
SessionHL(memgraph::query::InterpreterContext *interpreter_context,
|
||||
const memgraph::communication::v2::ServerEndpoint &endpoint,
|
||||
memgraph::communication::v2::ServerEndpoint endpoint,
|
||||
memgraph::communication::v2::InputStream *input_stream,
|
||||
memgraph::communication::v2::OutputStream *output_stream,
|
||||
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth
|
||||
|
||||
@@ -127,6 +127,8 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage
|
||||
return Value(value.ValueLocalDateTime());
|
||||
case query::TypedValue::Type::Duration:
|
||||
return Value(value.ValueDuration());
|
||||
case query::TypedValue::Type::Function:
|
||||
throw communication::bolt::ValueException("Unsupported conversion from TypedValue::Function to Value");
|
||||
case query::TypedValue::Type::Graph:
|
||||
auto maybe_graph = ToBoltGraph(value.ValueGraph(), db, view);
|
||||
if (maybe_graph.HasError()) return maybe_graph.GetError();
|
||||
@@ -202,7 +204,7 @@ storage::Result<std::map<std::string, Value>> ToBoltGraph(const query::Graph &gr
|
||||
for (const auto &v : graph.vertices()) {
|
||||
auto maybe_vertex = ToBoltVertex(v, db, view);
|
||||
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
|
||||
vertices.emplace_back(Value(std::move(*maybe_vertex)));
|
||||
vertices.emplace_back(std::move(*maybe_vertex));
|
||||
}
|
||||
map.emplace("nodes", Value(vertices));
|
||||
|
||||
@@ -211,7 +213,7 @@ storage::Result<std::map<std::string, Value>> ToBoltGraph(const query::Graph &gr
|
||||
for (const auto &e : graph.edges()) {
|
||||
auto maybe_edge = ToBoltEdge(e, db, view);
|
||||
if (maybe_edge.HasError()) return maybe_edge.GetError();
|
||||
edges.emplace_back(Value(std::move(*maybe_edge)));
|
||||
edges.emplace_back(std::move(*maybe_edge));
|
||||
}
|
||||
map.emplace("edges", Value(edges));
|
||||
|
||||
|
||||
@@ -11,6 +11,8 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace memgraph::glue {
|
||||
extern const std::string run_id_;
|
||||
} // namespace memgraph::glue
|
||||
|
||||
@@ -31,7 +31,7 @@ inline void LoadConfig(const std::string &product_name) {
|
||||
std::vector<fs::path> configs = {fs::path("/etc/memgraph/memgraph.conf")};
|
||||
if (getenv("HOME") != nullptr) configs.emplace_back(fs::path(getenv("HOME")) / fs::path(".memgraph/config"));
|
||||
{
|
||||
auto memgraph_config = getenv("MEMGRAPH_CONFIG");
|
||||
auto *memgraph_config = getenv("MEMGRAPH_CONFIG");
|
||||
if (memgraph_config != nullptr) {
|
||||
auto path = fs::path(memgraph_config);
|
||||
MG_ASSERT(fs::exists(path), "MEMGRAPH_CONFIG environment variable set to nonexisting path: {}",
|
||||
|
||||
@@ -23,7 +23,6 @@
|
||||
#include <utils/event_counter.hpp>
|
||||
#include <utils/event_gauge.hpp>
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "utils/event_gauge.hpp"
|
||||
#include "utils/event_histogram.hpp"
|
||||
|
||||
namespace memgraph::http {
|
||||
@@ -65,7 +64,7 @@ class MetricsService {
|
||||
return MetricsResponse{.vertex_count = info.vertex_count,
|
||||
.edge_count = info.edge_count,
|
||||
.average_degree = info.average_degree,
|
||||
.memory_usage = info.memory_usage,
|
||||
.memory_usage = info.memory_res,
|
||||
.disk_usage = info.disk_usage,
|
||||
.event_counters = GetEventCounters(),
|
||||
.event_gauges = GetEventGauges(),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -171,10 +171,10 @@ class Consumer final : public RdKafka::EventCb {
|
||||
|
||||
class ConsumerRebalanceCb : public RdKafka::RebalanceCb {
|
||||
public:
|
||||
ConsumerRebalanceCb(std::string consumer_name);
|
||||
explicit ConsumerRebalanceCb(std::string consumer_name);
|
||||
|
||||
void rebalance_cb(RdKafka::KafkaConsumer *consumer, RdKafka::ErrorCode err,
|
||||
std::vector<RdKafka::TopicPartition *> &partitions) override final;
|
||||
std::vector<RdKafka::TopicPartition *> &partitions) final;
|
||||
|
||||
void set_offset(int64_t offset);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -67,7 +67,7 @@ utils::BasicResult<std::string, std::vector<Message>> GetBatch(TConsumer &consum
|
||||
return std::move(batch);
|
||||
case pulsar_client::Result::ResultOk:
|
||||
if (message.getMessageId() != last_message_id) {
|
||||
batch.emplace_back(Message{std::move(message)});
|
||||
batch.emplace_back(std::move(message));
|
||||
}
|
||||
break;
|
||||
default:
|
||||
|
||||
@@ -48,8 +48,8 @@ struct Endpoint {
|
||||
uint16_t port{0};
|
||||
IpFamily family{IpFamily::NONE};
|
||||
|
||||
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrAddress(
|
||||
const std::string &address, const std::optional<uint16_t> default_port);
|
||||
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrAddress(const std::string &address,
|
||||
std::optional<uint16_t> default_port);
|
||||
|
||||
/**
|
||||
* Tries to parse the given string as either a socket address or ip address.
|
||||
@@ -61,8 +61,8 @@ struct Endpoint {
|
||||
* it into an ip address and a port number; even if a default port is given,
|
||||
* it won't be used, as we expect that it is given in the address string.
|
||||
*/
|
||||
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrIpAddress(
|
||||
const std::string &address, const std::optional<uint16_t> default_port);
|
||||
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrIpAddress(const std::string &address,
|
||||
std::optional<uint16_t> default_port);
|
||||
|
||||
/**
|
||||
* Tries to parse given string as either socket address or hostname.
|
||||
@@ -72,7 +72,7 @@ struct Endpoint {
|
||||
* After we parse hostname and port we try to resolve the hostname into an ip_address.
|
||||
*/
|
||||
static std::optional<std::pair<std::string, uint16_t>> ParseHostname(const std::string &address,
|
||||
const std::optional<uint16_t> default_port);
|
||||
std::optional<uint16_t> default_port);
|
||||
|
||||
static IpFamily GetIpFamily(const std::string &address);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -32,7 +32,7 @@ class Epoll {
|
||||
public:
|
||||
using Event = struct epoll_event;
|
||||
|
||||
Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) {
|
||||
explicit Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) {
|
||||
// epoll_create1 returns an error if there is a logical error in our code
|
||||
// (for example invalid flags) or if there is irrecoverable error. In both
|
||||
// cases it is best to terminate.
|
||||
|
||||
@@ -216,7 +216,7 @@ bool Socket::Write(const uint8_t *data, size_t len, bool have_more) {
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Socket::Write(const std::string &s, bool have_more) {
|
||||
bool Socket::Write(std::string_view s, bool have_more) {
|
||||
return Write(reinterpret_cast<const uint8_t *>(s.data()), s.size(), have_more);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,6 +14,7 @@
|
||||
#include <functional>
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
|
||||
#include "io/network/endpoint.hpp"
|
||||
|
||||
@@ -149,7 +150,7 @@ class Socket {
|
||||
* false if write failed
|
||||
*/
|
||||
bool Write(const uint8_t *data, size_t len, bool have_more = false);
|
||||
bool Write(const std::string &s, bool have_more = false);
|
||||
bool Write(std::string_view s, bool have_more = false);
|
||||
|
||||
/**
|
||||
* Read data from the socket.
|
||||
@@ -201,7 +202,7 @@ class Socket {
|
||||
bool WaitForReadyWrite();
|
||||
|
||||
private:
|
||||
Socket(int fd, const Endpoint &endpoint) : socket_(fd), endpoint_(endpoint) {}
|
||||
Socket(int fd, Endpoint endpoint) : socket_(fd), endpoint_(std::move(endpoint)) {}
|
||||
|
||||
int socket_ = -1;
|
||||
Endpoint endpoint_;
|
||||
|
||||
@@ -128,7 +128,7 @@ KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix, b
|
||||
|
||||
KVStore::iterator::iterator(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); }
|
||||
|
||||
KVStore::iterator::~iterator() {}
|
||||
KVStore::iterator::~iterator() = default;
|
||||
|
||||
KVStore::iterator &KVStore::iterator::operator=(KVStore::iterator &&other) {
|
||||
pimpl_ = std::move(other.pimpl_);
|
||||
|
||||
@@ -42,7 +42,7 @@ void LicenseInfoSender::SendData() {
|
||||
license_info_.WithLock([&data, this](const auto &license_info) mutable {
|
||||
if (license_info && !license_info->organization_name.empty()) {
|
||||
const auto memory_info = utils::GetMemoryInfo();
|
||||
const auto memory_usage = utils::GetMemoryUsage();
|
||||
const auto memory_res = utils::GetMemoryRES();
|
||||
data = {{"run_id", uuid_},
|
||||
{"machine_id", machine_id_},
|
||||
{"type", "license-check"},
|
||||
@@ -52,7 +52,7 @@ void LicenseInfoSender::SendData() {
|
||||
{"valid", license_info->is_valid},
|
||||
{"physical_memory_size", memory_info.memory},
|
||||
{"swap_memory_size", memory_info.swap},
|
||||
{"memory_used", memory_usage},
|
||||
{"memory_used", memory_res},
|
||||
{"runtime_memory_limit", memory_limit_},
|
||||
{"license_memory_limit", license_info->license.memory_limit},
|
||||
{"timestamp", utils::Timestamp::Now().SecWithNsecSinceTheEpoch()}};
|
||||
|
||||
@@ -9,11 +9,13 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <cstdint>
|
||||
#include "audit/log.hpp"
|
||||
#include "communication/metrics.hpp"
|
||||
#include "communication/websocket/auth.hpp"
|
||||
#include "communication/websocket/server.hpp"
|
||||
#include "dbms/constants.hpp"
|
||||
#include "dbms/inmemory/replication_handlers.hpp"
|
||||
#include "flags/all.hpp"
|
||||
#include "flags/run_time_configurable.hpp"
|
||||
#include "glue/MonitoringServerT.hpp"
|
||||
@@ -43,9 +45,11 @@
|
||||
#include "query/auth_query_handler.hpp"
|
||||
#include "query/interpreter_context.hpp"
|
||||
|
||||
namespace {
|
||||
constexpr const char *kMgUser = "MEMGRAPH_USER";
|
||||
constexpr const char *kMgPassword = "MEMGRAPH_PASSWORD";
|
||||
constexpr const char *kMgPassfile = "MEMGRAPH_PASSFILE";
|
||||
constexpr uint64_t kMgVmMaxMapCount = 262144;
|
||||
|
||||
// TODO: move elsewhere so that we can remove need of interpreter.hpp
|
||||
void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbms::DatabaseAccess &db_acc,
|
||||
@@ -61,10 +65,13 @@ void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbm
|
||||
std::string line;
|
||||
while (std::getline(file, line)) {
|
||||
if (!line.empty()) {
|
||||
auto results = interpreter.Prepare(line, {}, {});
|
||||
memgraph::query::DiscardValueResultStream stream;
|
||||
interpreter.Pull(&stream, {}, results.qid);
|
||||
|
||||
try {
|
||||
auto results = interpreter.Prepare(line, {}, {});
|
||||
memgraph::query::DiscardValueResultStream stream;
|
||||
interpreter.Pull(&stream, {}, results.qid);
|
||||
} catch (const memgraph::query::UserAlreadyExistsException &e) {
|
||||
spdlog::warn("{} The rest of the init-file will be run.", e.what());
|
||||
}
|
||||
if (audit_log) {
|
||||
audit_log->Record("", "", line, {}, memgraph::dbms::kDefaultDB);
|
||||
}
|
||||
@@ -107,6 +114,7 @@ void InitSignalHandlers(const std::function<void()> &shutdown_fun) {
|
||||
block_shutdown_signals),
|
||||
"Unable to register SIGINT handler!");
|
||||
}
|
||||
} // namespace
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
memgraph::memory::SetHooks();
|
||||
@@ -204,6 +212,17 @@ int main(int argc, char **argv) {
|
||||
std::cout << "You are running Memgraph v" << gflags::VersionString() << std::endl;
|
||||
std::cout << "To get started with Memgraph, visit https://memgr.ph/start" << std::endl;
|
||||
|
||||
const auto vm_max_map_count = memgraph::utils::GetVmMaxMapCount();
|
||||
if (vm_max_map_count.has_value()) {
|
||||
if (vm_max_map_count.value() < kMgVmMaxMapCount) {
|
||||
std::cout << "Max virtual memory areas vm.max_map_count " << vm_max_map_count.value()
|
||||
<< " is too low, increase to at least " << kMgVmMaxMapCount << std::endl;
|
||||
}
|
||||
} else {
|
||||
std::cout << "Can't get info on vm.max_map_count, check whether it is too low, vm.max_map_count is at least "
|
||||
<< kMgVmMaxMapCount << std::endl;
|
||||
}
|
||||
|
||||
auto data_directory = std::filesystem::path(FLAGS_data_directory);
|
||||
|
||||
memgraph::utils::EnsureDirOrDie(data_directory);
|
||||
@@ -352,20 +371,16 @@ int main(int argc, char **argv) {
|
||||
std::unique_ptr<memgraph::query::AuthChecker> auth_checker;
|
||||
auth_glue(&auth_, auth_handler, auth_checker);
|
||||
|
||||
memgraph::dbms::DbmsHandler dbms_handler(db_config
|
||||
#ifdef MG_ENTERPRISE
|
||||
memgraph::dbms::DbmsHandler new_handler(db_config, &auth_, FLAGS_data_recovery_on_startup,
|
||||
FLAGS_storage_delete_on_drop);
|
||||
auto db_acc = new_handler.Get(memgraph::dbms::kDefaultDB);
|
||||
memgraph::query::InterpreterContext interpreter_context_(interp_config, &new_handler, auth_handler.get(),
|
||||
auth_checker.get());
|
||||
#else
|
||||
memgraph::utils::Gatekeeper<memgraph::dbms::Database> db_gatekeeper{db_config};
|
||||
auto db_acc_opt = db_gatekeeper.access();
|
||||
MG_ASSERT(db_acc_opt, "Failed to access the main database");
|
||||
auto &db_acc = *db_acc_opt;
|
||||
memgraph::query::InterpreterContext interpreter_context_(interp_config, &db_gatekeeper, auth_handler.get(),
|
||||
auth_checker.get());
|
||||
,
|
||||
&auth_, FLAGS_data_recovery_on_startup, FLAGS_storage_delete_on_drop
|
||||
#endif
|
||||
);
|
||||
auto db_acc = dbms_handler.Get();
|
||||
|
||||
memgraph::query::InterpreterContext interpreter_context_(
|
||||
interp_config, &dbms_handler, &dbms_handler.ReplicationState(), auth_handler.get(), auth_checker.get());
|
||||
MG_ASSERT(db_acc, "Failed to access the main database");
|
||||
|
||||
memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(memgraph::flags::ParseQueryModulesDirectory(),
|
||||
@@ -388,8 +403,8 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
new_handler.RestoreTriggers(&interpreter_context_);
|
||||
new_handler.RestoreStreams(&interpreter_context_);
|
||||
dbms_handler.RestoreTriggers(&interpreter_context_);
|
||||
dbms_handler.RestoreStreams(&interpreter_context_);
|
||||
#else
|
||||
{
|
||||
// Triggers can execute query procedures, so we need to reload the modules first and then
|
||||
@@ -432,11 +447,10 @@ int main(int argc, char **argv) {
|
||||
if (FLAGS_telemetry_enabled) {
|
||||
telemetry.emplace(telemetry_server, data_directory / "telemetry", memgraph::glue::run_id_, machine_id,
|
||||
service_name == "BoltS", FLAGS_data_directory, std::chrono::minutes(10));
|
||||
telemetry->AddStorageCollector(dbms_handler, auth_);
|
||||
#ifdef MG_ENTERPRISE
|
||||
telemetry->AddStorageCollector(new_handler, auth_);
|
||||
telemetry->AddDatabaseCollector(new_handler);
|
||||
telemetry->AddDatabaseCollector(dbms_handler);
|
||||
#else
|
||||
telemetry->AddStorageCollector(db_gatekeeper, auth_);
|
||||
telemetry->AddDatabaseCollector();
|
||||
#endif
|
||||
telemetry->AddClientCollector();
|
||||
@@ -496,17 +510,16 @@ int main(int argc, char **argv) {
|
||||
|
||||
if (!FLAGS_init_data_file.empty()) {
|
||||
spdlog::info("Running init data file.");
|
||||
auto db_acc = dbms_handler.Get();
|
||||
MG_ASSERT(db_acc, "Failed to gain access to the main database");
|
||||
#ifdef MG_ENTERPRISE
|
||||
auto db_acc = new_handler.Get(memgraph::dbms::kDefaultDB);
|
||||
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
InitFromCypherlFile(interpreter_context_, db_acc, FLAGS_init_data_file, &audit_log);
|
||||
} else {
|
||||
InitFromCypherlFile(interpreter_context_, db_acc, FLAGS_init_data_file);
|
||||
}
|
||||
#else
|
||||
auto db_acc_2 = db_gatekeeper.access();
|
||||
MG_ASSERT(db_acc_2, "Failed to gain access to the main database");
|
||||
InitFromCypherlFile(interpreter_context_, *db_acc_2, FLAGS_init_data_file);
|
||||
InitFromCypherlFile(interpreter_context_, db_acc, FLAGS_init_data_file);
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@@ -3,14 +3,14 @@ set(memory_src_files
|
||||
global_memory_control.cpp
|
||||
query_memory_control.cpp)
|
||||
|
||||
|
||||
|
||||
find_package(jemalloc REQUIRED)
|
||||
|
||||
add_library(mg-memory STATIC ${memory_src_files})
|
||||
target_link_libraries(mg-memory mg-utils fmt)
|
||||
|
||||
message(STATUS "ENABLE_JEMALLOC: ${ENABLE_JEMALLOC}")
|
||||
if (ENABLE_JEMALLOC)
|
||||
find_package(jemalloc REQUIRED)
|
||||
target_link_libraries(mg-memory Jemalloc::Jemalloc ${CMAKE_DL_LIBS})
|
||||
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1)
|
||||
else()
|
||||
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=0)
|
||||
endif()
|
||||
|
||||
@@ -61,11 +61,8 @@ void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t
|
||||
// This needs to be before, to throw exception in case of too big alloc
|
||||
if (*commit) [[likely]] {
|
||||
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Alloc(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -73,11 +70,8 @@ void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t
|
||||
if (ptr == nullptr) [[unlikely]] {
|
||||
if (*commit) {
|
||||
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Free(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
|
||||
}
|
||||
}
|
||||
return ptr;
|
||||
@@ -96,11 +90,8 @@ static bool my_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, boo
|
||||
if (committed) [[likely]] {
|
||||
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
|
||||
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Free(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,11 +101,8 @@ static bool my_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, boo
|
||||
static void my_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) {
|
||||
if (committed) [[likely]] {
|
||||
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Free(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,11 +118,8 @@ static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, siz
|
||||
}
|
||||
|
||||
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Alloc(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
|
||||
}
|
||||
|
||||
return false;
|
||||
@@ -150,11 +135,8 @@ static bool my_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, s
|
||||
}
|
||||
|
||||
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(length));
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Free(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
|
||||
}
|
||||
|
||||
return false;
|
||||
@@ -170,11 +152,8 @@ static bool my_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t siz
|
||||
}
|
||||
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(length));
|
||||
|
||||
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
|
||||
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
|
||||
if (memory_tracker != nullptr) [[likely]] {
|
||||
memory_tracker->Alloc(static_cast<int64_t>(size));
|
||||
}
|
||||
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
|
||||
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
|
||||
}
|
||||
|
||||
return false;
|
||||
@@ -200,8 +179,11 @@ void SetHooks() {
|
||||
return;
|
||||
}
|
||||
|
||||
// Needs init due to error we might encounter otherwise
|
||||
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=13684
|
||||
[[maybe_unused]] const auto &queries_memory_control = GetQueriesMemoryControl();
|
||||
|
||||
for (int i = 0; i < n_arenas; i++) {
|
||||
GetQueriesMemoryControl().InitializeArenaCounter(i);
|
||||
std::string func_name = "arena." + std::to_string(i) + ".extent_hooks";
|
||||
|
||||
size_t hooks_len = sizeof(old_hooks);
|
||||
@@ -261,7 +243,6 @@ void UnsetHooks() {
|
||||
}
|
||||
|
||||
for (int i = 0; i < n_arenas; i++) {
|
||||
GetQueriesMemoryControl().InitializeArenaCounter(i);
|
||||
std::string func_name = "arena." + std::to_string(i) + ".extent_hooks";
|
||||
|
||||
MG_ASSERT(old_hooks);
|
||||
|
||||
@@ -10,6 +10,7 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <atomic>
|
||||
#include <cassert>
|
||||
#include <cstdint>
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
@@ -21,6 +22,7 @@
|
||||
#include "query_memory_control.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/rw_spin_lock.hpp"
|
||||
|
||||
@@ -32,20 +34,6 @@ namespace memgraph::memory {
|
||||
|
||||
#if USE_JEMALLOC
|
||||
|
||||
unsigned QueriesMemoryControl::GetArenaForThread() {
|
||||
unsigned thread_arena{0};
|
||||
size_t size_thread_arena = sizeof(thread_arena);
|
||||
int err = mallctl("thread.arena", &thread_arena, &size_thread_arena, nullptr, 0);
|
||||
if (err) {
|
||||
LOG_FATAL("Can't get arena for thread.");
|
||||
}
|
||||
return thread_arena;
|
||||
}
|
||||
|
||||
void QueriesMemoryControl::AddTrackingOnArena(unsigned arena_id) { arena_tracking[arena_id].fetch_add(1); }
|
||||
|
||||
void QueriesMemoryControl::RemoveTrackingOnArena(unsigned arena_id) { arena_tracking[arena_id].fetch_sub(1); }
|
||||
|
||||
void QueriesMemoryControl::UpdateThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
|
||||
auto accessor = thread_id_to_transaction_id.access();
|
||||
accessor.insert({thread_id, transaction_id});
|
||||
@@ -58,34 +46,58 @@ void QueriesMemoryControl::EraseThreadToTransactionId(const std::thread::id &thr
|
||||
accessor.remove(thread_id);
|
||||
}
|
||||
|
||||
utils::MemoryTracker *QueriesMemoryControl::GetTrackerCurrentThread() {
|
||||
void QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {
|
||||
auto thread_id_to_transaction_id_accessor = thread_id_to_transaction_id.access();
|
||||
|
||||
// we might be just constructing mapping between thread id and transaction id
|
||||
// so we miss this allocation
|
||||
auto thread_id_to_transaction_id_elem = thread_id_to_transaction_id_accessor.find(std::this_thread::get_id());
|
||||
if (thread_id_to_transaction_id_elem == thread_id_to_transaction_id_accessor.end()) {
|
||||
return nullptr;
|
||||
return;
|
||||
}
|
||||
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
auto transaction_id_to_tracker =
|
||||
transaction_id_to_tracker_accessor.find(thread_id_to_transaction_id_elem->transaction_id);
|
||||
return &transaction_id_to_tracker->tracker;
|
||||
|
||||
// It can happen that some allocation happens between mapping thread to
|
||||
// transaction id, so we miss this allocation
|
||||
if (transaction_id_to_tracker == transaction_id_to_tracker_accessor.end()) [[unlikely]] {
|
||||
return;
|
||||
}
|
||||
auto &query_tracker = transaction_id_to_tracker->tracker;
|
||||
query_tracker.TrackAlloc(size);
|
||||
}
|
||||
|
||||
void QueriesMemoryControl::TrackFreeOnCurrentThread(size_t size) {
|
||||
auto thread_id_to_transaction_id_accessor = thread_id_to_transaction_id.access();
|
||||
|
||||
// we might be just constructing mapping between thread id and transaction id
|
||||
// so we miss this allocation
|
||||
auto thread_id_to_transaction_id_elem = thread_id_to_transaction_id_accessor.find(std::this_thread::get_id());
|
||||
if (thread_id_to_transaction_id_elem == thread_id_to_transaction_id_accessor.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
auto transaction_id_to_tracker =
|
||||
transaction_id_to_tracker_accessor.find(thread_id_to_transaction_id_elem->transaction_id);
|
||||
|
||||
// It can happen that some allocation happens between mapping thread to
|
||||
// transaction id, so we miss this allocation
|
||||
if (transaction_id_to_tracker == transaction_id_to_tracker_accessor.end()) [[unlikely]] {
|
||||
return;
|
||||
}
|
||||
auto &query_tracker = transaction_id_to_tracker->tracker;
|
||||
query_tracker.TrackFree(size);
|
||||
}
|
||||
|
||||
void QueriesMemoryControl::CreateTransactionIdTracker(uint64_t transaction_id, size_t inital_limit) {
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
|
||||
auto [elem, result] = transaction_id_to_tracker_accessor.insert({transaction_id, utils::MemoryTracker{}});
|
||||
auto [elem, result] = transaction_id_to_tracker_accessor.insert({transaction_id, utils::QueryMemoryTracker{}});
|
||||
|
||||
elem->tracker.SetMaximumHardLimit(inital_limit);
|
||||
elem->tracker.SetHardLimit(inital_limit);
|
||||
}
|
||||
|
||||
bool QueriesMemoryControl::CheckTransactionIdTrackerExists(uint64_t transaction_id) {
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
return transaction_id_to_tracker_accessor.contains(transaction_id);
|
||||
elem->tracker.SetQueryLimit(inital_limit);
|
||||
}
|
||||
|
||||
bool QueriesMemoryControl::EraseTransactionIdTracker(uint64_t transaction_id) {
|
||||
@@ -94,27 +106,68 @@ bool QueriesMemoryControl::EraseTransactionIdTracker(uint64_t transaction_id) {
|
||||
return removed;
|
||||
}
|
||||
|
||||
bool QueriesMemoryControl::IsArenaTracked(unsigned arena_ind) {
|
||||
return arena_tracking[arena_ind].load(std::memory_order_acquire) != 0;
|
||||
bool QueriesMemoryControl::CheckTransactionIdTrackerExists(uint64_t transaction_id) {
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
return transaction_id_to_tracker_accessor.contains(transaction_id);
|
||||
}
|
||||
|
||||
void QueriesMemoryControl::InitializeArenaCounter(unsigned arena_ind) {
|
||||
arena_tracking[arena_ind].store(0, std::memory_order_relaxed);
|
||||
void QueriesMemoryControl::TryCreateTransactionProcTracker(uint64_t transaction_id, int64_t procedure_id,
|
||||
size_t limit) {
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
auto query_tracker = transaction_id_to_tracker_accessor.find(transaction_id);
|
||||
|
||||
if (query_tracker == transaction_id_to_tracker_accessor.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
query_tracker->tracker.TryCreateProcTracker(procedure_id, limit);
|
||||
}
|
||||
|
||||
void QueriesMemoryControl::SetActiveProcIdTracker(uint64_t transaction_id, int64_t procedure_id) {
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
auto query_tracker = transaction_id_to_tracker_accessor.find(transaction_id);
|
||||
|
||||
if (query_tracker == transaction_id_to_tracker_accessor.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
query_tracker->tracker.SetActiveProc(procedure_id);
|
||||
}
|
||||
|
||||
void QueriesMemoryControl::PauseProcedureTracking(uint64_t transaction_id) {
|
||||
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
|
||||
auto query_tracker = transaction_id_to_tracker_accessor.find(transaction_id);
|
||||
|
||||
if (query_tracker == transaction_id_to_tracker_accessor.end()) {
|
||||
return;
|
||||
}
|
||||
|
||||
query_tracker->tracker.StopProcTracking();
|
||||
}
|
||||
|
||||
inline int &Get_Thread_Tracker() {
|
||||
// store variable in bss segment for each thread
|
||||
// https://cs-fundamentals.com/c-programming/memory-layout-of-c-program-code-data-segments#size-of-code-data-bss-segments
|
||||
static thread_local int is_thread_tracked{0};
|
||||
return is_thread_tracked;
|
||||
}
|
||||
|
||||
bool QueriesMemoryControl::IsThreadTracked() { return Get_Thread_Tracker() == 1; }
|
||||
|
||||
#endif
|
||||
|
||||
void StartTrackingCurrentThreadTransaction(uint64_t transaction_id) {
|
||||
#if USE_JEMALLOC
|
||||
Get_Thread_Tracker() = 0;
|
||||
GetQueriesMemoryControl().UpdateThreadToTransactionId(std::this_thread::get_id(), transaction_id);
|
||||
GetQueriesMemoryControl().AddTrackingOnArena(QueriesMemoryControl::GetArenaForThread());
|
||||
Get_Thread_Tracker() = 1;
|
||||
#endif
|
||||
}
|
||||
|
||||
void StopTrackingCurrentThreadTransaction(uint64_t transaction_id) {
|
||||
#if USE_JEMALLOC
|
||||
Get_Thread_Tracker() = 0;
|
||||
GetQueriesMemoryControl().EraseThreadToTransactionId(std::this_thread::get_id(), transaction_id);
|
||||
GetQueriesMemoryControl().RemoveTrackingOnArena(QueriesMemoryControl::GetArenaForThread());
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -137,4 +190,29 @@ void TryStopTrackingOnTransaction(uint64_t transaction_id) {
|
||||
#endif
|
||||
}
|
||||
|
||||
#if USE_JEMALLOC
|
||||
bool IsTransactionTracked(uint64_t transaction_id) {
|
||||
return GetQueriesMemoryControl().CheckTransactionIdTrackerExists(transaction_id);
|
||||
}
|
||||
#else
|
||||
bool IsTransactionTracked(uint64_t /*transaction_id*/) { return false; }
|
||||
#endif
|
||||
|
||||
void CreateOrContinueProcedureTracking(uint64_t transaction_id, int64_t procedure_id, size_t limit) {
|
||||
#if USE_JEMALLOC
|
||||
if (!GetQueriesMemoryControl().CheckTransactionIdTrackerExists(transaction_id)) {
|
||||
LOG_FATAL("Memory tracker for transaction was not set");
|
||||
}
|
||||
|
||||
GetQueriesMemoryControl().TryCreateTransactionProcTracker(transaction_id, procedure_id, limit);
|
||||
GetQueriesMemoryControl().SetActiveProcIdTracker(transaction_id, procedure_id);
|
||||
#endif
|
||||
}
|
||||
|
||||
void PauseProcedureTracking(uint64_t transaction_id) {
|
||||
#if USE_JEMALLOC
|
||||
GetQueriesMemoryControl().PauseProcedureTracking(transaction_id);
|
||||
#endif
|
||||
}
|
||||
|
||||
} // namespace memgraph::memory
|
||||
|
||||
@@ -16,10 +16,13 @@
|
||||
#include <unordered_map>
|
||||
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/query_memory_tracker.hpp"
|
||||
#include "utils/skip_list.hpp"
|
||||
|
||||
namespace memgraph::memory {
|
||||
|
||||
static constexpr int64_t UNLIMITED_MEMORY{0};
|
||||
|
||||
#if USE_JEMALLOC
|
||||
|
||||
// Track memory allocations per query.
|
||||
@@ -30,25 +33,6 @@ namespace memgraph::memory {
|
||||
// it is necessary to restart tracking at the beginning of new query for that transaction.
|
||||
class QueriesMemoryControl {
|
||||
public:
|
||||
/*
|
||||
Arena stats
|
||||
*/
|
||||
|
||||
static unsigned GetArenaForThread();
|
||||
|
||||
// Add counter on threads allocating inside arena
|
||||
void AddTrackingOnArena(unsigned);
|
||||
|
||||
// Remove counter on threads allocating in arena
|
||||
void RemoveTrackingOnArena(unsigned);
|
||||
|
||||
// Are any threads using current arena for allocations
|
||||
// Multiple threads can allocate inside one arena
|
||||
bool IsArenaTracked(unsigned);
|
||||
|
||||
// Initialize arena counter
|
||||
void InitializeArenaCounter(unsigned);
|
||||
|
||||
/*
|
||||
Transaction id <-> tracker
|
||||
*/
|
||||
@@ -75,20 +59,25 @@ class QueriesMemoryControl {
|
||||
// Important to reset if one thread gets reused for different transaction
|
||||
void EraseThreadToTransactionId(const std::thread::id &, uint64_t);
|
||||
|
||||
// C-API functionality for thread to transaction mapping
|
||||
void UpdateThreadToTransactionId(const char *, uint64_t);
|
||||
// Find tracker for current thread if exists, track
|
||||
// query allocation and procedure allocation if
|
||||
// necessary
|
||||
void TrackAllocOnCurrentThread(size_t size);
|
||||
|
||||
// C-API functionality for thread to transaction unmapping
|
||||
void EraseThreadToTransactionId(const char *, uint64_t);
|
||||
// Find tracker for current thread if exists, track
|
||||
// query allocation and procedure allocation if
|
||||
// necessary
|
||||
void TrackFreeOnCurrentThread(size_t size);
|
||||
|
||||
// Get tracker to current thread if exists, otherwise return
|
||||
// nullptr. This can happen only if tracker is still
|
||||
// being constructed.
|
||||
utils::MemoryTracker *GetTrackerCurrentThread();
|
||||
void TryCreateTransactionProcTracker(uint64_t, int64_t, size_t);
|
||||
|
||||
void SetActiveProcIdTracker(uint64_t, int64_t);
|
||||
|
||||
void PauseProcedureTracking(uint64_t);
|
||||
|
||||
bool IsThreadTracked();
|
||||
|
||||
private:
|
||||
std::unordered_map<unsigned, std::atomic<int>> arena_tracking;
|
||||
|
||||
struct ThreadIdToTransactionId {
|
||||
std::thread::id thread_id;
|
||||
uint64_t transaction_id;
|
||||
@@ -102,7 +91,7 @@ class QueriesMemoryControl {
|
||||
|
||||
struct TransactionIdToTracker {
|
||||
uint64_t transaction_id;
|
||||
utils::MemoryTracker tracker;
|
||||
utils::QueryMemoryTracker tracker;
|
||||
|
||||
bool operator<(const TransactionIdToTracker &other) const { return transaction_id < other.transaction_id; }
|
||||
bool operator==(const TransactionIdToTracker &other) const { return transaction_id == other.transaction_id; }
|
||||
@@ -138,4 +127,15 @@ void TryStartTrackingOnTransaction(uint64_t transaction_id, size_t limit);
|
||||
// Does nothing if jemalloc is not enabled. Does nothing if tracker doesn't exist
|
||||
void TryStopTrackingOnTransaction(uint64_t transaction_id);
|
||||
|
||||
// Is transaction with given id tracked in memory tracker
|
||||
bool IsTransactionTracked(uint64_t transaction_id);
|
||||
|
||||
// Creates tracker on procedure if doesn't exist. Sets query tracker
|
||||
// to track procedure with id.
|
||||
void CreateOrContinueProcedureTracking(uint64_t transaction_id, int64_t procedure_id, size_t limit);
|
||||
|
||||
// Pauses procedure tracking. This enables to continue
|
||||
// tracking on procedure once procedure execution resumes.
|
||||
void PauseProcedureTracking(uint64_t transaction_id);
|
||||
|
||||
} // namespace memgraph::memory
|
||||
|
||||
@@ -19,7 +19,9 @@
|
||||
#include <regex>
|
||||
#include <unordered_map>
|
||||
|
||||
#include "dbms/inmemory/storage_helper.hpp"
|
||||
#include "helpers.hpp"
|
||||
#include "replication/state.hpp"
|
||||
#include "storage/v2/config.hpp"
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/inmemory/storage.hpp"
|
||||
@@ -702,14 +704,16 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
|
||||
std::unordered_map<NodeId, memgraph::storage::Gid> node_id_map;
|
||||
auto store = std::make_unique<memgraph::storage::InMemoryStorage>(memgraph::storage::Config{
|
||||
memgraph::storage::Config config{
|
||||
|
||||
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
|
||||
.durability = {.storage_directory = FLAGS_data_directory,
|
||||
.recover_on_startup = false,
|
||||
.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::DISABLED,
|
||||
.snapshot_on_exit = true},
|
||||
});
|
||||
};
|
||||
memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)};
|
||||
auto store = memgraph::dbms::CreateInMemoryStorage(config, repl_state);
|
||||
|
||||
memgraph::utils::Timer load_timer;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -62,6 +62,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
|
||||
case TypedValue::Type::Edge:
|
||||
case TypedValue::Type::Path:
|
||||
case TypedValue::Type::Graph:
|
||||
case TypedValue::Type::Function:
|
||||
throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type());
|
||||
case TypedValue::Type::Null:
|
||||
LOG_FATAL("Invalid type");
|
||||
|
||||
@@ -41,7 +41,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b);
|
||||
/// the define how respective elements compare.
|
||||
class TypedValueVectorCompare final {
|
||||
public:
|
||||
TypedValueVectorCompare() {}
|
||||
TypedValueVectorCompare() = default;
|
||||
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
|
||||
|
||||
template <class TAllocator>
|
||||
@@ -147,8 +147,8 @@ concept AccessorWithUpdateProperties = requires(T accessor,
|
||||
///
|
||||
/// @throw QueryRuntimeException if value cannot be set as a property value
|
||||
template <AccessorWithUpdateProperties T>
|
||||
auto UpdatePropertiesChecked(T *record, std::map<storage::PropertyId, storage::PropertyValue> &properties) ->
|
||||
typename std::remove_reference<decltype(record->UpdateProperties(properties).GetValue())>::type {
|
||||
auto UpdatePropertiesChecked(T *record, std::map<storage::PropertyId, storage::PropertyValue> &properties)
|
||||
-> std::remove_reference_t<decltype(record->UpdateProperties(properties).GetValue())> {
|
||||
try {
|
||||
auto maybe_values = record->UpdateProperties(properties);
|
||||
if (maybe_values.HasError()) {
|
||||
|
||||
@@ -12,15 +12,16 @@
|
||||
#include "query/cypher_query_interpreter.hpp"
|
||||
#include "query/frontend/ast/cypher_main_visitor.hpp"
|
||||
#include "query/frontend/opencypher/parser.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
|
||||
DEFINE_VALIDATED_int32(query_plan_cache_max_size, 1000, "Maximum number of query plans to cache.",
|
||||
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
|
||||
|
||||
namespace memgraph::query {
|
||||
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
|
||||
PlanWrapper::PlanWrapper(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
|
||||
|
||||
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> ¶ms,
|
||||
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config) {
|
||||
@@ -127,28 +128,24 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
|
||||
std::move(symbol_table));
|
||||
}
|
||||
|
||||
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, utils::SkipList<PlanCacheEntry> *plan_cache,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
|
||||
std::shared_ptr<PlanWrapper> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, PlanCacheLRU *plan_cache,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
if (plan_cache) {
|
||||
plan_cache_access.emplace(plan_cache->access());
|
||||
auto it = plan_cache_access->find(hash);
|
||||
if (it != plan_cache_access->end()) {
|
||||
if (it->second->IsExpired()) {
|
||||
plan_cache_access->remove(hash);
|
||||
} else {
|
||||
return it->second;
|
||||
}
|
||||
auto existing_plan = plan_cache->WithLock([&](auto &cache) { return cache.get(hash); });
|
||||
if (existing_plan.has_value()) {
|
||||
return existing_plan.value();
|
||||
}
|
||||
}
|
||||
|
||||
auto plan = std::make_shared<CachedPlan>(
|
||||
auto plan = std::make_shared<PlanWrapper>(
|
||||
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
|
||||
if (plan_cache_access) {
|
||||
plan_cache_access->insert({hash, plan});
|
||||
|
||||
if (plan_cache) {
|
||||
plan_cache->WithLock([&](auto &cache) { cache.put(hash, plan); });
|
||||
}
|
||||
|
||||
return plan;
|
||||
}
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -11,18 +11,22 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include "query/config.hpp"
|
||||
#include "query/frontend/semantic/required_privileges.hpp"
|
||||
#include "query/frontend/semantic/symbol_generator.hpp"
|
||||
#include "query/frontend/stripped.hpp"
|
||||
#include "query/plan/planner.hpp"
|
||||
#include "utils/flag_validation.hpp"
|
||||
#include "utils/lru_cache.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
#include "utils/timer.hpp"
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_bool(query_cost_planner);
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_int32(query_plan_cache_ttl);
|
||||
DECLARE_int32(query_plan_cache_max_size);
|
||||
|
||||
namespace memgraph::query {
|
||||
|
||||
@@ -45,23 +49,17 @@ class LogicalPlan {
|
||||
virtual const AstStorage &GetAstStorage() const = 0;
|
||||
};
|
||||
|
||||
class CachedPlan {
|
||||
class PlanWrapper {
|
||||
public:
|
||||
explicit CachedPlan(std::unique_ptr<LogicalPlan> plan);
|
||||
explicit PlanWrapper(std::unique_ptr<LogicalPlan> plan);
|
||||
|
||||
const auto &plan() const { return plan_->GetRoot(); }
|
||||
double cost() const { return plan_->GetCost(); }
|
||||
const auto &symbol_table() const { return plan_->GetSymbolTable(); }
|
||||
const auto &ast_storage() const { return plan_->GetAstStorage(); }
|
||||
|
||||
bool IsExpired() const {
|
||||
// NOLINTNEXTLINE (modernize-use-nullptr)
|
||||
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
|
||||
};
|
||||
|
||||
private:
|
||||
std::unique_ptr<LogicalPlan> plan_;
|
||||
utils::Timer cache_timer_;
|
||||
};
|
||||
|
||||
struct CachedQuery {
|
||||
@@ -82,18 +80,6 @@ struct QueryCacheEntry {
|
||||
CachedQuery second;
|
||||
};
|
||||
|
||||
struct PlanCacheEntry {
|
||||
bool operator==(const PlanCacheEntry &other) const { return first == other.first; }
|
||||
bool operator<(const PlanCacheEntry &other) const { return first < other.first; }
|
||||
bool operator==(const uint64_t &other) const { return first == other; }
|
||||
bool operator<(const uint64_t &other) const { return first < other; }
|
||||
|
||||
uint64_t first;
|
||||
// TODO: Maybe store the query string here and use it as a key with the hash
|
||||
// so that we eliminate the risk of hash collisions.
|
||||
std::shared_ptr<CachedPlan> second;
|
||||
};
|
||||
|
||||
/**
|
||||
* A container for data related to the parsing of a query.
|
||||
*/
|
||||
@@ -114,8 +100,8 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
|
||||
class SingleNodeLogicalPlan final : public LogicalPlan {
|
||||
public:
|
||||
SingleNodeLogicalPlan(std::unique_ptr<plan::LogicalOperator> root, double cost, AstStorage storage,
|
||||
const SymbolTable &symbol_table)
|
||||
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {}
|
||||
SymbolTable symbol_table)
|
||||
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(std::move(symbol_table)) {}
|
||||
|
||||
const plan::LogicalOperator &GetRoot() const override { return *root_; }
|
||||
double GetCost() const override { return cost_; }
|
||||
@@ -129,6 +115,9 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
|
||||
SymbolTable symbol_table_;
|
||||
};
|
||||
|
||||
using PlanCacheLRU =
|
||||
utils::Synchronized<utils::LRUCache<uint64_t, std::shared_ptr<query::PlanWrapper>>, utils::RWSpinLock>;
|
||||
|
||||
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers);
|
||||
@@ -141,9 +130,9 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
|
||||
* If an identifier is contained there, we inject it at that place and remove it,
|
||||
* because a predefined identifier can be used only in one scope.
|
||||
*/
|
||||
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, utils::SkipList<PlanCacheEntry> *plan_cache,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers = {});
|
||||
std::shared_ptr<PlanWrapper> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, PlanCacheLRU *plan_cache,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers = {});
|
||||
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -139,6 +139,8 @@ std::optional<VertexAccessor> SubgraphDbAccessor::FindVertex(storage::Gid gid, s
|
||||
|
||||
query::Graph *SubgraphDbAccessor::getGraph() { return graph_; }
|
||||
|
||||
DbAccessor *SubgraphDbAccessor::GetAccessor() { return &db_accessor_; }
|
||||
|
||||
VertexAccessor SubgraphVertexAccessor::GetVertexAccessor() const { return impl_; }
|
||||
|
||||
storage::Result<EdgeVertexAccessorResult> SubgraphVertexAccessor::OutEdges(storage::View view) const {
|
||||
|
||||
@@ -40,7 +40,6 @@ class EdgeAccessor final {
|
||||
public:
|
||||
storage::EdgeAccessor impl_;
|
||||
|
||||
public:
|
||||
explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {}
|
||||
|
||||
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
|
||||
@@ -108,7 +107,6 @@ class VertexAccessor final {
|
||||
|
||||
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); }
|
||||
|
||||
public:
|
||||
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {}
|
||||
|
||||
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
|
||||
@@ -694,6 +692,8 @@ class SubgraphDbAccessor final {
|
||||
std::optional<VertexAccessor> FindVertex(storage::Gid gid, storage::View view);
|
||||
|
||||
Graph *getGraph();
|
||||
|
||||
DbAccessor *GetAccessor();
|
||||
};
|
||||
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -159,7 +159,7 @@ void DumpProperties(std::ostream *os, query::DbAccessor *dba,
|
||||
*os << "{";
|
||||
if (property_id) {
|
||||
*os << kInternalPropertyId << ": " << *property_id;
|
||||
if (store.size() > 0) *os << ", ";
|
||||
if (!store.empty()) *os << ", ";
|
||||
}
|
||||
utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) {
|
||||
os << EscapeName(dba->PropertyToName(kv.first)) << ": ";
|
||||
@@ -228,7 +228,7 @@ void DumpEdge(std::ostream *os, query::DbAccessor *dba, const query::EdgeAccesso
|
||||
throw query::QueryRuntimeException("Unexpected error when getting properties.");
|
||||
}
|
||||
}
|
||||
if (maybe_props->size() > 0) {
|
||||
if (!maybe_props->empty()) {
|
||||
*os << " ";
|
||||
DumpProperties(os, dba, *maybe_props);
|
||||
}
|
||||
|
||||
@@ -126,6 +126,12 @@ class InfoInMulticommandTxException : public QueryException {
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(InfoInMulticommandTxException)
|
||||
};
|
||||
|
||||
class UserAlreadyExistsException : public QueryException {
|
||||
public:
|
||||
using QueryException::QueryException;
|
||||
SPECIALIZE_GET_EXCEPTION_NAME(UserAlreadyExistsException)
|
||||
};
|
||||
|
||||
/**
|
||||
* An exception for an illegal operation that can not be detected
|
||||
* before the query starts executing over data.
|
||||
|
||||
@@ -8,58 +8,59 @@
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
#include <tuple>
|
||||
#include <utility>
|
||||
#include "query/typed_value.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/pmr/unordered_map.hpp"
|
||||
#include "utils/pmr/vector.hpp"
|
||||
namespace memgraph::query {
|
||||
|
||||
// Key is hash output, value is vector of unique elements
|
||||
using CachedType = utils::pmr::unordered_map<size_t, std::vector<TypedValue>>;
|
||||
using CachedType = utils::pmr::unordered_map<size_t, utils::pmr::vector<TypedValue>>;
|
||||
|
||||
struct CachedValue {
|
||||
using allocator_type = utils::Allocator<CachedValue>;
|
||||
|
||||
// Cached value, this can be probably templateized
|
||||
CachedType cache_;
|
||||
|
||||
explicit CachedValue(utils::MemoryResource *mem) : cache_(mem) {}
|
||||
explicit CachedValue(utils::MemoryResource *mem) : cache_{mem} {};
|
||||
CachedValue(const CachedValue &other, utils::MemoryResource *mem) : cache_(other.cache_, mem) {}
|
||||
CachedValue(CachedValue &&other, utils::MemoryResource *mem) : cache_(std::move(other.cache_), mem){};
|
||||
|
||||
CachedValue(CachedType &&cache, memgraph::utils::MemoryResource *memory) : cache_(std::move(cache), memory) {}
|
||||
CachedValue(CachedValue &&other) noexcept : CachedValue(std::move(other), other.GetMemoryResource()) {}
|
||||
|
||||
CachedValue(const CachedValue &other, memgraph::utils::MemoryResource *memory) : cache_(other.cache_, memory) {}
|
||||
CachedValue(const CachedValue &other)
|
||||
: CachedValue(other, std::allocator_traits<allocator_type>::select_on_container_copy_construction(
|
||||
other.GetMemoryResource())
|
||||
.GetMemoryResource()) {}
|
||||
|
||||
CachedValue(CachedValue &&other, memgraph::utils::MemoryResource *memory) : cache_(std::move(other.cache_), memory) {}
|
||||
|
||||
CachedValue(CachedValue &&other) noexcept = delete;
|
||||
|
||||
/// Copy construction without memgraph::utils::MemoryResource is not allowed.
|
||||
CachedValue(const CachedValue &) = delete;
|
||||
utils::MemoryResource *GetMemoryResource() const { return cache_.get_allocator().GetMemoryResource(); }
|
||||
|
||||
CachedValue &operator=(const CachedValue &) = delete;
|
||||
CachedValue &operator=(CachedValue &&) = delete;
|
||||
|
||||
~CachedValue() = default;
|
||||
|
||||
memgraph::utils::MemoryResource *GetMemoryResource() const noexcept {
|
||||
return cache_.get_allocator().GetMemoryResource();
|
||||
}
|
||||
|
||||
// Func to check if cache_ contains value
|
||||
bool CacheValue(const TypedValue &value) {
|
||||
if (!value.IsList()) {
|
||||
bool CacheValue(const TypedValue &maybe_list) {
|
||||
if (!maybe_list.IsList()) {
|
||||
return false;
|
||||
}
|
||||
const auto &list = value.ValueList();
|
||||
const auto &list = maybe_list.ValueList();
|
||||
TypedValue::Hash hash{};
|
||||
for (const TypedValue &element : list) {
|
||||
for (const auto &element : list) {
|
||||
const auto key = hash(element);
|
||||
auto &vector_values = cache_[key];
|
||||
if (!IsValueInVec(vector_values, element)) {
|
||||
vector_values.push_back(element);
|
||||
vector_values.emplace_back(element);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
// Func to cache_value inside cache_
|
||||
|
||||
// Func to check if cache_ contains value
|
||||
bool ContainsValue(const TypedValue &value) const {
|
||||
TypedValue::Hash hash{};
|
||||
const auto key = hash(value);
|
||||
@@ -70,7 +71,7 @@ struct CachedValue {
|
||||
}
|
||||
|
||||
private:
|
||||
bool IsValueInVec(const std::vector<TypedValue> &vec_values, const TypedValue &value) const {
|
||||
static bool IsValueInVec(const utils::pmr::vector<TypedValue> &vec_values, const TypedValue &value) {
|
||||
return std::any_of(vec_values.begin(), vec_values.end(), [&value](auto &vec_value) {
|
||||
const auto is_value_equal = vec_value == value;
|
||||
if (is_value_equal.IsNull()) return false;
|
||||
@@ -80,43 +81,72 @@ struct CachedValue {
|
||||
};
|
||||
|
||||
// Class tracks keys for which user can cache values which help with faster search or faster retrieval
|
||||
// in the future.
|
||||
// in the future. Used for IN LIST operator.
|
||||
class FrameChangeCollector {
|
||||
public:
|
||||
explicit FrameChangeCollector(utils::MemoryResource *mem) : tracked_values_(mem){};
|
||||
/** Allocator type so that STL containers are aware that we need one */
|
||||
using allocator_type = utils::Allocator<FrameChangeCollector>;
|
||||
|
||||
public:
|
||||
explicit FrameChangeCollector(utils::MemoryResource *mem = utils::NewDeleteResource()) : tracked_values_{mem} {}
|
||||
|
||||
FrameChangeCollector(FrameChangeCollector &&other, utils::MemoryResource *mem)
|
||||
: tracked_values_(std::move(other.tracked_values_), mem) {}
|
||||
FrameChangeCollector(const FrameChangeCollector &other, utils::MemoryResource *mem)
|
||||
: tracked_values_(other.tracked_values_, mem) {}
|
||||
|
||||
FrameChangeCollector(const FrameChangeCollector &other)
|
||||
: FrameChangeCollector(other, std::allocator_traits<allocator_type>::select_on_container_copy_construction(
|
||||
other.GetMemoryResource())
|
||||
.GetMemoryResource()){};
|
||||
|
||||
FrameChangeCollector(FrameChangeCollector &&other) noexcept
|
||||
: FrameChangeCollector(std::move(other), other.GetMemoryResource()) {}
|
||||
|
||||
/** Copy assign other, utils::MemoryResource of `this` is used */
|
||||
FrameChangeCollector &operator=(const FrameChangeCollector &) = default;
|
||||
|
||||
/** Move assign other, utils::MemoryResource of `this` is used. */
|
||||
FrameChangeCollector &operator=(FrameChangeCollector &&) noexcept = default;
|
||||
|
||||
utils::MemoryResource *GetMemoryResource() const { return tracked_values_.get_allocator().GetMemoryResource(); }
|
||||
|
||||
// Add tracking key to cache later value
|
||||
CachedValue &AddTrackingKey(const std::string &key) {
|
||||
const auto &[it, _] = tracked_values_.emplace(key, tracked_values_.get_allocator().GetMemoryResource());
|
||||
const auto &[it, _] = tracked_values_.emplace(
|
||||
std::piecewise_construct, std::forward_as_tuple(utils::pmr::string(key, utils::NewDeleteResource())),
|
||||
std::forward_as_tuple());
|
||||
return it->second;
|
||||
}
|
||||
|
||||
// Is key tracked
|
||||
bool IsKeyTracked(const std::string &key) const { return tracked_values_.contains(key); }
|
||||
|
||||
// Is value for given key cached
|
||||
bool IsKeyValueCached(const std::string &key) const {
|
||||
return tracked_values_.contains(key) && !tracked_values_.at(key).cache_.empty();
|
||||
bool IsKeyTracked(const std::string &key) const {
|
||||
return tracked_values_.contains(utils::pmr::string(key, utils::NewDeleteResource()));
|
||||
}
|
||||
|
||||
// Reset value for tracking key
|
||||
bool ResetTrackingValue(const std::string &key) {
|
||||
if (tracked_values_.contains(key)) {
|
||||
tracked_values_.erase(key);
|
||||
AddTrackingKey(key);
|
||||
}
|
||||
bool IsKeyValueCached(const std::string &key) const {
|
||||
return IsKeyTracked(key) && !tracked_values_.at(utils::pmr::string(key, utils::NewDeleteResource())).cache_.empty();
|
||||
}
|
||||
|
||||
bool ResetTrackingValue(const std::string &key) {
|
||||
if (!tracked_values_.contains(utils::pmr::string(key, utils::NewDeleteResource()))) {
|
||||
return false;
|
||||
}
|
||||
tracked_values_.erase(utils::pmr::string(key, utils::NewDeleteResource()));
|
||||
AddTrackingKey(key);
|
||||
return true;
|
||||
}
|
||||
|
||||
// Get value cached for tracking key, throws if key is not in tracked
|
||||
CachedValue &GetCachedValue(const std::string &key) { return tracked_values_.at(key); }
|
||||
CachedValue &GetCachedValue(const std::string &key) {
|
||||
return tracked_values_.at(utils::pmr::string(key, utils::NewDeleteResource()));
|
||||
}
|
||||
|
||||
// Checks for keys tracked
|
||||
bool IsTrackingValues() const { return !tracked_values_.empty(); }
|
||||
|
||||
~FrameChangeCollector() = default;
|
||||
|
||||
private:
|
||||
// Key is output of utils::GetFrameChangeId, value is utils::pmr::unordered_map
|
||||
memgraph::utils::pmr::unordered_map<std::string, CachedValue> tracked_values_;
|
||||
struct PmrStringHash {
|
||||
size_t operator()(const utils::pmr::string &key) const { return utils::Fnv(key); }
|
||||
};
|
||||
|
||||
utils::pmr::unordered_map<utils::pmr::string, CachedValue, PmrStringHash> tracked_values_;
|
||||
};
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -23,9 +23,7 @@
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
|
||||
namespace memgraph {
|
||||
|
||||
namespace query {
|
||||
namespace memgraph::query {
|
||||
|
||||
struct LabelIx {
|
||||
static const utils::TypeInfo kType;
|
||||
@@ -62,8 +60,8 @@ inline bool operator!=(const PropertyIx &a, const PropertyIx &b) { return !(a ==
|
||||
inline bool operator==(const EdgeTypeIx &a, const EdgeTypeIx &b) { return a.ix == b.ix && a.name == b.name; }
|
||||
|
||||
inline bool operator!=(const EdgeTypeIx &a, const EdgeTypeIx &b) { return !(a == b); }
|
||||
} // namespace query
|
||||
} // namespace memgraph
|
||||
} // namespace memgraph::query
|
||||
|
||||
namespace std {
|
||||
|
||||
template <>
|
||||
@@ -83,9 +81,7 @@ struct hash<memgraph::query::EdgeTypeIx> {
|
||||
|
||||
} // namespace std
|
||||
|
||||
namespace memgraph {
|
||||
|
||||
namespace query {
|
||||
namespace memgraph::query {
|
||||
|
||||
class Tree;
|
||||
|
||||
@@ -3029,7 +3025,7 @@ class ReplicationQuery : public memgraph::query::Query {
|
||||
|
||||
enum class SyncMode { SYNC, ASYNC };
|
||||
|
||||
enum class ReplicaState { READY, REPLICATING, RECOVERY, INVALID };
|
||||
enum class ReplicaState { READY, REPLICATING, RECOVERY, MAYBE_BEHIND };
|
||||
|
||||
ReplicationQuery() = default;
|
||||
|
||||
@@ -3577,5 +3573,4 @@ class ShowDatabasesQuery : public memgraph::query::Query {
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace query
|
||||
} // namespace memgraph
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -1216,10 +1216,6 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
|
||||
call_proc->memory_limit_ = memory_limit_info->first;
|
||||
call_proc->memory_scale_ = memory_limit_info->second;
|
||||
}
|
||||
} else {
|
||||
// Default to 100 MB
|
||||
call_proc->memory_limit_ = storage_->Create<PrimitiveLiteral>(TypedValue(100));
|
||||
call_proc->memory_scale_ = 1024U * 1024U;
|
||||
}
|
||||
|
||||
const auto &maybe_found =
|
||||
@@ -1240,11 +1236,13 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
|
||||
throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
|
||||
}
|
||||
}
|
||||
call_proc->is_write_ = maybe_found->second->info.is_write;
|
||||
if (maybe_found) {
|
||||
call_proc->is_write_ = maybe_found->second->info.is_write;
|
||||
}
|
||||
|
||||
auto *yield_ctx = ctx->yieldProcedureResults();
|
||||
if (!yield_ctx) {
|
||||
if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) {
|
||||
if ((maybe_found && !maybe_found->second->results.empty()) && !call_proc->void_procedure_) {
|
||||
throw SemanticException(
|
||||
"CALL without YIELD may only be used on procedures which do not "
|
||||
"return any result fields.");
|
||||
|
||||
@@ -105,7 +105,7 @@ void PrintObject(std::ostream *out, const std::map<K, V> &map);
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const T &arg) {
|
||||
static_assert(!std::is_convertible<T, Expression *>::value,
|
||||
static_assert(!std::is_convertible_v<T, Expression *>,
|
||||
"This overload shouldn't be called with pointers convertible "
|
||||
"to Expression *. This means your other PrintObject overloads aren't "
|
||||
"being called for certain AST nodes when they should (or perhaps such "
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -31,7 +31,7 @@ class Parser {
|
||||
* @param query incoming query that has to be compiled into query plan
|
||||
* the first step is to generate AST
|
||||
*/
|
||||
Parser(const std::string query) : query_(std::move(query)) {
|
||||
explicit Parser(const std::string query) : query_(std::move(query)) {
|
||||
parser_.removeErrorListeners();
|
||||
parser_.addErrorListener(&error_listener_);
|
||||
tree_ = parser_.cypher();
|
||||
|
||||
@@ -12,12 +12,11 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <utility>
|
||||
|
||||
#include "utils/typeinfo.hpp"
|
||||
|
||||
namespace memgraph {
|
||||
|
||||
namespace query {
|
||||
namespace memgraph::query {
|
||||
|
||||
class Symbol {
|
||||
public:
|
||||
@@ -34,9 +33,13 @@ class Symbol {
|
||||
return enum_string[static_cast<int>(type)];
|
||||
}
|
||||
|
||||
Symbol() {}
|
||||
Symbol(const std::string &name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1)
|
||||
: name_(name), position_(position), user_declared_(user_declared), type_(type), token_position_(token_position) {}
|
||||
Symbol() = default;
|
||||
Symbol(std::string name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1)
|
||||
: name_(std::move(name)),
|
||||
position_(position),
|
||||
user_declared_(user_declared),
|
||||
type_(type),
|
||||
token_position_(token_position) {}
|
||||
|
||||
bool operator==(const Symbol &other) const {
|
||||
return position_ == other.position_ && name_ == other.name_ && type_ == other.type_;
|
||||
@@ -57,8 +60,8 @@ class Symbol {
|
||||
int64_t token_position_{-1};
|
||||
};
|
||||
|
||||
} // namespace query
|
||||
} // namespace memgraph
|
||||
} // namespace memgraph::query
|
||||
|
||||
namespace std {
|
||||
|
||||
template <>
|
||||
|
||||
@@ -480,12 +480,20 @@ bool SymbolGenerator::PreVisit(None &none) {
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Reduce &reduce) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_reduce = true;
|
||||
reduce.initializer_->Accept(*this);
|
||||
reduce.list_->Accept(*this);
|
||||
VisitWithIdentifiers(reduce.expression_, {reduce.accumulator_, reduce.identifier_});
|
||||
return false;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PostVisit(Reduce & /*reduce*/) {
|
||||
auto &scope = scopes_.back();
|
||||
scope.in_reduce = false;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool SymbolGenerator::PreVisit(Extract &extract) {
|
||||
extract.list_->Accept(*this);
|
||||
VisitWithIdentifiers(extract.expression_, {extract.identifier_});
|
||||
@@ -496,15 +504,23 @@ bool SymbolGenerator::PreVisit(Exists &exists) {
|
||||
auto &scope = scopes_.back();
|
||||
|
||||
if (scope.in_set_property) {
|
||||
throw utils::NotYetImplemented("Set property can not be used with exists, but only during matching!");
|
||||
throw utils::NotYetImplemented("Exists cannot be used within SET clause.!");
|
||||
}
|
||||
|
||||
if (scope.in_with) {
|
||||
throw utils::NotYetImplemented("WITH can not be used with exists, but only during matching!");
|
||||
throw utils::NotYetImplemented("Exists cannot be used within WITH!");
|
||||
}
|
||||
|
||||
if (scope.in_return) {
|
||||
throw utils::NotYetImplemented("RETURN can not be used with exists, but only during matching!");
|
||||
throw utils::NotYetImplemented("Exists cannot be used within RETURN!");
|
||||
}
|
||||
|
||||
if (scope.in_reduce) {
|
||||
throw utils::NotYetImplemented("Exists cannot be used within REDUCE!");
|
||||
}
|
||||
|
||||
if (scope.num_if_operators) {
|
||||
throw utils::NotYetImplemented("IF operator cannot be used with exists, but only during matching!");
|
||||
}
|
||||
|
||||
scope.in_exists = true;
|
||||
|
||||
@@ -84,6 +84,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
bool PreVisit(Any &) override;
|
||||
bool PreVisit(None &) override;
|
||||
bool PreVisit(Reduce &) override;
|
||||
bool PostVisit(Reduce &) override;
|
||||
bool PreVisit(Extract &) override;
|
||||
bool PreVisit(Exists & /*exists*/) override;
|
||||
bool PostVisit(Exists & /*exists*/) override;
|
||||
@@ -123,6 +124,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
bool in_match{false};
|
||||
bool in_foreach{false};
|
||||
bool in_exists{false};
|
||||
bool in_reduce{false};
|
||||
bool in_set_property{false};
|
||||
bool in_call_subquery{false};
|
||||
bool has_return{false};
|
||||
@@ -181,7 +183,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
|
||||
/// If property lookup for one symbol is visited more times, it is better to fetch all properties
|
||||
class PropertyLookupEvaluationModeVisitor : public ExpressionVisitor<void> {
|
||||
public:
|
||||
explicit PropertyLookupEvaluationModeVisitor() {}
|
||||
explicit PropertyLookupEvaluationModeVisitor() = default;
|
||||
|
||||
using ExpressionVisitor<void>::Visit;
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ namespace memgraph::query {
|
||||
|
||||
class SymbolTable final {
|
||||
public:
|
||||
SymbolTable() {}
|
||||
SymbolTable() = default;
|
||||
const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
|
||||
int32_t token_position = -1) {
|
||||
MG_ASSERT(table_.size() <= std::numeric_limits<int32_t>::max(),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -16,6 +16,7 @@
|
||||
#include <iostream>
|
||||
#include <span>
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "query/exceptions.hpp"
|
||||
@@ -32,7 +33,7 @@ namespace memgraph::query::frontend {
|
||||
|
||||
using namespace lexer_constants;
|
||||
|
||||
StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
|
||||
StrippedQuery::StrippedQuery(std::string query) : original_(std::move(query)) {
|
||||
enum class Token {
|
||||
UNMATCHED,
|
||||
KEYWORD, // Including true, false and null.
|
||||
@@ -255,29 +256,29 @@ std::string GetFirstUtf8Symbol(const char *_s) {
|
||||
// According to
|
||||
// https://stackoverflow.com/questions/16260033/reinterpret-cast-between-char-and-stduint8-t-safe
|
||||
// this checks if casting from const char * to uint8_t is undefined behaviour.
|
||||
static_assert(std::is_same<std::uint8_t, unsigned char>::value,
|
||||
static_assert(std::is_same_v<std::uint8_t, unsigned char>,
|
||||
"This library requires std::uint8_t to be implemented as "
|
||||
"unsigned char.");
|
||||
const uint8_t *s = reinterpret_cast<const uint8_t *>(_s);
|
||||
if ((*s >> 7) == 0x00) return std::string(_s, _s + 1);
|
||||
if ((*s >> 5) == 0x06) {
|
||||
auto *s1 = s + 1;
|
||||
const auto *s1 = s + 1;
|
||||
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
return std::string(_s, _s + 2);
|
||||
}
|
||||
if ((*s >> 4) == 0x0e) {
|
||||
auto *s1 = s + 1;
|
||||
const auto *s1 = s + 1;
|
||||
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
auto *s2 = s + 2;
|
||||
const auto *s2 = s + 2;
|
||||
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
return std::string(_s, _s + 3);
|
||||
}
|
||||
if ((*s >> 3) == 0x1e) {
|
||||
auto *s1 = s + 1;
|
||||
const auto *s1 = s + 1;
|
||||
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
auto *s2 = s + 2;
|
||||
const auto *s2 = s + 2;
|
||||
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
auto *s3 = s + 3;
|
||||
const auto *s3 = s + 3;
|
||||
if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
return std::string(_s, _s + 4);
|
||||
}
|
||||
@@ -286,29 +287,29 @@ std::string GetFirstUtf8Symbol(const char *_s) {
|
||||
|
||||
// Return codepoint of first utf8 symbol and its encoded length.
|
||||
std::pair<int, int> GetFirstUtf8SymbolCodepoint(const char *_s) {
|
||||
static_assert(std::is_same<std::uint8_t, unsigned char>::value,
|
||||
static_assert(std::is_same_v<std::uint8_t, unsigned char>,
|
||||
"This library requires std::uint8_t to be implemented as "
|
||||
"unsigned char.");
|
||||
const uint8_t *s = reinterpret_cast<const uint8_t *>(_s);
|
||||
if ((*s >> 7) == 0x00) return {*s & 0x7f, 1};
|
||||
if ((*s >> 5) == 0x06) {
|
||||
auto *s1 = s + 1;
|
||||
const auto *s1 = s + 1;
|
||||
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
return {((*s & 0x1f) << 6) | (*s1 & 0x3f), 2};
|
||||
}
|
||||
if ((*s >> 4) == 0x0e) {
|
||||
auto *s1 = s + 1;
|
||||
const auto *s1 = s + 1;
|
||||
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
auto *s2 = s + 2;
|
||||
const auto *s2 = s + 2;
|
||||
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
return {((*s & 0x0f) << 12) | ((*s1 & 0x3f) << 6) | (*s2 & 0x3f), 3};
|
||||
}
|
||||
if ((*s >> 3) == 0x1e) {
|
||||
auto *s1 = s + 1;
|
||||
const auto *s1 = s + 1;
|
||||
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
auto *s2 = s + 2;
|
||||
const auto *s2 = s + 2;
|
||||
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
auto *s3 = s + 3;
|
||||
const auto *s3 = s + 3;
|
||||
if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character.");
|
||||
return {((*s & 0x07) << 18) | ((*s1 & 0x3f) << 12) | ((*s2 & 0x3f) << 6) | (*s3 & 0x3f), 4};
|
||||
}
|
||||
@@ -336,7 +337,7 @@ int StrippedQuery::MatchSpecial(int start) const { return kSpecialTokens.Match(o
|
||||
int StrippedQuery::MatchString(int start) const {
|
||||
if (original_[start] != '"' && original_[start] != '\'') return 0;
|
||||
char start_char = original_[start];
|
||||
for (auto *p = original_.data() + start + 1; *p; ++p) {
|
||||
for (const auto *p = original_.data() + start + 1; *p; ++p) {
|
||||
if (*p == start_char) return p - (original_.data() + start) + 1;
|
||||
if (*p == '\\') {
|
||||
++p;
|
||||
@@ -346,7 +347,7 @@ int StrippedQuery::MatchString(int start) const {
|
||||
continue;
|
||||
} else if (*p == 'U' || *p == 'u') {
|
||||
int cnt = 0;
|
||||
auto *r = p + 1;
|
||||
const auto *r = p + 1;
|
||||
while (isxdigit(*r) && cnt < 8) {
|
||||
++cnt;
|
||||
++r;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -40,7 +40,7 @@ class StrippedQuery {
|
||||
*
|
||||
* @param query Input query.
|
||||
*/
|
||||
explicit StrippedQuery(const std::string &query);
|
||||
explicit StrippedQuery(std::string query);
|
||||
|
||||
/**
|
||||
* Copy constructor is deleted because we don't want to make unnecessary
|
||||
|
||||
@@ -17,8 +17,7 @@
|
||||
#include <unordered_set>
|
||||
#include <vector>
|
||||
|
||||
namespace memgraph::query {
|
||||
namespace lexer_constants {
|
||||
namespace memgraph::query::lexer_constants {
|
||||
|
||||
namespace trie {
|
||||
|
||||
@@ -33,7 +32,7 @@ inline int Noop(int x) { return x; }
|
||||
|
||||
class Trie {
|
||||
public:
|
||||
Trie() {}
|
||||
Trie() = default;
|
||||
Trie(std::initializer_list<std::string> l) {
|
||||
for (const auto &s : l) {
|
||||
Insert(s);
|
||||
@@ -2934,5 +2933,4 @@ const trie::Trie kSpecialTokens = {";",
|
||||
"\xEF\xB9\x98", // u8"\ufe58"
|
||||
"\xEF\xB9\xA3", // u8"\ufe63"
|
||||
"\xEF\xBC\x8D"}; // u8"\uff0d"
|
||||
} // namespace lexer_constants
|
||||
} // namespace memgraph::query
|
||||
} // namespace memgraph::query::lexer_constants
|
||||
|
||||
@@ -593,6 +593,7 @@ TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContex
|
||||
case TypedValue::Type::Duration:
|
||||
return TypedValue("DURATION", ctx.memory);
|
||||
case TypedValue::Type::Graph:
|
||||
case TypedValue::Type::Function:
|
||||
throw QueryRuntimeException("Cannot fetch graph as it is not standardized openCypher type name");
|
||||
}
|
||||
}
|
||||
@@ -1156,11 +1157,16 @@ void MapNumericParameters(auto ¶meter_mappings, const auto &input_parameters
|
||||
}
|
||||
|
||||
TypedValue Date(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
|
||||
FType<Optional<Or<String, Map>>>("date", args, nargs);
|
||||
FType<Optional<Or<String, Map, LocalDateTime>>>("date", args, nargs);
|
||||
if (nargs == 0) {
|
||||
return TypedValue(utils::LocalDateTime(ctx.timestamp).date, ctx.memory);
|
||||
}
|
||||
|
||||
if (args[0].IsLocalDateTime()) {
|
||||
utils::Date date{args[0].ValueLocalDateTime().date};
|
||||
return TypedValue(date, ctx.memory);
|
||||
}
|
||||
|
||||
if (args[0].IsString()) {
|
||||
const auto &[date_parameters, is_extended] = utils::ParseDateParameters(args[0].ValueString());
|
||||
return TypedValue(utils::Date(date_parameters), ctx.memory);
|
||||
@@ -1178,12 +1184,17 @@ TypedValue Date(const TypedValue *args, int64_t nargs, const FunctionContext &ct
|
||||
}
|
||||
|
||||
TypedValue LocalTime(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
|
||||
FType<Optional<Or<String, Map>>>("localtime", args, nargs);
|
||||
FType<Optional<Or<String, Map, LocalDateTime>>>("localtime", args, nargs);
|
||||
|
||||
if (nargs == 0) {
|
||||
return TypedValue(utils::LocalDateTime(ctx.timestamp).local_time, ctx.memory);
|
||||
}
|
||||
|
||||
if (args[0].IsLocalDateTime()) {
|
||||
utils::LocalTime local_time{args[0].ValueLocalDateTime().local_time};
|
||||
return TypedValue(local_time, ctx.memory);
|
||||
}
|
||||
|
||||
if (args[0].IsString()) {
|
||||
const auto &[local_time_parameters, is_extended] = utils::ParseLocalTimeParameters(args[0].ValueString());
|
||||
return TypedValue(utils::LocalTime(local_time_parameters), ctx.memory);
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <regex>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
@@ -187,6 +188,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
|
||||
utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; }
|
||||
|
||||
void ClearPropertyLookupCache() { property_lookup_cache_.clear(); }
|
||||
|
||||
TypedValue Visit(NamedExpression &named_expression) override {
|
||||
const auto &symbol = symbol_table_->at(named_expression);
|
||||
auto value = named_expression.expression_->Accept(*this);
|
||||
@@ -270,36 +273,33 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
|
||||
TypedValue Visit(InListOperator &in_list) override {
|
||||
TypedValue *_list_ptr = nullptr;
|
||||
TypedValue _list;
|
||||
auto literal = in_list.expression1_->Accept(*this);
|
||||
|
||||
auto get_list_literal = [this, &in_list, &_list, &_list_ptr]() -> void {
|
||||
auto get_list_literal = [this, &in_list]() -> TypedValue {
|
||||
ReferenceExpressionEvaluator reference_expression_evaluator{frame_, symbol_table_, ctx_};
|
||||
_list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
|
||||
if (nullptr == _list_ptr) {
|
||||
_list = in_list.expression2_->Accept(*this);
|
||||
_list_ptr = &_list;
|
||||
auto *list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
|
||||
if (nullptr == list_ptr) {
|
||||
return in_list.expression2_->Accept(*this);
|
||||
}
|
||||
return *list_ptr;
|
||||
};
|
||||
|
||||
auto do_list_literal_checks = [this, &literal, &_list_ptr]() -> std::optional<TypedValue> {
|
||||
MG_ASSERT(_list_ptr, "List literal should have been defined");
|
||||
if (_list_ptr->IsNull()) {
|
||||
auto do_list_literal_checks = [this, &literal](const TypedValue &list) -> std::optional<TypedValue> {
|
||||
if (list.IsNull()) {
|
||||
return TypedValue(ctx_->memory);
|
||||
}
|
||||
// Exceptions have higher priority than returning nulls when list expression
|
||||
// is not null.
|
||||
if (_list_ptr->type() != TypedValue::Type::List) {
|
||||
throw QueryRuntimeException("IN expected a list, got {}.", _list_ptr->type());
|
||||
if (list.type() != TypedValue::Type::List) {
|
||||
throw QueryRuntimeException("IN expected a list, got {}.", list.type());
|
||||
}
|
||||
const auto &list = _list_ptr->ValueList();
|
||||
const auto &list_value = list.ValueList();
|
||||
|
||||
// If literal is NULL there is no need to try to compare it with every
|
||||
// element in the list since result of every comparison will be NULL. There
|
||||
// is one special case that we must test explicitly: if list is empty then
|
||||
// result is false since no comparison will be performed.
|
||||
if (list.empty()) return TypedValue(false, ctx_->memory);
|
||||
if (list_value.empty()) return TypedValue(false, ctx_->memory);
|
||||
if (literal.IsNull()) return TypedValue(ctx_->memory);
|
||||
return {};
|
||||
};
|
||||
@@ -312,14 +312,14 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
if (!frame_change_collector_->IsKeyValueCached(*cached_id)) {
|
||||
// Check only first time if everything is okay, later when we use
|
||||
// cache there is no need to check again as we did check first time
|
||||
get_list_literal();
|
||||
auto preoperational_checks = do_list_literal_checks();
|
||||
auto list = get_list_literal();
|
||||
auto preoperational_checks = do_list_literal_checks(list);
|
||||
if (preoperational_checks) {
|
||||
return std::move(*preoperational_checks);
|
||||
}
|
||||
auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
|
||||
cached_value.CacheValue(*_list_ptr);
|
||||
spdlog::trace("Value cached {}", *cached_id);
|
||||
// Don't move here because we don't want to remove the element from the frame
|
||||
cached_value.CacheValue(list);
|
||||
}
|
||||
const auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
|
||||
|
||||
@@ -334,16 +334,15 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
// When caching is not an option, we need to evaluate list literal every time
|
||||
// and do the checks
|
||||
get_list_literal();
|
||||
auto preoperational_checks = do_list_literal_checks();
|
||||
const auto list = get_list_literal();
|
||||
auto preoperational_checks = do_list_literal_checks(list);
|
||||
if (preoperational_checks) {
|
||||
return std::move(*preoperational_checks);
|
||||
}
|
||||
|
||||
const auto &list = _list.ValueList();
|
||||
spdlog::trace("Not using cache on IN LIST operator");
|
||||
const auto &list_value = list.ValueList();
|
||||
auto has_null = false;
|
||||
for (const auto &element : list) {
|
||||
for (const auto &element : list_value) {
|
||||
auto result = literal == element;
|
||||
if (result.IsNull()) {
|
||||
has_null = true;
|
||||
@@ -829,8 +828,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
throw QueryRuntimeException("'coalesce' requires at least one argument.");
|
||||
}
|
||||
|
||||
for (int64_t i = 0; i < exprs.size(); ++i) {
|
||||
TypedValue val(exprs[i]->Accept(*this), ctx_->memory);
|
||||
for (auto &expr : exprs) {
|
||||
TypedValue val(expr->Accept(*this), ctx_->memory);
|
||||
if (!val.IsNull()) {
|
||||
return val;
|
||||
}
|
||||
@@ -907,7 +906,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
return TypedValue(std::move(result), ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(Exists &exists) override { return TypedValue{frame_->at(symbol_table_->at(exists)), ctx_->memory}; }
|
||||
TypedValue Visit(Exists &exists) override {
|
||||
TypedValue &frame_exists_value = frame_->at(symbol_table_->at(exists));
|
||||
if (!frame_exists_value.IsFunction()) [[unlikely]] {
|
||||
throw QueryRuntimeException(
|
||||
"Unexpected behavior: Exists expected a function, got {}. Please report the problem on GitHub issues",
|
||||
frame_exists_value.type());
|
||||
}
|
||||
TypedValue result{ctx_->memory};
|
||||
frame_exists_value.ValueFunction()(&result);
|
||||
return result;
|
||||
}
|
||||
|
||||
TypedValue Visit(All &all) override {
|
||||
auto list_value = all.list_expression_->Accept(*this);
|
||||
|
||||
@@ -37,6 +37,7 @@
|
||||
#include "dbms/database.hpp"
|
||||
#include "dbms/dbms_handler.hpp"
|
||||
#include "dbms/global.hpp"
|
||||
#include "dbms/inmemory/storage_helper.hpp"
|
||||
#include "flags/run_time_configurable.hpp"
|
||||
#include "glue/communication.hpp"
|
||||
#include "license/license.hpp"
|
||||
@@ -94,16 +95,17 @@
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/readable_size.hpp"
|
||||
#include "utils/settings.hpp"
|
||||
#include "utils/stat.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/tsc.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
#include "dbms/dbms_handler.hpp"
|
||||
#include "dbms/replication_handler.hpp"
|
||||
#include "query/auth_query_handler.hpp"
|
||||
#include "query/interpreter_context.hpp"
|
||||
#include "replication/state.hpp"
|
||||
#include "storage/v2/replication/replication_handler.hpp"
|
||||
|
||||
namespace memgraph::metrics {
|
||||
extern Event ReadQuery;
|
||||
@@ -145,6 +147,8 @@ void memgraph::query::CurrentDB::CleanupDBTransaction(bool abort) {
|
||||
|
||||
namespace memgraph::query {
|
||||
|
||||
constexpr std::string_view kSchemaAssert = "SCHEMA.ASSERT";
|
||||
|
||||
template <typename>
|
||||
constexpr auto kAlwaysFalse = false;
|
||||
|
||||
@@ -270,7 +274,7 @@ inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode
|
||||
|
||||
class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
public:
|
||||
explicit ReplQueryHandler(storage::Storage *db) : db_(db), handler_{db_->repl_state_, *db_} {}
|
||||
explicit ReplQueryHandler(dbms::DbmsHandler *dbms_handler) : dbms_handler_(dbms_handler), handler_{*dbms_handler} {}
|
||||
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional<int64_t> port) override {
|
||||
@@ -314,10 +318,6 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
throw QueryRuntimeException("Replica can't register another replica!");
|
||||
}
|
||||
|
||||
if (name == memgraph::replication::kReservedReplicationRoleName) {
|
||||
throw QueryRuntimeException("This replica name is reserved and can not be used as replica name!");
|
||||
}
|
||||
|
||||
auto repl_mode = convertToReplicationMode(sync_mode);
|
||||
|
||||
auto maybe_ip_and_port =
|
||||
@@ -330,8 +330,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
.port = port,
|
||||
.replica_check_frequency = replica_check_frequency,
|
||||
.ssl = std::nullopt};
|
||||
using storage::RegistrationMode;
|
||||
auto ret = handler_.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, config);
|
||||
auto ret = handler_.RegisterReplica(config);
|
||||
if (ret.HasError()) {
|
||||
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
|
||||
}
|
||||
@@ -344,7 +343,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
void DropReplica(std::string_view replica_name) override {
|
||||
auto const result = handler_.UnregisterReplica(replica_name);
|
||||
switch (result) {
|
||||
using enum memgraph::storage::UnregisterReplicaResult;
|
||||
using enum memgraph::dbms::UnregisterReplicaResult;
|
||||
case NOT_MAIN:
|
||||
throw QueryRuntimeException("Replica can't unregister a replica!");
|
||||
case COULD_NOT_BE_PERSISTED:
|
||||
@@ -358,13 +357,22 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
|
||||
using Replica = ReplicationQueryHandler::Replica;
|
||||
std::vector<Replica> ShowReplicas() const override {
|
||||
auto const &replState = db_->repl_state_;
|
||||
if (replState.IsReplica()) {
|
||||
if (handler_.IsReplica()) {
|
||||
// replica can't show registered replicas (it shouldn't have any)
|
||||
throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!");
|
||||
}
|
||||
|
||||
auto repl_infos = db_->ReplicasInfo();
|
||||
// TODO: Combine results? Have a single place with clients???
|
||||
// Also authentication checks (replica + database visibility)
|
||||
std::vector<storage::ReplicaInfo> repl_infos{};
|
||||
dbms_handler_->ForOne([&repl_infos](dbms::Database *db) -> bool {
|
||||
auto infos = db->storage()->ReplicasInfo();
|
||||
if (!infos.empty()) {
|
||||
repl_infos = std::move(infos);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
std::vector<Replica> replicas;
|
||||
replicas.reserve(repl_infos.size());
|
||||
|
||||
@@ -395,8 +403,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
case storage::replication::ReplicaState::RECOVERY:
|
||||
replica.state = ReplicationQuery::ReplicaState::RECOVERY;
|
||||
break;
|
||||
case storage::replication::ReplicaState::INVALID:
|
||||
replica.state = ReplicationQuery::ReplicaState::INVALID;
|
||||
case storage::replication::ReplicaState::MAYBE_BEHIND:
|
||||
replica.state = ReplicationQuery::ReplicaState::MAYBE_BEHIND;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -408,8 +416,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
}
|
||||
|
||||
private:
|
||||
storage::Storage *db_;
|
||||
storage::ReplicationHandler handler_;
|
||||
dbms::DbmsHandler *dbms_handler_;
|
||||
dbms::ReplicationHandler handler_;
|
||||
};
|
||||
|
||||
/// returns false if the replication role can't be set
|
||||
@@ -418,7 +426,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
|
||||
Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_context, const Parameters ¶meters) {
|
||||
AuthQueryHandler *auth = interpreter_context->auth;
|
||||
#ifdef MG_ENTERPRISE
|
||||
auto *db_handler = interpreter_context->db_handler;
|
||||
auto *db_handler = interpreter_context->dbms_handler;
|
||||
#endif
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
// the argument to Callback.
|
||||
@@ -470,7 +478,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_
|
||||
MG_ASSERT(password.IsString() || password.IsNull());
|
||||
if (!auth->CreateUser(username, password.IsString() ? std::make_optional(std::string(password.ValueString()))
|
||||
: std::nullopt)) {
|
||||
throw QueryRuntimeException("User '{}' already exists.", username);
|
||||
throw UserAlreadyExistsException("User '{}' already exists.", username);
|
||||
}
|
||||
|
||||
// If the license is not valid we create users with admin access
|
||||
@@ -702,8 +710,9 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_
|
||||
}
|
||||
} // namespace
|
||||
|
||||
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters, storage::Storage *storage,
|
||||
const query::InterpreterConfig &config, std::vector<Notification> *notifications) {
|
||||
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters,
|
||||
dbms::DbmsHandler *dbms_handler, const query::InterpreterConfig &config,
|
||||
std::vector<Notification> *notifications) {
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
// the argument to Callback.
|
||||
EvaluationContext evaluation_context;
|
||||
@@ -723,7 +732,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
notifications->emplace_back(SeverityLevel::WARNING, NotificationCode::REPLICA_PORT_WARNING,
|
||||
"Be careful the replication port must be different from the memgraph port!");
|
||||
}
|
||||
callback.fn = [handler = ReplQueryHandler{storage}, role = repl_query->role_, maybe_port]() mutable {
|
||||
callback.fn = [handler = ReplQueryHandler{dbms_handler}, role = repl_query->role_, maybe_port]() mutable {
|
||||
handler.SetReplicationRole(role, maybe_port);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
};
|
||||
@@ -735,7 +744,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
}
|
||||
case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: {
|
||||
callback.header = {"replication role"};
|
||||
callback.fn = [handler = ReplQueryHandler{storage}] {
|
||||
callback.fn = [handler = ReplQueryHandler{dbms_handler}] {
|
||||
auto mode = handler.ShowReplicationRole();
|
||||
switch (mode) {
|
||||
case ReplicationQuery::ReplicationRole::MAIN: {
|
||||
@@ -754,7 +763,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
auto socket_address = repl_query->socket_address_->Accept(evaluator);
|
||||
const auto replica_check_frequency = config.replication_replica_check_frequency;
|
||||
|
||||
callback.fn = [handler = ReplQueryHandler{storage}, name, socket_address, sync_mode,
|
||||
callback.fn = [handler = ReplQueryHandler{dbms_handler}, name, socket_address, sync_mode,
|
||||
replica_check_frequency]() mutable {
|
||||
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, replica_check_frequency);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
@@ -765,7 +774,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
}
|
||||
case ReplicationQuery::Action::DROP_REPLICA: {
|
||||
const auto &name = repl_query->replica_name_;
|
||||
callback.fn = [handler = ReplQueryHandler{storage}, name]() mutable {
|
||||
callback.fn = [handler = ReplQueryHandler{dbms_handler}, name]() mutable {
|
||||
handler.DropReplica(name);
|
||||
return std::vector<std::vector<TypedValue>>();
|
||||
};
|
||||
@@ -777,7 +786,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
callback.header = {
|
||||
"name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master",
|
||||
"state"};
|
||||
callback.fn = [handler = ReplQueryHandler{storage}, replica_nfields = callback.header.size()] {
|
||||
callback.fn = [handler = ReplQueryHandler{dbms_handler}, replica_nfields = callback.header.size()] {
|
||||
const auto &replicas = handler.ShowReplicas();
|
||||
auto typed_replicas = std::vector<std::vector<TypedValue>>{};
|
||||
typed_replicas.reserve(replicas.size());
|
||||
@@ -785,34 +794,33 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
std::vector<TypedValue> typed_replica;
|
||||
typed_replica.reserve(replica_nfields);
|
||||
|
||||
typed_replica.emplace_back(TypedValue(replica.name));
|
||||
typed_replica.emplace_back(TypedValue(replica.socket_address));
|
||||
typed_replica.emplace_back(replica.name);
|
||||
typed_replica.emplace_back(replica.socket_address);
|
||||
|
||||
switch (replica.sync_mode) {
|
||||
case ReplicationQuery::SyncMode::SYNC:
|
||||
typed_replica.emplace_back(TypedValue("sync"));
|
||||
typed_replica.emplace_back("sync");
|
||||
break;
|
||||
case ReplicationQuery::SyncMode::ASYNC:
|
||||
typed_replica.emplace_back(TypedValue("async"));
|
||||
typed_replica.emplace_back("async");
|
||||
break;
|
||||
}
|
||||
|
||||
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.current_timestamp_of_replica)));
|
||||
typed_replica.emplace_back(
|
||||
TypedValue(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master)));
|
||||
typed_replica.emplace_back(static_cast<int64_t>(replica.current_timestamp_of_replica));
|
||||
typed_replica.emplace_back(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master));
|
||||
|
||||
switch (replica.state) {
|
||||
case ReplicationQuery::ReplicaState::READY:
|
||||
typed_replica.emplace_back(TypedValue("ready"));
|
||||
typed_replica.emplace_back("ready");
|
||||
break;
|
||||
case ReplicationQuery::ReplicaState::REPLICATING:
|
||||
typed_replica.emplace_back(TypedValue("replicating"));
|
||||
typed_replica.emplace_back("replicating");
|
||||
break;
|
||||
case ReplicationQuery::ReplicaState::RECOVERY:
|
||||
typed_replica.emplace_back(TypedValue("recovery"));
|
||||
typed_replica.emplace_back("recovery");
|
||||
break;
|
||||
case ReplicationQuery::ReplicaState::INVALID:
|
||||
typed_replica.emplace_back(TypedValue("invalid"));
|
||||
case ReplicationQuery::ReplicaState::MAYBE_BEHIND:
|
||||
typed_replica.emplace_back("invalid");
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -1203,7 +1211,7 @@ struct TxTimeout {
|
||||
};
|
||||
|
||||
struct PullPlan {
|
||||
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, bool is_profile_query,
|
||||
explicit PullPlan(std::shared_ptr<PlanWrapper> plan, const Parameters ¶meters, bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
|
||||
std::shared_ptr<utils::AsyncTimer> tx_timer,
|
||||
@@ -1216,7 +1224,7 @@ struct PullPlan {
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
|
||||
private:
|
||||
std::shared_ptr<CachedPlan> plan_ = nullptr;
|
||||
std::shared_ptr<PlanWrapper> plan_ = nullptr;
|
||||
plan::UniqueCursorPtr cursor_ = nullptr;
|
||||
Frame frame_;
|
||||
ExecutionContext ctx_;
|
||||
@@ -1243,7 +1251,7 @@ struct PullPlan {
|
||||
bool use_monotonic_memory_;
|
||||
};
|
||||
|
||||
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
PullPlan::PullPlan(const std::shared_ptr<PlanWrapper> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
|
||||
std::shared_ptr<utils::AsyncTimer> tx_timer, TriggerContextCollector *trigger_context_collector,
|
||||
@@ -1399,42 +1407,16 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
|
||||
|
||||
using RWType = plan::ReadWriteTypeChecker::RWType;
|
||||
|
||||
bool IsWriteQueryOnMainMemoryReplica(storage::Storage *storage,
|
||||
const query::plan::ReadWriteTypeChecker::RWType query_type) {
|
||||
if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL ||
|
||||
storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
|
||||
auto const &replState = storage->repl_state_;
|
||||
return replState.IsReplica() && (query_type == RWType::W || query_type == RWType::RW);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool IsReplica(storage::Storage *storage) {
|
||||
if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL ||
|
||||
storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
|
||||
auto const &replState = storage->repl_state_;
|
||||
return replState.IsReplica();
|
||||
}
|
||||
return false;
|
||||
bool IsQueryWrite(const query::plan::ReadWriteTypeChecker::RWType query_type) {
|
||||
return query_type == RWType::W || query_type == RWType::RW;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
#ifdef MG_ENTERPRISE
|
||||
InterpreterContext::InterpreterContext(InterpreterConfig interpreter_config, memgraph::dbms::DbmsHandler *handler,
|
||||
query::AuthQueryHandler *ah, query::AuthChecker *ac)
|
||||
: db_handler(handler), config(interpreter_config), auth(ah), auth_checker(ac) {}
|
||||
#else
|
||||
InterpreterContext::InterpreterContext(InterpreterConfig interpreter_config,
|
||||
memgraph::utils::Gatekeeper<memgraph::dbms::Database> *db_gatekeeper,
|
||||
query::AuthQueryHandler *ah, query::AuthChecker *ac)
|
||||
: db_gatekeeper(db_gatekeeper), config(interpreter_config), auth(ah), auth_checker(ac) {}
|
||||
#endif
|
||||
|
||||
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
|
||||
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
|
||||
#ifndef MG_ENTERPRISE
|
||||
auto db_acc = interpreter_context_->db_gatekeeper->access();
|
||||
auto db_acc = interpreter_context_->dbms_handler->Get();
|
||||
MG_ASSERT(db_acc, "Database accessor needs to be valid");
|
||||
current_db_.db_acc_ = std::move(db_acc);
|
||||
#endif
|
||||
@@ -1551,7 +1533,6 @@ inline static void TryCaching(const AstStorage &ast_storage, FrameChangeCollecto
|
||||
continue;
|
||||
}
|
||||
frame_change_collector->AddTrackingKey(*cached_id);
|
||||
spdlog::trace("Tracking {} operator, by id: {}", InListOperator::kType.name, *cached_id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1612,8 +1593,6 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
|
||||
auto use_monotonic_memory =
|
||||
!contains_csv && !IsCallBatchedProcedureQuery(clauses) && !IsAllShortestPathsQuery(clauses);
|
||||
spdlog::trace("PrepareCypher has {} encountered all shortest paths and will {} use of monotonic memory",
|
||||
IsAllShortestPathsQuery(clauses) ? "" : "not", use_monotonic_memory ? "" : "not");
|
||||
|
||||
MG_ASSERT(current_db.execution_db_accessor_, "Cypher query expects a current DB transaction");
|
||||
auto *dba =
|
||||
@@ -1979,11 +1958,11 @@ std::vector<std::vector<TypedValue>> AnalyzeGraphQueryHandler::AnalyzeGraphCreat
|
||||
result.reserve(kComputeStatisticsNumResults);
|
||||
|
||||
result.emplace_back(execution_db_accessor->LabelToName(stat_entry.first));
|
||||
result.emplace_back(TypedValue());
|
||||
result.emplace_back();
|
||||
result.emplace_back(static_cast<int64_t>(stat_entry.second.count));
|
||||
result.emplace_back(TypedValue());
|
||||
result.emplace_back(TypedValue());
|
||||
result.emplace_back(TypedValue());
|
||||
result.emplace_back();
|
||||
result.emplace_back();
|
||||
result.emplace_back();
|
||||
result.emplace_back(stat_entry.second.avg_degree);
|
||||
results.push_back(std::move(result));
|
||||
});
|
||||
@@ -2124,10 +2103,7 @@ PreparedQuery PrepareAnalyzeGraphQuery(ParsedQuery parsed_query, bool in_explici
|
||||
|
||||
// Creating an index influences computed plan costs.
|
||||
auto invalidate_plan_cache = [plan_cache = current_db.db_acc_->get()->plan_cache()] {
|
||||
auto access = plan_cache->access();
|
||||
for (auto &kv : access) {
|
||||
access.remove(kv.first);
|
||||
}
|
||||
plan_cache->WithLock([&](auto &cache) { cache.reset(); });
|
||||
};
|
||||
utils::OnScopeExit cache_invalidator(invalidate_plan_cache);
|
||||
|
||||
@@ -2172,10 +2148,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
|
||||
|
||||
// Creating an index influences computed plan costs.
|
||||
auto invalidate_plan_cache = [plan_cache = db_acc->plan_cache()] {
|
||||
auto access = plan_cache->access();
|
||||
for (auto &kv : access) {
|
||||
access.remove(kv.first);
|
||||
}
|
||||
plan_cache->WithLock([&](auto &cache) { cache.reset(); });
|
||||
};
|
||||
|
||||
auto *storage = db_acc->storage();
|
||||
@@ -2286,21 +2259,15 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
|
||||
}
|
||||
|
||||
PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::vector<Notification> *notifications, CurrentDB ¤t_db,
|
||||
std::vector<Notification> *notifications, dbms::DbmsHandler &dbms_handler,
|
||||
const InterpreterConfig &config) {
|
||||
if (in_explicit_transaction) {
|
||||
throw ReplicationModificationInMulticommandTxException();
|
||||
}
|
||||
|
||||
MG_ASSERT(current_db.db_acc_, "Replication query expects a current DB");
|
||||
storage::Storage *storage = current_db.db_acc_->get()->storage();
|
||||
|
||||
if (storage->GetStorageMode() == storage::StorageMode::ON_DISK_TRANSACTIONAL) {
|
||||
throw ReplicationDisabledOnDiskStorage();
|
||||
}
|
||||
|
||||
auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
|
||||
auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, storage, config, notifications);
|
||||
auto callback =
|
||||
HandleReplicationQuery(replication_query, parsed_query.parameters, &dbms_handler, config, notifications);
|
||||
|
||||
return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
|
||||
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
|
||||
@@ -2773,7 +2740,8 @@ PreparedQuery PrepareStorageModeQuery(ParsedQuery parsed_query, const bool in_ex
|
||||
"transactions using 'SHOW TRANSACTIONS' query and ensure no other transactions are active.");
|
||||
}
|
||||
|
||||
callback = [requested_mode, storage = db_acc->storage()]() -> std::function<void()> {
|
||||
callback = [requested_mode,
|
||||
storage = static_cast<storage::InMemoryStorage *>(db_acc->storage())]() -> std::function<void()> {
|
||||
// SetStorageMode will probably be handled at the Database level
|
||||
return [storage, requested_mode] { storage->SetStorageMode(requested_mode); };
|
||||
}();
|
||||
@@ -2836,15 +2804,11 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
|
||||
std::move(parsed_query.required_privileges),
|
||||
[storage](AnyStream * /*stream*/, std::optional<int> /*n*/) -> std::optional<QueryHandlerResult> {
|
||||
auto *mem_storage = static_cast<storage::InMemoryStorage *>(storage);
|
||||
if (auto maybe_error = mem_storage->CreateSnapshot(storage->repl_state_, {}); maybe_error.HasError()) {
|
||||
if (auto maybe_error = mem_storage->CreateSnapshot(); maybe_error.HasError()) {
|
||||
switch (maybe_error.GetError()) {
|
||||
case storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica:
|
||||
throw utils::BasicException(
|
||||
"Failed to create a snapshot. Replica instances are not allowed to create them.");
|
||||
case storage::InMemoryStorage::CreateSnapshotError::DisabledForAnalyticsPeriodicCommit:
|
||||
spdlog::warn(utils::MessageWithLink("Periodic snapshots are disabled for analytical mode.",
|
||||
"https://memgr.ph/replication"));
|
||||
break;
|
||||
case storage::InMemoryStorage::CreateSnapshotError::ReachedMaxNumTries:
|
||||
spdlog::warn("Failed to create snapshot. Reached max number of tries. Please contact support");
|
||||
break;
|
||||
@@ -2914,7 +2878,7 @@ auto ShowTransactions(const std::unordered_set<Interpreter *> &interpreters, con
|
||||
metadata_tv.emplace(md.first, TypedValue(md.second));
|
||||
}
|
||||
}
|
||||
results.back().push_back(TypedValue(metadata_tv));
|
||||
results.back().emplace_back(metadata_tv);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
@@ -3116,14 +3080,18 @@ PreparedQuery PrepareSystemInfoQuery(ParsedQuery parsed_query, bool in_explicit_
|
||||
handler = [storage = current_db.db_acc_->get()->storage(), interpreter_isolation_level,
|
||||
next_transaction_isolation_level] {
|
||||
auto info = storage->GetBaseInfo();
|
||||
const auto vm_max_map_count = utils::GetVmMaxMapCount();
|
||||
const int64_t vm_max_map_count_storage_info =
|
||||
vm_max_map_count.has_value() ? vm_max_map_count.value() : memgraph::utils::VM_MAX_MAP_COUNT_DEFAULT;
|
||||
std::vector<std::vector<TypedValue>> results{
|
||||
{TypedValue("name"), TypedValue(storage->id())},
|
||||
{TypedValue("vertex_count"), TypedValue(static_cast<int64_t>(info.vertex_count))},
|
||||
{TypedValue("edge_count"), TypedValue(static_cast<int64_t>(info.edge_count))},
|
||||
{TypedValue("average_degree"), TypedValue(info.average_degree)},
|
||||
{TypedValue("memory_usage"), TypedValue(utils::GetReadableSize(static_cast<double>(info.memory_usage)))},
|
||||
{TypedValue("vm_max_map_count"), TypedValue(vm_max_map_count_storage_info)},
|
||||
{TypedValue("memory_res"), TypedValue(utils::GetReadableSize(static_cast<double>(info.memory_res)))},
|
||||
{TypedValue("disk_usage"), TypedValue(utils::GetReadableSize(static_cast<double>(info.disk_usage)))},
|
||||
{TypedValue("memory_allocated"),
|
||||
{TypedValue("memory_tracked"),
|
||||
TypedValue(utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.Amount())))},
|
||||
{TypedValue("allocation_limit"),
|
||||
TypedValue(utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.HardLimit())))},
|
||||
@@ -3384,15 +3352,17 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
|
||||
}
|
||||
// TODO: Remove once replicas support multi-tenant replication
|
||||
if (!current_db.db_acc_) throw DatabaseContextRequiredException("Multi database queries require a defined database.");
|
||||
if (IsReplica(current_db.db_acc_->get()->storage())) {
|
||||
throw QueryException("Query forbidden on the replica!");
|
||||
}
|
||||
|
||||
auto *query = utils::Downcast<MultiDatabaseQuery>(parsed_query.query);
|
||||
auto *db_handler = interpreter_context->db_handler;
|
||||
auto *db_handler = interpreter_context->dbms_handler;
|
||||
|
||||
const bool is_replica = interpreter_context->repl_state->IsReplica();
|
||||
|
||||
switch (query->action_) {
|
||||
case MultiDatabaseQuery::Action::CREATE:
|
||||
if (is_replica) {
|
||||
throw QueryException("Query forbidden on the replica!");
|
||||
}
|
||||
return PreparedQuery{
|
||||
{"STATUS"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
@@ -3435,9 +3405,12 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
|
||||
if (current_db.in_explicit_db_) {
|
||||
throw QueryException("Database switching is prohibited if session explicitly defines the used database");
|
||||
}
|
||||
if (!dbms::allow_mt_repl && is_replica) {
|
||||
throw QueryException("Query forbidden on the replica!");
|
||||
}
|
||||
return PreparedQuery{{"STATUS"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[db_name = query->db_name_, db_handler, ¤t_db, on_change_cb](
|
||||
[db_name = query->db_name_, db_handler, ¤t_db, on_change = std::move(on_change_cb)](
|
||||
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
std::vector<std::vector<TypedValue>> status;
|
||||
std::string res;
|
||||
@@ -3447,7 +3420,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
|
||||
res = "Already using " + db_name;
|
||||
} else {
|
||||
auto tmp = db_handler->Get(db_name);
|
||||
if (on_change_cb) (*on_change_cb)(db_name); // Will trow if cb fails
|
||||
if (on_change) (*on_change)(db_name); // Will trow if cb fails
|
||||
current_db.SetCurrentDB(std::move(tmp), false);
|
||||
res = "Using " + db_name;
|
||||
}
|
||||
@@ -3466,6 +3439,9 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
|
||||
query->db_name_};
|
||||
|
||||
case MultiDatabaseQuery::Action::DROP:
|
||||
if (is_replica) {
|
||||
throw QueryException("Query forbidden on the replica!");
|
||||
}
|
||||
return PreparedQuery{
|
||||
{"STATUS"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
@@ -3473,11 +3449,9 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
|
||||
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
std::vector<std::vector<TypedValue>> status;
|
||||
|
||||
memgraph::dbms::DeleteResult success{};
|
||||
|
||||
try {
|
||||
// Remove database
|
||||
success = db_handler->Delete(db_name);
|
||||
auto success = db_handler->Delete(db_name);
|
||||
if (!success.HasError()) {
|
||||
// Remove from auth
|
||||
auth->DeleteDatabase(db_name);
|
||||
@@ -3526,14 +3500,9 @@ PreparedQuery PrepareShowDatabasesQuery(ParsedQuery parsed_query, CurrentDB &cur
|
||||
if (!license::global_license_checker.IsEnterpriseValidFast()) {
|
||||
throw QueryException("Trying to use enterprise feature without a valid license.");
|
||||
}
|
||||
// TODO: Remove once replicas support multi-tenant replication
|
||||
auto &replState = storage->repl_state_;
|
||||
if (replState.IsReplica()) {
|
||||
throw QueryException("SHOW DATABASES forbidden on the replica!");
|
||||
}
|
||||
|
||||
// TODO pick directly from ic
|
||||
auto *db_handler = interpreter_context->db_handler;
|
||||
auto *db_handler = interpreter_context->dbms_handler;
|
||||
AuthQueryHandler *auth = interpreter_context->auth;
|
||||
|
||||
Callback callback;
|
||||
@@ -3637,7 +3606,7 @@ void Interpreter::RollbackTransaction() {
|
||||
void Interpreter::SetCurrentDB(std::string_view db_name, bool in_explicit_db) {
|
||||
// Can throw
|
||||
// do we lock here?
|
||||
current_db_.SetCurrentDB(interpreter_context_->db_handler->Get(db_name), in_explicit_db);
|
||||
current_db_.SetCurrentDB(interpreter_context_->dbms_handler->Get(db_name), in_explicit_db);
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -3714,8 +3683,6 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
// Setup QueryExecution
|
||||
// its MemoryResource is mostly used for allocations done on Frame and storing `row`s
|
||||
if (usePool) {
|
||||
spdlog::trace("PrepareCypher has {} encountered all shortest paths, QueryExecution will use PoolResource",
|
||||
hasAllShortestPaths ? "" : "not");
|
||||
query_executions_.emplace_back(QueryExecution::Create(utils::PoolResource(128, kExecutionPoolMaxBlockSize)));
|
||||
} else {
|
||||
query_executions_.emplace_back(QueryExecution::Create(utils::MonotonicBufferResource(kExecutionMemoryBlockSize)));
|
||||
@@ -3746,7 +3713,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
// TODO: ATM only a single database, will change when we have multiple database transactions
|
||||
bool could_commit = utils::Downcast<CypherQuery>(parsed_query.query) != nullptr;
|
||||
bool unique = utils::Downcast<IndexQuery>(parsed_query.query) != nullptr ||
|
||||
utils::Downcast<ConstraintQuery>(parsed_query.query) != nullptr;
|
||||
utils::Downcast<ConstraintQuery>(parsed_query.query) != nullptr ||
|
||||
upper_case_query.find(kSchemaAssert) != std::string::npos;
|
||||
SetupDatabaseTransaction(could_commit, unique);
|
||||
}
|
||||
|
||||
@@ -3761,7 +3729,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
std::visit([](auto &execution_memory) -> utils::MemoryResource * { return &execution_memory; },
|
||||
query_execution->execution_memory);
|
||||
frame_change_collector_.reset();
|
||||
frame_change_collector_.emplace(memory_resource);
|
||||
frame_change_collector_.emplace();
|
||||
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
current_db_, memory_resource, &query_execution->notifications, username_,
|
||||
@@ -3796,7 +3764,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
/// TODO: make replication DB agnostic
|
||||
prepared_query =
|
||||
PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
|
||||
current_db_, interpreter_context_->config);
|
||||
*interpreter_context_->dbms_handler, interpreter_context_->config);
|
||||
} else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, current_db_);
|
||||
} else if (utils::Downcast<FreeMemoryQuery>(parsed_query.query)) {
|
||||
@@ -3859,7 +3827,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
|
||||
UpdateTypeCount(rw_type);
|
||||
|
||||
if (IsWriteQueryOnMainMemoryReplica(current_db_.db_acc_->get()->storage(), rw_type)) {
|
||||
if (interpreter_context_->repl_state->IsReplica() && IsQueryWrite(rw_type)) {
|
||||
query_execution = nullptr;
|
||||
throw QueryException("Write query forbidden on the replica!");
|
||||
}
|
||||
@@ -4088,7 +4056,8 @@ void Interpreter::Commit() {
|
||||
|
||||
auto commit_confirmed_by_all_sync_repplicas = true;
|
||||
|
||||
auto maybe_commit_error = current_db_.db_transactional_accessor_->Commit();
|
||||
auto maybe_commit_error =
|
||||
current_db_.db_transactional_accessor_->Commit(std::nullopt, interpreter_context_->repl_state->IsMain());
|
||||
if (maybe_commit_error.HasError()) {
|
||||
const auto &error = maybe_commit_error.GetError();
|
||||
|
||||
|
||||
@@ -174,7 +174,7 @@ struct CurrentDB {
|
||||
|
||||
class Interpreter final {
|
||||
public:
|
||||
Interpreter(InterpreterContext *interpreter_context);
|
||||
explicit Interpreter(InterpreterContext *interpreter_context);
|
||||
Interpreter(InterpreterContext *interpreter_context, memgraph::dbms::DatabaseAccess db);
|
||||
Interpreter(const Interpreter &) = delete;
|
||||
Interpreter &operator=(const Interpreter &) = delete;
|
||||
|
||||
@@ -13,6 +13,12 @@
|
||||
|
||||
#include "query/interpreter.hpp"
|
||||
namespace memgraph::query {
|
||||
|
||||
InterpreterContext::InterpreterContext(InterpreterConfig interpreter_config, dbms::DbmsHandler *dbms_handler,
|
||||
replication::ReplicationState *rs, query::AuthQueryHandler *ah,
|
||||
query::AuthChecker *ac)
|
||||
: dbms_handler(dbms_handler), config(interpreter_config), repl_state(rs), auth(ah), auth_checker(ac) {}
|
||||
|
||||
std::vector<std::vector<TypedValue>> InterpreterContext::TerminateTransactions(
|
||||
std::vector<std::string> maybe_kill_transaction_ids, const std::optional<std::string> &username,
|
||||
std::function<bool(std::string const &)> privilege_checker) {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user