Compare commits

..

51 Commits

Author SHA1 Message Date
Andi Skrgat
c5af23ee23 Add clearing of property lookup cache 2023-12-01 09:03:33 +01:00
hal-eisen-MG
7fc9b89634 Merge pull request #1530 from memgraph/1529-configure-sonarcloud-for-automatic-analysis
First draft of a sonarcloud properties file
2023-11-28 17:03:06 -08:00
hal-eisen-MG
ed5bdb841f Merge branch 'master' into 1529-configure-sonarcloud-for-automatic-analysis 2023-11-28 15:23:11 -08:00
Antonio Filipovic
bb2a7b8f21 Fix frame change collector incomplete pmr type (#1491) 2023-11-28 10:00:34 +01:00
hal-eisen-MG
6680f99e4d Merge branch 'master' into 1529-configure-sonarcloud-for-automatic-analysis 2023-11-27 10:00:09 -08:00
Antonio Filipovic
72d47fc3bf Implement short circuiting of exists evaluation (#1539) 2023-11-27 16:44:12 +01:00
hal-eisen-MG
3cf13e2deb Merge branch 'master' into 1529-configure-sonarcloud-for-automatic-analysis 2023-11-27 05:40:39 -08:00
Andi
7f5a55f1b2 Fix restarts when using init-file flag (#1465) 2023-11-24 13:11:47 +01:00
Marko Barišić
70db2fca56 Change package_all to package_memgraph (#1507)
Add the ability to pick a specific package to build
2023-11-23 12:46:04 +00:00
andrejtonev
8b9e1fa08b Replication refactor part 6 (#1484)
Single (instance level) connection to a replica (messages from all databases get multiplexed through it)
ReplicationClient split in two: ReplicationClient and ReplicationStorageClient
New ReplicationClient, moved under replication, handles the raw connection, owned by MainRoleData
ReplicationStorageClient handles the storage <-> replica state machine and holds to a stream
Removed epoch and storage from *Clients
rpc::Stream proactively aborts on error and sets itself to a defunct state
Removed HandleRpcFailure, instead we simply log the error and let the FrequentCheck handle re-connection
replica_state is now a synced variable
ReplicaStorageClient state machine bugfixes
Single FrequentCheck that goes through DBMS
Moved ReplicationState under DbmsHandler
Moved some replication startup logic under the DbmsHandler's constructor
Removed InMemoryReplicationClient
CreateReplicationClient has been removed from Storage
Simplified GetRecoverySteps and made safer

---------

Co-authored-by: Gareth Lloyd <gareth.lloyd@memgraph.io>
2023-11-23 11:02:35 +01:00
Gareth Andrew Lloyd
e4f94c15c6 Fixes for clang-tidy / sonar issues (#1536) 2023-11-22 13:05:02 +00:00
hal-eisen-MG
0ce6dcf194 Merge branch 'master' into 1529-configure-sonarcloud-for-automatic-analysis 2023-11-21 07:31:17 -08:00
Andi
1d90b60f56 Add schema.assert (#1485) 2023-11-21 09:19:50 +01:00
hal-eisen-MG
108d83cc8c Merge branch 'master' into 1529-configure-sonarcloud-for-automatic-analysis 2023-11-20 09:33:03 -08:00
Hal Eisen
eff857447a Refine scope to pull in 'include' and 'query_module' directories, in addition to src 2023-11-20 08:16:35 -08:00
Andi
d03fafcef6 Aggregations return empty result when used with group by (#1531) 2023-11-20 11:52:17 +01:00
Hal Eisen
c31a7f9648 First draft of a sonarcloud properties file 2023-11-17 17:25:11 -08:00
imilinovic
6053a91ef8 Fix flaky GC test (#1521) 2023-11-17 17:06:46 -05:00
Antonio Filipovic
645568a75b Remove default memory limit on procedures (#1506)
* remove default limit on procedures
* fix bug on GraphQL also
2023-11-16 15:01:44 +01:00
Antonio Filipovic
d3f4c35362 Add OOM enabler for MG procedure (#1401) 2023-11-15 12:42:04 +01:00
Josipmrden
c037cddb0e Add granular index and constraint recovery info (#1480) 2023-11-14 17:23:06 -05:00
imilinovic
ced08fd7bc Fix GC by adding periodic jemalloc purge (#1471) 2023-11-14 15:06:21 -05:00
Marko Barišić
9cc060c4b0 Fix error in upload-to-s3 job (#1504) 2023-11-13 13:01:01 +01:00
Marko Barišić
e671a0737e Fix package specific workflow file (#1503) 2023-11-13 12:54:19 +01:00
Marko Barišić
11be3972c4 Add workflow for packaging memgraph for specific target OS (#1502) 2023-11-13 12:20:49 +01:00
DavIvek
fdab42a023 Use static linking on c++ query modules for glibcxx (#1490) 2023-11-13 12:08:48 +01:00
Andi
e5b2c19ea2 Empty Collect() returns nothing (#1482) 2023-11-13 11:45:09 +01:00
Josipmrden
e907817854 Fix for in list segmentation fault (#1494) 2023-11-13 05:17:10 +01:00
Josipmrden
0756cd6898 Add fix indexed join crash (#1478) 2023-11-12 22:12:25 -05:00
Josipmrden
38ad5e2146 Fix parallel index loading (#1479) 2023-11-12 23:51:00 +01:00
Josipmrden
3c413a7e50 Fix hash join expression matching (#1496) 2023-11-12 14:45:02 -05:00
Antonio Filipovic
17915578f8 Fix race condition and arena tracking bug (#1468) 2023-11-09 18:56:36 +01:00
Marko Barišić
4e9a036881 Fix v2.12 release pipeline (#1445) 2023-11-08 07:09:02 -05:00
DavIvek
c8fe9ee7d9 Fix accessing a variable bound to a list within BFS function (#1380) 2023-11-07 20:34:50 +01:00
Ante Javor
e4afddf518 Fix compare results in mgbench (#1319) 2023-11-07 17:04:37 +01:00
Antonio Filipovic
4d5ea03dfa Use extent hooks for memory procedure limit (#1443) 2023-11-07 16:04:29 +01:00
DavIvek
ece4b0dba8 Fix cached plan not getting invalidated (#1348) 2023-11-07 13:34:03 +01:00
Andi
66487a6dce Durability improvements (#1385) 2023-11-07 11:37:54 +01:00
Andi
f4b97fc03d Fix missing statistics for SetPropertiesCursor (#1460) 2023-11-07 09:11:20 +01:00
Antonio Filipovic
58648d1a70 Revert license sender info (#1461) 2023-11-06 16:57:09 +01:00
Antonio Filipovic
1ab7f6ac78 Add notification for user on max map count (#1408) 2023-11-06 15:44:26 +01:00
andrejtonev
dbc6054689 Replication refactor (part 5) (#1378) 2023-11-06 11:50:49 +00:00
Aidar Samerkhanov
16b8c7b27c Fix Kafka flaky unit test (#1409) 2023-11-05 20:51:56 +01:00
Antonio Filipovic
48631d1e37 Rename memory usage and memory allocated (#1426) 2023-11-03 14:40:45 +01:00
Antonio Filipovic
93e6d058d2 Remove all_shortest paths unnecessary logs (#1425) 2023-11-03 12:46:06 +01:00
Andi
3e9f25b8e4 Support creating date and localtime from localdatetime (#1381) 2023-11-03 10:54:01 +01:00
Andi
c94201621a Support deleting paths (#1383) 2023-11-02 14:07:48 +01:00
Andi
fdbc390d53 Throw when reduce inside exists (#1392) 2023-11-02 12:18:15 +01:00
Andi
5e6c5618f5 Make FrameChangeCollector owning the memory resource (#1398) 2023-11-02 09:54:39 +01:00
Andi
4aacd45640 Throw when exists() combined with CASE (#1382) 2023-11-02 08:25:34 +01:00
Gareth Andrew Lloyd
157b36162b Speedup socket unit test (#1444)
Was testing a setup that wasn't used in production, it would
unnecessarily thrash small buffers.
2023-11-01 17:24:24 +00:00
319 changed files with 9284 additions and 4667 deletions

View File

@@ -1,4 +1,4 @@
name: Package All
name: Package memgraph
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
@@ -6,18 +6,56 @@ on:
workflow_dispatch:
inputs:
memgraph_version:
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'X.Y.Z'"
description: "Memgraph version to upload as. Leave this field empty if you don't want to upload binaries to S3. Format: 'X.Y.Z'"
required: false
build_type:
type: choice
description: "Memgraph Build type. Default value is Release."
description: "Memgraph Build type. Default value is Release"
default: 'Release'
options:
- Release
- RelWithDebInfo
target_os:
type: choice
description: "Target OS for which memgraph will be packaged. Select 'all' if you want to package for every listed OS. Default is Ubuntu 22.04"
default: 'ubuntu-22_04'
options:
- all
- amzn-2
- centos-7
- centos-9
- debian-10
- debian-11
- debian-11-arm
- debian-11-platform
- docker
- fedora-36
- ubuntu-18_04
- ubuntu-20_04
- ubuntu-22_04
- ubuntu-22_04-arm
jobs:
amzn-2:
if: ${{ github.event.inputs.target_os == 'amzn-2' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: amzn-2
path: build/output/amzn-2/memgraph*.rpm
centos-7:
if: ${{ github.event.inputs.target_os == 'centos-7' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -35,6 +73,7 @@ jobs:
path: build/output/centos-7/memgraph*.rpm
centos-9:
if: ${{ github.event.inputs.target_os == 'centos-9' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -52,6 +91,7 @@ jobs:
path: build/output/centos-9/memgraph*.rpm
debian-10:
if: ${{ github.event.inputs.target_os == 'debian-10' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -69,6 +109,7 @@ jobs:
path: build/output/debian-10/memgraph*.deb
debian-11:
if: ${{ github.event.inputs.target_os == 'debian-11' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -85,7 +126,44 @@ jobs:
name: debian-11
path: build/output/debian-11/memgraph*.deb
debian-11-arm:
if: ${{ github.event.inputs.target_os == 'debian-11-arm' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 120
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11-aarch64
path: build/output/debian-11-arm/memgraph*.deb
debian-11-platform:
if: ${{ github.event.inputs.target_os == 'debian-11-platform' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
docker:
if: ${{ github.event.inputs.target_os == 'docker' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -104,75 +182,8 @@ jobs:
name: docker
path: build/output/docker/memgraph*.tar.gz
ubuntu-1804:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-18.04
path: build/output/ubuntu-18.04/memgraph*.deb
ubuntu-2004:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-20.04
path: build/output/ubuntu-20.04/memgraph*.deb
ubuntu-2204:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04
path: build/output/ubuntu-22.04/memgraph*.deb
debian-11-platform:
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11 ${{ github.event.inputs.build_type }} --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
fedora-36:
if: ${{ github.event.inputs.target_os == 'fedora-36' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -189,7 +200,8 @@ jobs:
name: fedora-36
path: build/output/fedora-36/memgraph*.rpm
amzn-2:
ubuntu-18_04:
if: ${{ github.event.inputs.target_os == 'ubuntu-18_04' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
@@ -199,16 +211,17 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package amzn-2 ${{ github.event.inputs.build_type }}
./release/package/run.sh package ubuntu-18.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: amzn-2
path: build/output/amzn-2/memgraph*.rpm
name: ubuntu-18.04
path: build/output/ubuntu-18.04/memgraph*.deb
debian-11-arm:
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 120
ubuntu-20_04:
if: ${{ github.event.inputs.target_os == 'ubuntu-20_04' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
@@ -216,14 +229,33 @@ jobs:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package debian-11-arm ${{ github.event.inputs.build_type }}
./release/package/run.sh package ubuntu-20.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: debian-11-aarch64
path: build/output/debian-11-arm/memgraph*.deb
name: ubuntu-20.04
path: build/output/ubuntu-20.04/memgraph*.deb
ubuntu-2204-arm:
ubuntu-22_04:
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, X64]
timeout-minutes: 60
steps:
- name: "Set up repository"
uses: actions/checkout@v3
with:
fetch-depth: 0 # Required because of release/get_version.py
- name: "Build package"
run: |
./release/package/run.sh package ubuntu-22.04 ${{ github.event.inputs.build_type }}
- name: "Upload package"
uses: actions/upload-artifact@v3
with:
name: ubuntu-22.04
path: build/output/ubuntu-22.04/memgraph*.deb
ubuntu-22_04-arm:
if: ${{ github.event.inputs.target_os == 'ubuntu-22_04-arm' || github.event.inputs.target_os == 'all' }}
runs-on: [self-hosted, DockerMgBuild, ARM64, strange]
timeout-minutes: 120
steps:
@@ -243,7 +275,7 @@ jobs:
upload-to-s3:
# only run upload if we specified version. Allows for runs without upload
if: "${{ github.event.inputs.memgraph_version != '' }}"
needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm]
needs: [amzn-2, centos-7, centos-9, debian-10, debian-11, debian-11-arm, debian-11-platform, docker, fedora-36, ubuntu-18_04, ubuntu-20_04, ubuntu-22_04, ubuntu-22_04-arm]
runs-on: ubuntu-latest
steps:
- name: Download artifacts

22
.sonarcloud.properties Normal file
View File

@@ -0,0 +1,22 @@
# Path to sources
sonar.sources = .
# sonar.exclusions=
sonar.inclusions=src,include,query_modules
# Path to tests
sonar.tests = tests/
# sonar.test.exclusions=
# sonar.test.inclusions=
# Source encoding
# sonar.sourceEncoding=
# Exclusions for copy-paste detection
# sonar.cpd.exclusions=
# Python version (for python projects only)
# sonar.python.version=
# C++ standard version (for C++ projects only)
# If not specified, it defaults to the latest supported standard
# sonar.cfamily.reportingCppStandardOverride=c++98|c++11|c++14|c++17|c++20

View File

@@ -42,10 +42,6 @@ endif()
project(memgraph LANGUAGES C CXX)
# NOTE: once in a while this needs to be toggled to check headers are
# correct and PCH isn't masking any include issues
set(CMAKE_DISABLE_PRECOMPILE_HEADERS OFF)
#TODO: upgrade to cmake 3.24 + CheckIPOSupported
#cmake_policy(SET CMP0138 NEW)
#include(CheckIPOSupported)
@@ -296,7 +292,7 @@ if (MG_ENTERPRISE)
add_definitions(-DMG_ENTERPRISE)
endif()
set(ENABLE_JEMALLOC ON)
option(ENABLE_JEMALLOC "Use jemalloc" ON)
if (ASAN)
message(WARNING "Disabling jemalloc as it doesn't work well with ASAN")

View File

@@ -236,6 +236,54 @@ inline mgp_type *type_nullable(mgp_type *type) { return MgInvoke<mgp_type *>(mgp
// mgp_graph
inline bool create_label_index(mgp_graph *graph, const char *label) {
return MgInvoke<int>(mgp_create_label_index, graph, label);
}
inline bool drop_label_index(mgp_graph *graph, const char *label) {
return MgInvoke<int>(mgp_drop_label_index, graph, label);
}
inline mgp_list *list_all_label_indices(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_all_label_indices, graph, memory);
}
inline bool create_label_property_index(mgp_graph *graph, const char *label, const char *property) {
return MgInvoke<int>(mgp_create_label_property_index, graph, label, property);
}
inline bool drop_label_property_index(mgp_graph *graph, const char *label, const char *property) {
return MgInvoke<int>(mgp_drop_label_property_index, graph, label, property);
}
inline mgp_list *list_all_label_property_indices(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_all_label_property_indices, graph, memory);
}
inline bool create_existence_constraint(mgp_graph *graph, const char *label, const char *property) {
return MgInvoke<int>(mgp_create_existence_constraint, graph, label, property);
}
inline bool drop_existence_constraint(mgp_graph *graph, const char *label, const char *property) {
return MgInvoke<int>(mgp_drop_existence_constraint, graph, label, property);
}
inline mgp_list *list_all_existence_constraints(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_all_existence_constraints, graph, memory);
}
inline bool create_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) {
return MgInvoke<int>(mgp_create_unique_constraint, memgraph_graph, label, properties);
}
inline bool drop_unique_constraint(mgp_graph *memgraph_graph, const char *label, mgp_value *properties) {
return MgInvoke<int>(mgp_drop_unique_constraint, memgraph_graph, label, properties);
}
inline mgp_list *list_all_unique_constraints(mgp_graph *graph, mgp_memory *memory) {
return MgInvoke<mgp_list *>(mgp_list_all_unique_constraints, graph, memory);
}
inline bool graph_is_mutable(mgp_graph *graph) { return MgInvoke<int>(mgp_graph_is_mutable, graph); }
inline mgp_vertex *graph_create_vertex(mgp_graph *graph, mgp_memory *memory) {

View File

@@ -876,6 +876,65 @@ enum mgp_error mgp_edge_iter_properties(struct mgp_edge *e, struct mgp_memory *m
enum mgp_error mgp_graph_get_vertex_by_id(struct mgp_graph *g, struct mgp_vertex_id id, struct mgp_memory *memory,
struct mgp_vertex **result);
/// Creates label index for given label.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if label index already exists, result will be 0, otherwise 1.
enum mgp_error mgp_create_label_index(struct mgp_graph *graph, const char *label, int *result);
/// Drop label index.
enum mgp_error mgp_drop_label_index(struct mgp_graph *graph, const char *label, int *result);
/// List all label indices.
enum mgp_error mgp_list_all_label_indices(struct mgp_graph *graph, struct mgp_memory *memory, struct mgp_list **result);
/// Creates label-property index for given label and propery.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if label property index already exists, result will be 0, otherwise 1.
enum mgp_error mgp_create_label_property_index(struct mgp_graph *graph, const char *label, const char *property,
int *result);
/// Drops label-property index for given label and propery.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if dropping label property index failed, result will be 0, otherwise 1.
enum mgp_error mgp_drop_label_property_index(struct mgp_graph *graph, const char *label, const char *property,
int *result);
/// List all label+property indices.
enum mgp_error mgp_list_all_label_property_indices(struct mgp_graph *graph, struct mgp_memory *memory,
struct mgp_list **result);
/// Creates existence constraint for given label and property.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if creating existence constraint failed, result will be 0, otherwise 1.
enum mgp_error mgp_create_existence_constraint(struct mgp_graph *graph, const char *label, const char *property,
int *result);
/// Drops existence constraint for given label and property.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if dropping existence constraint failed, result will be 0, otherwise 1.
enum mgp_error mgp_drop_existence_constraint(struct mgp_graph *graph, const char *label, const char *property,
int *result);
/// List all existence constraints.
enum mgp_error mgp_list_all_existence_constraints(struct mgp_graph *graph, struct mgp_memory *memory,
struct mgp_list **result);
/// Creates unique constraint for given label and properties.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if creating unique constraint failed, result will be 0, otherwise 1.
enum mgp_error mgp_create_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties,
int *result);
/// Drops unique constraint for given label and properties.
/// mgp_error::MGP_ERROR_NO_ERROR is always returned.
/// if dropping unique constraint failed, result will be 0, otherwise 1.
enum mgp_error mgp_drop_unique_constraint(struct mgp_graph *graph, const char *label, struct mgp_value *properties,
int *result);
/// List all unique constraints
enum mgp_error mgp_list_all_unique_constraints(struct mgp_graph *graph, struct mgp_memory *memory,
struct mgp_list **result);
/// Result is non-zero if the graph can be modified.
/// If a graph is immutable, then vertices cannot be created or deleted, and all of the returned vertices will be
/// immutable also. The same applies for edges.

File diff suppressed because it is too large Load Diff

View File

@@ -13,6 +13,7 @@ string(TOLOWER ${CMAKE_BUILD_TYPE} lower_build_type)
add_library(example_c SHARED example.c)
target_include_directories(example_c PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_c PRIVATE -Wall)
target_link_libraries(example_c PRIVATE -static-libgcc -static-libstdc++)
# Strip C example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_c POST_BUILD
@@ -28,6 +29,7 @@ install(FILES example.c DESTINATION lib/memgraph/query_modules/src)
add_library(example_cpp SHARED example.cpp)
target_include_directories(example_cpp PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(example_cpp PRIVATE -Wall)
target_link_libraries(example_cpp PRIVATE -static-libgcc -static-libstdc++)
# Strip C++ example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET example_cpp POST_BUILD
@@ -43,6 +45,7 @@ install(FILES example.cpp DESTINATION lib/memgraph/query_modules/src)
add_library(schema SHARED schema.cpp)
target_include_directories(schema PRIVATE ${CMAKE_SOURCE_DIR}/include)
target_compile_options(schema PRIVATE -Wall)
target_link_libraries(schema PRIVATE -static-libgcc -static-libstdc++)
# Strip C++ example in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET schema POST_BUILD

View File

@@ -10,18 +10,33 @@
// licenses/APL.txt.
#include <mgp.hpp>
#include "utils/string.hpp"
#include <optional>
namespace Schema {
/*NodeTypeProperties and RelTypeProperties constants*/
constexpr std::string_view kStatusKept = "Kept";
constexpr std::string_view kStatusCreated = "Created";
constexpr std::string_view kStatusDropped = "Dropped";
constexpr std::string_view kReturnNodeType = "nodeType";
constexpr std::string_view kProcedureNodeType = "node_type_properties";
constexpr std::string_view kProcedureRelType = "rel_type_properties";
constexpr std::string_view kProcedureAssert = "assert";
constexpr std::string_view kReturnLabels = "nodeLabels";
constexpr std::string_view kReturnRelType = "relType";
constexpr std::string_view kReturnPropertyName = "propertyName";
constexpr std::string_view kReturnPropertyType = "propertyTypes";
constexpr std::string_view kReturnMandatory = "mandatory";
constexpr std::string_view kReturnLabel = "label";
constexpr std::string_view kReturnKey = "key";
constexpr std::string_view kReturnKeys = "keys";
constexpr std::string_view kReturnUnique = "unique";
constexpr std::string_view kReturnAction = "action";
constexpr std::string_view kParameterIndices = "indices";
constexpr std::string_view kParameterUniqueConstraints = "unique_constraints";
constexpr std::string_view kParameterExistenceConstraints = "existence_constraints";
constexpr std::string_view kParameterDropExisting = "drop_existing";
std::string TypeOf(const mgp::Type &type);
@@ -35,6 +50,7 @@ void ProcessPropertiesRel(mgp::Record &record, const std::string_view &type, con
void NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
void RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
void Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory);
} // namespace Schema
/*we have << operator for type in Cpp API, but in it we return somewhat different strings than I would like in this
@@ -92,21 +108,21 @@ void Schema::ProcessPropertiesRel(mgp::Record &record, const std::string_view &t
record.Insert(std::string(kReturnMandatory).c_str(), mandatory);
}
void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
void Schema::NodeTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result,
mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
;
const auto record_factory = mgp::RecordFactory(result);
try {
const mgp::Graph graph = mgp::Graph(memgraph_graph);
for (auto node : graph.Nodes()) {
std::string type = "";
std::string type;
mgp::List labels = mgp::List();
for (auto label : node.Labels()) {
labels.AppendExtend(mgp::Value(label));
type += ":`" + std::string(label) + "`";
}
if (node.Properties().size() == 0) {
if (node.Properties().empty()) {
auto record = record_factory.NewRecord();
ProcessPropertiesNode<std::string>(record, type, labels, "", "", false);
continue;
@@ -126,16 +142,15 @@ void Schema::NodeTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_r
}
}
void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
void Schema::RelTypeProperties(mgp_list * /*args*/, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
;
const auto record_factory = mgp::RecordFactory(result);
try {
const mgp::Graph graph = mgp::Graph(memgraph_graph);
for (auto rel : graph.Relationships()) {
std::string type = ":`" + std::string(rel.Type()) + "`";
if (rel.Properties().size() == 0) {
if (rel.Properties().empty()) {
auto record = record_factory.NewRecord();
ProcessPropertiesRel<std::string>(record, type, "", "", false);
continue;
@@ -155,29 +170,435 @@ void Schema::RelTypeProperties(mgp_list *args, mgp_graph *memgraph_graph, mgp_re
}
}
void InsertRecordForLabelIndex(const auto &record_factory, const std::string_view label,
const std::string_view status) {
auto record = record_factory.NewRecord();
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
record.Insert(std::string(Schema::kReturnKey).c_str(), "");
record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List());
record.Insert(std::string(Schema::kReturnUnique).c_str(), false);
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
}
void InsertRecordForUniqueConstraint(const auto &record_factory, const std::string_view label,
const mgp::List &properties, const std::string_view status) {
auto record = record_factory.NewRecord();
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
record.Insert(std::string(Schema::kReturnKey).c_str(), properties.ToString());
record.Insert(std::string(Schema::kReturnKeys).c_str(), properties);
record.Insert(std::string(Schema::kReturnUnique).c_str(), true);
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
}
void InsertRecordForLabelPropertyIndexAndExistenceConstraint(const auto &record_factory, const std::string_view label,
const std::string_view property,
const std::string_view status) {
auto record = record_factory.NewRecord();
record.Insert(std::string(Schema::kReturnLabel).c_str(), label);
record.Insert(std::string(Schema::kReturnKey).c_str(), property);
record.Insert(std::string(Schema::kReturnKeys).c_str(), mgp::List({mgp::Value(property)}));
record.Insert(std::string(Schema::kReturnUnique).c_str(), false);
record.Insert(std::string(Schema::kReturnAction).c_str(), status);
}
void ProcessCreatingLabelIndex(const std::string_view label, const std::set<std::string_view> &existing_label_indices,
mgp_graph *memgraph_graph, const auto &record_factory) {
if (existing_label_indices.contains(label)) {
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusKept);
} else if (mgp::CreateLabelIndex(memgraph_graph, label)) {
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated);
}
}
template <typename TFunc>
void ProcessCreatingLabelPropertyIndexAndExistenceConstraint(const std::string_view label,
const std::string_view property,
const std::set<std::string_view> &existing_collection,
const TFunc &func_creation, mgp_graph *memgraph_graph,
const auto &record_factory) {
const auto label_property_search_key = std::string(label) + ":" + std::string(property);
if (existing_collection.contains(label_property_search_key)) {
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusKept);
} else if (func_creation(memgraph_graph, label, property)) {
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusCreated);
}
}
/// We collect properties for which index was created.
using AssertedIndices = std::set<std::string, std::less<>>;
AssertedIndices CreateIndicesForLabel(const std::string_view label, const mgp::Value &properties_val,
mgp_graph *memgraph_graph, const auto &record_factory,
const std::set<std::string_view> &existing_label_indices,
const std::set<std::string_view> &existing_label_property_indices) {
AssertedIndices asserted_indices;
if (!properties_val.IsList()) {
return {};
}
if (const auto properties = properties_val.ValueList();
properties.Empty() && mgp::CreateLabelIndex(memgraph_graph, label)) {
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusCreated);
asserted_indices.emplace("");
} else {
std::for_each(properties.begin(), properties.end(),
[&label, &existing_label_indices, &existing_label_property_indices, &memgraph_graph, &record_factory,
&asserted_indices](const mgp::Value &property) {
if (!property.IsString()) {
return;
}
const auto property_str = property.ValueString();
if (property_str.empty()) {
ProcessCreatingLabelIndex(label, existing_label_indices, memgraph_graph, record_factory);
asserted_indices.emplace("");
} else {
ProcessCreatingLabelPropertyIndexAndExistenceConstraint(
label, property_str, existing_label_property_indices, mgp::CreateLabelPropertyIndex,
memgraph_graph, record_factory);
asserted_indices.emplace(property_str);
}
});
}
return asserted_indices;
}
void ProcessIndices(const mgp::Map &indices_map, mgp_graph *memgraph_graph, const auto &record_factory,
bool drop_existing) {
auto mgp_existing_label_indices = mgp::ListAllLabelIndices(memgraph_graph);
auto mgp_existing_label_property_indices = mgp::ListAllLabelPropertyIndices(memgraph_graph);
std::set<std::string_view> existing_label_indices;
std::transform(mgp_existing_label_indices.begin(), mgp_existing_label_indices.end(),
std::inserter(existing_label_indices, existing_label_indices.begin()),
[](const mgp::Value &index) { return index.ValueString(); });
std::set<std::string_view> existing_label_property_indices;
std::transform(mgp_existing_label_property_indices.begin(), mgp_existing_label_property_indices.end(),
std::inserter(existing_label_property_indices, existing_label_property_indices.begin()),
[](const mgp::Value &index) { return index.ValueString(); });
std::set<std::string> asserted_label_indices;
std::set<std::string> asserted_label_property_indices;
auto merge_label_property = [](const std::string &label, const std::string &property) {
return label + ":" + property;
};
for (const auto &index : indices_map) {
const std::string_view label = index.key;
const mgp::Value &properties_val = index.value;
AssertedIndices asserted_indices_new = CreateIndicesForLabel(
label, properties_val, memgraph_graph, record_factory, existing_label_indices, existing_label_property_indices);
if (!drop_existing) {
continue;
}
std::ranges::for_each(asserted_indices_new, [&asserted_label_indices, &asserted_label_property_indices, label,
&merge_label_property](const std::string &property) {
if (property.empty()) {
asserted_label_indices.emplace(label);
} else {
asserted_label_property_indices.emplace(merge_label_property(std::string(label), property));
}
});
}
if (!drop_existing) {
return;
}
std::set<std::string_view> label_indices_to_drop;
std::ranges::set_difference(existing_label_indices, asserted_label_indices,
std::inserter(label_indices_to_drop, label_indices_to_drop.begin()));
std::ranges::for_each(label_indices_to_drop, [memgraph_graph, &record_factory](const std::string_view label) {
if (mgp::DropLabelIndex(memgraph_graph, label)) {
InsertRecordForLabelIndex(record_factory, label, Schema::kStatusDropped);
}
});
std::set<std::string_view> label_property_indices_to_drop;
std::ranges::set_difference(existing_label_property_indices, asserted_label_property_indices,
std::inserter(label_property_indices_to_drop, label_property_indices_to_drop.begin()));
auto decouple_label_property = [](const std::string_view label_property) {
const auto label_size = label_property.find(':');
const auto label = std::string(label_property.substr(0, label_size));
const auto property = std::string(label_property.substr(label_size + 1));
return std::make_pair(label, property);
};
std::ranges::for_each(label_property_indices_to_drop, [memgraph_graph, &record_factory, decouple_label_property](
const std::string_view label_property) {
const auto [label, property] = decouple_label_property(label_property);
if (mgp::DropLabelPropertyIndex(memgraph_graph, label, property)) {
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped);
}
});
}
using ExistenceConstraintsStorage = std::set<std::string_view>;
ExistenceConstraintsStorage CreateExistenceConstraintsForLabel(
const std::string_view label, const mgp::Value &properties_val, mgp_graph *memgraph_graph,
const auto &record_factory, const std::set<std::string_view> &existing_existence_constraints) {
ExistenceConstraintsStorage asserted_existence_constraints;
if (!properties_val.IsList()) {
return asserted_existence_constraints;
}
auto validate_property = [](const mgp::Value &property) -> bool {
return property.IsString() && !property.ValueString().empty();
};
const auto &properties = properties_val.ValueList();
std::for_each(properties.begin(), properties.end(),
[&label, &existing_existence_constraints, &asserted_existence_constraints, &memgraph_graph,
&record_factory, &validate_property](const mgp::Value &property) {
if (!validate_property(property)) {
return;
}
const std::string_view property_str = property.ValueString();
asserted_existence_constraints.emplace(property_str);
ProcessCreatingLabelPropertyIndexAndExistenceConstraint(
label, property_str, existing_existence_constraints, mgp::CreateExistenceConstraint,
memgraph_graph, record_factory);
});
return asserted_existence_constraints;
}
void ProcessExistenceConstraints(const mgp::Map &existence_constraints_map, mgp_graph *memgraph_graph,
const auto &record_factory, bool drop_existing) {
auto mgp_existing_existence_constraints = mgp::ListAllExistenceConstraints(memgraph_graph);
std::set<std::string_view> existing_existence_constraints;
std::transform(mgp_existing_existence_constraints.begin(), mgp_existing_existence_constraints.end(),
std::inserter(existing_existence_constraints, existing_existence_constraints.begin()),
[](const mgp::Value &constraint) { return constraint.ValueString(); });
auto merge_label_property = [](const std::string_view label, const std::string_view property) {
auto str = std::string(label) + ":";
str += property;
return str;
};
ExistenceConstraintsStorage asserted_existence_constraints;
for (const auto &existing_constraint : existence_constraints_map) {
const std::string_view label = existing_constraint.key;
const mgp::Value &properties_val = existing_constraint.value;
auto asserted_existence_constraints_new = CreateExistenceConstraintsForLabel(
label, properties_val, memgraph_graph, record_factory, existing_existence_constraints);
if (!drop_existing) {
continue;
}
std::ranges::for_each(asserted_existence_constraints_new, [&asserted_existence_constraints, &merge_label_property,
label](const std::string_view property) {
asserted_existence_constraints.emplace(merge_label_property(label, property));
});
}
if (!drop_existing) {
return;
}
std::set<std::string_view> existence_constraints_to_drop;
std::ranges::set_difference(existing_existence_constraints, asserted_existence_constraints,
std::inserter(existence_constraints_to_drop, existence_constraints_to_drop.begin()));
auto decouple_label_property = [](const std::string_view label_property) {
const auto label_size = label_property.find(':');
const auto label = std::string(label_property.substr(0, label_size));
const auto property = std::string(label_property.substr(label_size + 1));
return std::make_pair(label, property);
};
std::ranges::for_each(existence_constraints_to_drop, [&](const std::string_view label_property) {
const auto [label, property] = decouple_label_property(label_property);
if (mgp::DropExistenceConstraint(memgraph_graph, label, property)) {
InsertRecordForLabelPropertyIndexAndExistenceConstraint(record_factory, label, property, Schema::kStatusDropped);
}
});
}
using AssertedUniqueConstraintsStorage = std::set<std::set<std::string_view>>;
AssertedUniqueConstraintsStorage CreateUniqueConstraintsForLabel(
const std::string_view label, const mgp::Value &unique_props_nested,
const std::map<std::string_view, AssertedUniqueConstraintsStorage> &existing_unique_constraints,
mgp_graph *memgraph_graph, const auto &record_factory) {
AssertedUniqueConstraintsStorage asserted_unique_constraints;
if (!unique_props_nested.IsList()) {
return asserted_unique_constraints;
}
auto validate_unique_constraint_props = [](const mgp::Value &properties) -> bool {
if (!properties.IsList()) {
return false;
}
const auto &properties_list = properties.ValueList();
if (properties_list.Empty()) {
return false;
}
return std::all_of(properties_list.begin(), properties_list.end(), [](const mgp::Value &property) {
return property.IsString() && !property.ValueString().empty();
});
};
auto unique_constraint_exists =
[](const std::string_view label, const std::set<std::string_view> &properties,
const std::map<std::string_view, AssertedUniqueConstraintsStorage> &existing_unique_constraints) -> bool {
auto iter = existing_unique_constraints.find(label);
if (iter == existing_unique_constraints.end()) {
return false;
}
return iter->second.find(properties) != iter->second.end();
};
for (const auto unique_props_nested_list = unique_props_nested.ValueList();
const auto &properties : unique_props_nested_list) {
if (!validate_unique_constraint_props(properties)) {
continue;
}
const auto properties_list = properties.ValueList();
std::set<std::string_view> properties_coll;
std::transform(properties_list.begin(), properties_list.end(),
std::inserter(properties_coll, properties_coll.begin()),
[](const mgp::Value &property) { return property.ValueString(); });
if (unique_constraint_exists(label, properties_coll, existing_unique_constraints)) {
InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusKept);
} else if (mgp::CreateUniqueConstraint(memgraph_graph, label, properties.ptr())) {
InsertRecordForUniqueConstraint(record_factory, label, properties_list, Schema::kStatusCreated);
}
asserted_unique_constraints.emplace(std::move(properties_coll));
}
return asserted_unique_constraints;
}
void ProcessUniqueConstraints(const mgp::Map &unique_constraints_map, mgp_graph *memgraph_graph,
const auto &record_factory, bool drop_existing) {
auto mgp_existing_unique_constraints = mgp::ListAllUniqueConstraints(memgraph_graph);
// label-unique_constraints pair
std::map<std::string_view, AssertedUniqueConstraintsStorage> existing_unique_constraints;
for (const auto &constraint : mgp_existing_unique_constraints) {
auto constraint_list = constraint.ValueList();
std::set<std::string_view> properties;
for (int i = 1; i < constraint_list.Size(); i++) {
properties.emplace(constraint_list[i].ValueString());
}
const std::string_view label = constraint_list[0].ValueString();
auto [it, inserted] = existing_unique_constraints.try_emplace(label, AssertedUniqueConstraintsStorage{properties});
if (!inserted) {
it->second.emplace(std::move(properties));
}
}
std::map<std::string_view, AssertedUniqueConstraintsStorage> asserted_unique_constraints;
for (const auto &[label, unique_props_nested] : unique_constraints_map) {
auto asserted_unique_constraints_new = CreateUniqueConstraintsForLabel(
label, unique_props_nested, existing_unique_constraints, memgraph_graph, record_factory);
if (drop_existing) {
asserted_unique_constraints.emplace(label, std::move(asserted_unique_constraints_new));
}
}
if (!drop_existing) {
return;
}
std::vector<std::pair<std::string_view, std::set<std::string_view>>> unique_constraints_to_drop;
// Check for each label for we found existing constraint in the DB whether it was asserted.
// If no unique constraint was found with label, we can drop all unique constraints for this label. (if branch)
// If some unique constraint was found with label, we can drop only those unique constraints that were not asserted.
// (else branch.)
std::ranges::for_each(existing_unique_constraints, [&asserted_unique_constraints, &unique_constraints_to_drop](
const auto &existing_label_unique_constraints) {
const auto &label = existing_label_unique_constraints.first;
const auto &existing_unique_constraints_for_label = existing_label_unique_constraints.second;
const auto &asserted_unique_constraints_for_label = asserted_unique_constraints.find(label);
if (asserted_unique_constraints_for_label == asserted_unique_constraints.end()) {
std::ranges::for_each(
std::make_move_iterator(existing_unique_constraints_for_label.begin()),
std::make_move_iterator(existing_unique_constraints_for_label.end()),
[&unique_constraints_to_drop, &label](std::set<std::string_view> existing_unique_constraint_for_label) {
unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label));
});
} else {
const auto &asserted_unique_constraints_for_label_coll = asserted_unique_constraints_for_label->second;
std::ranges::for_each(
std::make_move_iterator(existing_unique_constraints_for_label.begin()),
std::make_move_iterator(existing_unique_constraints_for_label.end()),
[&unique_constraints_to_drop, &label, &asserted_unique_constraints_for_label_coll](
std::set<std::string_view> existing_unique_constraint_for_label) {
if (!asserted_unique_constraints_for_label_coll.contains(existing_unique_constraint_for_label)) {
unique_constraints_to_drop.emplace_back(label, std::move(existing_unique_constraint_for_label));
}
});
}
});
std::ranges::for_each(
unique_constraints_to_drop, [memgraph_graph, &record_factory](const auto &label_unique_constraint) {
const auto &[label, unique_constraint] = label_unique_constraint;
auto unique_constraint_list = mgp::List();
std::ranges::for_each(unique_constraint, [&unique_constraint_list](const std::string_view &property) {
unique_constraint_list.AppendExtend(mgp::Value(property));
});
if (mgp::DropUniqueConstraint(memgraph_graph, label, mgp::Value(unique_constraint_list).ptr())) {
InsertRecordForUniqueConstraint(record_factory, label, unique_constraint_list, Schema::kStatusDropped);
}
});
}
void Schema::Assert(mgp_list *args, mgp_graph *memgraph_graph, mgp_result *result, mgp_memory *memory) {
mgp::MemoryDispatcherGuard guard{memory};
const auto record_factory = mgp::RecordFactory(result);
auto arguments = mgp::List(args);
auto indices_map = arguments[0].ValueMap();
auto unique_constraints_map = arguments[1].ValueMap();
auto existence_constraints_map = arguments[2].ValueMap();
auto drop_existing = arguments[3].ValueBool();
ProcessIndices(indices_map, memgraph_graph, record_factory, drop_existing);
ProcessExistenceConstraints(existence_constraints_map, memgraph_graph, record_factory, drop_existing);
ProcessUniqueConstraints(unique_constraints_map, memgraph_graph, record_factory, drop_existing);
}
extern "C" int mgp_init_module(struct mgp_module *module, struct mgp_memory *memory) {
try {
mgp::MemoryDispatcherGuard guard{memory};
;
AddProcedure(Schema::NodeTypeProperties, std::string(Schema::kProcedureNodeType).c_str(), mgp::ProcedureType::Read,
{},
{mgp::Return(std::string(Schema::kReturnNodeType).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnLabels).c_str(), {mgp::Type::List, mgp::Type::String}),
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
AddProcedure(Schema::NodeTypeProperties, Schema::kProcedureNodeType, mgp::ProcedureType::Read, {},
{mgp::Return(Schema::kReturnNodeType, mgp::Type::String),
mgp::Return(Schema::kReturnLabels, {mgp::Type::List, mgp::Type::String}),
mgp::Return(Schema::kReturnPropertyName, mgp::Type::String),
mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any),
mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)},
module, memory);
AddProcedure(Schema::RelTypeProperties, std::string(Schema::kProcedureRelType).c_str(), mgp::ProcedureType::Read,
{},
{mgp::Return(std::string(Schema::kReturnRelType).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnPropertyName).c_str(), mgp::Type::String),
mgp::Return(std::string(Schema::kReturnPropertyType).c_str(), mgp::Type::Any),
mgp::Return(std::string(Schema::kReturnMandatory).c_str(), mgp::Type::Bool)},
AddProcedure(Schema::RelTypeProperties, Schema::kProcedureRelType, mgp::ProcedureType::Read, {},
{mgp::Return(Schema::kReturnRelType, mgp::Type::String),
mgp::Return(Schema::kReturnPropertyName, mgp::Type::String),
mgp::Return(Schema::kReturnPropertyType, mgp::Type::Any),
mgp::Return(Schema::kReturnMandatory, mgp::Type::Bool)},
module, memory);
AddProcedure(
Schema::Assert, Schema::kProcedureAssert, mgp::ProcedureType::Read,
{
mgp::Parameter(Schema::kParameterIndices, {mgp::Type::Map, mgp::Type::Any}),
mgp::Parameter(Schema::kParameterUniqueConstraints, {mgp::Type::Map, mgp::Type::Any}),
mgp::Parameter(Schema::kParameterExistenceConstraints, {mgp::Type::Map, mgp::Type::Any},
mgp::Value(mgp::Map{})),
mgp::Parameter(Schema::kParameterDropExisting, mgp::Type::Bool, mgp::Value(true)),
},
{mgp::Return(Schema::kReturnLabel, mgp::Type::String), mgp::Return(Schema::kReturnKey, mgp::Type::String),
mgp::Return(Schema::kReturnKeys, {mgp::Type::List, mgp::Type::String}),
mgp::Return(Schema::kReturnUnique, mgp::Type::Bool), mgp::Return(Schema::kReturnAction, mgp::Type::String)},
module, memory);
} catch (const std::exception &e) {
std::cerr << "Error while initializing query module: " << e.what() << std::endl;
return 1;
}

View File

@@ -104,7 +104,9 @@ def retry(retry_limit, timeout=100):
except Exception:
time.sleep(timeout)
return func(*args, **kwargs)
return wrapper
return inner_func
@@ -200,19 +202,19 @@ if args.version:
try:
current_branch = get_output("git", "rev-parse", "--abbrev-ref", "HEAD")
if current_branch != "master":
branches = get_output("git", "branch")
if "master" in branches:
branches = get_output("git", "branch", "-r", "--list", "origin/master")
if "origin/master" in branches:
# If master is present locally, the fetch is allowed to fail
# because this script will still be able to compare against the
# master branch.
try:
get_output("git", "fetch", "origin", "master:master")
get_output("git", "fetch", "origin", "master")
except Exception:
pass
else:
# If master is not present locally, the fetch command has to
# succeed because something else will fail otherwise.
get_output("git", "fetch", "origin", "master:master")
get_output("git", "fetch", "origin", "master")
except Exception:
print("Fatal error while ensuring local master branch.")
sys.exit(1)
@@ -232,7 +234,7 @@ for branch in branches:
match = branch_regex.match(branch)
if match is not None:
version = tuple(map(int, match.group(1).split(".")))
master_branch_merge = get_output("git", "merge-base", "master", branch)
master_branch_merge = get_output("git", "merge-base", "origin/master", branch)
versions.append((version, branch, master_branch_merge))
versions.sort(reverse=True)
@@ -243,7 +245,7 @@ current_version = None
for version in versions:
version_tuple, branch, master_branch_merge = version
current_branch_merge = get_output("git", "merge-base", current_hash, branch)
master_current_merge = get_output("git", "merge-base", current_hash, "master")
master_current_merge = get_output("git", "merge-base", current_hash, "origin/master")
# The first check checks whether this commit is a child of `master` and
# the version branch was created before us.
# The second check checks whether this commit is a child of the version

View File

@@ -143,7 +143,7 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
# Memgraph CSV Import Tool Executable
# ----------------------------------------------------------------------------
add_executable(mg_import_csv mg_import_csv.cpp)
target_link_libraries(mg_import_csv mg-storage-v2)
target_link_libraries(mg_import_csv mg-storage-v2 mg-dbms)
# Strip the executable in release build.
if(lower_build_type STREQUAL "release")

View File

@@ -13,6 +13,7 @@
#include <fmt/format.h>
#include <json/json.hpp>
#include <utility>
#include "storage/v2/temporal.hpp"
#include "utils/logging.hpp"
@@ -87,8 +88,8 @@ inline nlohmann::json PropertyValueToJson(const storage::PropertyValue &pv) {
return ret;
}
Log::Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
: storage_directory_(storage_directory),
Log::Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis)
: storage_directory_(std::move(storage_directory)),
buffer_size_(buffer_size),
buffer_flush_interval_millis_(buffer_flush_interval_millis),
started_(false) {}

View File

@@ -36,7 +36,7 @@ class Log {
};
public:
Log(const std::filesystem::path &storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
Log(std::filesystem::path storage_directory, int32_t buffer_size, int32_t buffer_flush_interval_millis);
~Log();

View File

@@ -10,6 +10,7 @@
#include <cstdint>
#include <regex>
#include <utility>
#include <gflags/gflags.h>
@@ -560,20 +561,20 @@ Databases Databases::Deserialize(const nlohmann::json &data) {
}
#endif
User::User() {}
User::User() = default;
User::User(const std::string &username) : username_(utils::ToLowerCase(username)) {}
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(password_hash), permissions_(permissions) {}
User::User(const std::string &username, std::string password_hash, const Permissions &permissions)
: username_(utils::ToLowerCase(username)), password_hash_(std::move(password_hash)), permissions_(permissions) {}
#ifdef MG_ENTERPRISE
User::User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
User::User(const std::string &username, std::string password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access)
: username_(utils::ToLowerCase(username)),
password_hash_(password_hash),
password_hash_(std::move(password_hash)),
permissions_(permissions),
fine_grained_access_handler_(std::move(fine_grained_access_handler)),
database_access_(db_access) {}
database_access_(std::move(db_access)) {}
#endif
bool User::CheckPassword(const std::string &password) {

View File

@@ -14,6 +14,7 @@
#include <unordered_map>
#include <json/json.hpp>
#include <utility>
#include "dbms/constants.hpp"
#include "utils/logging.hpp"
@@ -301,8 +302,8 @@ class Databases final {
bool Contains(const std::string &db) const;
bool GetAllowAll() const { return allow_all_; }
const std::set<std::string> &GetGrants() const { return grants_dbs_; }
const std::set<std::string> &GetDenies() const { return denies_dbs_; }
const std::set<std::string, std::less<>> &GetGrants() const { return grants_dbs_; }
const std::set<std::string, std::less<>> &GetDenies() const { return denies_dbs_; }
const std::string &GetDefault() const;
nlohmann::json Serialize() const;
@@ -310,14 +311,17 @@ class Databases final {
static Databases Deserialize(const nlohmann::json &data);
private:
Databases(bool allow_all, std::set<std::string> grant, std::set<std::string> deny,
const std::string &default_db = dbms::kDefaultDB)
: grants_dbs_(grant), denies_dbs_(deny), allow_all_(allow_all), default_db_(default_db) {}
Databases(bool allow_all, std::set<std::string, std::less<>> grant, std::set<std::string, std::less<>> deny,
std::string default_db = dbms::kDefaultDB)
: grants_dbs_(std::move(grant)),
denies_dbs_(std::move(deny)),
allow_all_(allow_all),
default_db_(std::move(default_db)) {}
std::set<std::string> grants_dbs_; //!< set of databases with granted access
std::set<std::string> denies_dbs_; //!< set of databases with denied access
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
std::string default_db_; //!< user's default database
std::set<std::string, std::less<>> grants_dbs_; //!< set of databases with granted access
std::set<std::string, std::less<>> denies_dbs_; //!< set of databases with denied access
bool allow_all_; //!< flag to allow access to everything (denied overrides this)
std::string default_db_; //!< user's default database
};
#endif
@@ -327,9 +331,9 @@ class User final {
User();
explicit User(const std::string &username);
User(const std::string &username, const std::string &password_hash, const Permissions &permissions);
User(const std::string &username, std::string password_hash, const Permissions &permissions);
#ifdef MG_ENTERPRISE
User(const std::string &username, const std::string &password_hash, const Permissions &permissions,
User(const std::string &username, std::string password_hash, const Permissions &permissions,
FineGrainedAccessHandler fine_grained_access_handler, Databases db_access = {});
#endif
User(const User &) = default;

View File

@@ -25,4 +25,3 @@ target_link_libraries(mg-communication Boost::headers Threads::Threads mg-utils
find_package(OpenSSL REQUIRED)
target_link_libraries(mg-communication ${OPENSSL_LIBRARIES})
target_include_directories(mg-communication SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
target_precompile_headers(mg-communication INTERFACE http/server.hpp <boost/beast/websocket.hpp> bolt/v1/session.hpp)

View File

@@ -14,6 +14,7 @@
#include <map>
#include <optional>
#include <string>
#include <utility>
#include <vector>
#include "communication/bolt/v1/codes.hpp"
@@ -34,8 +35,8 @@ class FailureResponseException : public utils::BasicException {
explicit FailureResponseException(const std::string &message) : utils::BasicException{message} {}
FailureResponseException(const std::string &code, const std::string &message)
: utils::BasicException{message}, code_{code} {}
FailureResponseException(std::string code, const std::string &message)
: utils::BasicException{message}, code_{std::move(code)} {}
const std::string &code() const { return code_; }
SPECIALIZE_GET_EXCEPTION_NAME(FailureResponseException)

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,7 +51,7 @@ enum class ChunkState : uint8_t {
template <typename TBuffer>
class ChunkedDecoderBuffer {
public:
ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
explicit ChunkedDecoderBuffer(TBuffer &buffer) : buffer_(buffer) { data_.reserve(kChunkMaxDataSize); }
/**
* Reads data from the internal buffer.

View File

@@ -401,11 +401,11 @@ class Decoder {
}
auto &labels = dv.ValueList();
vertex.labels.reserve(labels.size());
for (size_t i = 0; i < labels.size(); ++i) {
if (labels[i].type() != Value::Type::String) {
for (auto &label : labels) {
if (label.type() != Value::Type::String) {
return false;
}
vertex.labels.emplace_back(std::move(labels[i].ValueString()));
vertex.labels.emplace_back(std::move(label.ValueString()));
}
// read properties

View File

@@ -111,12 +111,12 @@ class BaseEncoder {
void WriteList(const std::vector<Value> &value) {
WriteTypeSize(value.size(), MarkerList);
for (auto &x : value) WriteValue(x);
for (const auto &x : value) WriteValue(x);
}
void WriteMap(const std::map<std::string, Value> &value) {
WriteTypeSize(value.size(), MarkerMap);
for (auto &x : value) {
for (const auto &x : value) {
WriteString(x.first);
WriteValue(x.second);
}
@@ -205,11 +205,11 @@ class BaseEncoder {
WriteRAW(utils::UnderlyingCast(Marker::TinyStruct) + 3);
WriteRAW(utils::UnderlyingCast(Signature::Path));
WriteTypeSize(path.vertices.size(), MarkerList);
for (auto &v : path.vertices) WriteVertex(v);
for (const auto &v : path.vertices) WriteVertex(v);
WriteTypeSize(path.edges.size(), MarkerList);
for (auto &e : path.edges) WriteEdge(e);
for (const auto &e : path.edges) WriteEdge(e);
WriteTypeSize(path.indices.size(), MarkerList);
for (auto &i : path.indices) WriteInt(i);
for (const auto &i : path.indices) WriteInt(i);
}
void WriteDate(const utils::Date &date) {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -48,7 +48,7 @@ namespace memgraph::communication::bolt {
template <class TOutputStream>
class ChunkedEncoderBuffer {
public:
ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
explicit ChunkedEncoderBuffer(TOutputStream &output_stream) : output_stream_(output_stream) {}
/**
* Writes n values into the buffer. If n is bigger than whole chunk size

View File

@@ -39,7 +39,7 @@ class ClientEncoder : private BaseEncoder<Buffer> {
using BaseEncoder<Buffer>::buffer_;
public:
ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
explicit ClientEncoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
using BaseEncoder<Buffer>::UpdateVersion;

View File

@@ -32,7 +32,7 @@ class Encoder : private BaseEncoder<Buffer> {
using BaseEncoder<Buffer>::buffer_;
public:
Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
explicit Encoder(Buffer &buffer) : BaseEncoder<Buffer>(buffer) {}
using BaseEncoder<Buffer>::UpdateVersion;

View File

@@ -93,7 +93,7 @@ State HandlePullDiscard(TSession &session, std::optional<int> n, std::optional<i
return State::Close;
}
if (summary.count("has_more") && summary.at("has_more").ValueBool()) {
if (summary.contains("has_more") && summary.at("has_more").ValueBool()) {
return State::Result;
}
@@ -148,13 +148,13 @@ State HandlePullDiscardV4(TSession &session, const State state, const Marker mar
spdlog::trace("Couldn't read extra field!");
}
const auto &extra_map = extra.ValueMap();
if (extra_map.count("n")) {
if (extra_map.contains("n")) {
if (const auto n_value = extra_map.at("n").ValueInt(); n_value != kPullAll) {
n = n_value;
}
}
if (extra_map.count("qid")) {
if (extra_map.contains("qid")) {
if (const auto qid_value = extra_map.at("qid").ValueInt(); qid_value != kPullLast) {
qid = qid_value;
}
@@ -209,7 +209,11 @@ State HandleRunV1(TSession &session, const State state, const Marker marker) {
DMG_ASSERT(!session.encoder_buffer_.HasData(), "There should be no data to write in this state");
#if MG_ENTERPRISE
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
#else
spdlog::debug("[Run] '{}'", query.ValueString());
#endif
// Increment number of queries in the metrics
IncrementQueryMetrics(session);
@@ -276,7 +280,11 @@ State HandleRunV4(TSession &session, const State state, const Marker marker) {
return HandleFailure(session, e);
}
#if MG_ENTERPRISE
spdlog::debug("[Run - {}] '{}'", session.GetCurrentDB(), query.ValueString());
#else
spdlog::debug("[Run] '{}'", query.ValueString());
#endif
// Increment number of queries in the metrics
IncrementQueryMetrics(session);

View File

@@ -42,11 +42,11 @@ std::optional<State> AuthenticateUser(TSession &session, Value &metadata) {
std::string username;
std::string password;
if (data["scheme"].ValueString() == "basic") {
if (!data.count("principal")) { // Special case principal = ""
if (!data.contains("principal")) { // Special case principal = ""
spdlog::warn("The client didn't supply the principal field! Trying with \"\"...");
data["principal"] = "";
}
if (!data.count("credentials")) { // Special case credentials = ""
if (!data.contains("credentials")) { // Special case credentials = ""
spdlog::warn("The client didn't supply the credentials field! Trying with \"\"...");
data["credentials"] = "";
}
@@ -118,7 +118,7 @@ std::optional<Value> GetMetadataV4(TSession &session, const Marker marker) {
}
auto &data = metadata.ValueMap();
if (!data.count("user_agent")) {
if (!data.contains("user_agent")) {
spdlog::warn("The client didn't supply the user agent!");
return std::nullopt;
}
@@ -142,7 +142,7 @@ std::optional<Value> GetInitDataV5(TSession &session, const Marker marker) {
}
const auto &data = metadata.ValueMap();
if (!data.count("user_agent")) {
if (!data.contains("user_agent")) {
spdlog::warn("The client didn't supply the user agent!");
return std::nullopt;
}

View File

@@ -91,7 +91,7 @@ struct UnboundedEdge {
* The decoder writes data into this structure.
*/
struct Path {
Path() {}
Path() = default;
Path(const std::vector<Vertex> &vertices, const std::vector<Edge> &edges) {
// Helper function. Looks for the given element in the collection. If found,

View File

@@ -132,7 +132,7 @@ class Client final {
*/
class ClientInputStream final {
public:
ClientInputStream(Client &client);
explicit ClientInputStream(Client &client);
ClientInputStream(const ClientInputStream &) = delete;
ClientInputStream(ClientInputStream &&) = delete;
@@ -156,7 +156,7 @@ class ClientInputStream final {
*/
class ClientOutputStream final {
public:
ClientOutputStream(Client &client);
explicit ClientOutputStream(Client &client);
ClientOutputStream(const ClientOutputStream &) = delete;
ClientOutputStream(ClientOutputStream &&) = delete;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -34,7 +34,7 @@ ClientContext::ClientContext(bool use_ssl) : use_ssl_(use_ssl), ctx_(nullptr) {
}
ClientContext::ClientContext(const std::string &key_file, const std::string &cert_file) : ClientContext(true) {
if (key_file != "" && cert_file != "") {
if (!key_file.empty() && !cert_file.empty()) {
MG_ASSERT(SSL_CTX_use_certificate_file(ctx_, cert_file.c_str(), SSL_FILETYPE_PEM) == 1,
"Couldn't load client certificate from file: {}", cert_file);
MG_ASSERT(SSL_CTX_use_PrivateKey_file(ctx_, key_file.c_str(), SSL_FILETYPE_PEM) == 1,
@@ -124,7 +124,7 @@ ServerContext &ServerContext::operator=(ServerContext &&other) noexcept {
return *this;
}
ServerContext::~ServerContext() {}
ServerContext::~ServerContext() = default;
SSL_CTX *ServerContext::context() {
MG_ASSERT(ctx_);

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,7 +15,7 @@
namespace memgraph::communication {
const std::string SslGetLastError() {
std::string SslGetLastError() {
char buff[2048];
auto err = ERR_get_error();
ERR_error_string_n(err, buff, sizeof(buff));

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,6 +18,6 @@ namespace memgraph::communication {
/**
* This function reads and returns a string describing the last OpenSSL error.
*/
const std::string SslGetLastError();
std::string SslGetLastError();
} // namespace memgraph::communication

View File

@@ -38,7 +38,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TRequestHand
Listener(Listener &&) = delete;
Listener &operator=(const Listener &) = delete;
Listener &operator=(Listener &&) = delete;
~Listener() {}
~Listener() = default;
template <typename... Args>
static std::shared_ptr<Listener> Create(Args &&...args) {

View File

@@ -17,6 +17,7 @@
#include <memory>
#include <mutex>
#include <thread>
#include <utility>
#include <gflags/gflags.h>
@@ -51,13 +52,13 @@ class Listener final {
using SessionHandler = Session<TSession, TSessionContext>;
public:
Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, const std::string &service_name,
Listener(TSessionContext *data, ServerContext *context, int inactivity_timeout_sec, std::string service_name,
size_t workers_count)
: data_(data),
alive_(false),
context_(context),
inactivity_timeout_sec_(inactivity_timeout_sec),
service_name_(service_name),
service_name_(std::move(service_name)),
workers_count_(workers_count) {}
~Listener() {

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -80,7 +80,7 @@ class ResultStreamFaker {
std::transform(header.begin(), header.end(), column_widths.begin(), [](const auto &s) { return s.size(); });
// convert all the results into strings, and track max column width
auto &results_data = results.GetResults();
const auto &results_data = results.GetResults();
std::vector<std::vector<std::string>> result_strings(results_data.size(),
std::vector<std::string>(column_widths.size()));
for (int row_ind = 0; row_ind < static_cast<int>(results_data.size()); ++row_ind) {

View File

@@ -17,6 +17,7 @@
#include <memory>
#include <mutex>
#include <thread>
#include <utility>
#include <openssl/bio.h>
#include <openssl/err.h>
@@ -51,7 +52,8 @@ using InputStream = Buffer::ReadEnd;
*/
class OutputStream final {
public:
OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function) : write_function_(write_function) {}
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
: write_function_(std::move(write_function)) {}
OutputStream(const OutputStream &) = delete;
OutputStream(OutputStream &&) = delete;

View File

@@ -47,7 +47,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
Listener(Listener &&) = delete;
Listener &operator=(const Listener &) = delete;
Listener &operator=(Listener &&) = delete;
~Listener() {}
~Listener() = default;
template <typename... Args>
static std::shared_ptr<Listener> Create(Args &&...args) {

View File

@@ -76,7 +76,7 @@ using tcp = boost::asio::ip::tcp;
class OutputStream final {
public:
explicit OutputStream(std::function<bool(const uint8_t *, size_t, bool)> write_function)
: write_function_(write_function) {}
: write_function_(std::move(write_function)) {}
OutputStream(const OutputStream &) = delete;
OutputStream(OutputStream &&) = delete;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -16,6 +16,7 @@
#include <spdlog/sinks/base_sink.h>
#include <boost/asio/io_context.hpp>
#include <boost/asio/ip/tcp.hpp>
#include <utility>
#include "communication/websocket/listener.hpp"
#include "io/network/endpoint.hpp"
@@ -45,7 +46,7 @@ class Server final {
class LoggingSink : public spdlog::sinks::base_sink<std::mutex> {
public:
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(listener) {}
explicit LoggingSink(std::weak_ptr<Listener> listener) : listener_(std::move(listener)) {}
private:
void sink_it_(const spdlog::details::log_msg &msg) override;

View File

@@ -1,3 +1,3 @@
add_library(mg-dbms STATIC database.cpp)
add_library(mg-dbms STATIC dbms_handler.cpp database.cpp replication_handler.cpp replication_client.cpp inmemory/replication_handlers.cpp)
target_link_libraries(mg-dbms mg-utils mg-storage-v2 mg-query)

View File

@@ -15,4 +15,10 @@ namespace memgraph::dbms {
constexpr static const char *kDefaultDB = "memgraph"; //!< Name of the default database
#ifdef MG_EXPERIMENTAL_REPLICATION_MULTITENANCY
constexpr bool allow_mt_repl = true;
#else
constexpr bool allow_mt_repl = false;
#endif
} // namespace memgraph::dbms

View File

@@ -10,6 +10,8 @@
// licenses/APL.txt.
#include "dbms/database.hpp"
#include "dbms/inmemory/storage_helper.hpp"
#include "dbms/replication_handler.hpp"
#include "flags/storage_mode.hpp"
#include "storage/v2/disk/storage.hpp"
#include "storage/v2/inmemory/storage.hpp"
@@ -19,14 +21,16 @@ template struct memgraph::utils::Gatekeeper<memgraph::dbms::Database>;
namespace memgraph::dbms {
Database::Database(const storage::Config &config)
Database::Database(storage::Config config, replication::ReplicationState &repl_state)
: trigger_store_(config.durability.storage_directory / "triggers"),
streams_{config.durability.storage_directory / "streams"} {
streams_{config.durability.storage_directory / "streams"},
plan_cache_{FLAGS_query_plan_cache_max_size},
repl_state_(&repl_state) {
if (config.storage_mode == memgraph::storage::StorageMode::ON_DISK_TRANSACTIONAL || config.force_on_disk ||
utils::DirExists(config.disk.main_storage_directory)) {
storage_ = std::make_unique<storage::DiskStorage>(config);
storage_ = std::make_unique<storage::DiskStorage>(std::move(config));
} else {
storage_ = std::make_unique<storage::InMemoryStorage>(config, config.storage_mode);
storage_ = dbms::CreateInMemoryStorage(std::move(config), repl_state);
}
}

View File

@@ -24,6 +24,8 @@
#include "query/trigger.hpp"
#include "storage/v2/storage.hpp"
#include "utils/gatekeeper.hpp"
#include "utils/lru_cache.hpp"
#include "utils/synchronized.hpp"
namespace memgraph::dbms {
@@ -46,7 +48,7 @@ class Database {
*
* @param config storage configuration
*/
explicit Database(const storage::Config &config);
explicit Database(storage::Config config, replication::ReplicationState &repl_state);
/**
* @brief Returns the raw storage pointer.
@@ -56,6 +58,7 @@ class Database {
* @return storage::Storage*
*/
storage::Storage *storage() { return storage_.get(); }
storage::Storage const *storage() const { return storage_.get(); }
/**
* @brief Storage's Accessor
@@ -65,12 +68,12 @@ class Database {
*/
std::unique_ptr<storage::Storage::Accessor> Access(
std::optional<storage::IsolationLevel> override_isolation_level = {}) {
return storage_->Access(override_isolation_level);
return storage_->Access(override_isolation_level, repl_state_->IsMain());
}
std::unique_ptr<storage::Storage::Accessor> UniqueAccess(
std::optional<storage::IsolationLevel> override_isolation_level = {}) {
return storage_->UniqueAccess(override_isolation_level);
return storage_->UniqueAccess(override_isolation_level, repl_state_->IsMain());
}
/**
@@ -145,9 +148,9 @@ class Database {
/**
* @brief Returns the PlanCache vector raw pointer
*
* @return utils::SkipList<query::PlanCacheEntry>*
* @return utils::Synchronized<utils::LRUCache<uint64_t, std::shared_ptr<PlanWrapper>>, utils::RWSpinLock>
*/
utils::SkipList<query::PlanCacheEntry> *plan_cache() { return &plan_cache_; }
query::PlanCacheLRU *plan_cache() { return &plan_cache_; }
private:
std::unique_ptr<storage::Storage> storage_; //!< Underlying storage
@@ -156,7 +159,9 @@ class Database {
query::stream::Streams streams_; //!< Streams associated with the storage
// TODO: Move to a better place
utils::SkipList<query::PlanCacheEntry> plan_cache_; //!< Plan cache associated with the storage
query::PlanCacheLRU plan_cache_; //!< Plan cache associated with the storage
const replication::ReplicationState *repl_state_;
};
} // namespace memgraph::dbms

View File

@@ -51,7 +51,7 @@ class DatabaseHandler : public Handler<Database> {
* @param config Storage configuration
* @return HandlerT::NewResult
*/
HandlerT::NewResult New(std::string_view name, storage::Config config) {
HandlerT::NewResult New(std::string_view name, storage::Config config, replication::ReplicationState &repl_state) {
// Control that no one is using the same data directory
if (std::any_of(begin(), end(), [&](auto &elem) {
auto db_acc = elem.second.access();
@@ -62,7 +62,7 @@ class DatabaseHandler : public Handler<Database> {
return NewError::EXISTS;
}
config.name = name; // Set storage id via config
return HandlerT::New(std::piecewise_construct, name, config);
return HandlerT::New(std::piecewise_construct, name, config, repl_state);
}
/**
@@ -93,5 +93,4 @@ class DatabaseHandler : public Handler<Database> {
};
} // namespace memgraph::dbms
#endif

75
src/dbms/dbms_handler.cpp Normal file
View File

@@ -0,0 +1,75 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "dbms/dbms_handler.hpp"
namespace memgraph::dbms {
#ifdef MG_ENTERPRISE
DbmsHandler::DbmsHandler(
storage::Config config,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
bool recovery_on_startup, bool delete_on_drop)
: default_config_{std::move(config)},
delete_on_drop_(delete_on_drop),
repl_state_{ReplicationStateRootPath(default_config_)} {
// TODO: Decouple storage config from dbms config
// TODO: Save individual db configs inside the kvstore and restore from there
storage::UpdatePaths(default_config_, default_config_.durability.storage_directory / "databases");
const auto &db_dir = default_config_.durability.storage_directory;
const auto durability_dir = db_dir / ".durability";
utils::EnsureDirOrDie(db_dir);
utils::EnsureDirOrDie(durability_dir);
durability_ = std::make_unique<kvstore::KVStore>(durability_dir);
// Generate the default database
MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB.");
// Recover previous databases
if (recovery_on_startup) {
for (const auto &[name, _] : *durability_) {
if (name == kDefaultDB) continue; // Already set
spdlog::info("Restoring database {}.", name);
MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name);
spdlog::info("Database {} restored.", name);
}
} else { // Clear databases from the durability list and auth
auto locked_auth = auth->Lock();
for (const auto &[name, _] : *durability_) {
if (name == kDefaultDB) continue;
locked_auth->DeleteDatabase(name);
durability_->Delete(name);
}
}
// Startup replication state (if recovered at startup)
auto replica = [this](replication::RoleReplicaData const &data) {
// Register handlers
InMemoryReplicationHandlers::Register(this, *data.server);
if (!data.server->Start()) {
spdlog::error("Unable to start the replication server.");
return false;
}
return true;
};
// Replication frequent check start
auto main = [this](replication::RoleMainData &data) {
for (auto &client : data.registered_replicas_) {
StartReplicaClient(*this, client);
}
return true;
};
// Startup proccess for main/replica
MG_ASSERT(std::visit(memgraph::utils::Overloaded{replica, main}, repl_state_.ReplicationData()),
"Replica recovery failure!");
}
#endif
} // namespace memgraph::dbms

View File

@@ -26,7 +26,11 @@
#include "auth/auth.hpp"
#include "constants.hpp"
#include "dbms/database.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#ifdef MG_ENTERPRISE
#include "dbms/database_handler.hpp"
#endif
#include "dbms/replication_client.hpp"
#include "global.hpp"
#include "query/config.hpp"
#include "query/interpreter_context.hpp"
@@ -81,58 +85,44 @@ static inline nlohmann::json ToJson(const Statistics &stats) {
return res;
}
#ifdef MG_ENTERPRISE
using DeleteResult = utils::BasicResult<DeleteError>;
/**
* @brief Multi-database session contexts handler.
*/
class DbmsHandler {
public:
using LockT = utils::RWLock;
#ifdef MG_ENTERPRISE
using NewResultT = utils::BasicResult<NewError, DatabaseAccess>;
using DeleteResult = utils::BasicResult<DeleteError>;
/**
* @brief Initialize the handler.
*
* @param configs storage and interpreter configurations
* @param configs storage configuration
* @param auth pointer to the global authenticator
* @param recovery_on_startup restore databases (and its content) and authentication data
* @param delete_on_drop when dropping delete any associated directories on disk
*/
DbmsHandler(storage::Config config, auto *auth, bool recovery_on_startup, bool delete_on_drop)
: lock_{utils::RWLock::Priority::READ}, default_config_{std::move(config)}, delete_on_drop_(delete_on_drop) {
// TODO: Decouple storage config from dbms config
// TODO: Save individual db configs inside the kvstore and restore from there
storage::UpdatePaths(*default_config_, default_config_->durability.storage_directory / "databases");
const auto &db_dir = default_config_->durability.storage_directory;
const auto durability_dir = db_dir / ".durability";
utils::EnsureDirOrDie(db_dir);
utils::EnsureDirOrDie(durability_dir);
durability_ = std::make_unique<kvstore::KVStore>(durability_dir);
// Generate the default database
MG_ASSERT(!NewDefault_().HasError(), "Failed while creating the default DB.");
// Recover previous databases
if (recovery_on_startup) {
for (const auto &[name, _] : *durability_) {
if (name == kDefaultDB) continue; // Already set
spdlog::info("Restoring database {}.", name);
MG_ASSERT(!New_(name).HasError(), "Failed while creating database {}.", name);
spdlog::info("Database {} restored.", name);
}
} else { // Clear databases from the durability list and auth
auto locked_auth = auth->Lock();
for (const auto &[name, _] : *durability_) {
if (name == kDefaultDB) continue;
locked_auth->DeleteDatabase(name);
durability_->Delete(name);
}
}
}
DbmsHandler(storage::Config config,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth,
bool recovery_on_startup, bool delete_on_drop); // TODO If more arguments are added use a config strut
#else
/**
* @brief Initialize the handler. A single database is supported in community edition.
*
* @param configs storage configuration
*/
DbmsHandler(storage::Config config)
: repl_state_{ReplicationStateRootPath(config)},
db_gatekeeper_{[&] {
config.name = kDefaultDB;
return std::move(config);
}(),
repl_state_} {}
#endif
#ifdef MG_ENTERPRISE
/**
* @brief Create a new Database associated with the "name" database
*
@@ -151,11 +141,24 @@ class DbmsHandler {
* @return DatabaseAccess
* @throw UnknownDatabaseException if database not found
*/
DatabaseAccess Get(std::string_view name) {
DatabaseAccess Get(std::string_view name = kDefaultDB) {
std::shared_lock<LockT> rd(lock_);
return Get_(name);
}
#else
/**
* @brief Get the context associated with the default database
*
* @return DatabaseAccess
*/
DatabaseAccess Get() {
auto acc = db_gatekeeper_.access();
MG_ASSERT(acc, "Failed to get default database!");
return *acc;
}
#endif
#ifdef MG_ENTERPRISE
/**
* @brief Delete database.
*
@@ -201,6 +204,7 @@ class DbmsHandler {
return {}; // Success
}
#endif
/**
* @brief Return all active databases.
@@ -208,10 +212,20 @@ class DbmsHandler {
* @return std::vector<std::string>
*/
std::vector<std::string> All() const {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
return db_handler_.All();
#else
return {db_gatekeeper_.access()->get()->id()};
#endif
}
replication::ReplicationState &ReplicationState() { return repl_state_; }
replication::ReplicationState const &ReplicationState() const { return repl_state_; }
bool IsMain() const { return repl_state_.IsMain(); }
bool IsReplica() const { return repl_state_.IsReplica(); }
/**
* @brief Return the statistics all databases.
*
@@ -220,24 +234,30 @@ class DbmsHandler {
Statistics Stats() {
Statistics stats{};
// TODO: Handle overflow?
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue;
auto &db_acc = *db_acc_opt;
const auto &info = db_acc->GetInfo();
const auto &storage_info = info.storage_info;
stats.num_vertex += storage_info.vertex_count;
stats.num_edges += storage_info.edge_count;
stats.triggers += info.triggers;
stats.streams += info.streams;
++stats.num_databases;
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
++stats.storage_modes[(int)storage_info.storage_mode];
++stats.isolation_levels[(int)storage_info.isolation_level];
stats.snapshot_enabled += storage_info.durability_snapshot_enabled;
stats.wal_enabled += storage_info.durability_wal_enabled;
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
const auto &info = db_acc->GetInfo();
const auto &storage_info = info.storage_info;
stats.num_vertex += storage_info.vertex_count;
stats.num_edges += storage_info.edge_count;
stats.triggers += info.triggers;
stats.streams += info.streams;
++stats.num_databases;
stats.indices += storage_info.label_indices + storage_info.label_property_indices;
stats.constraints += storage_info.existence_constraints + storage_info.unique_constraints;
++stats.storage_modes[(int)storage_info.storage_mode];
++stats.isolation_levels[(int)storage_info.isolation_level];
stats.snapshot_enabled += storage_info.durability_snapshot_enabled;
stats.wal_enabled += storage_info.durability_wal_enabled;
}
}
return stats;
}
@@ -249,13 +269,19 @@ class DbmsHandler {
*/
std::vector<DatabaseInfo> Info() {
std::vector<DatabaseInfo> res;
res.reserve(std::distance(db_handler_.cbegin(), db_handler_.cend()));
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
res.reserve(std::distance(db_handler_.cbegin(), db_handler_.cend()));
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue;
auto &db_acc = *db_acc_opt;
res.push_back(db_acc->GetInfo());
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
res.push_back(db_acc->GetInfo());
}
}
return res;
}
@@ -267,15 +293,21 @@ class DbmsHandler {
* @param ic global InterpreterContext
*/
void RestoreTriggers(query::InterpreterContext *ic) {
#ifdef MG_ENTERPRISE
std::lock_guard<LockT> wr(lock_);
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc_opt = db_gk.access();
if (!db_acc_opt) continue;
auto &db_acc = *db_acc_opt;
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id());
auto storage_accessor = db_acc->Access();
auto dba = memgraph::query::DbAccessor{storage_accessor.get()};
db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker);
if (db_acc_opt) {
auto &db_acc = *db_acc_opt;
spdlog::debug("Restoring trigger for database \"{}\"", db_acc->id());
auto storage_accessor = db_acc->Access();
auto dba = memgraph::query::DbAccessor{storage_accessor.get()};
db_acc->trigger_store()->RestoreTriggers(&ic->ast_cache, &dba, ic->config.query, ic->auth_checker);
}
}
}
@@ -286,17 +318,67 @@ class DbmsHandler {
* @param ic global InterpreterContext
*/
void RestoreStreams(query::InterpreterContext *ic) {
#ifdef MG_ENTERPRISE
std::lock_guard<LockT> wr(lock_);
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc = db_gk.access();
if (!db_acc) continue;
auto *db = db_acc->get();
spdlog::debug("Restoring streams for database \"{}\"", db->id());
db->streams()->RestoreStreams(*db_acc, ic);
if (db_acc) {
auto *db = db_acc->get();
spdlog::debug("Restoring streams for database \"{}\"", db->id());
db->streams()->RestoreStreams(*db_acc, ic);
}
}
}
/**
* @brief todo
*
* @param f
*/
void ForEach(auto f) {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
#else
{
auto &db_gk = db_gatekeeper_;
#endif
auto db_acc = db_gk.access();
if (db_acc) { // This isn't an error, just a defunct db
f(db_acc->get());
}
}
}
/**
* @brief todo
*
* @param f
*/
void ForOne(auto f) {
#ifdef MG_ENTERPRISE
std::shared_lock<LockT> rd(lock_);
for (auto &[_, db_gk] : db_handler_) {
auto db_acc = db_gk.access();
if (db_acc) { // This isn't an error, just a defunct db
if (f(db_acc->get())) break; // Run until the first successful one
}
}
#else
{
auto db_acc = db_gatekeeper_.access();
MG_ASSERT(db_acc, "Should always have the database");
f(db_acc->get());
}
#endif
}
private:
#ifdef MG_ENTERPRISE
/**
* @brief return the storage directory of the associated database
*
@@ -328,13 +410,9 @@ class DbmsHandler {
* @return NewResultT context on success, error on failure
*/
NewResultT New_(const std::string &name, std::filesystem::path storage_subdir) {
if (default_config_) {
auto config_copy = *default_config_;
storage::UpdatePaths(config_copy, default_config_->durability.storage_directory / storage_subdir);
return New_(name, config_copy);
}
spdlog::info("Trying to generate session context without any configurations.");
return NewError::NO_CONFIGS;
auto config_copy = default_config_;
storage::UpdatePaths(config_copy, default_config_.durability.storage_directory / storage_subdir);
return New_(name, config_copy);
}
/**
@@ -351,7 +429,7 @@ class DbmsHandler {
return NewError::DEFUNCT;
}
auto new_db = db_handler_.New(name, storage_config);
auto new_db = db_handler_.New(name, storage_config, repl_state_);
if (new_db.HasValue()) {
// Success
if (durability_) durability_->Put(name, "ok"); // TODO: Serialize the configuration?
@@ -436,14 +514,17 @@ class DbmsHandler {
throw UnknownDatabaseException("Tried to retrieve an unknown database \"{}\".", name);
}
// Should storage objects ever be deleted?
mutable LockT lock_; //!< protective lock
DatabaseHandler db_handler_; //!< multi-tenancy storage handler
std::optional<storage::Config> default_config_; //!< Storage configuration used when creating new databases
std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation)
std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures
bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory
};
mutable LockT lock_{utils::RWLock::Priority::READ}; //!< protective lock
storage::Config default_config_; //!< Storage configuration used when creating new databases
DatabaseHandler db_handler_; //!< multi-tenancy storage handler
std::unique_ptr<kvstore::KVStore> durability_; //!< list of active dbs (pointer so we can postpone its creation)
bool delete_on_drop_; //!< Flag defining if dropping storage also deletes its directory
std::set<std::string> defunct_dbs_; //!< Databases that are in an unknown state due to various failures
#endif
replication::ReplicationState repl_state_; //!< Global replication state
#ifndef MG_ENTERPRISE
mutable utils::Gatekeeper<Database> db_gatekeeper_; //!< Single databases gatekeeper
#endif
};
} // namespace memgraph::dbms

View File

@@ -19,6 +19,7 @@
namespace memgraph::dbms {
#ifdef MG_ENTERPRISE
enum class DeleteError : uint8_t {
DEFAULT_DB,
USING,
@@ -34,11 +35,7 @@ enum class NewError : uint8_t {
GENERIC,
};
enum class SetForResult : uint8_t {
SUCCESS,
ALREADY_SET,
FAIL,
};
#endif
/**
* UnknownSession Exception

View File

@@ -38,7 +38,7 @@ class Handler {
* @brief Empty Handler constructor.
*
*/
Handler() {}
Handler() = default;
/**
* @brief Generate a new context and corresponding configuration.
@@ -49,7 +49,7 @@ class Handler {
* @return NewResult
*/
template <typename... Args>
NewResult New(std::piecewise_construct_t /* marker */, std::string_view name, Args... args) {
NewResult New(std::piecewise_construct_t /* marker */, std::string_view name, Args &&...args) {
// Make sure the emplace will succeed, since we don't want to create temporary objects that could break something
if (!Has(name)) {
auto [itr, _] = items_.emplace(std::piecewise_construct, std::forward_as_tuple(name),

View File

@@ -9,7 +9,11 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "storage/v2/inmemory/replication/replication_server.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "dbms/constants.hpp"
#include "dbms/dbms_handler.hpp"
#include "replication/replication_server.hpp"
#include "spdlog/spdlog.h"
#include "storage/v2/durability/durability.hpp"
#include "storage/v2/durability/snapshot.hpp"
#include "storage/v2/durability/version.hpp"
@@ -17,9 +21,20 @@
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/inmemory/unique_constraints.hpp"
namespace memgraph::storage {
using memgraph::storage::Delta;
using memgraph::storage::EdgeAccessor;
using memgraph::storage::EdgeRef;
using memgraph::storage::EdgeTypeId;
using memgraph::storage::LabelIndexStats;
using memgraph::storage::LabelPropertyIndexStats;
using memgraph::storage::PropertyId;
using memgraph::storage::UniqueConstraints;
using memgraph::storage::View;
using memgraph::storage::durability::WalDeltaData;
namespace memgraph::dbms {
namespace {
std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder *decoder) {
std::pair<uint64_t, WalDeltaData> ReadDelta(storage::durability::BaseDecoder *decoder) {
try {
auto timestamp = ReadWalDeltaHeader(decoder);
SPDLOG_INFO(" Timestamp {}", timestamp);
@@ -27,78 +42,114 @@ std::pair<uint64_t, durability::WalDeltaData> ReadDelta(durability::BaseDecoder
return {timestamp, delta};
} catch (const slk::SlkReaderException &) {
throw utils::BasicException("Missing data!");
} catch (const durability::RecoveryFailure &) {
} catch (const storage::durability::RecoveryFailure &) {
throw utils::BasicException("Invalid data!");
}
};
std::optional<DatabaseAccess> GetDatabaseAccessor(dbms::DbmsHandler *dbms_handler, std::string_view db_name) {
try {
#ifdef MG_ENTERPRISE
auto acc = dbms_handler->Get(db_name);
#else
if (db_name != dbms::kDefaultDB) {
spdlog::warn("Trying to replicate a non-default database on a community replica.");
return std::nullopt;
}
auto acc = dbms_handler->Get();
#endif
if (!acc) {
spdlog::error("Failed to get access to ", db_name);
return std::nullopt;
}
auto *inmem_storage = dynamic_cast<storage::InMemoryStorage *>(acc.get()->storage());
if (!inmem_storage || inmem_storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
spdlog::error("Database \"{}\" is not IN_MEMORY_TRANSACTIONAL.", db_name);
return std::nullopt;
}
return std::optional{std::move(acc)};
} catch (const dbms::UnknownDatabaseException &e) {
spdlog::warn("No database \"{}\" on replica!", db_name);
return std::nullopt;
}
}
} // namespace
InMemoryReplicationServer::InMemoryReplicationServer(InMemoryStorage *storage,
const memgraph::replication::ReplicationServerConfig &config,
memgraph::replication::ReplicationEpoch *repl_epoch)
: ReplicationServer{config}, storage_(storage), repl_epoch_{repl_epoch} {
rpc_server_.Register<replication::HeartbeatRpc>([this](auto *req_reader, auto *res_builder) {
void InMemoryReplicationHandlers::Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server) {
server.rpc_server_.Register<storage::replication::HeartbeatRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received HeartbeatRpc");
this->HeartbeatHandler(req_reader, res_builder);
InMemoryReplicationHandlers::HeartbeatHandler(dbms_handler, req_reader, res_builder);
});
rpc_server_.Register<replication::AppendDeltasRpc>([this](auto *req_reader, auto *res_builder) {
spdlog::debug("Received AppendDeltasRpc");
this->AppendDeltasHandler(req_reader, res_builder);
});
rpc_server_.Register<replication::SnapshotRpc>([this](auto *req_reader, auto *res_builder) {
server.rpc_server_.Register<storage::replication::AppendDeltasRpc>(
[dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received AppendDeltasRpc");
InMemoryReplicationHandlers::AppendDeltasHandler(dbms_handler, req_reader, res_builder);
});
server.rpc_server_.Register<storage::replication::SnapshotRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received SnapshotRpc");
this->SnapshotHandler(req_reader, res_builder);
InMemoryReplicationHandlers::SnapshotHandler(dbms_handler, req_reader, res_builder);
});
rpc_server_.Register<replication::WalFilesRpc>([this](auto *req_reader, auto *res_builder) {
server.rpc_server_.Register<storage::replication::WalFilesRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received WalFilesRpc");
this->WalFilesHandler(req_reader, res_builder);
InMemoryReplicationHandlers::WalFilesHandler(dbms_handler, req_reader, res_builder);
});
rpc_server_.Register<replication::CurrentWalRpc>([this](auto *req_reader, auto *res_builder) {
server.rpc_server_.Register<storage::replication::CurrentWalRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received CurrentWalRpc");
this->CurrentWalHandler(req_reader, res_builder);
InMemoryReplicationHandlers::CurrentWalHandler(dbms_handler, req_reader, res_builder);
});
rpc_server_.Register<replication::TimestampRpc>([this](auto *req_reader, auto *res_builder) {
server.rpc_server_.Register<storage::replication::TimestampRpc>([dbms_handler](auto *req_reader, auto *res_builder) {
spdlog::debug("Received TimestampRpc");
this->TimestampHandler(req_reader, res_builder);
InMemoryReplicationHandlers::TimestampHandler(dbms_handler, req_reader, res_builder);
});
}
void InMemoryReplicationServer::HeartbeatHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::HeartbeatReq req;
void InMemoryReplicationHandlers::HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
storage::replication::HeartbeatReq req;
slk::Load(&req, req_reader);
replication::HeartbeatRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load(),
std::string{repl_epoch_->id()}};
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
auto const *storage = db_acc->get()->storage();
storage::replication::HeartbeatRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load(),
std::string{storage->repl_storage_state_.epoch_.id()}};
slk::Save(res, res_builder);
}
void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::AppendDeltasReq req;
void InMemoryReplicationHandlers::AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
storage::replication::AppendDeltasReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::Decoder decoder(req_reader);
storage::replication::Decoder decoder(req_reader);
auto maybe_epoch_id = decoder.ReadString();
MG_ASSERT(maybe_epoch_id, "Invalid replication message");
auto &repl_storage_state = storage_->repl_storage_state_;
if (*maybe_epoch_id != repl_epoch_->id()) {
auto prev_epoch = repl_epoch_->SetEpoch(*maybe_epoch_id);
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
auto &repl_storage_state = storage->repl_storage_state_;
if (*maybe_epoch_id != storage->repl_storage_state_.epoch_.id()) {
auto prev_epoch = storage->repl_storage_state_.epoch_.SetEpoch(*maybe_epoch_id);
repl_storage_state.AddEpochToHistoryForce(prev_epoch);
}
if (storage_->wal_file_) {
if (req.seq_num > storage_->wal_file_->SequenceNumber() || *maybe_epoch_id != repl_epoch_->id()) {
storage_->wal_file_->FinalizeWal();
storage_->wal_file_.reset();
storage_->wal_seq_num_ = req.seq_num;
if (storage->wal_file_) {
if (req.seq_num > storage->wal_file_->SequenceNumber() ||
*maybe_epoch_id != storage->repl_storage_state_.epoch_.id()) {
storage->wal_file_->FinalizeWal();
storage->wal_file_.reset();
storage->wal_seq_num_ = req.seq_num;
spdlog::trace("Finalized WAL file");
} else {
MG_ASSERT(storage_->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
storage_->wal_seq_num_ = req.seq_num + 1;
MG_ASSERT(storage->wal_file_->SequenceNumber() == req.seq_num, "Invalid sequence number of current wal file");
storage->wal_seq_num_ = req.seq_num + 1;
}
} else {
storage_->wal_seq_num_ = req.seq_num;
storage->wal_seq_num_ = req.seq_num;
}
if (req.previous_commit_timestamp != repl_storage_state.last_commit_timestamp_.load()) {
@@ -107,144 +158,162 @@ void InMemoryReplicationServer::AppendDeltasHandler(slk::Reader *req_reader, slk
while (!transaction_complete) {
SPDLOG_INFO("Skipping delta");
const auto [timestamp, delta] = ReadDelta(&decoder);
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(
delta.type, durability::kVersion); // TODO: Check if we are always using the latest version when replicating
transaction_complete = storage::durability::IsWalDeltaDataTypeTransactionEnd(
delta.type,
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
}
replication::AppendDeltasRes res{false, repl_storage_state.last_commit_timestamp_.load()};
storage::replication::AppendDeltasRes res{storage->id(), false, repl_storage_state.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
return;
}
ReadAndApplyDelta(storage_, &decoder,
durability::kVersion); // TODO: Check if we are always using the latest version when replicating
ReadAndApplyDelta(
storage, &decoder,
storage::durability::kVersion); // TODO: Check if we are always using the latest version when replicating
replication::AppendDeltasRes res{true, repl_storage_state.last_commit_timestamp_.load()};
storage::replication::AppendDeltasRes res{storage->id(), true, repl_storage_state.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::debug("Replication recovery from append deltas finished, replica is now up to date!");
}
void InMemoryReplicationServer::SnapshotHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::SnapshotReq req;
void InMemoryReplicationHandlers::SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
storage::replication::SnapshotReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::Decoder decoder(req_reader);
storage::replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->snapshot_directory_);
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
utils::EnsureDirOrDie(storage->snapshot_directory_);
const auto maybe_snapshot_path = decoder.ReadFile(storage_->snapshot_directory_);
const auto maybe_snapshot_path = decoder.ReadFile(storage->snapshot_directory_);
MG_ASSERT(maybe_snapshot_path, "Failed to load snapshot!");
spdlog::info("Received snapshot saved to {}", *maybe_snapshot_path);
auto storage_guard = std::unique_lock{storage_->main_lock_};
auto storage_guard = std::unique_lock{storage->main_lock_};
spdlog::trace("Clearing database since recovering from snapshot.");
// Clear the database
storage_->vertices_.clear();
storage_->edges_.clear();
storage->vertices_.clear();
storage->edges_.clear();
storage_->constraints_.existence_constraints_ = std::make_unique<ExistenceConstraints>();
storage_->constraints_.unique_constraints_ = std::make_unique<InMemoryUniqueConstraints>();
storage_->indices_.label_index_ = std::make_unique<InMemoryLabelIndex>();
storage_->indices_.label_property_index_ = std::make_unique<InMemoryLabelPropertyIndex>();
storage->constraints_.existence_constraints_ = std::make_unique<storage::ExistenceConstraints>();
storage->constraints_.unique_constraints_ = std::make_unique<storage::InMemoryUniqueConstraints>();
storage->indices_.label_index_ = std::make_unique<storage::InMemoryLabelIndex>();
storage->indices_.label_property_index_ = std::make_unique<storage::InMemoryLabelPropertyIndex>();
try {
spdlog::debug("Loading snapshot");
auto recovered_snapshot = durability::LoadSnapshot(
*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_, &storage_->repl_storage_state_.history,
storage_->name_id_mapper_.get(), &storage_->edge_count_, storage_->config_);
auto recovered_snapshot = storage::durability::LoadSnapshot(
*maybe_snapshot_path, &storage->vertices_, &storage->edges_, &storage->repl_storage_state_.history,
storage->name_id_mapper_.get(), &storage->edge_count_, storage->config_);
spdlog::debug("Snapshot loaded successfully");
// If this step is present it should always be the first step of
// the recovery so we use the UUID we read from snasphost
storage_->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
repl_epoch_->SetEpoch(std::move(recovered_snapshot.snapshot_info.epoch_id));
storage->uuid_ = std::move(recovered_snapshot.snapshot_info.uuid);
storage->repl_storage_state_.epoch_.SetEpoch(std::move(recovered_snapshot.snapshot_info.epoch_id));
const auto &recovery_info = recovered_snapshot.recovery_info;
storage_->vertex_id_ = recovery_info.next_vertex_id;
storage_->edge_id_ = recovery_info.next_edge_id;
storage_->timestamp_ = std::max(storage_->timestamp_, recovery_info.next_timestamp);
storage->vertex_id_ = recovery_info.next_vertex_id;
storage->edge_id_ = recovery_info.next_edge_id;
storage->timestamp_ = std::max(storage->timestamp_, recovery_info.next_timestamp);
spdlog::trace("Recovering indices and constraints from snapshot.");
durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage_->indices_,
&storage_->constraints_, &storage_->vertices_);
} catch (const durability::RecoveryFailure &e) {
storage::durability::RecoverIndicesAndConstraints(recovered_snapshot.indices_constraints, &storage->indices_,
&storage->constraints_, &storage->vertices_,
storage->name_id_mapper_.get());
} catch (const storage::durability::RecoveryFailure &e) {
LOG_FATAL("Couldn't load the snapshot because of: {}", e.what());
}
storage_guard.unlock();
replication::SnapshotRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::SnapshotRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::trace("Deleting old snapshot files due to snapshot recovery.");
// Delete other durability files
auto snapshot_files = durability::GetSnapshotFiles(storage_->snapshot_directory_, storage_->uuid_);
auto snapshot_files = storage::durability::GetSnapshotFiles(storage->snapshot_directory_, storage->uuid_);
for (const auto &[path, uuid, _] : snapshot_files) {
if (path != *maybe_snapshot_path) {
spdlog::trace("Deleting snapshot file {}", path);
storage_->file_retainer_.DeleteFile(path);
storage->file_retainer_.DeleteFile(path);
}
}
spdlog::trace("Deleting old WAL files due to snapshot recovery.");
auto wal_files = durability::GetWalFiles(storage_->wal_directory_, storage_->uuid_);
auto wal_files = storage::durability::GetWalFiles(storage->wal_directory_, storage->uuid_);
if (wal_files) {
for (const auto &wal_file : *wal_files) {
spdlog::trace("Deleting WAL file {}", wal_file.path);
storage_->file_retainer_.DeleteFile(wal_file.path);
storage->file_retainer_.DeleteFile(wal_file.path);
}
storage_->wal_file_.reset();
storage->wal_file_.reset();
}
spdlog::debug("Replication recovery from snapshot finished!");
}
void InMemoryReplicationServer::WalFilesHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::WalFilesReq req;
void InMemoryReplicationHandlers::WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
storage::replication::WalFilesReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
const auto wal_file_number = req.file_number;
spdlog::debug("Received WAL files: {}", wal_file_number);
replication::Decoder decoder(req_reader);
storage::replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->wal_directory_);
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
utils::EnsureDirOrDie(storage->wal_directory_);
for (auto i = 0; i < wal_file_number; ++i) {
LoadWal(storage_, *repl_epoch_, &decoder);
LoadWal(storage, &decoder);
}
replication::WalFilesRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::WalFilesRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::debug("Replication recovery from WAL files ended successfully, replica is now up to date!");
}
void InMemoryReplicationServer::CurrentWalHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::CurrentWalReq req;
void InMemoryReplicationHandlers::CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
storage::replication::CurrentWalReq req;
slk::Load(&req, req_reader);
auto db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::Decoder decoder(req_reader);
storage::replication::Decoder decoder(req_reader);
utils::EnsureDirOrDie(storage_->wal_directory_);
auto *storage = static_cast<storage::InMemoryStorage *>(db_acc->get()->storage());
utils::EnsureDirOrDie(storage->wal_directory_);
LoadWal(storage_, *repl_epoch_, &decoder);
LoadWal(storage, &decoder);
replication::CurrentWalRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
storage::replication::CurrentWalRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
spdlog::debug("Replication recovery from current WAL ended successfully, replica is now up to date!");
}
void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::replication::ReplicationEpoch &epoch,
replication::Decoder *decoder) {
const auto temp_wal_directory = std::filesystem::temp_directory_path() / "memgraph" / durability::kWalDirectory;
void InMemoryReplicationHandlers::LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder) {
const auto temp_wal_directory =
std::filesystem::temp_directory_path() / "memgraph" / storage::durability::kWalDirectory;
utils::EnsureDir(temp_wal_directory);
auto maybe_wal_path = decoder->ReadFile(temp_wal_directory);
MG_ASSERT(maybe_wal_path, "Failed to load WAL!");
spdlog::trace("Received WAL saved to {}", *maybe_wal_path);
try {
auto wal_info = durability::ReadWalInfo(*maybe_wal_path);
auto wal_info = storage::durability::ReadWalInfo(*maybe_wal_path);
if (wal_info.seq_num == 0) {
storage->uuid_ = wal_info.uuid;
}
if (wal_info.epoch_id != epoch.id()) {
auto prev_epoch = epoch.SetEpoch(wal_info.epoch_id);
auto &replica_epoch = storage->repl_storage_state_.epoch_;
if (wal_info.epoch_id != replica_epoch.id()) {
auto prev_epoch = replica_epoch.SetEpoch(wal_info.epoch_id);
storage->repl_storage_state_.AddEpochToHistoryForce(prev_epoch);
}
@@ -259,11 +328,12 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::repl
storage->wal_seq_num_ = wal_info.seq_num;
}
spdlog::trace("Loading WAL deltas from {}", *maybe_wal_path);
durability::Decoder wal;
const auto version = wal.Initialize(*maybe_wal_path, durability::kWalMagic);
storage::durability::Decoder wal;
const auto version = wal.Initialize(*maybe_wal_path, storage::durability::kWalMagic);
spdlog::debug("WAL file {} loaded successfully", *maybe_wal_path);
if (!version) throw durability::RecoveryFailure("Couldn't read WAL magic and/or version!");
if (!durability::IsVersionSupported(*version)) throw durability::RecoveryFailure("Invalid WAL version!");
if (!version) throw storage::durability::RecoveryFailure("Couldn't read WAL magic and/or version!");
if (!storage::durability::IsVersionSupported(*version))
throw storage::durability::RecoveryFailure("Invalid WAL version!");
wal.SetPosition(wal_info.offset_deltas);
for (size_t i = 0; i < wal_info.num_deltas;) {
@@ -271,38 +341,46 @@ void InMemoryReplicationServer::LoadWal(InMemoryStorage *storage, memgraph::repl
}
spdlog::debug("Replication from current WAL successful!");
} catch (const durability::RecoveryFailure &e) {
} catch (const storage::durability::RecoveryFailure &e) {
LOG_FATAL("Couldn't recover WAL deltas from {} because of: {}", *maybe_wal_path, e.what());
}
}
void InMemoryReplicationServer::TimestampHandler(slk::Reader *req_reader, slk::Builder *res_builder) {
replication::TimestampReq req;
void InMemoryReplicationHandlers::TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader,
slk::Builder *res_builder) {
storage::replication::TimestampReq req;
slk::Load(&req, req_reader);
auto const db_acc = GetDatabaseAccessor(dbms_handler, req.db_name);
if (!db_acc) return;
replication::TimestampRes res{true, storage_->repl_storage_state_.last_commit_timestamp_.load()};
// TODO: this handler is agnostic of InMemory, move to be reused by on-disk
auto const *storage = db_acc->get()->storage();
storage::replication::TimestampRes res{storage->id(), true,
storage->repl_storage_state_.last_commit_timestamp_.load()};
slk::Save(res, res_builder);
}
uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage, durability::BaseDecoder *decoder,
const uint64_t version) {
uint64_t InMemoryReplicationHandlers::ReadAndApplyDelta(storage::InMemoryStorage *storage,
storage::durability::BaseDecoder *decoder,
const uint64_t version) {
auto edge_acc = storage->edges_.access();
auto vertex_acc = storage->vertices_.access();
constexpr bool kUniqueAccess = true;
constexpr bool kSharedAccess = false;
std::optional<std::pair<uint64_t, InMemoryStorage::ReplicationAccessor>> commit_timestamp_and_accessor;
std::optional<std::pair<uint64_t, storage::InMemoryStorage::ReplicationAccessor>> commit_timestamp_and_accessor;
auto get_transaction = [storage, &commit_timestamp_and_accessor](uint64_t commit_timestamp,
bool unique = !kUniqueAccess) {
bool unique = kSharedAccess) {
if (!commit_timestamp_and_accessor) {
std::unique_ptr<Storage::Accessor> acc = nullptr;
std::unique_ptr<storage::Storage::Accessor> acc = nullptr;
if (unique) {
acc = storage->UniqueAccess(std::nullopt);
acc = storage->UniqueAccess(std::nullopt, false /*not main*/);
} else {
acc = storage->Access(std::nullopt);
acc = storage->Access(std::nullopt, false /*not main*/);
}
auto inmem_acc = std::unique_ptr<InMemoryStorage::InMemoryAccessor>(
static_cast<InMemoryStorage::InMemoryAccessor *>(acc.release()));
auto inmem_acc = std::unique_ptr<storage::InMemoryStorage::InMemoryAccessor>(
static_cast<storage::InMemoryStorage::InMemoryAccessor *>(acc.release()));
commit_timestamp_and_accessor.emplace(commit_timestamp, std::move(*inmem_acc));
} else if (commit_timestamp_and_accessor->first != commit_timestamp) {
throw utils::BasicException("Received more than one transaction!");
@@ -319,7 +397,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
max_commit_timestamp = timestamp;
}
transaction_complete = durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version);
transaction_complete = storage::durability::IsWalDeltaDataTypeTransactionEnd(delta.type, version);
if (timestamp < storage->timestamp_) {
continue;
@@ -327,13 +405,13 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
SPDLOG_INFO(" Delta {}", applied_deltas);
switch (delta.type) {
case durability::WalDeltaData::Type::VERTEX_CREATE: {
case WalDeltaData::Type::VERTEX_CREATE: {
spdlog::trace(" Create vertex {}", delta.vertex_create_delete.gid.AsUint());
auto *transaction = get_transaction(timestamp);
transaction->CreateVertexEx(delta.vertex_create_delete.gid);
break;
}
case durability::WalDeltaData::Type::VERTEX_DELETE: {
case WalDeltaData::Type::VERTEX_DELETE: {
spdlog::trace(" Delete vertex {}", delta.vertex_create_delete.gid.AsUint());
auto *transaction = get_transaction(timestamp);
auto vertex = transaction->FindVertex(delta.vertex_create_delete.gid, View::NEW);
@@ -342,7 +420,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::VERTEX_ADD_LABEL: {
case WalDeltaData::Type::VERTEX_ADD_LABEL: {
spdlog::trace(" Vertex {} add label {}", delta.vertex_add_remove_label.gid.AsUint(),
delta.vertex_add_remove_label.label);
auto *transaction = get_transaction(timestamp);
@@ -352,7 +430,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
case WalDeltaData::Type::VERTEX_REMOVE_LABEL: {
spdlog::trace(" Vertex {} remove label {}", delta.vertex_add_remove_label.gid.AsUint(),
delta.vertex_add_remove_label.label);
auto *transaction = get_transaction(timestamp);
@@ -362,7 +440,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError() || !ret.GetValue()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::VERTEX_SET_PROPERTY: {
case WalDeltaData::Type::VERTEX_SET_PROPERTY: {
spdlog::trace(" Vertex {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
auto *transaction = get_transaction(timestamp);
@@ -373,7 +451,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EDGE_CREATE: {
case WalDeltaData::Type::EDGE_CREATE: {
spdlog::trace(" Create edge {} of type {} from vertex {} to vertex {}",
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
@@ -388,7 +466,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (edge.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EDGE_DELETE: {
case WalDeltaData::Type::EDGE_DELETE: {
spdlog::trace(" Delete edge {} of type {} from vertex {} to vertex {}",
delta.edge_create_delete.gid.AsUint(), delta.edge_create_delete.edge_type,
delta.edge_create_delete.from_vertex.AsUint(), delta.edge_create_delete.to_vertex.AsUint());
@@ -406,7 +484,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EDGE_SET_PROPERTY: {
case WalDeltaData::Type::EDGE_SET_PROPERTY: {
spdlog::trace(" Edge {} set property {} to {}", delta.vertex_edge_set_property.gid.AsUint(),
delta.vertex_edge_set_property.property, delta.vertex_edge_set_property.value);
if (!storage->config_.items.properties_on_edges)
@@ -469,17 +547,18 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
break;
}
case durability::WalDeltaData::Type::TRANSACTION_END: {
case WalDeltaData::Type::TRANSACTION_END: {
spdlog::trace(" Transaction end");
if (!commit_timestamp_and_accessor || commit_timestamp_and_accessor->first != timestamp)
throw utils::BasicException("Invalid commit data!");
auto ret = commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first);
auto ret =
commit_timestamp_and_accessor->second.Commit(commit_timestamp_and_accessor->first, false /* not main */);
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
commit_timestamp_and_accessor = std::nullopt;
break;
}
case durability::WalDeltaData::Type::LABEL_INDEX_CREATE: {
case WalDeltaData::Type::LABEL_INDEX_CREATE: {
spdlog::trace(" Create label index on :{}", delta.operation_label.label);
// Need to send the timestamp
auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -487,14 +566,14 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_INDEX_DROP: {
case WalDeltaData::Type::LABEL_INDEX_DROP: {
spdlog::trace(" Drop label index on :{}", delta.operation_label.label);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
if (transaction->DropIndex(storage->NameToLabel(delta.operation_label.label)).HasError())
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_INDEX_STATS_SET: {
case WalDeltaData::Type::LABEL_INDEX_STATS_SET: {
spdlog::trace(" Set label index statistics on :{}", delta.operation_label_stats.label);
// Need to send the timestamp
auto *transaction = get_transaction(timestamp);
@@ -506,7 +585,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->SetIndexStats(label, stats);
break;
}
case durability::WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: {
case WalDeltaData::Type::LABEL_INDEX_STATS_CLEAR: {
const auto &info = delta.operation_label;
spdlog::trace(" Clear label index statistics on :{}", info.label);
// Need to send the timestamp
@@ -514,7 +593,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->DeleteLabelIndexStats(storage->NameToLabel(info.label));
break;
}
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_CREATE: {
spdlog::trace(" Create label+property index on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -525,7 +604,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_DROP: {
spdlog::trace(" Drop label+property index on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -536,7 +615,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: {
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_SET: {
const auto &info = delta.operation_label_property_stats;
spdlog::trace(" Set label-property index statistics on :{}", info.label);
// Need to send the timestamp
@@ -550,7 +629,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->SetIndexStats(label, property, stats);
break;
}
case durability::WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: {
case WalDeltaData::Type::LABEL_PROPERTY_INDEX_STATS_CLEAR: {
const auto &info = delta.operation_label;
spdlog::trace(" Clear label-property index statistics on :{}", info.label);
// Need to send the timestamp
@@ -558,7 +637,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
transaction->DeleteLabelPropertyIndexStats(storage->NameToLabel(info.label));
break;
}
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_CREATE: {
spdlog::trace(" Create existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -568,7 +647,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
if (ret.HasError()) throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
case WalDeltaData::Type::EXISTENCE_CONSTRAINT_DROP: {
spdlog::trace(" Drop existence constraint on :{} ({})", delta.operation_label_property.label,
delta.operation_label_property.property);
auto *transaction = get_transaction(timestamp, kUniqueAccess);
@@ -579,7 +658,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
case WalDeltaData::Type::UNIQUE_CONSTRAINT_CREATE: {
std::stringstream ss;
utils::PrintIterable(ss, delta.operation_label_properties.properties);
spdlog::trace(" Create unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
@@ -594,7 +673,7 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
throw utils::BasicException("Invalid transaction!");
break;
}
case durability::WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
case WalDeltaData::Type::UNIQUE_CONSTRAINT_DROP: {
std::stringstream ss;
utils::PrintIterable(ss, delta.operation_label_properties.properties);
spdlog::trace(" Drop unique constraint on :{} ({})", delta.operation_label_properties.label, ss.str());
@@ -621,4 +700,4 @@ uint64_t InMemoryReplicationServer::ReadAndApplyDelta(InMemoryStorage *storage,
return applied_deltas;
}
} // namespace memgraph::storage
} // namespace memgraph::dbms

View File

@@ -0,0 +1,49 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "replication/replication_server.hpp"
#include "replication/state.hpp"
#include "storage/v2/replication/serialization.hpp"
namespace memgraph::storage {
class InMemoryStorage;
}
namespace memgraph::dbms {
class DbmsHandler;
class InMemoryReplicationHandlers {
public:
static void Register(dbms::DbmsHandler *dbms_handler, replication::ReplicationServer &server);
private:
// RPC handlers
static void HeartbeatHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void AppendDeltasHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void SnapshotHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void WalFilesHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void CurrentWalHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void TimestampHandler(dbms::DbmsHandler *dbms_handler, slk::Reader *req_reader, slk::Builder *res_builder);
static void LoadWal(storage::InMemoryStorage *storage, storage::replication::Decoder *decoder);
static uint64_t ReadAndApplyDelta(storage::InMemoryStorage *storage, storage::durability::BaseDecoder *decoder,
uint64_t version);
};
} // namespace memgraph::dbms

View File

@@ -0,0 +1,61 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <variant>
#include "dbms/constants.hpp"
#include "dbms/replication_handler.hpp"
#include "replication/state.hpp"
#include "storage/v2/config.hpp"
#include "storage/v2/inmemory/storage.hpp"
#include "storage/v2/storage.hpp"
namespace memgraph::dbms {
inline std::unique_ptr<storage::Storage> CreateInMemoryStorage(storage::Config config,
::memgraph::replication::ReplicationState &repl_state) {
const auto wal_mode = config.durability.snapshot_wal_mode;
const auto name = config.name;
auto storage = std::make_unique<storage::InMemoryStorage>(std::move(config));
// Connect replication state and storage
storage->CreateSnapshotHandler(
[storage = storage.get(), &repl_state]() -> utils::BasicResult<storage::InMemoryStorage::CreateSnapshotError> {
if (repl_state.IsReplica()) {
return storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica;
}
return storage->CreateSnapshot();
});
if (allow_mt_repl || name == dbms::kDefaultDB) {
// Handle global replication state
spdlog::info("Replication configuration will be stored and will be automatically restored in case of a crash.");
// RECOVER REPLICA CONNECTIONS
memgraph::dbms::RestoreReplication(repl_state, *storage);
} else if (const ::memgraph::replication::RoleMainData *data =
std::get_if<::memgraph::replication::RoleMainData>(&repl_state.ReplicationData());
data && !data->registered_replicas_.empty()) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
if (wal_mode == storage::Config::Durability::SnapshotWalMode::DISABLED && repl_state.IsMain()) {
spdlog::warn(
"The instance has the MAIN replication role, but durability logs and snapshots are disabled. Please consider "
"enabling durability by using --storage-snapshot-interval-sec and --storage-wal-enabled flags because "
"without write-ahead logs this instance is not replicating any data.");
}
return std::move(storage);
}
} // namespace memgraph::dbms

View File

@@ -0,0 +1,34 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "dbms/replication_client.hpp"
namespace memgraph::dbms {
void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client) {
// No client error, start instance level client
auto const &endpoint = client.rpc_client_.Endpoint();
spdlog::trace("Replication client started at: {}:{}", endpoint.address, endpoint.port);
client.StartFrequentCheck([&dbms_handler](std::string_view name) {
// Working connection, check if any database has been left behind
dbms_handler.ForEach([name](dbms::Database *db) {
// Specific database <-> replica client
db->storage()->repl_storage_state_.WithClient(name, [&](storage::ReplicationStorageClient *client) {
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
// Database <-> replica might be behind, check and recover
client->TryCheckReplicaStateAsync(db->storage());
}
});
});
});
}
} // namespace memgraph::dbms

View File

@@ -0,0 +1,21 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "dbms/dbms_handler.hpp"
#include "replication/replication_client.hpp"
namespace memgraph::dbms {
void StartReplicaClient(DbmsHandler &dbms_handler, replication::ReplicationClient &client);
} // namespace memgraph::dbms

View File

@@ -0,0 +1,262 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "dbms/replication_handler.hpp"
#include "dbms/constants.hpp"
#include "dbms/dbms_handler.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "dbms/inmemory/storage_helper.hpp"
#include "dbms/replication_client.hpp"
#include "replication/state.hpp"
using memgraph::replication::ReplicationClientConfig;
using memgraph::replication::ReplicationState;
using memgraph::replication::RoleMainData;
using memgraph::replication::RoleReplicaData;
namespace memgraph::dbms {
namespace {
std::string RegisterReplicaErrorToString(RegisterReplicaError error) {
switch (error) {
using enum RegisterReplicaError;
case NAME_EXISTS:
return "NAME_EXISTS";
case END_POINT_EXISTS:
return "END_POINT_EXISTS";
case CONNECTION_FAILED:
return "CONNECTION_FAILED";
case COULD_NOT_BE_PERSISTED:
return "COULD_NOT_BE_PERSISTED";
}
}
} // namespace
ReplicationHandler::ReplicationHandler(DbmsHandler &dbms_handler) : dbms_handler_(dbms_handler) {}
bool ReplicationHandler::SetReplicationRoleMain() {
auto const main_handler = [](RoleMainData const &) {
// If we are already MAIN, we don't want to change anything
return false;
};
auto const replica_handler = [this](RoleReplicaData const &) {
// STEP 1) bring down all REPLICA servers
dbms_handler_.ForEach([](Database *db) {
auto *storage = db->storage();
// Remember old epoch + storage timestamp association
storage->PrepareForNewEpoch();
});
// STEP 2) Change to MAIN
// TODO: restore replication servers if false?
if (!dbms_handler_.ReplicationState().SetReplicationRoleMain()) {
// TODO: Handle recovery on failure???
return false;
}
// STEP 3) We are now MAIN, update storage local epoch
const auto &epoch =
std::get<RoleMainData>(std::as_const(dbms_handler_.ReplicationState()).ReplicationData()).epoch_;
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
storage->repl_storage_state_.epoch_ = epoch;
});
return true;
};
// TODO: under lock
return std::visit(utils::Overloaded{main_handler, replica_handler},
dbms_handler_.ReplicationState().ReplicationData());
}
bool ReplicationHandler::SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config) {
// We don't want to restart the server if we're already a REPLICA
if (dbms_handler_.ReplicationState().IsReplica()) {
return false;
}
// TODO StorageState needs to be synched. Could have a dangling reference if someone adds a database as we are
// deleting the replica.
// Remove database specific clients
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
storage->repl_storage_state_.replication_clients_.WithLock([](auto &clients) { clients.clear(); });
});
// Remove instance level clients
std::get<RoleMainData>(dbms_handler_.ReplicationState().ReplicationData()).registered_replicas_.clear();
// Creates the server
dbms_handler_.ReplicationState().SetReplicationRoleReplica(config);
// Start
const auto success =
std::visit(utils::Overloaded{[](RoleMainData const &) {
// ASSERT
return false;
},
[this](RoleReplicaData const &data) {
// Register handlers
InMemoryReplicationHandlers::Register(&dbms_handler_, *data.server);
if (!data.server->Start()) {
spdlog::error("Unable to start the replication server.");
return false;
}
return true;
}},
dbms_handler_.ReplicationState().ReplicationData());
// TODO Handle error (restore to main?)
return success;
}
auto ReplicationHandler::RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
-> memgraph::utils::BasicResult<RegisterReplicaError> {
MG_ASSERT(dbms_handler_.ReplicationState().IsMain(), "Only main instance can register a replica!");
auto instance_client = dbms_handler_.ReplicationState().RegisterReplica(config);
if (instance_client.HasError()) switch (instance_client.GetError()) {
case memgraph::replication::RegisterReplicaError::NOT_MAIN:
MG_ASSERT(false, "Only main instance can register a replica!");
return {};
case memgraph::replication::RegisterReplicaError::NAME_EXISTS:
return memgraph::dbms::RegisterReplicaError::NAME_EXISTS;
case memgraph::replication::RegisterReplicaError::END_POINT_EXISTS:
return memgraph::dbms::RegisterReplicaError::END_POINT_EXISTS;
case memgraph::replication::RegisterReplicaError::COULD_NOT_BE_PERSISTED:
return memgraph::dbms::RegisterReplicaError::COULD_NOT_BE_PERSISTED;
case memgraph::replication::RegisterReplicaError::SUCCESS:
break;
}
if (!allow_mt_repl && dbms_handler_.All().size() > 1) {
spdlog::warn("Multi-tenant replication is currently not supported!");
}
bool all_clients_good = true;
// Add database specific clients (NOTE Currently all databases are connected to each replica)
dbms_handler_.ForEach([&](Database *db) {
auto *storage = db->storage();
if (!allow_mt_repl && storage->id() != kDefaultDB) {
return;
}
// TODO: ATM only IN_MEMORY_TRANSACTIONAL, fix other modes
if (storage->storage_mode_ != storage::StorageMode::IN_MEMORY_TRANSACTIONAL) return;
all_clients_good &=
storage->repl_storage_state_.replication_clients_.WithLock([storage, &instance_client](auto &storage_clients) {
auto client = std::make_unique<storage::ReplicationStorageClient>(*instance_client.GetValue());
client->Start(storage);
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
// Failed to start due to branching of MAIN and REPLICA
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
return false;
}
storage_clients.push_back(std::move(client));
return true;
});
});
// NOTE Currently if any databases fails, we revert back
if (!all_clients_good) {
spdlog::error("Failed to register all databases to the REPLICA \"{}\"", config.name);
UnregisterReplica(config.name);
return RegisterReplicaError::CONNECTION_FAILED;
}
// No client error, start instance level client
StartReplicaClient(dbms_handler_, *instance_client.GetValue());
return {};
}
auto ReplicationHandler::UnregisterReplica(std::string_view name) -> UnregisterReplicaResult {
auto const replica_handler = [](RoleReplicaData const &) -> UnregisterReplicaResult {
return UnregisterReplicaResult::NOT_MAIN;
};
auto const main_handler = [this, name](RoleMainData &mainData) -> UnregisterReplicaResult {
if (!dbms_handler_.ReplicationState().TryPersistUnregisterReplica(name)) {
return UnregisterReplicaResult::COULD_NOT_BE_PERSISTED;
}
// Remove database specific clients
dbms_handler_.ForEach([name](Database *db) {
db->storage()->repl_storage_state_.replication_clients_.WithLock([&name](auto &clients) {
std::erase_if(clients, [name](const auto &client) { return client->Name() == name; });
});
});
// Remove instance level clients
auto const n_unregistered =
std::erase_if(mainData.registered_replicas_, [name](auto const &client) { return client.name_ == name; });
return n_unregistered != 0 ? UnregisterReplicaResult::SUCCESS : UnregisterReplicaResult::CAN_NOT_UNREGISTER;
};
return std::visit(utils::Overloaded{main_handler, replica_handler},
dbms_handler_.ReplicationState().ReplicationData());
}
auto ReplicationHandler::GetRole() const -> memgraph::replication::ReplicationRole {
return dbms_handler_.ReplicationState().GetRole();
}
bool ReplicationHandler::IsMain() const { return dbms_handler_.ReplicationState().IsMain(); }
bool ReplicationHandler::IsReplica() const { return dbms_handler_.ReplicationState().IsReplica(); }
// Per storage
// NOTE Storage will connect to all replicas. Future work might change this
void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage) {
spdlog::info("Restoring replication role.");
/// MAIN
auto const recover_main = [&storage](RoleMainData &mainData) {
// Each individual client has already been restored and started. Here we just go through each database and start its
// client
for (auto &instance_client : mainData.registered_replicas_) {
spdlog::info("Replica {} restoration started for {}.", instance_client.name_, storage.id());
const auto &ret = storage.repl_storage_state_.replication_clients_.WithLock(
[&](auto &storage_clients) -> utils::BasicResult<RegisterReplicaError> {
auto client = std::make_unique<storage::ReplicationStorageClient>(instance_client);
client->Start(&storage);
// After start the storage <-> replica state should be READY or RECOVERING (if correctly started)
// MAYBE_BEHIND isn't a statement of the current state, this is the default value
// Failed to start due to branching of MAIN and REPLICA
if (client->State() == storage::replication::ReplicaState::MAYBE_BEHIND) {
spdlog::warn("Connection failed when registering replica {}. Replica will still be registered.",
instance_client.name_);
}
storage_clients.push_back(std::move(client));
return {};
});
if (ret.HasError()) {
MG_ASSERT(RegisterReplicaError::CONNECTION_FAILED != ret.GetError());
LOG_FATAL("Failure when restoring replica {}: {}.", instance_client.name_,
RegisterReplicaErrorToString(ret.GetError()));
}
spdlog::info("Replica {} restored for {}.", instance_client.name_, storage.id());
}
spdlog::info("Replication role restored to MAIN.");
};
/// REPLICA
auto const recover_replica = [](RoleReplicaData const &data) { /*nothing to do*/ };
std::visit(
utils::Overloaded{
recover_main,
recover_replica,
},
repl_state.ReplicationData());
}
} // namespace memgraph::dbms

View File

@@ -12,6 +12,7 @@
#pragma once
#include "replication/role.hpp"
#include "storage/v2/storage.hpp"
#include "utils/result.hpp"
// BEGIN fwd declares
@@ -20,14 +21,10 @@ struct ReplicationState;
struct ReplicationServerConfig;
struct ReplicationClientConfig;
} // namespace memgraph::replication
namespace memgraph::storage {
class Storage;
}
// END fwd declares
namespace memgraph::storage {
namespace memgraph::dbms {
class DbmsHandler;
enum class RegistrationMode : std::uint8_t { MUST_BE_INSTANTLY_VALID, RESTORE };
enum class RegisterReplicaError : uint8_t { NAME_EXISTS, END_POINT_EXISTS, CONNECTION_FAILED, COULD_NOT_BE_PERSISTED };
enum class UnregisterReplicaResult : uint8_t {
NOT_MAIN,
@@ -39,8 +36,7 @@ enum class UnregisterReplicaResult : uint8_t {
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
/// TODO: extend to do multiple storages
struct ReplicationHandler {
ReplicationHandler(memgraph::replication::ReplicationState &replState, Storage &storage)
: repl_state_(replState), storage_(storage) {}
explicit ReplicationHandler(DbmsHandler &dbms_handler);
// as REPLICA, become MAIN
bool SetReplicationRoleMain();
@@ -49,23 +45,23 @@ struct ReplicationHandler {
bool SetReplicationRoleReplica(const memgraph::replication::ReplicationServerConfig &config);
// as MAIN, define and connect to REPLICAs
auto RegisterReplica(RegistrationMode registration_mode, const memgraph::replication::ReplicationClientConfig &config)
auto RegisterReplica(const memgraph::replication::ReplicationClientConfig &config)
-> utils::BasicResult<RegisterReplicaError>;
// as MAIN, remove a REPLICA connection
auto UnregisterReplica(std::string_view name) -> UnregisterReplicaResult;
// Generic restoration
// TODO: decouple storage restoration from epoch restoration
void RestoreReplication();
// Helper pass-through (TODO: remove)
auto GetRole() const -> memgraph::replication::ReplicationRole;
bool IsMain() const;
bool IsReplica() const;
private:
memgraph::replication::ReplicationState &repl_state_;
Storage &storage_;
DbmsHandler &dbms_handler_;
};
} // namespace memgraph::storage
/// A handler type that keep in sync current ReplicationState and the MAIN/REPLICA-ness of Storage
/// TODO: extend to do multiple storages
void RestoreReplication(replication::ReplicationState &repl_state, storage::Storage &storage);
} // namespace memgraph::dbms

View File

@@ -65,7 +65,7 @@ DEFINE_bool(allow_load_csv, true, "Controls whether LOAD CSV clause is allowed i
// Storage flags.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_uint64(storage_gc_cycle_sec, 30, "Storage garbage collector interval (in seconds).",
FLAG_IN_RANGE(1, 24 * 3600));
FLAG_IN_RANGE(1, 24UL * 3600));
// NOTE: The `storage_properties_on_edges` flag must be the same here and in
// `mg_import_csv`. If you change it, make sure to change it there as well.
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)

View File

@@ -8,4 +8,3 @@ target_sources(mg-glue PRIVATE auth.cpp
MonitoringServerT.cpp
run_id.cpp)
target_link_libraries(mg-glue mg-query mg-auth mg-audit mg-flags)
target_precompile_headers(mg-glue INTERFACE auth_checker.hpp auth_handler.hpp)

View File

@@ -10,6 +10,7 @@
// licenses/APL.txt.
#include <optional>
#include <utility>
#include "gflags/gflags.h"
#include "audit/log.hpp"
@@ -272,7 +273,7 @@ void SessionHL::Configure(const std::map<std::string, memgraph::communication::b
#endif
}
SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context,
const memgraph::communication::v2::ServerEndpoint &endpoint,
memgraph::communication::v2::ServerEndpoint endpoint,
memgraph::communication::v2::InputStream *input_stream,
memgraph::communication::v2::OutputStream *output_stream,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth
@@ -289,7 +290,7 @@ SessionHL::SessionHL(memgraph::query::InterpreterContext *interpreter_context,
audit_log_(audit_log),
#endif
auth_(auth),
endpoint_(endpoint),
endpoint_(std::move(endpoint)),
implicit_db_(dbms::kDefaultDB) {
// Metrics update
memgraph::metrics::IncrementCounter(memgraph::metrics::ActiveBoltSessions);

View File

@@ -23,7 +23,7 @@ class SessionHL final : public memgraph::communication::bolt::Session<memgraph::
memgraph::communication::v2::OutputStream> {
public:
SessionHL(memgraph::query::InterpreterContext *interpreter_context,
const memgraph::communication::v2::ServerEndpoint &endpoint,
memgraph::communication::v2::ServerEndpoint endpoint,
memgraph::communication::v2::InputStream *input_stream,
memgraph::communication::v2::OutputStream *output_stream,
memgraph::utils::Synchronized<memgraph::auth::Auth, memgraph::utils::WritePrioritizedRWLock> *auth

View File

@@ -127,6 +127,8 @@ storage::Result<Value> ToBoltValue(const query::TypedValue &value, const storage
return Value(value.ValueLocalDateTime());
case query::TypedValue::Type::Duration:
return Value(value.ValueDuration());
case query::TypedValue::Type::Function:
throw communication::bolt::ValueException("Unsupported conversion from TypedValue::Function to Value");
case query::TypedValue::Type::Graph:
auto maybe_graph = ToBoltGraph(value.ValueGraph(), db, view);
if (maybe_graph.HasError()) return maybe_graph.GetError();
@@ -202,7 +204,7 @@ storage::Result<std::map<std::string, Value>> ToBoltGraph(const query::Graph &gr
for (const auto &v : graph.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(Value(std::move(*maybe_vertex)));
vertices.emplace_back(std::move(*maybe_vertex));
}
map.emplace("nodes", Value(vertices));
@@ -211,7 +213,7 @@ storage::Result<std::map<std::string, Value>> ToBoltGraph(const query::Graph &gr
for (const auto &e : graph.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(Value(std::move(*maybe_edge)));
edges.emplace_back(std::move(*maybe_edge));
}
map.emplace("edges", Value(edges));

View File

@@ -11,6 +11,8 @@
#pragma once
#include <string>
namespace memgraph::glue {
extern const std::string run_id_;
} // namespace memgraph::glue

View File

@@ -31,7 +31,7 @@ inline void LoadConfig(const std::string &product_name) {
std::vector<fs::path> configs = {fs::path("/etc/memgraph/memgraph.conf")};
if (getenv("HOME") != nullptr) configs.emplace_back(fs::path(getenv("HOME")) / fs::path(".memgraph/config"));
{
auto memgraph_config = getenv("MEMGRAPH_CONFIG");
auto *memgraph_config = getenv("MEMGRAPH_CONFIG");
if (memgraph_config != nullptr) {
auto path = fs::path(memgraph_config);
MG_ASSERT(fs::exists(path), "MEMGRAPH_CONFIG environment variable set to nonexisting path: {}",

View File

@@ -23,7 +23,6 @@
#include <utils/event_counter.hpp>
#include <utils/event_gauge.hpp>
#include "storage/v2/storage.hpp"
#include "utils/event_gauge.hpp"
#include "utils/event_histogram.hpp"
namespace memgraph::http {
@@ -65,7 +64,7 @@ class MetricsService {
return MetricsResponse{.vertex_count = info.vertex_count,
.edge_count = info.edge_count,
.average_degree = info.average_degree,
.memory_usage = info.memory_usage,
.memory_usage = info.memory_res,
.disk_usage = info.disk_usage,
.event_counters = GetEventCounters(),
.event_gauges = GetEventGauges(),

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -171,10 +171,10 @@ class Consumer final : public RdKafka::EventCb {
class ConsumerRebalanceCb : public RdKafka::RebalanceCb {
public:
ConsumerRebalanceCb(std::string consumer_name);
explicit ConsumerRebalanceCb(std::string consumer_name);
void rebalance_cb(RdKafka::KafkaConsumer *consumer, RdKafka::ErrorCode err,
std::vector<RdKafka::TopicPartition *> &partitions) override final;
std::vector<RdKafka::TopicPartition *> &partitions) final;
void set_offset(int64_t offset);

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -67,7 +67,7 @@ utils::BasicResult<std::string, std::vector<Message>> GetBatch(TConsumer &consum
return std::move(batch);
case pulsar_client::Result::ResultOk:
if (message.getMessageId() != last_message_id) {
batch.emplace_back(Message{std::move(message)});
batch.emplace_back(std::move(message));
}
break;
default:

View File

@@ -48,8 +48,8 @@ struct Endpoint {
uint16_t port{0};
IpFamily family{IpFamily::NONE};
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrAddress(
const std::string &address, const std::optional<uint16_t> default_port);
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrAddress(const std::string &address,
std::optional<uint16_t> default_port);
/**
* Tries to parse the given string as either a socket address or ip address.
@@ -61,8 +61,8 @@ struct Endpoint {
* it into an ip address and a port number; even if a default port is given,
* it won't be used, as we expect that it is given in the address string.
*/
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrIpAddress(
const std::string &address, const std::optional<uint16_t> default_port);
static std::optional<std::pair<std::string, uint16_t>> ParseSocketOrIpAddress(const std::string &address,
std::optional<uint16_t> default_port);
/**
* Tries to parse given string as either socket address or hostname.
@@ -72,7 +72,7 @@ struct Endpoint {
* After we parse hostname and port we try to resolve the hostname into an ip_address.
*/
static std::optional<std::pair<std::string, uint16_t>> ParseHostname(const std::string &address,
const std::optional<uint16_t> default_port);
std::optional<uint16_t> default_port);
static IpFamily GetIpFamily(const std::string &address);

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -32,7 +32,7 @@ class Epoll {
public:
using Event = struct epoll_event;
Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) {
explicit Epoll(bool set_cloexec = false) : epoll_fd_(epoll_create1(set_cloexec ? EPOLL_CLOEXEC : 0)) {
// epoll_create1 returns an error if there is a logical error in our code
// (for example invalid flags) or if there is irrecoverable error. In both
// cases it is best to terminate.

View File

@@ -216,7 +216,7 @@ bool Socket::Write(const uint8_t *data, size_t len, bool have_more) {
return true;
}
bool Socket::Write(const std::string &s, bool have_more) {
bool Socket::Write(std::string_view s, bool have_more) {
return Write(reinterpret_cast<const uint8_t *>(s.data()), s.size(), have_more);
}

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,6 +14,7 @@
#include <functional>
#include <iostream>
#include <optional>
#include <utility>
#include "io/network/endpoint.hpp"
@@ -149,7 +150,7 @@ class Socket {
* false if write failed
*/
bool Write(const uint8_t *data, size_t len, bool have_more = false);
bool Write(const std::string &s, bool have_more = false);
bool Write(std::string_view s, bool have_more = false);
/**
* Read data from the socket.
@@ -201,7 +202,7 @@ class Socket {
bool WaitForReadyWrite();
private:
Socket(int fd, const Endpoint &endpoint) : socket_(fd), endpoint_(endpoint) {}
Socket(int fd, Endpoint endpoint) : socket_(fd), endpoint_(std::move(endpoint)) {}
int socket_ = -1;
Endpoint endpoint_;

View File

@@ -128,7 +128,7 @@ KVStore::iterator::iterator(const KVStore *kvstore, const std::string &prefix, b
KVStore::iterator::iterator(KVStore::iterator &&other) { pimpl_ = std::move(other.pimpl_); }
KVStore::iterator::~iterator() {}
KVStore::iterator::~iterator() = default;
KVStore::iterator &KVStore::iterator::operator=(KVStore::iterator &&other) {
pimpl_ = std::move(other.pimpl_);

View File

@@ -42,7 +42,7 @@ void LicenseInfoSender::SendData() {
license_info_.WithLock([&data, this](const auto &license_info) mutable {
if (license_info && !license_info->organization_name.empty()) {
const auto memory_info = utils::GetMemoryInfo();
const auto memory_usage = utils::GetMemoryUsage();
const auto memory_res = utils::GetMemoryRES();
data = {{"run_id", uuid_},
{"machine_id", machine_id_},
{"type", "license-check"},
@@ -52,7 +52,7 @@ void LicenseInfoSender::SendData() {
{"valid", license_info->is_valid},
{"physical_memory_size", memory_info.memory},
{"swap_memory_size", memory_info.swap},
{"memory_used", memory_usage},
{"memory_used", memory_res},
{"runtime_memory_limit", memory_limit_},
{"license_memory_limit", license_info->license.memory_limit},
{"timestamp", utils::Timestamp::Now().SecWithNsecSinceTheEpoch()}};

View File

@@ -9,11 +9,13 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <cstdint>
#include "audit/log.hpp"
#include "communication/metrics.hpp"
#include "communication/websocket/auth.hpp"
#include "communication/websocket/server.hpp"
#include "dbms/constants.hpp"
#include "dbms/inmemory/replication_handlers.hpp"
#include "flags/all.hpp"
#include "flags/run_time_configurable.hpp"
#include "glue/MonitoringServerT.hpp"
@@ -43,9 +45,11 @@
#include "query/auth_query_handler.hpp"
#include "query/interpreter_context.hpp"
namespace {
constexpr const char *kMgUser = "MEMGRAPH_USER";
constexpr const char *kMgPassword = "MEMGRAPH_PASSWORD";
constexpr const char *kMgPassfile = "MEMGRAPH_PASSFILE";
constexpr uint64_t kMgVmMaxMapCount = 262144;
// TODO: move elsewhere so that we can remove need of interpreter.hpp
void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbms::DatabaseAccess &db_acc,
@@ -61,10 +65,13 @@ void InitFromCypherlFile(memgraph::query::InterpreterContext &ctx, memgraph::dbm
std::string line;
while (std::getline(file, line)) {
if (!line.empty()) {
auto results = interpreter.Prepare(line, {}, {});
memgraph::query::DiscardValueResultStream stream;
interpreter.Pull(&stream, {}, results.qid);
try {
auto results = interpreter.Prepare(line, {}, {});
memgraph::query::DiscardValueResultStream stream;
interpreter.Pull(&stream, {}, results.qid);
} catch (const memgraph::query::UserAlreadyExistsException &e) {
spdlog::warn("{} The rest of the init-file will be run.", e.what());
}
if (audit_log) {
audit_log->Record("", "", line, {}, memgraph::dbms::kDefaultDB);
}
@@ -107,6 +114,7 @@ void InitSignalHandlers(const std::function<void()> &shutdown_fun) {
block_shutdown_signals),
"Unable to register SIGINT handler!");
}
} // namespace
int main(int argc, char **argv) {
memgraph::memory::SetHooks();
@@ -204,6 +212,17 @@ int main(int argc, char **argv) {
std::cout << "You are running Memgraph v" << gflags::VersionString() << std::endl;
std::cout << "To get started with Memgraph, visit https://memgr.ph/start" << std::endl;
const auto vm_max_map_count = memgraph::utils::GetVmMaxMapCount();
if (vm_max_map_count.has_value()) {
if (vm_max_map_count.value() < kMgVmMaxMapCount) {
std::cout << "Max virtual memory areas vm.max_map_count " << vm_max_map_count.value()
<< " is too low, increase to at least " << kMgVmMaxMapCount << std::endl;
}
} else {
std::cout << "Can't get info on vm.max_map_count, check whether it is too low, vm.max_map_count is at least "
<< kMgVmMaxMapCount << std::endl;
}
auto data_directory = std::filesystem::path(FLAGS_data_directory);
memgraph::utils::EnsureDirOrDie(data_directory);
@@ -352,20 +371,16 @@ int main(int argc, char **argv) {
std::unique_ptr<memgraph::query::AuthChecker> auth_checker;
auth_glue(&auth_, auth_handler, auth_checker);
memgraph::dbms::DbmsHandler dbms_handler(db_config
#ifdef MG_ENTERPRISE
memgraph::dbms::DbmsHandler new_handler(db_config, &auth_, FLAGS_data_recovery_on_startup,
FLAGS_storage_delete_on_drop);
auto db_acc = new_handler.Get(memgraph::dbms::kDefaultDB);
memgraph::query::InterpreterContext interpreter_context_(interp_config, &new_handler, auth_handler.get(),
auth_checker.get());
#else
memgraph::utils::Gatekeeper<memgraph::dbms::Database> db_gatekeeper{db_config};
auto db_acc_opt = db_gatekeeper.access();
MG_ASSERT(db_acc_opt, "Failed to access the main database");
auto &db_acc = *db_acc_opt;
memgraph::query::InterpreterContext interpreter_context_(interp_config, &db_gatekeeper, auth_handler.get(),
auth_checker.get());
,
&auth_, FLAGS_data_recovery_on_startup, FLAGS_storage_delete_on_drop
#endif
);
auto db_acc = dbms_handler.Get();
memgraph::query::InterpreterContext interpreter_context_(
interp_config, &dbms_handler, &dbms_handler.ReplicationState(), auth_handler.get(), auth_checker.get());
MG_ASSERT(db_acc, "Failed to access the main database");
memgraph::query::procedure::gModuleRegistry.SetModulesDirectory(memgraph::flags::ParseQueryModulesDirectory(),
@@ -388,8 +403,8 @@ int main(int argc, char **argv) {
}
#ifdef MG_ENTERPRISE
new_handler.RestoreTriggers(&interpreter_context_);
new_handler.RestoreStreams(&interpreter_context_);
dbms_handler.RestoreTriggers(&interpreter_context_);
dbms_handler.RestoreStreams(&interpreter_context_);
#else
{
// Triggers can execute query procedures, so we need to reload the modules first and then
@@ -432,11 +447,10 @@ int main(int argc, char **argv) {
if (FLAGS_telemetry_enabled) {
telemetry.emplace(telemetry_server, data_directory / "telemetry", memgraph::glue::run_id_, machine_id,
service_name == "BoltS", FLAGS_data_directory, std::chrono::minutes(10));
telemetry->AddStorageCollector(dbms_handler, auth_);
#ifdef MG_ENTERPRISE
telemetry->AddStorageCollector(new_handler, auth_);
telemetry->AddDatabaseCollector(new_handler);
telemetry->AddDatabaseCollector(dbms_handler);
#else
telemetry->AddStorageCollector(db_gatekeeper, auth_);
telemetry->AddDatabaseCollector();
#endif
telemetry->AddClientCollector();
@@ -496,17 +510,16 @@ int main(int argc, char **argv) {
if (!FLAGS_init_data_file.empty()) {
spdlog::info("Running init data file.");
auto db_acc = dbms_handler.Get();
MG_ASSERT(db_acc, "Failed to gain access to the main database");
#ifdef MG_ENTERPRISE
auto db_acc = new_handler.Get(memgraph::dbms::kDefaultDB);
if (memgraph::license::global_license_checker.IsEnterpriseValidFast()) {
InitFromCypherlFile(interpreter_context_, db_acc, FLAGS_init_data_file, &audit_log);
} else {
InitFromCypherlFile(interpreter_context_, db_acc, FLAGS_init_data_file);
}
#else
auto db_acc_2 = db_gatekeeper.access();
MG_ASSERT(db_acc_2, "Failed to gain access to the main database");
InitFromCypherlFile(interpreter_context_, *db_acc_2, FLAGS_init_data_file);
InitFromCypherlFile(interpreter_context_, db_acc, FLAGS_init_data_file);
#endif
}

View File

@@ -3,14 +3,14 @@ set(memory_src_files
global_memory_control.cpp
query_memory_control.cpp)
find_package(jemalloc REQUIRED)
add_library(mg-memory STATIC ${memory_src_files})
target_link_libraries(mg-memory mg-utils fmt)
message(STATUS "ENABLE_JEMALLOC: ${ENABLE_JEMALLOC}")
if (ENABLE_JEMALLOC)
find_package(jemalloc REQUIRED)
target_link_libraries(mg-memory Jemalloc::Jemalloc ${CMAKE_DL_LIBS})
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=1)
else()
target_compile_definitions(mg-memory PRIVATE USE_JEMALLOC=0)
endif()

View File

@@ -61,11 +61,8 @@ void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t
// This needs to be before, to throw exception in case of too big alloc
if (*commit) [[likely]] {
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(size));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Alloc(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackAllocOnCurrentThread(size);
}
}
@@ -73,11 +70,8 @@ void *my_alloc(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t
if (ptr == nullptr) [[unlikely]] {
if (*commit) {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Free(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
}
return ptr;
@@ -96,11 +90,8 @@ static bool my_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, boo
if (committed) [[likely]] {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Free(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
}
@@ -110,11 +101,8 @@ static bool my_dalloc(extent_hooks_t *extent_hooks, void *addr, size_t size, boo
static void my_destroy(extent_hooks_t *extent_hooks, void *addr, size_t size, bool committed, unsigned arena_ind) {
if (committed) [[likely]] {
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(size));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Free(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
}
@@ -130,11 +118,8 @@ static bool my_commit(extent_hooks_t *extent_hooks, void *addr, size_t size, siz
}
memgraph::utils::total_memory_tracker.Alloc(static_cast<int64_t>(length));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Alloc(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
return false;
@@ -150,11 +135,8 @@ static bool my_decommit(extent_hooks_t *extent_hooks, void *addr, size_t size, s
}
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(length));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Free(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
return false;
@@ -170,11 +152,8 @@ static bool my_purge_forced(extent_hooks_t *extent_hooks, void *addr, size_t siz
}
memgraph::utils::total_memory_tracker.Free(static_cast<int64_t>(length));
if (GetQueriesMemoryControl().IsArenaTracked(arena_ind)) [[unlikely]] {
auto *memory_tracker = GetQueriesMemoryControl().GetTrackerCurrentThread();
if (memory_tracker != nullptr) [[likely]] {
memory_tracker->Alloc(static_cast<int64_t>(size));
}
if (GetQueriesMemoryControl().IsThreadTracked()) [[unlikely]] {
GetQueriesMemoryControl().TrackFreeOnCurrentThread(size);
}
return false;
@@ -200,8 +179,11 @@ void SetHooks() {
return;
}
// Needs init due to error we might encounter otherwise
// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=13684
[[maybe_unused]] const auto &queries_memory_control = GetQueriesMemoryControl();
for (int i = 0; i < n_arenas; i++) {
GetQueriesMemoryControl().InitializeArenaCounter(i);
std::string func_name = "arena." + std::to_string(i) + ".extent_hooks";
size_t hooks_len = sizeof(old_hooks);
@@ -261,7 +243,6 @@ void UnsetHooks() {
}
for (int i = 0; i < n_arenas; i++) {
GetQueriesMemoryControl().InitializeArenaCounter(i);
std::string func_name = "arena." + std::to_string(i) + ".extent_hooks";
MG_ASSERT(old_hooks);

View File

@@ -10,6 +10,7 @@
// licenses/APL.txt.
#include <atomic>
#include <cassert>
#include <cstdint>
#include <iostream>
#include <optional>
@@ -21,6 +22,7 @@
#include "query_memory_control.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
#include "utils/memory_tracker.hpp"
#include "utils/rw_spin_lock.hpp"
@@ -32,20 +34,6 @@ namespace memgraph::memory {
#if USE_JEMALLOC
unsigned QueriesMemoryControl::GetArenaForThread() {
unsigned thread_arena{0};
size_t size_thread_arena = sizeof(thread_arena);
int err = mallctl("thread.arena", &thread_arena, &size_thread_arena, nullptr, 0);
if (err) {
LOG_FATAL("Can't get arena for thread.");
}
return thread_arena;
}
void QueriesMemoryControl::AddTrackingOnArena(unsigned arena_id) { arena_tracking[arena_id].fetch_add(1); }
void QueriesMemoryControl::RemoveTrackingOnArena(unsigned arena_id) { arena_tracking[arena_id].fetch_sub(1); }
void QueriesMemoryControl::UpdateThreadToTransactionId(const std::thread::id &thread_id, uint64_t transaction_id) {
auto accessor = thread_id_to_transaction_id.access();
accessor.insert({thread_id, transaction_id});
@@ -58,34 +46,58 @@ void QueriesMemoryControl::EraseThreadToTransactionId(const std::thread::id &thr
accessor.remove(thread_id);
}
utils::MemoryTracker *QueriesMemoryControl::GetTrackerCurrentThread() {
void QueriesMemoryControl::TrackAllocOnCurrentThread(size_t size) {
auto thread_id_to_transaction_id_accessor = thread_id_to_transaction_id.access();
// we might be just constructing mapping between thread id and transaction id
// so we miss this allocation
auto thread_id_to_transaction_id_elem = thread_id_to_transaction_id_accessor.find(std::this_thread::get_id());
if (thread_id_to_transaction_id_elem == thread_id_to_transaction_id_accessor.end()) {
return nullptr;
return;
}
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
auto transaction_id_to_tracker =
transaction_id_to_tracker_accessor.find(thread_id_to_transaction_id_elem->transaction_id);
return &transaction_id_to_tracker->tracker;
// It can happen that some allocation happens between mapping thread to
// transaction id, so we miss this allocation
if (transaction_id_to_tracker == transaction_id_to_tracker_accessor.end()) [[unlikely]] {
return;
}
auto &query_tracker = transaction_id_to_tracker->tracker;
query_tracker.TrackAlloc(size);
}
void QueriesMemoryControl::TrackFreeOnCurrentThread(size_t size) {
auto thread_id_to_transaction_id_accessor = thread_id_to_transaction_id.access();
// we might be just constructing mapping between thread id and transaction id
// so we miss this allocation
auto thread_id_to_transaction_id_elem = thread_id_to_transaction_id_accessor.find(std::this_thread::get_id());
if (thread_id_to_transaction_id_elem == thread_id_to_transaction_id_accessor.end()) {
return;
}
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
auto transaction_id_to_tracker =
transaction_id_to_tracker_accessor.find(thread_id_to_transaction_id_elem->transaction_id);
// It can happen that some allocation happens between mapping thread to
// transaction id, so we miss this allocation
if (transaction_id_to_tracker == transaction_id_to_tracker_accessor.end()) [[unlikely]] {
return;
}
auto &query_tracker = transaction_id_to_tracker->tracker;
query_tracker.TrackFree(size);
}
void QueriesMemoryControl::CreateTransactionIdTracker(uint64_t transaction_id, size_t inital_limit) {
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
auto [elem, result] = transaction_id_to_tracker_accessor.insert({transaction_id, utils::MemoryTracker{}});
auto [elem, result] = transaction_id_to_tracker_accessor.insert({transaction_id, utils::QueryMemoryTracker{}});
elem->tracker.SetMaximumHardLimit(inital_limit);
elem->tracker.SetHardLimit(inital_limit);
}
bool QueriesMemoryControl::CheckTransactionIdTrackerExists(uint64_t transaction_id) {
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
return transaction_id_to_tracker_accessor.contains(transaction_id);
elem->tracker.SetQueryLimit(inital_limit);
}
bool QueriesMemoryControl::EraseTransactionIdTracker(uint64_t transaction_id) {
@@ -94,27 +106,68 @@ bool QueriesMemoryControl::EraseTransactionIdTracker(uint64_t transaction_id) {
return removed;
}
bool QueriesMemoryControl::IsArenaTracked(unsigned arena_ind) {
return arena_tracking[arena_ind].load(std::memory_order_acquire) != 0;
bool QueriesMemoryControl::CheckTransactionIdTrackerExists(uint64_t transaction_id) {
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
return transaction_id_to_tracker_accessor.contains(transaction_id);
}
void QueriesMemoryControl::InitializeArenaCounter(unsigned arena_ind) {
arena_tracking[arena_ind].store(0, std::memory_order_relaxed);
void QueriesMemoryControl::TryCreateTransactionProcTracker(uint64_t transaction_id, int64_t procedure_id,
size_t limit) {
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
auto query_tracker = transaction_id_to_tracker_accessor.find(transaction_id);
if (query_tracker == transaction_id_to_tracker_accessor.end()) {
return;
}
query_tracker->tracker.TryCreateProcTracker(procedure_id, limit);
}
void QueriesMemoryControl::SetActiveProcIdTracker(uint64_t transaction_id, int64_t procedure_id) {
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
auto query_tracker = transaction_id_to_tracker_accessor.find(transaction_id);
if (query_tracker == transaction_id_to_tracker_accessor.end()) {
return;
}
query_tracker->tracker.SetActiveProc(procedure_id);
}
void QueriesMemoryControl::PauseProcedureTracking(uint64_t transaction_id) {
auto transaction_id_to_tracker_accessor = transaction_id_to_tracker.access();
auto query_tracker = transaction_id_to_tracker_accessor.find(transaction_id);
if (query_tracker == transaction_id_to_tracker_accessor.end()) {
return;
}
query_tracker->tracker.StopProcTracking();
}
inline int &Get_Thread_Tracker() {
// store variable in bss segment for each thread
// https://cs-fundamentals.com/c-programming/memory-layout-of-c-program-code-data-segments#size-of-code-data-bss-segments
static thread_local int is_thread_tracked{0};
return is_thread_tracked;
}
bool QueriesMemoryControl::IsThreadTracked() { return Get_Thread_Tracker() == 1; }
#endif
void StartTrackingCurrentThreadTransaction(uint64_t transaction_id) {
#if USE_JEMALLOC
Get_Thread_Tracker() = 0;
GetQueriesMemoryControl().UpdateThreadToTransactionId(std::this_thread::get_id(), transaction_id);
GetQueriesMemoryControl().AddTrackingOnArena(QueriesMemoryControl::GetArenaForThread());
Get_Thread_Tracker() = 1;
#endif
}
void StopTrackingCurrentThreadTransaction(uint64_t transaction_id) {
#if USE_JEMALLOC
Get_Thread_Tracker() = 0;
GetQueriesMemoryControl().EraseThreadToTransactionId(std::this_thread::get_id(), transaction_id);
GetQueriesMemoryControl().RemoveTrackingOnArena(QueriesMemoryControl::GetArenaForThread());
#endif
}
@@ -137,4 +190,29 @@ void TryStopTrackingOnTransaction(uint64_t transaction_id) {
#endif
}
#if USE_JEMALLOC
bool IsTransactionTracked(uint64_t transaction_id) {
return GetQueriesMemoryControl().CheckTransactionIdTrackerExists(transaction_id);
}
#else
bool IsTransactionTracked(uint64_t /*transaction_id*/) { return false; }
#endif
void CreateOrContinueProcedureTracking(uint64_t transaction_id, int64_t procedure_id, size_t limit) {
#if USE_JEMALLOC
if (!GetQueriesMemoryControl().CheckTransactionIdTrackerExists(transaction_id)) {
LOG_FATAL("Memory tracker for transaction was not set");
}
GetQueriesMemoryControl().TryCreateTransactionProcTracker(transaction_id, procedure_id, limit);
GetQueriesMemoryControl().SetActiveProcIdTracker(transaction_id, procedure_id);
#endif
}
void PauseProcedureTracking(uint64_t transaction_id) {
#if USE_JEMALLOC
GetQueriesMemoryControl().PauseProcedureTracking(transaction_id);
#endif
}
} // namespace memgraph::memory

View File

@@ -16,10 +16,13 @@
#include <unordered_map>
#include "utils/memory_tracker.hpp"
#include "utils/query_memory_tracker.hpp"
#include "utils/skip_list.hpp"
namespace memgraph::memory {
static constexpr int64_t UNLIMITED_MEMORY{0};
#if USE_JEMALLOC
// Track memory allocations per query.
@@ -30,25 +33,6 @@ namespace memgraph::memory {
// it is necessary to restart tracking at the beginning of new query for that transaction.
class QueriesMemoryControl {
public:
/*
Arena stats
*/
static unsigned GetArenaForThread();
// Add counter on threads allocating inside arena
void AddTrackingOnArena(unsigned);
// Remove counter on threads allocating in arena
void RemoveTrackingOnArena(unsigned);
// Are any threads using current arena for allocations
// Multiple threads can allocate inside one arena
bool IsArenaTracked(unsigned);
// Initialize arena counter
void InitializeArenaCounter(unsigned);
/*
Transaction id <-> tracker
*/
@@ -75,20 +59,25 @@ class QueriesMemoryControl {
// Important to reset if one thread gets reused for different transaction
void EraseThreadToTransactionId(const std::thread::id &, uint64_t);
// C-API functionality for thread to transaction mapping
void UpdateThreadToTransactionId(const char *, uint64_t);
// Find tracker for current thread if exists, track
// query allocation and procedure allocation if
// necessary
void TrackAllocOnCurrentThread(size_t size);
// C-API functionality for thread to transaction unmapping
void EraseThreadToTransactionId(const char *, uint64_t);
// Find tracker for current thread if exists, track
// query allocation and procedure allocation if
// necessary
void TrackFreeOnCurrentThread(size_t size);
// Get tracker to current thread if exists, otherwise return
// nullptr. This can happen only if tracker is still
// being constructed.
utils::MemoryTracker *GetTrackerCurrentThread();
void TryCreateTransactionProcTracker(uint64_t, int64_t, size_t);
void SetActiveProcIdTracker(uint64_t, int64_t);
void PauseProcedureTracking(uint64_t);
bool IsThreadTracked();
private:
std::unordered_map<unsigned, std::atomic<int>> arena_tracking;
struct ThreadIdToTransactionId {
std::thread::id thread_id;
uint64_t transaction_id;
@@ -102,7 +91,7 @@ class QueriesMemoryControl {
struct TransactionIdToTracker {
uint64_t transaction_id;
utils::MemoryTracker tracker;
utils::QueryMemoryTracker tracker;
bool operator<(const TransactionIdToTracker &other) const { return transaction_id < other.transaction_id; }
bool operator==(const TransactionIdToTracker &other) const { return transaction_id == other.transaction_id; }
@@ -138,4 +127,15 @@ void TryStartTrackingOnTransaction(uint64_t transaction_id, size_t limit);
// Does nothing if jemalloc is not enabled. Does nothing if tracker doesn't exist
void TryStopTrackingOnTransaction(uint64_t transaction_id);
// Is transaction with given id tracked in memory tracker
bool IsTransactionTracked(uint64_t transaction_id);
// Creates tracker on procedure if doesn't exist. Sets query tracker
// to track procedure with id.
void CreateOrContinueProcedureTracking(uint64_t transaction_id, int64_t procedure_id, size_t limit);
// Pauses procedure tracking. This enables to continue
// tracking on procedure once procedure execution resumes.
void PauseProcedureTracking(uint64_t transaction_id);
} // namespace memgraph::memory

View File

@@ -19,7 +19,9 @@
#include <regex>
#include <unordered_map>
#include "dbms/inmemory/storage_helper.hpp"
#include "helpers.hpp"
#include "replication/state.hpp"
#include "storage/v2/config.hpp"
#include "storage/v2/edge_accessor.hpp"
#include "storage/v2/inmemory/storage.hpp"
@@ -702,14 +704,16 @@ int main(int argc, char *argv[]) {
}
std::unordered_map<NodeId, memgraph::storage::Gid> node_id_map;
auto store = std::make_unique<memgraph::storage::InMemoryStorage>(memgraph::storage::Config{
memgraph::storage::Config config{
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
.durability = {.storage_directory = FLAGS_data_directory,
.recover_on_startup = false,
.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::DISABLED,
.snapshot_on_exit = true},
});
};
memgraph::replication::ReplicationState repl_state{memgraph::storage::ReplicationStateRootPath(config)};
auto store = memgraph::dbms::CreateInMemoryStorage(config, repl_state);
memgraph::utils::Timer load_timer;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -62,6 +62,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b) {
case TypedValue::Type::Edge:
case TypedValue::Type::Path:
case TypedValue::Type::Graph:
case TypedValue::Type::Function:
throw QueryRuntimeException("Comparison is not defined for values of type {}.", a.type());
case TypedValue::Type::Null:
LOG_FATAL("Invalid type");

View File

@@ -41,7 +41,7 @@ bool TypedValueCompare(const TypedValue &a, const TypedValue &b);
/// the define how respective elements compare.
class TypedValueVectorCompare final {
public:
TypedValueVectorCompare() {}
TypedValueVectorCompare() = default;
explicit TypedValueVectorCompare(const std::vector<Ordering> &ordering) : ordering_(ordering) {}
template <class TAllocator>
@@ -147,8 +147,8 @@ concept AccessorWithUpdateProperties = requires(T accessor,
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <AccessorWithUpdateProperties T>
auto UpdatePropertiesChecked(T *record, std::map<storage::PropertyId, storage::PropertyValue> &properties) ->
typename std::remove_reference<decltype(record->UpdateProperties(properties).GetValue())>::type {
auto UpdatePropertiesChecked(T *record, std::map<storage::PropertyId, storage::PropertyValue> &properties)
-> std::remove_reference_t<decltype(record->UpdateProperties(properties).GetValue())> {
try {
auto maybe_values = record->UpdateProperties(properties);
if (maybe_values.HasError()) {

View File

@@ -12,15 +12,16 @@
#include "query/cypher_query_interpreter.hpp"
#include "query/frontend/ast/cypher_main_visitor.hpp"
#include "query/frontend/opencypher/parser.hpp"
#include "utils/synchronized.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
DEFINE_VALIDATED_int32(query_plan_cache_max_size, 1000, "Maximum number of query plans to cache.",
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
namespace memgraph::query {
CachedPlan::CachedPlan(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
PlanWrapper::PlanWrapper(std::unique_ptr<LogicalPlan> plan) : plan_(std::move(plan)) {}
ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::string, storage::PropertyValue> &params,
utils::SkipList<QueryCacheEntry> *cache, const InterpreterConfig::Query &query_config) {
@@ -127,28 +128,24 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
std::move(symbol_table));
}
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
std::shared_ptr<PlanWrapper> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, PlanCacheLRU *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
if (plan_cache) {
plan_cache_access.emplace(plan_cache->access());
auto it = plan_cache_access->find(hash);
if (it != plan_cache_access->end()) {
if (it->second->IsExpired()) {
plan_cache_access->remove(hash);
} else {
return it->second;
}
auto existing_plan = plan_cache->WithLock([&](auto &cache) { return cache.get(hash); });
if (existing_plan.has_value()) {
return existing_plan.value();
}
}
auto plan = std::make_shared<CachedPlan>(
auto plan = std::make_shared<PlanWrapper>(
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
if (plan_cache_access) {
plan_cache_access->insert({hash, plan});
if (plan_cache) {
plan_cache->WithLock([&](auto &cache) { cache.put(hash, plan); });
}
return plan;
}
} // namespace memgraph::query

View File

@@ -11,18 +11,22 @@
#pragma once
#include <utility>
#include "query/config.hpp"
#include "query/frontend/semantic/required_privileges.hpp"
#include "query/frontend/semantic/symbol_generator.hpp"
#include "query/frontend/stripped.hpp"
#include "query/plan/planner.hpp"
#include "utils/flag_validation.hpp"
#include "utils/lru_cache.hpp"
#include "utils/synchronized.hpp"
#include "utils/timer.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(query_cost_planner);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int32(query_plan_cache_ttl);
DECLARE_int32(query_plan_cache_max_size);
namespace memgraph::query {
@@ -45,23 +49,17 @@ class LogicalPlan {
virtual const AstStorage &GetAstStorage() const = 0;
};
class CachedPlan {
class PlanWrapper {
public:
explicit CachedPlan(std::unique_ptr<LogicalPlan> plan);
explicit PlanWrapper(std::unique_ptr<LogicalPlan> plan);
const auto &plan() const { return plan_->GetRoot(); }
double cost() const { return plan_->GetCost(); }
const auto &symbol_table() const { return plan_->GetSymbolTable(); }
const auto &ast_storage() const { return plan_->GetAstStorage(); }
bool IsExpired() const {
// NOLINTNEXTLINE (modernize-use-nullptr)
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
};
private:
std::unique_ptr<LogicalPlan> plan_;
utils::Timer cache_timer_;
};
struct CachedQuery {
@@ -82,18 +80,6 @@ struct QueryCacheEntry {
CachedQuery second;
};
struct PlanCacheEntry {
bool operator==(const PlanCacheEntry &other) const { return first == other.first; }
bool operator<(const PlanCacheEntry &other) const { return first < other.first; }
bool operator==(const uint64_t &other) const { return first == other; }
bool operator<(const uint64_t &other) const { return first < other; }
uint64_t first;
// TODO: Maybe store the query string here and use it as a key with the hash
// so that we eliminate the risk of hash collisions.
std::shared_ptr<CachedPlan> second;
};
/**
* A container for data related to the parsing of a query.
*/
@@ -114,8 +100,8 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
class SingleNodeLogicalPlan final : public LogicalPlan {
public:
SingleNodeLogicalPlan(std::unique_ptr<plan::LogicalOperator> root, double cost, AstStorage storage,
const SymbolTable &symbol_table)
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(symbol_table) {}
SymbolTable symbol_table)
: root_(std::move(root)), cost_(cost), storage_(std::move(storage)), symbol_table_(std::move(symbol_table)) {}
const plan::LogicalOperator &GetRoot() const override { return *root_; }
double GetCost() const override { return cost_; }
@@ -129,6 +115,9 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
SymbolTable symbol_table_;
};
using PlanCacheLRU =
utils::Synchronized<utils::LRUCache<uint64_t, std::shared_ptr<query::PlanWrapper>>, utils::RWSpinLock>;
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers);
@@ -141,9 +130,9 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
* If an identifier is contained there, we inject it at that place and remove it,
* because a predefined identifier can be used only in one scope.
*/
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers = {});
std::shared_ptr<PlanWrapper> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, PlanCacheLRU *plan_cache,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers = {});
} // namespace memgraph::query

View File

@@ -139,6 +139,8 @@ std::optional<VertexAccessor> SubgraphDbAccessor::FindVertex(storage::Gid gid, s
query::Graph *SubgraphDbAccessor::getGraph() { return graph_; }
DbAccessor *SubgraphDbAccessor::GetAccessor() { return &db_accessor_; }
VertexAccessor SubgraphVertexAccessor::GetVertexAccessor() const { return impl_; }
storage::Result<EdgeVertexAccessorResult> SubgraphVertexAccessor::OutEdges(storage::View view) const {

View File

@@ -40,7 +40,6 @@ class EdgeAccessor final {
public:
storage::EdgeAccessor impl_;
public:
explicit EdgeAccessor(storage::EdgeAccessor impl) : impl_(std::move(impl)) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
@@ -108,7 +107,6 @@ class VertexAccessor final {
static EdgeAccessor MakeEdgeAccessor(const storage::EdgeAccessor impl) { return EdgeAccessor(impl); }
public:
explicit VertexAccessor(storage::VertexAccessor impl) : impl_(impl) {}
bool IsVisible(storage::View view) const { return impl_.IsVisible(view); }
@@ -694,6 +692,8 @@ class SubgraphDbAccessor final {
std::optional<VertexAccessor> FindVertex(storage::Gid gid, storage::View view);
Graph *getGraph();
DbAccessor *GetAccessor();
};
} // namespace memgraph::query

View File

@@ -159,7 +159,7 @@ void DumpProperties(std::ostream *os, query::DbAccessor *dba,
*os << "{";
if (property_id) {
*os << kInternalPropertyId << ": " << *property_id;
if (store.size() > 0) *os << ", ";
if (!store.empty()) *os << ", ";
}
utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) {
os << EscapeName(dba->PropertyToName(kv.first)) << ": ";
@@ -228,7 +228,7 @@ void DumpEdge(std::ostream *os, query::DbAccessor *dba, const query::EdgeAccesso
throw query::QueryRuntimeException("Unexpected error when getting properties.");
}
}
if (maybe_props->size() > 0) {
if (!maybe_props->empty()) {
*os << " ";
DumpProperties(os, dba, *maybe_props);
}

View File

@@ -126,6 +126,12 @@ class InfoInMulticommandTxException : public QueryException {
SPECIALIZE_GET_EXCEPTION_NAME(InfoInMulticommandTxException)
};
class UserAlreadyExistsException : public QueryException {
public:
using QueryException::QueryException;
SPECIALIZE_GET_EXCEPTION_NAME(UserAlreadyExistsException)
};
/**
* An exception for an illegal operation that can not be detected
* before the query starts executing over data.

View File

@@ -8,58 +8,59 @@
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <tuple>
#include <utility>
#include "query/typed_value.hpp"
#include "utils/fnv.hpp"
#include "utils/memory.hpp"
#include "utils/pmr/unordered_map.hpp"
#include "utils/pmr/vector.hpp"
namespace memgraph::query {
// Key is hash output, value is vector of unique elements
using CachedType = utils::pmr::unordered_map<size_t, std::vector<TypedValue>>;
using CachedType = utils::pmr::unordered_map<size_t, utils::pmr::vector<TypedValue>>;
struct CachedValue {
using allocator_type = utils::Allocator<CachedValue>;
// Cached value, this can be probably templateized
CachedType cache_;
explicit CachedValue(utils::MemoryResource *mem) : cache_(mem) {}
explicit CachedValue(utils::MemoryResource *mem) : cache_{mem} {};
CachedValue(const CachedValue &other, utils::MemoryResource *mem) : cache_(other.cache_, mem) {}
CachedValue(CachedValue &&other, utils::MemoryResource *mem) : cache_(std::move(other.cache_), mem){};
CachedValue(CachedType &&cache, memgraph::utils::MemoryResource *memory) : cache_(std::move(cache), memory) {}
CachedValue(CachedValue &&other) noexcept : CachedValue(std::move(other), other.GetMemoryResource()) {}
CachedValue(const CachedValue &other, memgraph::utils::MemoryResource *memory) : cache_(other.cache_, memory) {}
CachedValue(const CachedValue &other)
: CachedValue(other, std::allocator_traits<allocator_type>::select_on_container_copy_construction(
other.GetMemoryResource())
.GetMemoryResource()) {}
CachedValue(CachedValue &&other, memgraph::utils::MemoryResource *memory) : cache_(std::move(other.cache_), memory) {}
CachedValue(CachedValue &&other) noexcept = delete;
/// Copy construction without memgraph::utils::MemoryResource is not allowed.
CachedValue(const CachedValue &) = delete;
utils::MemoryResource *GetMemoryResource() const { return cache_.get_allocator().GetMemoryResource(); }
CachedValue &operator=(const CachedValue &) = delete;
CachedValue &operator=(CachedValue &&) = delete;
~CachedValue() = default;
memgraph::utils::MemoryResource *GetMemoryResource() const noexcept {
return cache_.get_allocator().GetMemoryResource();
}
// Func to check if cache_ contains value
bool CacheValue(const TypedValue &value) {
if (!value.IsList()) {
bool CacheValue(const TypedValue &maybe_list) {
if (!maybe_list.IsList()) {
return false;
}
const auto &list = value.ValueList();
const auto &list = maybe_list.ValueList();
TypedValue::Hash hash{};
for (const TypedValue &element : list) {
for (const auto &element : list) {
const auto key = hash(element);
auto &vector_values = cache_[key];
if (!IsValueInVec(vector_values, element)) {
vector_values.push_back(element);
vector_values.emplace_back(element);
}
}
return true;
}
// Func to cache_value inside cache_
// Func to check if cache_ contains value
bool ContainsValue(const TypedValue &value) const {
TypedValue::Hash hash{};
const auto key = hash(value);
@@ -70,7 +71,7 @@ struct CachedValue {
}
private:
bool IsValueInVec(const std::vector<TypedValue> &vec_values, const TypedValue &value) const {
static bool IsValueInVec(const utils::pmr::vector<TypedValue> &vec_values, const TypedValue &value) {
return std::any_of(vec_values.begin(), vec_values.end(), [&value](auto &vec_value) {
const auto is_value_equal = vec_value == value;
if (is_value_equal.IsNull()) return false;
@@ -80,43 +81,72 @@ struct CachedValue {
};
// Class tracks keys for which user can cache values which help with faster search or faster retrieval
// in the future.
// in the future. Used for IN LIST operator.
class FrameChangeCollector {
public:
explicit FrameChangeCollector(utils::MemoryResource *mem) : tracked_values_(mem){};
/** Allocator type so that STL containers are aware that we need one */
using allocator_type = utils::Allocator<FrameChangeCollector>;
public:
explicit FrameChangeCollector(utils::MemoryResource *mem = utils::NewDeleteResource()) : tracked_values_{mem} {}
FrameChangeCollector(FrameChangeCollector &&other, utils::MemoryResource *mem)
: tracked_values_(std::move(other.tracked_values_), mem) {}
FrameChangeCollector(const FrameChangeCollector &other, utils::MemoryResource *mem)
: tracked_values_(other.tracked_values_, mem) {}
FrameChangeCollector(const FrameChangeCollector &other)
: FrameChangeCollector(other, std::allocator_traits<allocator_type>::select_on_container_copy_construction(
other.GetMemoryResource())
.GetMemoryResource()){};
FrameChangeCollector(FrameChangeCollector &&other) noexcept
: FrameChangeCollector(std::move(other), other.GetMemoryResource()) {}
/** Copy assign other, utils::MemoryResource of `this` is used */
FrameChangeCollector &operator=(const FrameChangeCollector &) = default;
/** Move assign other, utils::MemoryResource of `this` is used. */
FrameChangeCollector &operator=(FrameChangeCollector &&) noexcept = default;
utils::MemoryResource *GetMemoryResource() const { return tracked_values_.get_allocator().GetMemoryResource(); }
// Add tracking key to cache later value
CachedValue &AddTrackingKey(const std::string &key) {
const auto &[it, _] = tracked_values_.emplace(key, tracked_values_.get_allocator().GetMemoryResource());
const auto &[it, _] = tracked_values_.emplace(
std::piecewise_construct, std::forward_as_tuple(utils::pmr::string(key, utils::NewDeleteResource())),
std::forward_as_tuple());
return it->second;
}
// Is key tracked
bool IsKeyTracked(const std::string &key) const { return tracked_values_.contains(key); }
// Is value for given key cached
bool IsKeyValueCached(const std::string &key) const {
return tracked_values_.contains(key) && !tracked_values_.at(key).cache_.empty();
bool IsKeyTracked(const std::string &key) const {
return tracked_values_.contains(utils::pmr::string(key, utils::NewDeleteResource()));
}
// Reset value for tracking key
bool ResetTrackingValue(const std::string &key) {
if (tracked_values_.contains(key)) {
tracked_values_.erase(key);
AddTrackingKey(key);
}
bool IsKeyValueCached(const std::string &key) const {
return IsKeyTracked(key) && !tracked_values_.at(utils::pmr::string(key, utils::NewDeleteResource())).cache_.empty();
}
bool ResetTrackingValue(const std::string &key) {
if (!tracked_values_.contains(utils::pmr::string(key, utils::NewDeleteResource()))) {
return false;
}
tracked_values_.erase(utils::pmr::string(key, utils::NewDeleteResource()));
AddTrackingKey(key);
return true;
}
// Get value cached for tracking key, throws if key is not in tracked
CachedValue &GetCachedValue(const std::string &key) { return tracked_values_.at(key); }
CachedValue &GetCachedValue(const std::string &key) {
return tracked_values_.at(utils::pmr::string(key, utils::NewDeleteResource()));
}
// Checks for keys tracked
bool IsTrackingValues() const { return !tracked_values_.empty(); }
~FrameChangeCollector() = default;
private:
// Key is output of utils::GetFrameChangeId, value is utils::pmr::unordered_map
memgraph::utils::pmr::unordered_map<std::string, CachedValue> tracked_values_;
struct PmrStringHash {
size_t operator()(const utils::pmr::string &key) const { return utils::Fnv(key); }
};
utils::pmr::unordered_map<utils::pmr::string, CachedValue, PmrStringHash> tracked_values_;
};
} // namespace memgraph::query

View File

@@ -23,9 +23,7 @@
#include "storage/v2/property_value.hpp"
#include "utils/typeinfo.hpp"
namespace memgraph {
namespace query {
namespace memgraph::query {
struct LabelIx {
static const utils::TypeInfo kType;
@@ -62,8 +60,8 @@ inline bool operator!=(const PropertyIx &a, const PropertyIx &b) { return !(a ==
inline bool operator==(const EdgeTypeIx &a, const EdgeTypeIx &b) { return a.ix == b.ix && a.name == b.name; }
inline bool operator!=(const EdgeTypeIx &a, const EdgeTypeIx &b) { return !(a == b); }
} // namespace query
} // namespace memgraph
} // namespace memgraph::query
namespace std {
template <>
@@ -83,9 +81,7 @@ struct hash<memgraph::query::EdgeTypeIx> {
} // namespace std
namespace memgraph {
namespace query {
namespace memgraph::query {
class Tree;
@@ -3029,7 +3025,7 @@ class ReplicationQuery : public memgraph::query::Query {
enum class SyncMode { SYNC, ASYNC };
enum class ReplicaState { READY, REPLICATING, RECOVERY, INVALID };
enum class ReplicaState { READY, REPLICATING, RECOVERY, MAYBE_BEHIND };
ReplicationQuery() = default;
@@ -3577,5 +3573,4 @@ class ShowDatabasesQuery : public memgraph::query::Query {
}
};
} // namespace query
} // namespace memgraph
} // namespace memgraph::query

View File

@@ -1216,10 +1216,6 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
call_proc->memory_limit_ = memory_limit_info->first;
call_proc->memory_scale_ = memory_limit_info->second;
}
} else {
// Default to 100 MB
call_proc->memory_limit_ = storage_->Create<PrimitiveLiteral>(TypedValue(100));
call_proc->memory_scale_ = 1024U * 1024U;
}
const auto &maybe_found =
@@ -1240,11 +1236,13 @@ antlrcpp::Any CypherMainVisitor::visitCallProcedure(MemgraphCypher::CallProcedur
throw SemanticException("There is no procedure named '{}'.", call_proc->procedure_name_);
}
}
call_proc->is_write_ = maybe_found->second->info.is_write;
if (maybe_found) {
call_proc->is_write_ = maybe_found->second->info.is_write;
}
auto *yield_ctx = ctx->yieldProcedureResults();
if (!yield_ctx) {
if (!maybe_found->second->results.empty() && !call_proc->void_procedure_) {
if ((maybe_found && !maybe_found->second->results.empty()) && !call_proc->void_procedure_) {
throw SemanticException(
"CALL without YIELD may only be used on procedures which do not "
"return any result fields.");

View File

@@ -105,7 +105,7 @@ void PrintObject(std::ostream *out, const std::map<K, V> &map);
template <typename T>
void PrintObject(std::ostream *out, const T &arg) {
static_assert(!std::is_convertible<T, Expression *>::value,
static_assert(!std::is_convertible_v<T, Expression *>,
"This overload shouldn't be called with pointers convertible "
"to Expression *. This means your other PrintObject overloads aren't "
"being called for certain AST nodes when they should (or perhaps such "

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -31,7 +31,7 @@ class Parser {
* @param query incoming query that has to be compiled into query plan
* the first step is to generate AST
*/
Parser(const std::string query) : query_(std::move(query)) {
explicit Parser(const std::string query) : query_(std::move(query)) {
parser_.removeErrorListeners();
parser_.addErrorListener(&error_listener_);
tree_ = parser_.cypher();

View File

@@ -12,12 +12,11 @@
#pragma once
#include <string>
#include <utility>
#include "utils/typeinfo.hpp"
namespace memgraph {
namespace query {
namespace memgraph::query {
class Symbol {
public:
@@ -34,9 +33,13 @@ class Symbol {
return enum_string[static_cast<int>(type)];
}
Symbol() {}
Symbol(const std::string &name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1)
: name_(name), position_(position), user_declared_(user_declared), type_(type), token_position_(token_position) {}
Symbol() = default;
Symbol(std::string name, int position, bool user_declared, Type type = Type::ANY, int token_position = -1)
: name_(std::move(name)),
position_(position),
user_declared_(user_declared),
type_(type),
token_position_(token_position) {}
bool operator==(const Symbol &other) const {
return position_ == other.position_ && name_ == other.name_ && type_ == other.type_;
@@ -57,8 +60,8 @@ class Symbol {
int64_t token_position_{-1};
};
} // namespace query
} // namespace memgraph
} // namespace memgraph::query
namespace std {
template <>

View File

@@ -480,12 +480,20 @@ bool SymbolGenerator::PreVisit(None &none) {
}
bool SymbolGenerator::PreVisit(Reduce &reduce) {
auto &scope = scopes_.back();
scope.in_reduce = true;
reduce.initializer_->Accept(*this);
reduce.list_->Accept(*this);
VisitWithIdentifiers(reduce.expression_, {reduce.accumulator_, reduce.identifier_});
return false;
}
bool SymbolGenerator::PostVisit(Reduce & /*reduce*/) {
auto &scope = scopes_.back();
scope.in_reduce = false;
return true;
}
bool SymbolGenerator::PreVisit(Extract &extract) {
extract.list_->Accept(*this);
VisitWithIdentifiers(extract.expression_, {extract.identifier_});
@@ -496,15 +504,23 @@ bool SymbolGenerator::PreVisit(Exists &exists) {
auto &scope = scopes_.back();
if (scope.in_set_property) {
throw utils::NotYetImplemented("Set property can not be used with exists, but only during matching!");
throw utils::NotYetImplemented("Exists cannot be used within SET clause.!");
}
if (scope.in_with) {
throw utils::NotYetImplemented("WITH can not be used with exists, but only during matching!");
throw utils::NotYetImplemented("Exists cannot be used within WITH!");
}
if (scope.in_return) {
throw utils::NotYetImplemented("RETURN can not be used with exists, but only during matching!");
throw utils::NotYetImplemented("Exists cannot be used within RETURN!");
}
if (scope.in_reduce) {
throw utils::NotYetImplemented("Exists cannot be used within REDUCE!");
}
if (scope.num_if_operators) {
throw utils::NotYetImplemented("IF operator cannot be used with exists, but only during matching!");
}
scope.in_exists = true;

View File

@@ -84,6 +84,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
bool PreVisit(Any &) override;
bool PreVisit(None &) override;
bool PreVisit(Reduce &) override;
bool PostVisit(Reduce &) override;
bool PreVisit(Extract &) override;
bool PreVisit(Exists & /*exists*/) override;
bool PostVisit(Exists & /*exists*/) override;
@@ -123,6 +124,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
bool in_match{false};
bool in_foreach{false};
bool in_exists{false};
bool in_reduce{false};
bool in_set_property{false};
bool in_call_subquery{false};
bool has_return{false};
@@ -181,7 +183,7 @@ class SymbolGenerator : public HierarchicalTreeVisitor {
/// If property lookup for one symbol is visited more times, it is better to fetch all properties
class PropertyLookupEvaluationModeVisitor : public ExpressionVisitor<void> {
public:
explicit PropertyLookupEvaluationModeVisitor() {}
explicit PropertyLookupEvaluationModeVisitor() = default;
using ExpressionVisitor<void>::Visit;

View File

@@ -22,7 +22,7 @@ namespace memgraph::query {
class SymbolTable final {
public:
SymbolTable() {}
SymbolTable() = default;
const Symbol &CreateSymbol(const std::string &name, bool user_declared, Symbol::Type type = Symbol::Type::ANY,
int32_t token_position = -1) {
MG_ASSERT(table_.size() <= std::numeric_limits<int32_t>::max(),

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -16,6 +16,7 @@
#include <iostream>
#include <span>
#include <string>
#include <utility>
#include <vector>
#include "query/exceptions.hpp"
@@ -32,7 +33,7 @@ namespace memgraph::query::frontend {
using namespace lexer_constants;
StrippedQuery::StrippedQuery(const std::string &query) : original_(query) {
StrippedQuery::StrippedQuery(std::string query) : original_(std::move(query)) {
enum class Token {
UNMATCHED,
KEYWORD, // Including true, false and null.
@@ -255,29 +256,29 @@ std::string GetFirstUtf8Symbol(const char *_s) {
// According to
// https://stackoverflow.com/questions/16260033/reinterpret-cast-between-char-and-stduint8-t-safe
// this checks if casting from const char * to uint8_t is undefined behaviour.
static_assert(std::is_same<std::uint8_t, unsigned char>::value,
static_assert(std::is_same_v<std::uint8_t, unsigned char>,
"This library requires std::uint8_t to be implemented as "
"unsigned char.");
const uint8_t *s = reinterpret_cast<const uint8_t *>(_s);
if ((*s >> 7) == 0x00) return std::string(_s, _s + 1);
if ((*s >> 5) == 0x06) {
auto *s1 = s + 1;
const auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
return std::string(_s, _s + 2);
}
if ((*s >> 4) == 0x0e) {
auto *s1 = s + 1;
const auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
const auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
return std::string(_s, _s + 3);
}
if ((*s >> 3) == 0x1e) {
auto *s1 = s + 1;
const auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
const auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s3 = s + 3;
const auto *s3 = s + 3;
if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character.");
return std::string(_s, _s + 4);
}
@@ -286,29 +287,29 @@ std::string GetFirstUtf8Symbol(const char *_s) {
// Return codepoint of first utf8 symbol and its encoded length.
std::pair<int, int> GetFirstUtf8SymbolCodepoint(const char *_s) {
static_assert(std::is_same<std::uint8_t, unsigned char>::value,
static_assert(std::is_same_v<std::uint8_t, unsigned char>,
"This library requires std::uint8_t to be implemented as "
"unsigned char.");
const uint8_t *s = reinterpret_cast<const uint8_t *>(_s);
if ((*s >> 7) == 0x00) return {*s & 0x7f, 1};
if ((*s >> 5) == 0x06) {
auto *s1 = s + 1;
const auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
return {((*s & 0x1f) << 6) | (*s1 & 0x3f), 2};
}
if ((*s >> 4) == 0x0e) {
auto *s1 = s + 1;
const auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
const auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
return {((*s & 0x0f) << 12) | ((*s1 & 0x3f) << 6) | (*s2 & 0x3f), 3};
}
if ((*s >> 3) == 0x1e) {
auto *s1 = s + 1;
const auto *s1 = s + 1;
if ((*s1 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s2 = s + 2;
const auto *s2 = s + 2;
if ((*s2 >> 6) != 0x02) throw LexingException("Invalid character.");
auto *s3 = s + 3;
const auto *s3 = s + 3;
if ((*s3 >> 6) != 0x02) throw LexingException("Invalid character.");
return {((*s & 0x07) << 18) | ((*s1 & 0x3f) << 12) | ((*s2 & 0x3f) << 6) | (*s3 & 0x3f), 4};
}
@@ -336,7 +337,7 @@ int StrippedQuery::MatchSpecial(int start) const { return kSpecialTokens.Match(o
int StrippedQuery::MatchString(int start) const {
if (original_[start] != '"' && original_[start] != '\'') return 0;
char start_char = original_[start];
for (auto *p = original_.data() + start + 1; *p; ++p) {
for (const auto *p = original_.data() + start + 1; *p; ++p) {
if (*p == start_char) return p - (original_.data() + start) + 1;
if (*p == '\\') {
++p;
@@ -346,7 +347,7 @@ int StrippedQuery::MatchString(int start) const {
continue;
} else if (*p == 'U' || *p == 'u') {
int cnt = 0;
auto *r = p + 1;
const auto *r = p + 1;
while (isxdigit(*r) && cnt < 8) {
++cnt;
++r;

View File

@@ -1,4 +1,4 @@
// Copyright 2022 Memgraph Ltd.
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -40,7 +40,7 @@ class StrippedQuery {
*
* @param query Input query.
*/
explicit StrippedQuery(const std::string &query);
explicit StrippedQuery(std::string query);
/**
* Copy constructor is deleted because we don't want to make unnecessary

View File

@@ -17,8 +17,7 @@
#include <unordered_set>
#include <vector>
namespace memgraph::query {
namespace lexer_constants {
namespace memgraph::query::lexer_constants {
namespace trie {
@@ -33,7 +32,7 @@ inline int Noop(int x) { return x; }
class Trie {
public:
Trie() {}
Trie() = default;
Trie(std::initializer_list<std::string> l) {
for (const auto &s : l) {
Insert(s);
@@ -2934,5 +2933,4 @@ const trie::Trie kSpecialTokens = {";",
"\xEF\xB9\x98", // u8"\ufe58"
"\xEF\xB9\xA3", // u8"\ufe63"
"\xEF\xBC\x8D"}; // u8"\uff0d"
} // namespace lexer_constants
} // namespace memgraph::query
} // namespace memgraph::query::lexer_constants

View File

@@ -593,6 +593,7 @@ TypedValue ValueType(const TypedValue *args, int64_t nargs, const FunctionContex
case TypedValue::Type::Duration:
return TypedValue("DURATION", ctx.memory);
case TypedValue::Type::Graph:
case TypedValue::Type::Function:
throw QueryRuntimeException("Cannot fetch graph as it is not standardized openCypher type name");
}
}
@@ -1156,11 +1157,16 @@ void MapNumericParameters(auto &parameter_mappings, const auto &input_parameters
}
TypedValue Date(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
FType<Optional<Or<String, Map>>>("date", args, nargs);
FType<Optional<Or<String, Map, LocalDateTime>>>("date", args, nargs);
if (nargs == 0) {
return TypedValue(utils::LocalDateTime(ctx.timestamp).date, ctx.memory);
}
if (args[0].IsLocalDateTime()) {
utils::Date date{args[0].ValueLocalDateTime().date};
return TypedValue(date, ctx.memory);
}
if (args[0].IsString()) {
const auto &[date_parameters, is_extended] = utils::ParseDateParameters(args[0].ValueString());
return TypedValue(utils::Date(date_parameters), ctx.memory);
@@ -1178,12 +1184,17 @@ TypedValue Date(const TypedValue *args, int64_t nargs, const FunctionContext &ct
}
TypedValue LocalTime(const TypedValue *args, int64_t nargs, const FunctionContext &ctx) {
FType<Optional<Or<String, Map>>>("localtime", args, nargs);
FType<Optional<Or<String, Map, LocalDateTime>>>("localtime", args, nargs);
if (nargs == 0) {
return TypedValue(utils::LocalDateTime(ctx.timestamp).local_time, ctx.memory);
}
if (args[0].IsLocalDateTime()) {
utils::LocalTime local_time{args[0].ValueLocalDateTime().local_time};
return TypedValue(local_time, ctx.memory);
}
if (args[0].IsString()) {
const auto &[local_time_parameters, is_extended] = utils::ParseLocalTimeParameters(args[0].ValueString());
return TypedValue(utils::LocalTime(local_time_parameters), ctx.memory);

View File

@@ -18,6 +18,7 @@
#include <map>
#include <optional>
#include <regex>
#include <stdexcept>
#include <string>
#include <vector>
@@ -187,6 +188,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
utils::MemoryResource *GetMemoryResource() const { return ctx_->memory; }
void ClearPropertyLookupCache() { property_lookup_cache_.clear(); }
TypedValue Visit(NamedExpression &named_expression) override {
const auto &symbol = symbol_table_->at(named_expression);
auto value = named_expression.expression_->Accept(*this);
@@ -270,36 +273,33 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(InListOperator &in_list) override {
TypedValue *_list_ptr = nullptr;
TypedValue _list;
auto literal = in_list.expression1_->Accept(*this);
auto get_list_literal = [this, &in_list, &_list, &_list_ptr]() -> void {
auto get_list_literal = [this, &in_list]() -> TypedValue {
ReferenceExpressionEvaluator reference_expression_evaluator{frame_, symbol_table_, ctx_};
_list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
if (nullptr == _list_ptr) {
_list = in_list.expression2_->Accept(*this);
_list_ptr = &_list;
auto *list_ptr = in_list.expression2_->Accept(reference_expression_evaluator);
if (nullptr == list_ptr) {
return in_list.expression2_->Accept(*this);
}
return *list_ptr;
};
auto do_list_literal_checks = [this, &literal, &_list_ptr]() -> std::optional<TypedValue> {
MG_ASSERT(_list_ptr, "List literal should have been defined");
if (_list_ptr->IsNull()) {
auto do_list_literal_checks = [this, &literal](const TypedValue &list) -> std::optional<TypedValue> {
if (list.IsNull()) {
return TypedValue(ctx_->memory);
}
// Exceptions have higher priority than returning nulls when list expression
// is not null.
if (_list_ptr->type() != TypedValue::Type::List) {
throw QueryRuntimeException("IN expected a list, got {}.", _list_ptr->type());
if (list.type() != TypedValue::Type::List) {
throw QueryRuntimeException("IN expected a list, got {}.", list.type());
}
const auto &list = _list_ptr->ValueList();
const auto &list_value = list.ValueList();
// If literal is NULL there is no need to try to compare it with every
// element in the list since result of every comparison will be NULL. There
// is one special case that we must test explicitly: if list is empty then
// result is false since no comparison will be performed.
if (list.empty()) return TypedValue(false, ctx_->memory);
if (list_value.empty()) return TypedValue(false, ctx_->memory);
if (literal.IsNull()) return TypedValue(ctx_->memory);
return {};
};
@@ -312,14 +312,14 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
if (!frame_change_collector_->IsKeyValueCached(*cached_id)) {
// Check only first time if everything is okay, later when we use
// cache there is no need to check again as we did check first time
get_list_literal();
auto preoperational_checks = do_list_literal_checks();
auto list = get_list_literal();
auto preoperational_checks = do_list_literal_checks(list);
if (preoperational_checks) {
return std::move(*preoperational_checks);
}
auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
cached_value.CacheValue(*_list_ptr);
spdlog::trace("Value cached {}", *cached_id);
// Don't move here because we don't want to remove the element from the frame
cached_value.CacheValue(list);
}
const auto &cached_value = frame_change_collector_->GetCachedValue(*cached_id);
@@ -334,16 +334,15 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
// When caching is not an option, we need to evaluate list literal every time
// and do the checks
get_list_literal();
auto preoperational_checks = do_list_literal_checks();
const auto list = get_list_literal();
auto preoperational_checks = do_list_literal_checks(list);
if (preoperational_checks) {
return std::move(*preoperational_checks);
}
const auto &list = _list.ValueList();
spdlog::trace("Not using cache on IN LIST operator");
const auto &list_value = list.ValueList();
auto has_null = false;
for (const auto &element : list) {
for (const auto &element : list_value) {
auto result = literal == element;
if (result.IsNull()) {
has_null = true;
@@ -829,8 +828,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
throw QueryRuntimeException("'coalesce' requires at least one argument.");
}
for (int64_t i = 0; i < exprs.size(); ++i) {
TypedValue val(exprs[i]->Accept(*this), ctx_->memory);
for (auto &expr : exprs) {
TypedValue val(expr->Accept(*this), ctx_->memory);
if (!val.IsNull()) {
return val;
}
@@ -907,7 +906,17 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
return TypedValue(std::move(result), ctx_->memory);
}
TypedValue Visit(Exists &exists) override { return TypedValue{frame_->at(symbol_table_->at(exists)), ctx_->memory}; }
TypedValue Visit(Exists &exists) override {
TypedValue &frame_exists_value = frame_->at(symbol_table_->at(exists));
if (!frame_exists_value.IsFunction()) [[unlikely]] {
throw QueryRuntimeException(
"Unexpected behavior: Exists expected a function, got {}. Please report the problem on GitHub issues",
frame_exists_value.type());
}
TypedValue result{ctx_->memory};
frame_exists_value.ValueFunction()(&result);
return result;
}
TypedValue Visit(All &all) override {
auto list_value = all.list_expression_->Accept(*this);

View File

@@ -37,6 +37,7 @@
#include "dbms/database.hpp"
#include "dbms/dbms_handler.hpp"
#include "dbms/global.hpp"
#include "dbms/inmemory/storage_helper.hpp"
#include "flags/run_time_configurable.hpp"
#include "glue/communication.hpp"
#include "license/license.hpp"
@@ -94,16 +95,17 @@
#include "utils/on_scope_exit.hpp"
#include "utils/readable_size.hpp"
#include "utils/settings.hpp"
#include "utils/stat.hpp"
#include "utils/string.hpp"
#include "utils/tsc.hpp"
#include "utils/typeinfo.hpp"
#include "utils/variant_helpers.hpp"
#include "dbms/dbms_handler.hpp"
#include "dbms/replication_handler.hpp"
#include "query/auth_query_handler.hpp"
#include "query/interpreter_context.hpp"
#include "replication/state.hpp"
#include "storage/v2/replication/replication_handler.hpp"
namespace memgraph::metrics {
extern Event ReadQuery;
@@ -145,6 +147,8 @@ void memgraph::query::CurrentDB::CleanupDBTransaction(bool abort) {
namespace memgraph::query {
constexpr std::string_view kSchemaAssert = "SCHEMA.ASSERT";
template <typename>
constexpr auto kAlwaysFalse = false;
@@ -270,7 +274,7 @@ inline auto convertToReplicationMode(const ReplicationQuery::SyncMode &sync_mode
class ReplQueryHandler final : public query::ReplicationQueryHandler {
public:
explicit ReplQueryHandler(storage::Storage *db) : db_(db), handler_{db_->repl_state_, *db_} {}
explicit ReplQueryHandler(dbms::DbmsHandler *dbms_handler) : dbms_handler_(dbms_handler), handler_{*dbms_handler} {}
/// @throw QueryRuntimeException if an error ocurred.
void SetReplicationRole(ReplicationQuery::ReplicationRole replication_role, std::optional<int64_t> port) override {
@@ -314,10 +318,6 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
throw QueryRuntimeException("Replica can't register another replica!");
}
if (name == memgraph::replication::kReservedReplicationRoleName) {
throw QueryRuntimeException("This replica name is reserved and can not be used as replica name!");
}
auto repl_mode = convertToReplicationMode(sync_mode);
auto maybe_ip_and_port =
@@ -330,8 +330,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
.port = port,
.replica_check_frequency = replica_check_frequency,
.ssl = std::nullopt};
using storage::RegistrationMode;
auto ret = handler_.RegisterReplica(RegistrationMode::MUST_BE_INSTANTLY_VALID, config);
auto ret = handler_.RegisterReplica(config);
if (ret.HasError()) {
throw QueryRuntimeException(fmt::format("Couldn't register replica '{}'!", name));
}
@@ -344,7 +343,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
void DropReplica(std::string_view replica_name) override {
auto const result = handler_.UnregisterReplica(replica_name);
switch (result) {
using enum memgraph::storage::UnregisterReplicaResult;
using enum memgraph::dbms::UnregisterReplicaResult;
case NOT_MAIN:
throw QueryRuntimeException("Replica can't unregister a replica!");
case COULD_NOT_BE_PERSISTED:
@@ -358,13 +357,22 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
using Replica = ReplicationQueryHandler::Replica;
std::vector<Replica> ShowReplicas() const override {
auto const &replState = db_->repl_state_;
if (replState.IsReplica()) {
if (handler_.IsReplica()) {
// replica can't show registered replicas (it shouldn't have any)
throw QueryRuntimeException("Replica can't show registered replicas (it shouldn't have any)!");
}
auto repl_infos = db_->ReplicasInfo();
// TODO: Combine results? Have a single place with clients???
// Also authentication checks (replica + database visibility)
std::vector<storage::ReplicaInfo> repl_infos{};
dbms_handler_->ForOne([&repl_infos](dbms::Database *db) -> bool {
auto infos = db->storage()->ReplicasInfo();
if (!infos.empty()) {
repl_infos = std::move(infos);
return true;
}
return false;
});
std::vector<Replica> replicas;
replicas.reserve(repl_infos.size());
@@ -395,8 +403,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
case storage::replication::ReplicaState::RECOVERY:
replica.state = ReplicationQuery::ReplicaState::RECOVERY;
break;
case storage::replication::ReplicaState::INVALID:
replica.state = ReplicationQuery::ReplicaState::INVALID;
case storage::replication::ReplicaState::MAYBE_BEHIND:
replica.state = ReplicationQuery::ReplicaState::MAYBE_BEHIND;
break;
}
@@ -408,8 +416,8 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
}
private:
storage::Storage *db_;
storage::ReplicationHandler handler_;
dbms::DbmsHandler *dbms_handler_;
dbms::ReplicationHandler handler_;
};
/// returns false if the replication role can't be set
@@ -418,7 +426,7 @@ class ReplQueryHandler final : public query::ReplicationQueryHandler {
Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_context, const Parameters &parameters) {
AuthQueryHandler *auth = interpreter_context->auth;
#ifdef MG_ENTERPRISE
auto *db_handler = interpreter_context->db_handler;
auto *db_handler = interpreter_context->dbms_handler;
#endif
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
// the argument to Callback.
@@ -470,7 +478,7 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_
MG_ASSERT(password.IsString() || password.IsNull());
if (!auth->CreateUser(username, password.IsString() ? std::make_optional(std::string(password.ValueString()))
: std::nullopt)) {
throw QueryRuntimeException("User '{}' already exists.", username);
throw UserAlreadyExistsException("User '{}' already exists.", username);
}
// If the license is not valid we create users with admin access
@@ -702,8 +710,9 @@ Callback HandleAuthQuery(AuthQuery *auth_query, InterpreterContext *interpreter_
}
} // namespace
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &parameters, storage::Storage *storage,
const query::InterpreterConfig &config, std::vector<Notification> *notifications) {
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &parameters,
dbms::DbmsHandler *dbms_handler, const query::InterpreterConfig &config,
std::vector<Notification> *notifications) {
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
// the argument to Callback.
EvaluationContext evaluation_context;
@@ -723,7 +732,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
notifications->emplace_back(SeverityLevel::WARNING, NotificationCode::REPLICA_PORT_WARNING,
"Be careful the replication port must be different from the memgraph port!");
}
callback.fn = [handler = ReplQueryHandler{storage}, role = repl_query->role_, maybe_port]() mutable {
callback.fn = [handler = ReplQueryHandler{dbms_handler}, role = repl_query->role_, maybe_port]() mutable {
handler.SetReplicationRole(role, maybe_port);
return std::vector<std::vector<TypedValue>>();
};
@@ -735,7 +744,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
}
case ReplicationQuery::Action::SHOW_REPLICATION_ROLE: {
callback.header = {"replication role"};
callback.fn = [handler = ReplQueryHandler{storage}] {
callback.fn = [handler = ReplQueryHandler{dbms_handler}] {
auto mode = handler.ShowReplicationRole();
switch (mode) {
case ReplicationQuery::ReplicationRole::MAIN: {
@@ -754,7 +763,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
auto socket_address = repl_query->socket_address_->Accept(evaluator);
const auto replica_check_frequency = config.replication_replica_check_frequency;
callback.fn = [handler = ReplQueryHandler{storage}, name, socket_address, sync_mode,
callback.fn = [handler = ReplQueryHandler{dbms_handler}, name, socket_address, sync_mode,
replica_check_frequency]() mutable {
handler.RegisterReplica(name, std::string(socket_address.ValueString()), sync_mode, replica_check_frequency);
return std::vector<std::vector<TypedValue>>();
@@ -765,7 +774,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
}
case ReplicationQuery::Action::DROP_REPLICA: {
const auto &name = repl_query->replica_name_;
callback.fn = [handler = ReplQueryHandler{storage}, name]() mutable {
callback.fn = [handler = ReplQueryHandler{dbms_handler}, name]() mutable {
handler.DropReplica(name);
return std::vector<std::vector<TypedValue>>();
};
@@ -777,7 +786,7 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
callback.header = {
"name", "socket_address", "sync_mode", "current_timestamp_of_replica", "number_of_timestamp_behind_master",
"state"};
callback.fn = [handler = ReplQueryHandler{storage}, replica_nfields = callback.header.size()] {
callback.fn = [handler = ReplQueryHandler{dbms_handler}, replica_nfields = callback.header.size()] {
const auto &replicas = handler.ShowReplicas();
auto typed_replicas = std::vector<std::vector<TypedValue>>{};
typed_replicas.reserve(replicas.size());
@@ -785,34 +794,33 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
std::vector<TypedValue> typed_replica;
typed_replica.reserve(replica_nfields);
typed_replica.emplace_back(TypedValue(replica.name));
typed_replica.emplace_back(TypedValue(replica.socket_address));
typed_replica.emplace_back(replica.name);
typed_replica.emplace_back(replica.socket_address);
switch (replica.sync_mode) {
case ReplicationQuery::SyncMode::SYNC:
typed_replica.emplace_back(TypedValue("sync"));
typed_replica.emplace_back("sync");
break;
case ReplicationQuery::SyncMode::ASYNC:
typed_replica.emplace_back(TypedValue("async"));
typed_replica.emplace_back("async");
break;
}
typed_replica.emplace_back(TypedValue(static_cast<int64_t>(replica.current_timestamp_of_replica)));
typed_replica.emplace_back(
TypedValue(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master)));
typed_replica.emplace_back(static_cast<int64_t>(replica.current_timestamp_of_replica));
typed_replica.emplace_back(static_cast<int64_t>(replica.current_number_of_timestamp_behind_master));
switch (replica.state) {
case ReplicationQuery::ReplicaState::READY:
typed_replica.emplace_back(TypedValue("ready"));
typed_replica.emplace_back("ready");
break;
case ReplicationQuery::ReplicaState::REPLICATING:
typed_replica.emplace_back(TypedValue("replicating"));
typed_replica.emplace_back("replicating");
break;
case ReplicationQuery::ReplicaState::RECOVERY:
typed_replica.emplace_back(TypedValue("recovery"));
typed_replica.emplace_back("recovery");
break;
case ReplicationQuery::ReplicaState::INVALID:
typed_replica.emplace_back(TypedValue("invalid"));
case ReplicationQuery::ReplicaState::MAYBE_BEHIND:
typed_replica.emplace_back("invalid");
break;
}
@@ -1203,7 +1211,7 @@ struct TxTimeout {
};
struct PullPlan {
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
explicit PullPlan(std::shared_ptr<PlanWrapper> plan, const Parameters &parameters, bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
std::shared_ptr<utils::AsyncTimer> tx_timer,
@@ -1216,7 +1224,7 @@ struct PullPlan {
std::map<std::string, TypedValue> *summary);
private:
std::shared_ptr<CachedPlan> plan_ = nullptr;
std::shared_ptr<PlanWrapper> plan_ = nullptr;
plan::UniqueCursorPtr cursor_ = nullptr;
Frame frame_;
ExecutionContext ctx_;
@@ -1243,7 +1251,7 @@ struct PullPlan {
bool use_monotonic_memory_;
};
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
PullPlan::PullPlan(const std::shared_ptr<PlanWrapper> plan, const Parameters &parameters, const bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
std::shared_ptr<utils::AsyncTimer> tx_timer, TriggerContextCollector *trigger_context_collector,
@@ -1399,42 +1407,16 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
using RWType = plan::ReadWriteTypeChecker::RWType;
bool IsWriteQueryOnMainMemoryReplica(storage::Storage *storage,
const query::plan::ReadWriteTypeChecker::RWType query_type) {
if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL ||
storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
auto const &replState = storage->repl_state_;
return replState.IsReplica() && (query_type == RWType::W || query_type == RWType::RW);
}
return false;
}
bool IsReplica(storage::Storage *storage) {
if (auto storage_mode = storage->GetStorageMode(); storage_mode == storage::StorageMode::IN_MEMORY_ANALYTICAL ||
storage_mode == storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
auto const &replState = storage->repl_state_;
return replState.IsReplica();
}
return false;
bool IsQueryWrite(const query::plan::ReadWriteTypeChecker::RWType query_type) {
return query_type == RWType::W || query_type == RWType::RW;
}
} // namespace
#ifdef MG_ENTERPRISE
InterpreterContext::InterpreterContext(InterpreterConfig interpreter_config, memgraph::dbms::DbmsHandler *handler,
query::AuthQueryHandler *ah, query::AuthChecker *ac)
: db_handler(handler), config(interpreter_config), auth(ah), auth_checker(ac) {}
#else
InterpreterContext::InterpreterContext(InterpreterConfig interpreter_config,
memgraph::utils::Gatekeeper<memgraph::dbms::Database> *db_gatekeeper,
query::AuthQueryHandler *ah, query::AuthChecker *ac)
: db_gatekeeper(db_gatekeeper), config(interpreter_config), auth(ah), auth_checker(ac) {}
#endif
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
#ifndef MG_ENTERPRISE
auto db_acc = interpreter_context_->db_gatekeeper->access();
auto db_acc = interpreter_context_->dbms_handler->Get();
MG_ASSERT(db_acc, "Database accessor needs to be valid");
current_db_.db_acc_ = std::move(db_acc);
#endif
@@ -1551,7 +1533,6 @@ inline static void TryCaching(const AstStorage &ast_storage, FrameChangeCollecto
continue;
}
frame_change_collector->AddTrackingKey(*cached_id);
spdlog::trace("Tracking {} operator, by id: {}", InListOperator::kType.name, *cached_id);
}
}
@@ -1612,8 +1593,6 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
auto use_monotonic_memory =
!contains_csv && !IsCallBatchedProcedureQuery(clauses) && !IsAllShortestPathsQuery(clauses);
spdlog::trace("PrepareCypher has {} encountered all shortest paths and will {} use of monotonic memory",
IsAllShortestPathsQuery(clauses) ? "" : "not", use_monotonic_memory ? "" : "not");
MG_ASSERT(current_db.execution_db_accessor_, "Cypher query expects a current DB transaction");
auto *dba =
@@ -1979,11 +1958,11 @@ std::vector<std::vector<TypedValue>> AnalyzeGraphQueryHandler::AnalyzeGraphCreat
result.reserve(kComputeStatisticsNumResults);
result.emplace_back(execution_db_accessor->LabelToName(stat_entry.first));
result.emplace_back(TypedValue());
result.emplace_back();
result.emplace_back(static_cast<int64_t>(stat_entry.second.count));
result.emplace_back(TypedValue());
result.emplace_back(TypedValue());
result.emplace_back(TypedValue());
result.emplace_back();
result.emplace_back();
result.emplace_back();
result.emplace_back(stat_entry.second.avg_degree);
results.push_back(std::move(result));
});
@@ -2124,10 +2103,7 @@ PreparedQuery PrepareAnalyzeGraphQuery(ParsedQuery parsed_query, bool in_explici
// Creating an index influences computed plan costs.
auto invalidate_plan_cache = [plan_cache = current_db.db_acc_->get()->plan_cache()] {
auto access = plan_cache->access();
for (auto &kv : access) {
access.remove(kv.first);
}
plan_cache->WithLock([&](auto &cache) { cache.reset(); });
};
utils::OnScopeExit cache_invalidator(invalidate_plan_cache);
@@ -2172,10 +2148,7 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
// Creating an index influences computed plan costs.
auto invalidate_plan_cache = [plan_cache = db_acc->plan_cache()] {
auto access = plan_cache->access();
for (auto &kv : access) {
access.remove(kv.first);
}
plan_cache->WithLock([&](auto &cache) { cache.reset(); });
};
auto *storage = db_acc->storage();
@@ -2286,21 +2259,15 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
}
PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::vector<Notification> *notifications, CurrentDB &current_db,
std::vector<Notification> *notifications, dbms::DbmsHandler &dbms_handler,
const InterpreterConfig &config) {
if (in_explicit_transaction) {
throw ReplicationModificationInMulticommandTxException();
}
MG_ASSERT(current_db.db_acc_, "Replication query expects a current DB");
storage::Storage *storage = current_db.db_acc_->get()->storage();
if (storage->GetStorageMode() == storage::StorageMode::ON_DISK_TRANSACTIONAL) {
throw ReplicationDisabledOnDiskStorage();
}
auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, storage, config, notifications);
auto callback =
HandleReplicationQuery(replication_query, parsed_query.parameters, &dbms_handler, config, notifications);
return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -2773,7 +2740,8 @@ PreparedQuery PrepareStorageModeQuery(ParsedQuery parsed_query, const bool in_ex
"transactions using 'SHOW TRANSACTIONS' query and ensure no other transactions are active.");
}
callback = [requested_mode, storage = db_acc->storage()]() -> std::function<void()> {
callback = [requested_mode,
storage = static_cast<storage::InMemoryStorage *>(db_acc->storage())]() -> std::function<void()> {
// SetStorageMode will probably be handled at the Database level
return [storage, requested_mode] { storage->SetStorageMode(requested_mode); };
}();
@@ -2836,15 +2804,11 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
std::move(parsed_query.required_privileges),
[storage](AnyStream * /*stream*/, std::optional<int> /*n*/) -> std::optional<QueryHandlerResult> {
auto *mem_storage = static_cast<storage::InMemoryStorage *>(storage);
if (auto maybe_error = mem_storage->CreateSnapshot(storage->repl_state_, {}); maybe_error.HasError()) {
if (auto maybe_error = mem_storage->CreateSnapshot(); maybe_error.HasError()) {
switch (maybe_error.GetError()) {
case storage::InMemoryStorage::CreateSnapshotError::DisabledForReplica:
throw utils::BasicException(
"Failed to create a snapshot. Replica instances are not allowed to create them.");
case storage::InMemoryStorage::CreateSnapshotError::DisabledForAnalyticsPeriodicCommit:
spdlog::warn(utils::MessageWithLink("Periodic snapshots are disabled for analytical mode.",
"https://memgr.ph/replication"));
break;
case storage::InMemoryStorage::CreateSnapshotError::ReachedMaxNumTries:
spdlog::warn("Failed to create snapshot. Reached max number of tries. Please contact support");
break;
@@ -2914,7 +2878,7 @@ auto ShowTransactions(const std::unordered_set<Interpreter *> &interpreters, con
metadata_tv.emplace(md.first, TypedValue(md.second));
}
}
results.back().push_back(TypedValue(metadata_tv));
results.back().emplace_back(metadata_tv);
}
}
return results;
@@ -3116,14 +3080,18 @@ PreparedQuery PrepareSystemInfoQuery(ParsedQuery parsed_query, bool in_explicit_
handler = [storage = current_db.db_acc_->get()->storage(), interpreter_isolation_level,
next_transaction_isolation_level] {
auto info = storage->GetBaseInfo();
const auto vm_max_map_count = utils::GetVmMaxMapCount();
const int64_t vm_max_map_count_storage_info =
vm_max_map_count.has_value() ? vm_max_map_count.value() : memgraph::utils::VM_MAX_MAP_COUNT_DEFAULT;
std::vector<std::vector<TypedValue>> results{
{TypedValue("name"), TypedValue(storage->id())},
{TypedValue("vertex_count"), TypedValue(static_cast<int64_t>(info.vertex_count))},
{TypedValue("edge_count"), TypedValue(static_cast<int64_t>(info.edge_count))},
{TypedValue("average_degree"), TypedValue(info.average_degree)},
{TypedValue("memory_usage"), TypedValue(utils::GetReadableSize(static_cast<double>(info.memory_usage)))},
{TypedValue("vm_max_map_count"), TypedValue(vm_max_map_count_storage_info)},
{TypedValue("memory_res"), TypedValue(utils::GetReadableSize(static_cast<double>(info.memory_res)))},
{TypedValue("disk_usage"), TypedValue(utils::GetReadableSize(static_cast<double>(info.disk_usage)))},
{TypedValue("memory_allocated"),
{TypedValue("memory_tracked"),
TypedValue(utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.Amount())))},
{TypedValue("allocation_limit"),
TypedValue(utils::GetReadableSize(static_cast<double>(utils::total_memory_tracker.HardLimit())))},
@@ -3384,15 +3352,17 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
}
// TODO: Remove once replicas support multi-tenant replication
if (!current_db.db_acc_) throw DatabaseContextRequiredException("Multi database queries require a defined database.");
if (IsReplica(current_db.db_acc_->get()->storage())) {
throw QueryException("Query forbidden on the replica!");
}
auto *query = utils::Downcast<MultiDatabaseQuery>(parsed_query.query);
auto *db_handler = interpreter_context->db_handler;
auto *db_handler = interpreter_context->dbms_handler;
const bool is_replica = interpreter_context->repl_state->IsReplica();
switch (query->action_) {
case MultiDatabaseQuery::Action::CREATE:
if (is_replica) {
throw QueryException("Query forbidden on the replica!");
}
return PreparedQuery{
{"STATUS"},
std::move(parsed_query.required_privileges),
@@ -3435,9 +3405,12 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
if (current_db.in_explicit_db_) {
throw QueryException("Database switching is prohibited if session explicitly defines the used database");
}
if (!dbms::allow_mt_repl && is_replica) {
throw QueryException("Query forbidden on the replica!");
}
return PreparedQuery{{"STATUS"},
std::move(parsed_query.required_privileges),
[db_name = query->db_name_, db_handler, &current_db, on_change_cb](
[db_name = query->db_name_, db_handler, &current_db, on_change = std::move(on_change_cb)](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
std::vector<std::vector<TypedValue>> status;
std::string res;
@@ -3447,7 +3420,7 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
res = "Already using " + db_name;
} else {
auto tmp = db_handler->Get(db_name);
if (on_change_cb) (*on_change_cb)(db_name); // Will trow if cb fails
if (on_change) (*on_change)(db_name); // Will trow if cb fails
current_db.SetCurrentDB(std::move(tmp), false);
res = "Using " + db_name;
}
@@ -3466,6 +3439,9 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
query->db_name_};
case MultiDatabaseQuery::Action::DROP:
if (is_replica) {
throw QueryException("Query forbidden on the replica!");
}
return PreparedQuery{
{"STATUS"},
std::move(parsed_query.required_privileges),
@@ -3473,11 +3449,9 @@ PreparedQuery PrepareMultiDatabaseQuery(ParsedQuery parsed_query, CurrentDB &cur
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
std::vector<std::vector<TypedValue>> status;
memgraph::dbms::DeleteResult success{};
try {
// Remove database
success = db_handler->Delete(db_name);
auto success = db_handler->Delete(db_name);
if (!success.HasError()) {
// Remove from auth
auth->DeleteDatabase(db_name);
@@ -3526,14 +3500,9 @@ PreparedQuery PrepareShowDatabasesQuery(ParsedQuery parsed_query, CurrentDB &cur
if (!license::global_license_checker.IsEnterpriseValidFast()) {
throw QueryException("Trying to use enterprise feature without a valid license.");
}
// TODO: Remove once replicas support multi-tenant replication
auto &replState = storage->repl_state_;
if (replState.IsReplica()) {
throw QueryException("SHOW DATABASES forbidden on the replica!");
}
// TODO pick directly from ic
auto *db_handler = interpreter_context->db_handler;
auto *db_handler = interpreter_context->dbms_handler;
AuthQueryHandler *auth = interpreter_context->auth;
Callback callback;
@@ -3637,7 +3606,7 @@ void Interpreter::RollbackTransaction() {
void Interpreter::SetCurrentDB(std::string_view db_name, bool in_explicit_db) {
// Can throw
// do we lock here?
current_db_.SetCurrentDB(interpreter_context_->db_handler->Get(db_name), in_explicit_db);
current_db_.SetCurrentDB(interpreter_context_->dbms_handler->Get(db_name), in_explicit_db);
}
#endif
@@ -3714,8 +3683,6 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
// Setup QueryExecution
// its MemoryResource is mostly used for allocations done on Frame and storing `row`s
if (usePool) {
spdlog::trace("PrepareCypher has {} encountered all shortest paths, QueryExecution will use PoolResource",
hasAllShortestPaths ? "" : "not");
query_executions_.emplace_back(QueryExecution::Create(utils::PoolResource(128, kExecutionPoolMaxBlockSize)));
} else {
query_executions_.emplace_back(QueryExecution::Create(utils::MonotonicBufferResource(kExecutionMemoryBlockSize)));
@@ -3746,7 +3713,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
// TODO: ATM only a single database, will change when we have multiple database transactions
bool could_commit = utils::Downcast<CypherQuery>(parsed_query.query) != nullptr;
bool unique = utils::Downcast<IndexQuery>(parsed_query.query) != nullptr ||
utils::Downcast<ConstraintQuery>(parsed_query.query) != nullptr;
utils::Downcast<ConstraintQuery>(parsed_query.query) != nullptr ||
upper_case_query.find(kSchemaAssert) != std::string::npos;
SetupDatabaseTransaction(could_commit, unique);
}
@@ -3761,7 +3729,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
std::visit([](auto &execution_memory) -> utils::MemoryResource * { return &execution_memory; },
query_execution->execution_memory);
frame_change_collector_.reset();
frame_change_collector_.emplace(memory_resource);
frame_change_collector_.emplace();
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
current_db_, memory_resource, &query_execution->notifications, username_,
@@ -3796,7 +3764,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
/// TODO: make replication DB agnostic
prepared_query =
PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
current_db_, interpreter_context_->config);
*interpreter_context_->dbms_handler, interpreter_context_->config);
} else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, current_db_);
} else if (utils::Downcast<FreeMemoryQuery>(parsed_query.query)) {
@@ -3859,7 +3827,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
UpdateTypeCount(rw_type);
if (IsWriteQueryOnMainMemoryReplica(current_db_.db_acc_->get()->storage(), rw_type)) {
if (interpreter_context_->repl_state->IsReplica() && IsQueryWrite(rw_type)) {
query_execution = nullptr;
throw QueryException("Write query forbidden on the replica!");
}
@@ -4088,7 +4056,8 @@ void Interpreter::Commit() {
auto commit_confirmed_by_all_sync_repplicas = true;
auto maybe_commit_error = current_db_.db_transactional_accessor_->Commit();
auto maybe_commit_error =
current_db_.db_transactional_accessor_->Commit(std::nullopt, interpreter_context_->repl_state->IsMain());
if (maybe_commit_error.HasError()) {
const auto &error = maybe_commit_error.GetError();

View File

@@ -174,7 +174,7 @@ struct CurrentDB {
class Interpreter final {
public:
Interpreter(InterpreterContext *interpreter_context);
explicit Interpreter(InterpreterContext *interpreter_context);
Interpreter(InterpreterContext *interpreter_context, memgraph::dbms::DatabaseAccess db);
Interpreter(const Interpreter &) = delete;
Interpreter &operator=(const Interpreter &) = delete;

View File

@@ -13,6 +13,12 @@
#include "query/interpreter.hpp"
namespace memgraph::query {
InterpreterContext::InterpreterContext(InterpreterConfig interpreter_config, dbms::DbmsHandler *dbms_handler,
replication::ReplicationState *rs, query::AuthQueryHandler *ah,
query::AuthChecker *ac)
: dbms_handler(dbms_handler), config(interpreter_config), repl_state(rs), auth(ah), auth_checker(ac) {}
std::vector<std::vector<TypedValue>> InterpreterContext::TerminateTransactions(
std::vector<std::string> maybe_kill_transaction_ids, const std::optional<std::string> &username,
std::function<bool(std::string const &)> privilege_checker) {

Some files were not shown because too many files have changed in this diff Show More