Compare commits
32 Commits
disk-stora
...
accessors-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
58f4878483 | ||
|
|
271dd7d8dc | ||
|
|
32ed4f1235 | ||
|
|
688a5855b2 | ||
|
|
ffbcf882de | ||
|
|
fa0c2bc97f | ||
|
|
1cf90b9fbd | ||
|
|
d40126c79f | ||
|
|
b8f282468d | ||
|
|
614e321ebb | ||
|
|
9a70476872 | ||
|
|
d47372b455 | ||
|
|
ab38161cd2 | ||
|
|
3a5f140c2b | ||
|
|
eead0f79fc | ||
|
|
91017b7f36 | ||
|
|
00f8d54249 | ||
|
|
4fcdd52f88 | ||
|
|
6c947947eb | ||
|
|
64fd281b2e | ||
|
|
97e250129e | ||
|
|
b02b201129 | ||
|
|
2c6a55775d | ||
|
|
940bf6722c | ||
|
|
49b5343238 | ||
|
|
26a0866938 | ||
|
|
6545283dac | ||
|
|
69c735934c | ||
|
|
64e837b355 | ||
|
|
a586f2f98d | ||
|
|
9fc51f74a0 | ||
|
|
128771a6ec |
2
.github/workflows/daily_benchmark.yaml
vendored
2
.github/workflows/daily_benchmark.yaml
vendored
@@ -67,7 +67,7 @@ jobs:
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
|
||||
2
.github/workflows/diff.yaml
vendored
2
.github/workflows/diff.yaml
vendored
@@ -437,7 +437,7 @@ jobs:
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
./benchmark.py vendor-native --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
|
||||
39
.github/workflows/package_all.yaml
vendored
39
.github/workflows/package_all.yaml
vendored
@@ -2,7 +2,12 @@ name: Package All
|
||||
|
||||
# TODO(gitbuda): Cleanup docker container if GHA job was canceled.
|
||||
|
||||
on: workflow_dispatch
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
memgraph_version:
|
||||
description: "Memgraph version to upload as. If empty upload is skipped. Format: 'vX.Y.Z'"
|
||||
required: false
|
||||
|
||||
jobs:
|
||||
centos-7:
|
||||
@@ -106,7 +111,7 @@ jobs:
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-1804
|
||||
name: ubuntu-18.04
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
|
||||
ubuntu-2004:
|
||||
@@ -123,7 +128,7 @@ jobs:
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-2004
|
||||
name: ubuntu-20.04
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
|
||||
ubuntu-2204:
|
||||
@@ -140,7 +145,7 @@ jobs:
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-2204
|
||||
name: ubuntu-22.04
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
|
||||
debian-11-platform:
|
||||
@@ -208,7 +213,7 @@ jobs:
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: debian-11-arm
|
||||
name: debian-11-aarch64
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
ubuntu-2204-arm:
|
||||
@@ -225,5 +230,27 @@ jobs:
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: ubuntu-22.04-arm
|
||||
name: ubuntu-22.04-aarch64
|
||||
path: build/output/ubuntu-22.04-arm/memgraph*.deb
|
||||
|
||||
upload-to-s3:
|
||||
# only run upload if we specified version. Allows for runs without upload
|
||||
if: "${{ github.event.inputs.memgraph_version != '' }}"
|
||||
needs: [centos-7, centos-9, debian-10, debian-11, docker, ubuntu-1804, ubuntu-2004, ubuntu-2204, debian-11-platform, fedora-36, amzn-2, debian-11-arm, ubuntu-2204-arm]
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v3
|
||||
with:
|
||||
# name: # if name input parameter is not provided, all artifacts are downloaded
|
||||
# and put in directories named after each one.
|
||||
path: build/output/release
|
||||
- name: Upload to S3
|
||||
uses: jakejarvis/s3-sync-action@v0.5.1
|
||||
env:
|
||||
AWS_S3_BUCKET: "download.memgraph.com"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.S3_AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.S3_AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_REGION: "eu-west-1"
|
||||
SOURCE_DIR: "build/output/release"
|
||||
DEST_DIR: "memgraph/${{ github.event.inputs.memgraph_version }}/"
|
||||
|
||||
4
.github/workflows/release_docker.yaml
vendored
4
.github/workflows/release_docker.yaml
vendored
@@ -31,8 +31,8 @@ jobs:
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKER_USERNAME }}
|
||||
password: ${{ secrets.DOCKER_PASSWORD }}
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Download memgraph binary
|
||||
run: |
|
||||
|
||||
63
.github/workflows/release_mgbench_client.yaml
vendored
Normal file
63
.github/workflows/release_mgbench_client.yaml
vendored
Normal file
@@ -0,0 +1,63 @@
|
||||
name: "Mgbench Bolt Client Publish Docker Image"
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: "Mgbench bolt client version to publish on Dockerhub."
|
||||
required: true
|
||||
force_release:
|
||||
type: boolean
|
||||
required: false
|
||||
default: false
|
||||
|
||||
jobs:
|
||||
mgbench_docker_publish:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
DOCKER_ORGANIZATION_NAME: memgraph
|
||||
DOCKER_REPOSITORY_NAME: mgbench-client
|
||||
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v2
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
id: buildx
|
||||
uses: docker/setup-buildx-action@v2
|
||||
|
||||
- name: Log in to Docker Hub
|
||||
uses: docker/login-action@v2
|
||||
with:
|
||||
username: ${{ secrets.DOCKERHUB_USERNAME }}
|
||||
password: ${{ secrets.DOCKERHUB_TOKEN }}
|
||||
|
||||
- name: Check if specified version is already pushed
|
||||
run: |
|
||||
EXISTS=$(docker manifest inspect $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} > /dev/null; echo $?)
|
||||
echo $EXISTS
|
||||
if [[ ${EXISTS} -eq 0 ]]; then
|
||||
echo 'The specified version has been already released to DockerHub.'
|
||||
if [[ ${{ github.event.inputs.force_release }} = true ]]; then
|
||||
echo 'Forcing the release!'
|
||||
else
|
||||
echo 'Stopping the release!'
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
echo 'All good the specified version has not been release to DockerHub.'
|
||||
fi
|
||||
|
||||
- name: Build & push docker images
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
docker buildx build \
|
||||
--build-arg TOOLCHAIN_VERSION=toolchain-v4 \
|
||||
--platform linux/amd64,linux/arm64 \
|
||||
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:${{ github.event.inputs.version }} \
|
||||
--tag $DOCKER_ORGANIZATION_NAME/$DOCKER_REPOSITORY_NAME:latest \
|
||||
--file Dockerfile.mgbench_client \
|
||||
--push .
|
||||
@@ -103,6 +103,10 @@ modifications:
|
||||
value: "true"
|
||||
override: false
|
||||
|
||||
- name: "storage_parallel_index_recovery"
|
||||
value: "false"
|
||||
override: true
|
||||
|
||||
undocumented:
|
||||
- "flag_file"
|
||||
- "also_log_to_stderr"
|
||||
|
||||
2
init
2
init
@@ -109,7 +109,7 @@ done;
|
||||
# Install precommit hook except on old operating systems because we don't
|
||||
# develop on them -> pre-commit hook not required -> we can use latest
|
||||
# packages.
|
||||
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ]; then
|
||||
if [ "${DISTRO}" != "centos-7" ] && [ "$DISTRO" != "debian-10" ] && [ "${DISTRO}" != "ubuntu-18.04" ] && [ "${DISTRO}" != "amzn-2" ]; then
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pre_commit install
|
||||
# Install py format tools for usage during the development.
|
||||
|
||||
@@ -117,7 +117,7 @@ declare -A primary_urls=(
|
||||
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
|
||||
["spdlog"]="http://$local_cache_host/git/spdlog"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
|
||||
["neo4j"]="http://$local_cache_host/file/neo4j-community-5.6.0-unix.tar.gz"
|
||||
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
|
||||
["protobuf"]="http://$local_cache_host/git/protobuf.git"
|
||||
["pulsar"]="http://$local_cache_host/git/pulsar.git"
|
||||
@@ -142,7 +142,7 @@ declare -A secondary_urls=(
|
||||
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
|
||||
["spdlog"]="https://github.com/gabime/spdlog"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
|
||||
["neo4j"]="https://dist.neo4j.org/neo4j-community-5.6.0-unix.tar.gz"
|
||||
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
|
||||
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
|
||||
["pulsar"]="https://github.com/apache/pulsar.git"
|
||||
@@ -180,9 +180,9 @@ repo_clone_try_double "${primary_urls[libbcrypt]}" "${secondary_urls[libbcrypt]}
|
||||
|
||||
# neo4j
|
||||
file_get_try_double "${primary_urls[neo4j]}" "${secondary_urls[neo4j]}"
|
||||
tar -xzf neo4j-community-3.2.3-unix.tar.gz
|
||||
mv neo4j-community-3.2.3 neo4j
|
||||
rm neo4j-community-3.2.3-unix.tar.gz
|
||||
tar -xzf neo4j-community-5.6.0-unix.tar.gz
|
||||
mv neo4j-community-5.6.0 neo4j
|
||||
rm neo4j-community-5.6.0-unix.tar.gz
|
||||
|
||||
# nlohmann json
|
||||
# We wget header instead of cloning repo since repo is huge (lots of test data).
|
||||
|
||||
@@ -36,7 +36,7 @@ ADDITIONAL USE GRANT: You may use the Licensed Work in accordance with the
|
||||
3. using the Licensed Work to create a work or solution
|
||||
which competes (or might reasonably be expected to
|
||||
compete) with the Licensed Work.
|
||||
CHANGE DATE: 2027-08-03
|
||||
CHANGE DATE: 2027-05-04
|
||||
CHANGE LICENSE: Apache License, Version 2.0
|
||||
|
||||
For information about alternative licensing arrangements, please visit: https://memgraph.com/legal.
|
||||
|
||||
202
licenses/third-party/ldbc/LICENSE
vendored
Normal file
202
licenses/third-party/ldbc/LICENSE
vendored
Normal file
@@ -0,0 +1,202 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -16,6 +16,10 @@ target_link_libraries(mg-auth mg-utils mg-kvstore mg-license )
|
||||
target_link_libraries(mg-auth ${Seccomp_LIBRARIES})
|
||||
target_include_directories(mg-auth SYSTEM PRIVATE ${Seccomp_INCLUDE_DIRS})
|
||||
|
||||
find_package(OpenSSL REQUIRED)
|
||||
target_link_libraries(mg-auth ${OPENSSL_LIBRARIES})
|
||||
target_include_directories(mg-auth SYSTEM PUBLIC ${OPENSSL_INCLUDE_DIR})
|
||||
|
||||
# Install reference auth modules and their configuration files.
|
||||
install(PROGRAMS ${CMAKE_CURRENT_SOURCE_DIR}/reference_modules/example.py
|
||||
DESTINATION lib/memgraph/auth_module)
|
||||
|
||||
@@ -1,19 +1,64 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
// this file except in compliance with the License. You may obtain a copy of the License at https://memgraph.com/legal.
|
||||
//
|
||||
//
|
||||
|
||||
#include "auth/crypto.hpp"
|
||||
|
||||
#include <iomanip>
|
||||
#include <sstream>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include <libbcrypt/bcrypt.h>
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/opensslv.h>
|
||||
#include <openssl/sha.h>
|
||||
|
||||
#include "auth/exceptions.hpp"
|
||||
#include "utils/enum.hpp"
|
||||
#include "utils/flag_validation.hpp"
|
||||
|
||||
namespace {
|
||||
using namespace std::literals;
|
||||
inline constexpr std::array password_encryption_mappings{
|
||||
std::pair{"bcrypt"sv, memgraph::auth::PasswordEncryptionAlgorithm::BCRYPT},
|
||||
std::pair{"sha256"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256},
|
||||
std::pair{"sha256-multiple"sv, memgraph::auth::PasswordEncryptionAlgorithm::SHA256_MULTIPLE}};
|
||||
|
||||
inline constexpr uint64_t ONE_SHA_ITERATION = 1;
|
||||
inline constexpr uint64_t MULTIPLE_SHA_ITERATIONS = 1024;
|
||||
} // namespace
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables,misc-unused-parameters)
|
||||
DEFINE_VALIDATED_string(password_encryption_algorithm, "bcrypt",
|
||||
"The password encryption algorithm used for authentication.", {
|
||||
if (const auto result =
|
||||
memgraph::utils::IsValidEnumValueString(value, password_encryption_mappings);
|
||||
result.HasError()) {
|
||||
const auto error = result.GetError();
|
||||
switch (error) {
|
||||
case memgraph::utils::ValidationError::EmptyValue: {
|
||||
std::cout << "Password encryption algorithm cannot be empty." << std::endl;
|
||||
break;
|
||||
}
|
||||
case memgraph::utils::ValidationError::InvalidValue: {
|
||||
std::cout << "Invalid value for password encryption algorithm. Allowed values: "
|
||||
<< memgraph::utils::GetAllowedEnumValuesString(password_encryption_mappings)
|
||||
<< std::endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
});
|
||||
|
||||
namespace memgraph::auth {
|
||||
const std::string EncryptPassword(const std::string &password) {
|
||||
namespace BCrypt {
|
||||
std::string EncryptPassword(const std::string &password) {
|
||||
char salt[BCRYPT_HASHSIZE];
|
||||
char hash[BCRYPT_HASHSIZE];
|
||||
|
||||
@@ -28,7 +73,7 @@ const std::string EncryptPassword(const std::string &password) {
|
||||
throw AuthException("Couldn't hash password!");
|
||||
}
|
||||
|
||||
return std::string(hash);
|
||||
return {hash};
|
||||
}
|
||||
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash) {
|
||||
@@ -38,5 +83,102 @@ bool VerifyPassword(const std::string &password, const std::string &hash) {
|
||||
}
|
||||
return ret == 0;
|
||||
}
|
||||
} // namespace BCrypt
|
||||
|
||||
namespace SHA {
|
||||
#if OPENSSL_VERSION_MAJOR >= 3
|
||||
std::string EncryptPasswordOpenSSL3(const std::string &password, const uint64_t number_of_iterations) {
|
||||
unsigned char hash[SHA256_DIGEST_LENGTH];
|
||||
|
||||
EVP_MD_CTX *ctx = EVP_MD_CTX_new();
|
||||
EVP_MD *md = EVP_MD_fetch(nullptr, "SHA2-256", nullptr);
|
||||
|
||||
EVP_DigestInit_ex(ctx, md, nullptr);
|
||||
for (auto i = 0; i < number_of_iterations; i++) {
|
||||
EVP_DigestUpdate(ctx, password.c_str(), password.size());
|
||||
}
|
||||
EVP_DigestFinal_ex(ctx, hash, nullptr);
|
||||
|
||||
EVP_MD_free(md);
|
||||
EVP_MD_CTX_free(ctx);
|
||||
|
||||
std::stringstream result_stream;
|
||||
for (auto hash_char : hash) {
|
||||
result_stream << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
|
||||
}
|
||||
|
||||
return result_stream.str();
|
||||
}
|
||||
#else
|
||||
std::string EncryptPasswordOpenSSL1_1(const std::string &password, const uint64_t number_of_iterations) {
|
||||
unsigned char hash[SHA256_DIGEST_LENGTH];
|
||||
|
||||
SHA256_CTX sha256;
|
||||
SHA256_Init(&sha256);
|
||||
for (auto i = 0; i < number_of_iterations; i++) {
|
||||
SHA256_Update(&sha256, password.c_str(), password.size());
|
||||
}
|
||||
SHA256_Final(hash, &sha256);
|
||||
|
||||
std::stringstream ss;
|
||||
for (auto hash_char : hash) {
|
||||
ss << std::hex << std::setw(2) << std::setfill('0') << (int)hash_char;
|
||||
}
|
||||
|
||||
return ss.str();
|
||||
}
|
||||
#endif
|
||||
|
||||
std::string EncryptPassword(const std::string &password, const uint64_t number_of_iterations) {
|
||||
#if OPENSSL_VERSION_MAJOR >= 3
|
||||
return EncryptPasswordOpenSSL3(password, number_of_iterations);
|
||||
#else
|
||||
return EncryptPasswordOpenSSL1_1(password, number_of_iterations);
|
||||
#endif
|
||||
}
|
||||
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash, const uint64_t number_of_iterations) {
|
||||
auto password_hash = EncryptPassword(password, number_of_iterations);
|
||||
return password_hash == hash;
|
||||
}
|
||||
} // namespace SHA
|
||||
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash) {
|
||||
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
|
||||
FLAGS_password_encryption_algorithm, password_encryption_mappings);
|
||||
|
||||
if (!password_encryption_algorithm.has_value()) {
|
||||
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
|
||||
}
|
||||
|
||||
switch (password_encryption_algorithm.value()) {
|
||||
case PasswordEncryptionAlgorithm::BCRYPT:
|
||||
return BCrypt::VerifyPassword(password, hash);
|
||||
case PasswordEncryptionAlgorithm::SHA256:
|
||||
return SHA::VerifyPassword(password, hash, ONE_SHA_ITERATION);
|
||||
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
|
||||
return SHA::VerifyPassword(password, hash, MULTIPLE_SHA_ITERATIONS);
|
||||
}
|
||||
|
||||
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
|
||||
}
|
||||
|
||||
std::string EncryptPassword(const std::string &password) {
|
||||
const auto password_encryption_algorithm = utils::StringToEnum<PasswordEncryptionAlgorithm>(
|
||||
FLAGS_password_encryption_algorithm, password_encryption_mappings);
|
||||
|
||||
if (!password_encryption_algorithm.has_value()) {
|
||||
throw AuthException("Invalid password encryption flag '{}'!", FLAGS_password_encryption_algorithm);
|
||||
}
|
||||
|
||||
switch (password_encryption_algorithm.value()) {
|
||||
case PasswordEncryptionAlgorithm::BCRYPT:
|
||||
return BCrypt::EncryptPassword(password);
|
||||
case PasswordEncryptionAlgorithm::SHA256:
|
||||
return SHA::EncryptPassword(password, ONE_SHA_ITERATION);
|
||||
case PasswordEncryptionAlgorithm::SHA256_MULTIPLE:
|
||||
return SHA::EncryptPassword(password, MULTIPLE_SHA_ITERATIONS);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Licensed as a Memgraph Enterprise file under the Memgraph Enterprise
|
||||
// License (the "License"); by using this file, you agree to be bound by the terms of the License, and you may not use
|
||||
@@ -11,10 +11,11 @@
|
||||
#include <string>
|
||||
|
||||
namespace memgraph::auth {
|
||||
enum class PasswordEncryptionAlgorithm : uint8_t { BCRYPT, SHA256, SHA256_MULTIPLE };
|
||||
|
||||
/// @throw AuthException if unable to encrypt the password.
|
||||
const std::string EncryptPassword(const std::string &password);
|
||||
std::string EncryptPassword(const std::string &password);
|
||||
|
||||
/// @throw AuthException if unable to verify the password.
|
||||
bool VerifyPassword(const std::string &password, const std::string &hash);
|
||||
|
||||
} // namespace memgraph::auth
|
||||
|
||||
@@ -45,7 +45,9 @@ const std::vector<Permission> kPermissionsAll = {Permission::MATCH, Permis
|
||||
Permission::FREE_MEMORY, Permission::TRIGGER,
|
||||
Permission::CONFIG, Permission::STREAM,
|
||||
Permission::MODULE_READ, Permission::MODULE_WRITE,
|
||||
Permission::WEBSOCKET, Permission::TRANSACTION_MANAGEMENT};
|
||||
Permission::WEBSOCKET, Permission::TRANSACTION_MANAGEMENT,
|
||||
Permission::STORAGE_MODE};
|
||||
|
||||
} // namespace
|
||||
|
||||
std::string PermissionToString(Permission permission) {
|
||||
@@ -94,6 +96,8 @@ std::string PermissionToString(Permission permission) {
|
||||
return "WEBSOCKET";
|
||||
case Permission::TRANSACTION_MANAGEMENT:
|
||||
return "TRANSACTION_MANAGEMENT";
|
||||
case Permission::STORAGE_MODE:
|
||||
return "STORAGE_MODE";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -40,7 +40,8 @@ enum class Permission : uint64_t {
|
||||
MODULE_READ = 1U << 18U,
|
||||
MODULE_WRITE = 1U << 19U,
|
||||
WEBSOCKET = 1U << 20U,
|
||||
TRANSACTION_MANAGEMENT = 1U << 21U
|
||||
TRANSACTION_MANAGEMENT = 1U << 21U,
|
||||
STORAGE_MODE = 1U << 22U
|
||||
};
|
||||
// clang-format on
|
||||
|
||||
|
||||
@@ -58,6 +58,8 @@ auth::Permission PrivilegeToPermission(query::AuthQuery::Privilege privilege) {
|
||||
return auth::Permission::MODULE_WRITE;
|
||||
case query::AuthQuery::Privilege::WEBSOCKET:
|
||||
return auth::Permission::WEBSOCKET;
|
||||
case query::AuthQuery::Privilege::STORAGE_MODE:
|
||||
return auth::Permission::STORAGE_MODE;
|
||||
case query::AuthQuery::Privilege::TRANSACTION_MANAGEMENT:
|
||||
return auth::Permission::TRANSACTION_MANAGEMENT;
|
||||
}
|
||||
|
||||
@@ -120,9 +120,9 @@ bool AuthChecker::IsUserAuthorized(const memgraph::auth::User &user,
|
||||
FineGrainedAuthChecker::FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba)
|
||||
: user_{std::move(user)}, dba_(dba){};
|
||||
|
||||
bool FineGrainedAuthChecker::Has(const memgraph::query::VertexAccessor &vertex, const memgraph::storage::View view,
|
||||
bool FineGrainedAuthChecker::Has(const memgraph::query::VertexAccessor *vertex, const memgraph::storage::View view,
|
||||
const memgraph::query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
|
||||
auto maybe_labels = vertex.Labels(view);
|
||||
auto maybe_labels = vertex->Labels(view);
|
||||
if (maybe_labels.HasError()) {
|
||||
switch (maybe_labels.GetError()) {
|
||||
case memgraph::storage::Error::DELETED_OBJECT:
|
||||
|
||||
@@ -43,9 +43,14 @@ class FineGrainedAuthChecker : public query::FineGrainedAuthChecker {
|
||||
public:
|
||||
explicit FineGrainedAuthChecker(auth::User user, const memgraph::query::DbAccessor *dba);
|
||||
|
||||
bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
|
||||
bool Has(const query::VertexAccessor *vertex, memgraph::storage::View view,
|
||||
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
|
||||
|
||||
bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
|
||||
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
|
||||
return Has(&vertex, view, fine_grained_privilege);
|
||||
}
|
||||
|
||||
bool Has(const query::EdgeAccessor &edge,
|
||||
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const override;
|
||||
|
||||
|
||||
@@ -58,6 +58,7 @@
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
#include "telemetry/telemetry.hpp"
|
||||
#include "utils/enum.hpp"
|
||||
#include "utils/event_counter.hpp"
|
||||
#include "utils/file.hpp"
|
||||
#include "utils/flag_validation.hpp"
|
||||
@@ -104,42 +105,6 @@ constexpr const char *kMgUser = "MEMGRAPH_USER";
|
||||
constexpr const char *kMgPassword = "MEMGRAPH_PASSWORD";
|
||||
constexpr const char *kMgPassfile = "MEMGRAPH_PASSFILE";
|
||||
|
||||
namespace {
|
||||
std::string GetAllowedEnumValuesString(const auto &mappings) {
|
||||
std::vector<std::string> allowed_values;
|
||||
allowed_values.reserve(mappings.size());
|
||||
std::transform(mappings.begin(), mappings.end(), std::back_inserter(allowed_values),
|
||||
[](const auto &mapping) { return std::string(mapping.first); });
|
||||
return memgraph::utils::Join(allowed_values, ", ");
|
||||
}
|
||||
|
||||
enum class ValidationError : uint8_t { EmptyValue, InvalidValue };
|
||||
|
||||
memgraph::utils::BasicResult<ValidationError> IsValidEnumValueString(const auto &value, const auto &mappings) {
|
||||
if (value.empty()) {
|
||||
return ValidationError::EmptyValue;
|
||||
}
|
||||
|
||||
if (std::find_if(mappings.begin(), mappings.end(), [&](const auto &mapping) { return mapping.first == value; }) ==
|
||||
mappings.cend()) {
|
||||
return ValidationError::InvalidValue;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
|
||||
template <typename Enum>
|
||||
std::optional<Enum> StringToEnum(const auto &value, const auto &mappings) {
|
||||
const auto mapping_iter =
|
||||
std::find_if(mappings.begin(), mappings.end(), [&](const auto &mapping) { return mapping.first == value; });
|
||||
if (mapping_iter == mappings.cend()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return mapping_iter->second;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
// Short help flag.
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_HIDDEN_bool(h, false, "Print usage and exit.");
|
||||
@@ -228,6 +193,20 @@ DEFINE_VALIDATED_uint64(storage_wal_file_flush_every_n_tx,
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(storage_snapshot_on_exit, false, "Controls whether the storage creates another snapshot on exit.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint64(storage_items_per_batch, memgraph::storage::Config::Durability().items_per_batch,
|
||||
"The number of edges and vertices stored in a batch in a snapshot file.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(storage_parallel_index_recovery, false,
|
||||
"Controls whether the index creation can be done in a multithreaded fashion.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint64(storage_recovery_thread_count,
|
||||
std::max(static_cast<uint64_t>(std::thread::hardware_concurrency()),
|
||||
memgraph::storage::Config::Durability().recovery_thread_count),
|
||||
"The number of threads used to recover persisted data from disk.");
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(telemetry_enabled, false,
|
||||
"Set to true to enable telemetry. We collect information about the "
|
||||
@@ -291,21 +270,21 @@ inline constexpr std::array isolation_level_mappings{
|
||||
|
||||
const std::string isolation_level_help_string =
|
||||
fmt::format("Default isolation level used for the transactions. Allowed values: {}",
|
||||
GetAllowedEnumValuesString(isolation_level_mappings));
|
||||
memgraph::utils::GetAllowedEnumValuesString(isolation_level_mappings));
|
||||
} // namespace
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_VALIDATED_string(isolation_level, "SNAPSHOT_ISOLATION", isolation_level_help_string.c_str(), {
|
||||
if (const auto result = IsValidEnumValueString(value, isolation_level_mappings); result.HasError()) {
|
||||
if (const auto result = memgraph::utils::IsValidEnumValueString(value, isolation_level_mappings); result.HasError()) {
|
||||
const auto error = result.GetError();
|
||||
switch (error) {
|
||||
case ValidationError::EmptyValue: {
|
||||
case memgraph::utils::ValidationError::EmptyValue: {
|
||||
std::cout << "Isolation level cannot be empty." << std::endl;
|
||||
break;
|
||||
}
|
||||
case ValidationError::InvalidValue: {
|
||||
case memgraph::utils::ValidationError::InvalidValue: {
|
||||
std::cout << "Invalid value for isolation level. Allowed values: "
|
||||
<< GetAllowedEnumValuesString(isolation_level_mappings) << std::endl;
|
||||
<< memgraph::utils::GetAllowedEnumValuesString(isolation_level_mappings) << std::endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -318,7 +297,7 @@ DEFINE_VALIDATED_string(isolation_level, "SNAPSHOT_ISOLATION", isolation_level_h
|
||||
namespace {
|
||||
memgraph::storage::IsolationLevel ParseIsolationLevel() {
|
||||
const auto isolation_level =
|
||||
StringToEnum<memgraph::storage::IsolationLevel>(FLAGS_isolation_level, isolation_level_mappings);
|
||||
memgraph::utils::StringToEnum<memgraph::storage::IsolationLevel>(FLAGS_isolation_level, isolation_level_mappings);
|
||||
MG_ASSERT(isolation_level, "Invalid isolation level");
|
||||
return *isolation_level;
|
||||
}
|
||||
@@ -378,21 +357,21 @@ inline constexpr std::array log_level_mappings{
|
||||
std::pair{"INFO"sv, spdlog::level::info}, std::pair{"WARNING"sv, spdlog::level::warn},
|
||||
std::pair{"ERROR"sv, spdlog::level::err}, std::pair{"CRITICAL"sv, spdlog::level::critical}};
|
||||
|
||||
const std::string log_level_help_string =
|
||||
fmt::format("Minimum log level. Allowed values: {}", GetAllowedEnumValuesString(log_level_mappings));
|
||||
const std::string log_level_help_string = fmt::format("Minimum log level. Allowed values: {}",
|
||||
memgraph::utils::GetAllowedEnumValuesString(log_level_mappings));
|
||||
} // namespace
|
||||
|
||||
DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
|
||||
if (const auto result = IsValidEnumValueString(value, log_level_mappings); result.HasError()) {
|
||||
if (const auto result = memgraph::utils::IsValidEnumValueString(value, log_level_mappings); result.HasError()) {
|
||||
const auto error = result.GetError();
|
||||
switch (error) {
|
||||
case ValidationError::EmptyValue: {
|
||||
case memgraph::utils::ValidationError::EmptyValue: {
|
||||
std::cout << "Log level cannot be empty." << std::endl;
|
||||
break;
|
||||
}
|
||||
case ValidationError::InvalidValue: {
|
||||
std::cout << "Invalid value for log level. Allowed values: " << GetAllowedEnumValuesString(log_level_mappings)
|
||||
<< std::endl;
|
||||
case memgraph::utils::ValidationError::InvalidValue: {
|
||||
std::cout << "Invalid value for log level. Allowed values: "
|
||||
<< memgraph::utils::GetAllowedEnumValuesString(log_level_mappings) << std::endl;
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -404,7 +383,7 @@ DEFINE_VALIDATED_string(log_level, "WARNING", log_level_help_string.c_str(), {
|
||||
|
||||
namespace {
|
||||
spdlog::level::level_enum ParseLogLevel() {
|
||||
const auto log_level = StringToEnum<spdlog::level::level_enum>(FLAGS_log_level, log_level_mappings);
|
||||
const auto log_level = memgraph::utils::StringToEnum<spdlog::level::level_enum>(FLAGS_log_level, log_level_mappings);
|
||||
MG_ASSERT(log_level, "Invalid log level");
|
||||
return *log_level;
|
||||
}
|
||||
@@ -888,7 +867,10 @@ int main(int argc, char **argv) {
|
||||
.wal_file_size_kibibytes = FLAGS_storage_wal_file_size_kib,
|
||||
.wal_file_flush_every_n_tx = FLAGS_storage_wal_file_flush_every_n_tx,
|
||||
.snapshot_on_exit = FLAGS_storage_snapshot_on_exit,
|
||||
.restore_replicas_on_startup = true},
|
||||
.restore_replicas_on_startup = true,
|
||||
.items_per_batch = FLAGS_storage_items_per_batch,
|
||||
.recovery_thread_count = FLAGS_storage_recovery_thread_count,
|
||||
.allow_parallel_index_creation = FLAGS_storage_parallel_index_recovery},
|
||||
.transaction = {.isolation_level = ParseIsolationLevel()}};
|
||||
if (FLAGS_storage_snapshot_interval_sec == 0) {
|
||||
if (FLAGS_storage_wal_enabled) {
|
||||
@@ -908,7 +890,7 @@ int main(int argc, char **argv) {
|
||||
}
|
||||
db_config.durability.snapshot_interval = std::chrono::seconds(FLAGS_storage_snapshot_interval_sec);
|
||||
}
|
||||
auto db = std::unique_ptr<memgraph::storage::Storage>(new memgraph::storage::InMemoryStorage(db_config));
|
||||
std::unique_ptr<memgraph::storage::Storage> db = std::make_unique<memgraph::storage::InMemoryStorage>(db_config);
|
||||
|
||||
memgraph::query::InterpreterContext interpreter_context{
|
||||
db.get(),
|
||||
|
||||
@@ -699,13 +699,14 @@ int main(int argc, char *argv[]) {
|
||||
}
|
||||
|
||||
std::unordered_map<NodeId, memgraph::storage::Gid> node_id_map;
|
||||
std::unique_ptr<memgraph::storage::Storage> store{new memgraph::storage::InMemoryStorage{{
|
||||
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
|
||||
.durability = {.storage_directory = FLAGS_data_directory,
|
||||
.recover_on_startup = false,
|
||||
.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::DISABLED,
|
||||
.snapshot_on_exit = true},
|
||||
}}};
|
||||
std::unique_ptr<memgraph::storage::Storage> store =
|
||||
std::make_unique<memgraph::storage::InMemoryStorage>(memgraph::storage::Config{
|
||||
.items = {.properties_on_edges = FLAGS_storage_properties_on_edges},
|
||||
.durability = {.storage_directory = FLAGS_data_directory,
|
||||
.recover_on_startup = false,
|
||||
.snapshot_wal_mode = memgraph::storage::Config::Durability::SnapshotWalMode::DISABLED,
|
||||
.snapshot_on_exit = true},
|
||||
});
|
||||
|
||||
memgraph::utils::Timer load_timer;
|
||||
|
||||
|
||||
@@ -36,9 +36,14 @@ class FineGrainedAuthChecker {
|
||||
public:
|
||||
virtual ~FineGrainedAuthChecker() = default;
|
||||
|
||||
[[nodiscard]] virtual bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
|
||||
[[nodiscard]] virtual bool Has(const query::VertexAccessor *vertex, memgraph::storage::View view,
|
||||
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const = 0;
|
||||
|
||||
bool Has(const query::VertexAccessor &vertex, memgraph::storage::View view,
|
||||
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const {
|
||||
return Has(&vertex, view, fine_grained_privilege);
|
||||
}
|
||||
|
||||
[[nodiscard]] virtual bool Has(const query::EdgeAccessor &edge,
|
||||
query::AuthQuery::FineGrainedPrivilege fine_grained_privilege) const = 0;
|
||||
|
||||
@@ -57,7 +62,7 @@ class FineGrainedAuthChecker {
|
||||
|
||||
class AllowEverythingFineGrainedAuthChecker final : public query::FineGrainedAuthChecker {
|
||||
public:
|
||||
bool Has(const VertexAccessor & /*vertex*/, const memgraph::storage::View /*view*/,
|
||||
bool Has(const VertexAccessor * /*vertex*/, const memgraph::storage::View /*view*/,
|
||||
const query::AuthQuery::FineGrainedPrivilege /*fine_grained_privilege*/) const override {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -122,14 +122,32 @@ class VertexAccessor final {
|
||||
// It can affect performance if we use std::unique_ptr here.
|
||||
std::unique_ptr<storage::VertexAccessor> impl_;
|
||||
|
||||
explicit VertexAccessor(std::unique_ptr<storage::VertexAccessor> impl) : impl_(std::move(impl)) {}
|
||||
VertexAccessor(const VertexAccessor &impl) : impl_(impl.impl_->Copy()){};
|
||||
explicit VertexAccessor(VertexAccessor *impl) : VertexAccessor(*impl) {}
|
||||
VertexAccessor() : impl_(nullptr) {}
|
||||
explicit VertexAccessor(std::unique_ptr<storage::VertexAccessor> va) : impl_(std::move(va)) {}
|
||||
VertexAccessor(const VertexAccessor &va) : impl_(va.impl_ ? va.impl_->Copy() : nullptr){};
|
||||
VertexAccessor(VertexAccessor &&va) noexcept : impl_(va.impl_ ? std::move(va.impl_) : nullptr){};
|
||||
explicit VertexAccessor(const VertexAccessor *va) : VertexAccessor(*va) {}
|
||||
|
||||
~VertexAccessor() = default;
|
||||
|
||||
VertexAccessor &operator=(const VertexAccessor &other) {
|
||||
impl_ = other.impl_->Copy();
|
||||
if (this == &other) {
|
||||
return *this;
|
||||
}
|
||||
if (other.impl_) {
|
||||
impl_ = other.impl_->Copy();
|
||||
} else {
|
||||
impl_ = nullptr;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
VertexAccessor &operator=(VertexAccessor &&other) {
|
||||
if (other.impl_) {
|
||||
impl_ = std::move(other.impl_);
|
||||
} else {
|
||||
impl_ = nullptr;
|
||||
}
|
||||
return *this;
|
||||
}
|
||||
|
||||
@@ -285,23 +303,58 @@ class VerticesIterable final {
|
||||
utils::Allocator<VertexAccessor>>::iterator>
|
||||
it_;
|
||||
|
||||
VertexAccessor curr_vertex_accessor_;
|
||||
bool is_initialized_;
|
||||
|
||||
public:
|
||||
explicit Iterator(storage::VerticesIterable::Iterator it) : it_(it) {}
|
||||
explicit Iterator(storage::VerticesIterable::Iterator it) : it_(it), is_initialized_(false) {
|
||||
std::visit(memgraph::utils::Overloaded{
|
||||
[this](storage::VerticesIterable::Iterator &it_) {
|
||||
if (*it_ != nullptr) {
|
||||
curr_vertex_accessor_ = VertexAccessor((*it_)->Copy());
|
||||
is_initialized_ = true;
|
||||
}
|
||||
},
|
||||
[this](std::unordered_set<VertexAccessor, std::hash<VertexAccessor>, std::equal_to<void>,
|
||||
utils::Allocator<VertexAccessor>>::iterator &it_) {
|
||||
curr_vertex_accessor_ = *it_;
|
||||
is_initialized_ = true;
|
||||
}},
|
||||
it_);
|
||||
}
|
||||
explicit Iterator(std::unordered_set<VertexAccessor, std::hash<VertexAccessor>, std::equal_to<void>,
|
||||
utils::Allocator<VertexAccessor>>::iterator it)
|
||||
: it_(it) {}
|
||||
: it_(it), is_initialized_(true) {
|
||||
std::visit(memgraph::utils::Overloaded{
|
||||
[this](storage::VerticesIterable::Iterator it_) {
|
||||
curr_vertex_accessor_ = VertexAccessor((*it_)->Copy());
|
||||
},
|
||||
[this](std::unordered_set<VertexAccessor, std::hash<VertexAccessor>, std::equal_to<void>,
|
||||
utils::Allocator<VertexAccessor>>::iterator it_) {
|
||||
curr_vertex_accessor_ = *it_;
|
||||
}},
|
||||
it_);
|
||||
}
|
||||
|
||||
VertexAccessor operator*() const {
|
||||
return std::visit(
|
||||
memgraph::utils::Overloaded{
|
||||
[](storage::VerticesIterable::Iterator it_) { return VertexAccessor((*it_)->Copy()); },
|
||||
[](std::unordered_set<VertexAccessor, std::hash<VertexAccessor>, std::equal_to<void>,
|
||||
utils::Allocator<VertexAccessor>>::iterator it_) { return VertexAccessor(*it_); }},
|
||||
it_);
|
||||
VertexAccessor *operator*() {
|
||||
if (!is_initialized_) {
|
||||
std::visit(memgraph::utils::Overloaded{
|
||||
[this](storage::VerticesIterable::Iterator it_) {
|
||||
curr_vertex_accessor_ = VertexAccessor((*it_)->Copy());
|
||||
},
|
||||
[this](std::unordered_set<VertexAccessor, std::hash<VertexAccessor>, std::equal_to<void>,
|
||||
utils::Allocator<VertexAccessor>>::iterator it_) {
|
||||
curr_vertex_accessor_ = *it_;
|
||||
}},
|
||||
it_);
|
||||
is_initialized_ = true;
|
||||
}
|
||||
return &curr_vertex_accessor_;
|
||||
}
|
||||
|
||||
Iterator &operator++() {
|
||||
std::visit([this](auto it_) { this->it_ = ++it_; }, it_);
|
||||
std::visit([](auto &it_) { ++it_; }, it_);
|
||||
is_initialized_ = false;
|
||||
return *this;
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -168,10 +168,10 @@ void DumpProperties(std::ostream *os, query::DbAccessor *dba,
|
||||
*os << "}";
|
||||
}
|
||||
|
||||
void DumpVertex(std::ostream *os, query::DbAccessor *dba, const query::VertexAccessor &vertex) {
|
||||
void DumpVertex(std::ostream *os, query::DbAccessor *dba, const query::VertexAccessor *vertex) {
|
||||
*os << "CREATE (";
|
||||
*os << ":" << kInternalVertexLabel;
|
||||
auto maybe_labels = vertex.Labels(storage::View::OLD);
|
||||
auto maybe_labels = vertex->Labels(storage::View::OLD);
|
||||
if (maybe_labels.HasError()) {
|
||||
switch (maybe_labels.GetError()) {
|
||||
case storage::Error::DELETED_OBJECT:
|
||||
@@ -188,7 +188,7 @@ void DumpVertex(std::ostream *os, query::DbAccessor *dba, const query::VertexAcc
|
||||
*os << ":" << EscapeName(dba->LabelToName(label));
|
||||
}
|
||||
*os << " ";
|
||||
auto maybe_props = vertex.Properties(storage::View::OLD);
|
||||
auto maybe_props = vertex->Properties(storage::View::OLD);
|
||||
if (maybe_props.HasError()) {
|
||||
switch (maybe_props.GetError()) {
|
||||
case storage::Error::DELETED_OBJECT:
|
||||
@@ -201,7 +201,7 @@ void DumpVertex(std::ostream *os, query::DbAccessor *dba, const query::VertexAcc
|
||||
throw query::QueryRuntimeException("Unexpected error when getting properties.");
|
||||
}
|
||||
}
|
||||
DumpProperties(os, dba, *maybe_props, vertex.CypherId());
|
||||
DumpProperties(os, dba, *maybe_props, vertex->CypherId());
|
||||
*os << ");";
|
||||
}
|
||||
|
||||
@@ -478,11 +478,11 @@ PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
|
||||
auto ¤t_vertex_iter{*maybe_current_vertex_iter};
|
||||
size_t local_counter = 0U;
|
||||
for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) {
|
||||
const auto &vertex = *current_vertex_iter;
|
||||
const auto *vertex = *current_vertex_iter;
|
||||
// If we have a saved iterable from a previous pull
|
||||
// we need to use the same iterable
|
||||
if (!maybe_edge_iterable) {
|
||||
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::View::OLD));
|
||||
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex->OutEdges(storage::View::OLD));
|
||||
}
|
||||
auto &maybe_edges = *maybe_edge_iterable;
|
||||
MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
|
||||
|
||||
@@ -211,6 +211,21 @@ class IsolationLevelModificationInMulticommandTxException : public QueryExceptio
|
||||
: QueryException("Isolation level cannot be modified in multicommand transactions.") {}
|
||||
};
|
||||
|
||||
class IsolationLevelModificationInAnalyticsException : public QueryException {
|
||||
public:
|
||||
IsolationLevelModificationInAnalyticsException()
|
||||
: QueryException(
|
||||
"Isolation level cannot be modified when storage mode is set to IN_MEMORY_ANALYTICAL."
|
||||
"IN_MEMORY_ANALYTICAL mode doesn't provide any isolation guarantees, "
|
||||
"you can think about it as an equivalent to READ_UNCOMMITED.") {}
|
||||
};
|
||||
|
||||
class StorageModeModificationInMulticommandTxException : public QueryException {
|
||||
public:
|
||||
StorageModeModificationInMulticommandTxException()
|
||||
: QueryException("Storage mode cannot be modified in multicommand transactions.") {}
|
||||
};
|
||||
|
||||
class CreateSnapshotInMulticommandTxException final : public QueryException {
|
||||
public:
|
||||
CreateSnapshotInMulticommandTxException()
|
||||
|
||||
@@ -243,6 +243,9 @@ constexpr utils::TypeInfo query::TriggerQuery::kType{utils::TypeId::AST_TRIGGER_
|
||||
constexpr utils::TypeInfo query::IsolationLevelQuery::kType{utils::TypeId::AST_ISOLATION_LEVEL_QUERY,
|
||||
"IsolationLevelQuery", &query::Query::kType};
|
||||
|
||||
constexpr utils::TypeInfo query::StorageModeQuery::kType{utils::TypeId::AST_STORAGE_MODE_QUERY, "StorageModeQuery",
|
||||
&query::Query::kType};
|
||||
|
||||
constexpr utils::TypeInfo query::CreateSnapshotQuery::kType{utils::TypeId::AST_CREATE_SNAPSHOT_QUERY,
|
||||
"CreateSnapshotQuery", &query::Query::kType};
|
||||
|
||||
|
||||
@@ -2714,6 +2714,7 @@ class AuthQuery : public memgraph::query::Query {
|
||||
MODULE_READ,
|
||||
MODULE_WRITE,
|
||||
WEBSOCKET,
|
||||
STORAGE_MODE,
|
||||
TRANSACTION_MANAGEMENT
|
||||
};
|
||||
|
||||
@@ -2777,7 +2778,8 @@ const std::vector<AuthQuery::Privilege> kPrivilegesAll = {
|
||||
AuthQuery::Privilege::FREE_MEMORY, AuthQuery::Privilege::TRIGGER,
|
||||
AuthQuery::Privilege::CONFIG, AuthQuery::Privilege::STREAM,
|
||||
AuthQuery::Privilege::MODULE_READ, AuthQuery::Privilege::MODULE_WRITE,
|
||||
AuthQuery::Privilege::WEBSOCKET, AuthQuery::Privilege::TRANSACTION_MANAGEMENT};
|
||||
AuthQuery::Privilege::WEBSOCKET, AuthQuery::Privilege::TRANSACTION_MANAGEMENT,
|
||||
AuthQuery::Privilege::STORAGE_MODE};
|
||||
|
||||
class InfoQuery : public memgraph::query::Query {
|
||||
public:
|
||||
@@ -3046,6 +3048,29 @@ class IsolationLevelQuery : public memgraph::query::Query {
|
||||
friend class AstStorage;
|
||||
};
|
||||
|
||||
class StorageModeQuery : public memgraph::query::Query {
|
||||
public:
|
||||
static const utils::TypeInfo kType;
|
||||
const utils::TypeInfo &GetTypeInfo() const override { return kType; }
|
||||
|
||||
enum class StorageMode { IN_MEMORY_TRANSACTIONAL, IN_MEMORY_ANALYTICAL };
|
||||
|
||||
StorageModeQuery() = default;
|
||||
|
||||
DEFVISITABLE(QueryVisitor<void>);
|
||||
|
||||
memgraph::query::StorageModeQuery::StorageMode storage_mode_;
|
||||
|
||||
StorageModeQuery *Clone(AstStorage *storage) const override {
|
||||
StorageModeQuery *object = storage->Create<StorageModeQuery>();
|
||||
object->storage_mode_ = storage_mode_;
|
||||
return object;
|
||||
}
|
||||
|
||||
private:
|
||||
friend class AstStorage;
|
||||
};
|
||||
|
||||
class CreateSnapshotQuery : public memgraph::query::Query {
|
||||
public:
|
||||
static const utils::TypeInfo kType;
|
||||
|
||||
@@ -89,6 +89,7 @@ class LoadCsv;
|
||||
class FreeMemoryQuery;
|
||||
class TriggerQuery;
|
||||
class IsolationLevelQuery;
|
||||
class StorageModeQuery;
|
||||
class CreateSnapshotQuery;
|
||||
class StreamQuery;
|
||||
class SettingQuery;
|
||||
@@ -134,6 +135,6 @@ class QueryVisitor
|
||||
: public utils::Visitor<TResult, CypherQuery, ExplainQuery, ProfileQuery, IndexQuery, AuthQuery, InfoQuery,
|
||||
ConstraintQuery, DumpQuery, ReplicationQuery, LockPathQuery, FreeMemoryQuery, TriggerQuery,
|
||||
IsolationLevelQuery, CreateSnapshotQuery, StreamQuery, SettingQuery, VersionQuery,
|
||||
ShowConfigQuery, TransactionQueueQuery, AnalyzeGraphQuery> {};
|
||||
ShowConfigQuery, TransactionQueueQuery, StorageModeQuery, AnalyzeGraphQuery> {};
|
||||
|
||||
} // namespace memgraph::query
|
||||
|
||||
@@ -487,6 +487,20 @@ antlrcpp::Any CypherMainVisitor::visitIsolationLevelQuery(MemgraphCypher::Isolat
|
||||
return isolation_level_query;
|
||||
}
|
||||
|
||||
antlrcpp::Any CypherMainVisitor::visitStorageModeQuery(MemgraphCypher::StorageModeQueryContext *ctx) {
|
||||
auto *storage_mode_query = storage_->Create<StorageModeQuery>();
|
||||
|
||||
storage_mode_query->storage_mode_ = std::invoke([mode = ctx->storageMode()]() {
|
||||
if (mode->IN_MEMORY_ANALYTICAL()) {
|
||||
return StorageModeQuery::StorageMode::IN_MEMORY_ANALYTICAL;
|
||||
}
|
||||
return StorageModeQuery::StorageMode::IN_MEMORY_TRANSACTIONAL;
|
||||
});
|
||||
|
||||
query_ = storage_mode_query;
|
||||
return storage_mode_query;
|
||||
}
|
||||
|
||||
antlrcpp::Any CypherMainVisitor::visitCreateSnapshotQuery(MemgraphCypher::CreateSnapshotQueryContext *ctx) {
|
||||
query_ = storage_->Create<CreateSnapshotQuery>();
|
||||
return query_;
|
||||
@@ -1511,6 +1525,7 @@ antlrcpp::Any CypherMainVisitor::visitPrivilege(MemgraphCypher::PrivilegeContext
|
||||
if (ctx->MODULE_WRITE()) return AuthQuery::Privilege::MODULE_WRITE;
|
||||
if (ctx->WEBSOCKET()) return AuthQuery::Privilege::WEBSOCKET;
|
||||
if (ctx->TRANSACTION_MANAGEMENT()) return AuthQuery::Privilege::TRANSACTION_MANAGEMENT;
|
||||
if (ctx->STORAGE_MODE()) return AuthQuery::Privilege::STORAGE_MODE;
|
||||
LOG_FATAL("Should not get here - unknown privilege!");
|
||||
}
|
||||
|
||||
|
||||
@@ -263,6 +263,11 @@ class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
|
||||
*/
|
||||
antlrcpp::Any visitIsolationLevelQuery(MemgraphCypher::IsolationLevelQueryContext *ctx) override;
|
||||
|
||||
/**
|
||||
* @return StorageModeQuery*
|
||||
*/
|
||||
antlrcpp::Any visitStorageModeQuery(MemgraphCypher::StorageModeQueryContext *ctx) override;
|
||||
|
||||
/**
|
||||
* @return CreateSnapshotQuery*
|
||||
*/
|
||||
|
||||
@@ -59,6 +59,8 @@ memgraphCypherKeyword : cypherKeyword
|
||||
| HEADER
|
||||
| IDENTIFIED
|
||||
| ISOLATION
|
||||
| IN_MEMORY_ANALYTICAL
|
||||
| IN_MEMORY_TRANSACTIONAL
|
||||
| KAFKA
|
||||
| LABELS
|
||||
| LEVEL
|
||||
@@ -88,6 +90,7 @@ memgraphCypherKeyword : cypherKeyword
|
||||
| SNAPSHOT
|
||||
| START
|
||||
| STATS
|
||||
| STORAGE
|
||||
| STREAM
|
||||
| STREAMS
|
||||
| SYNC
|
||||
@@ -127,6 +130,7 @@ query : cypherQuery
|
||||
| freeMemoryQuery
|
||||
| triggerQuery
|
||||
| isolationLevelQuery
|
||||
| storageModeQuery
|
||||
| createSnapshotQuery
|
||||
| streamQuery
|
||||
| settingQuery
|
||||
@@ -277,6 +281,7 @@ privilege : CREATE
|
||||
| MODULE_WRITE
|
||||
| WEBSOCKET
|
||||
| TRANSACTION_MANAGEMENT
|
||||
| STORAGE_MODE
|
||||
;
|
||||
|
||||
granularPrivilege : NOTHING | READ | UPDATE | CREATE_DELETE ;
|
||||
@@ -354,6 +359,10 @@ isolationLevelScope : GLOBAL | SESSION | NEXT ;
|
||||
|
||||
isolationLevelQuery : SET isolationLevelScope TRANSACTION ISOLATION LEVEL isolationLevel ;
|
||||
|
||||
storageMode : IN_MEMORY_ANALYTICAL | IN_MEMORY_TRANSACTIONAL ;
|
||||
|
||||
storageModeQuery : STORAGE MODE storageMode ;
|
||||
|
||||
createSnapshotQuery : CREATE SNAPSHOT ;
|
||||
|
||||
streamName : symbolicName ;
|
||||
|
||||
@@ -25,103 +25,107 @@ import CypherLexer ;
|
||||
|
||||
UNDERSCORE : '_' ;
|
||||
|
||||
AFTER : A F T E R ;
|
||||
ALTER : A L T E R ;
|
||||
ANALYZE : A N A L Y Z E ;
|
||||
ASYNC : A S Y N C ;
|
||||
AUTH : A U T H ;
|
||||
BAD : B A D ;
|
||||
BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ;
|
||||
BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ;
|
||||
BATCH_SIZE : B A T C H UNDERSCORE S I Z E ;
|
||||
BEFORE : B E F O R E ;
|
||||
BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ;
|
||||
CALL : C A L L ;
|
||||
CHECK : C H E C K ;
|
||||
CLEAR : C L E A R ;
|
||||
COMMIT : C O M M I T ;
|
||||
COMMITTED : C O M M I T T E D ;
|
||||
CONFIG : C O N F I G ;
|
||||
CONFIGS : C O N F I G S;
|
||||
CONSUMER_GROUP : C O N S U M E R UNDERSCORE G R O U P ;
|
||||
CREATE_DELETE : C R E A T E UNDERSCORE D E L E T E ;
|
||||
CREDENTIALS : C R E D E N T I A L S ;
|
||||
CSV : C S V ;
|
||||
DATA : D A T A ;
|
||||
DELIMITER : D E L I M I T E R ;
|
||||
DATABASE : D A T A B A S E ;
|
||||
DENY : D E N Y ;
|
||||
DIRECTORY : D I R E C T O R Y ;
|
||||
DROP : D R O P ;
|
||||
DUMP : D U M P ;
|
||||
DURABILITY : D U R A B I L I T Y ;
|
||||
EDGE_TYPES : E D G E UNDERSCORE T Y P E S ;
|
||||
EXECUTE : E X E C U T E ;
|
||||
FOR : F O R ;
|
||||
FOREACH : F O R E A C H;
|
||||
FREE : F R E E ;
|
||||
FREE_MEMORY : F R E E UNDERSCORE M E M O R Y ;
|
||||
FROM : F R O M ;
|
||||
GLOBAL : G L O B A L ;
|
||||
GRANT : G R A N T ;
|
||||
GRAPH : G R A P H ;
|
||||
GRANTS : G R A N T S ;
|
||||
HEADER : H E A D E R ;
|
||||
IDENTIFIED : I D E N T I F I E D ;
|
||||
IGNORE : I G N O R E ;
|
||||
ISOLATION : I S O L A T I O N ;
|
||||
KAFKA : K A F K A ;
|
||||
LABELS : L A B E L S ;
|
||||
LEVEL : L E V E L ;
|
||||
LOAD : L O A D ;
|
||||
LOCK : L O C K ;
|
||||
MAIN : M A I N ;
|
||||
MODE : M O D E ;
|
||||
MODULE_READ : M O D U L E UNDERSCORE R E A D ;
|
||||
MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ;
|
||||
NEXT : N E X T ;
|
||||
NO : N O ;
|
||||
NOTHING : N O T H I N G ;
|
||||
PASSWORD : P A S S W O R D ;
|
||||
PORT : P O R T ;
|
||||
PRIVILEGES : P R I V I L E G E S ;
|
||||
PULSAR : P U L S A R ;
|
||||
READ : R E A D ;
|
||||
READ_FILE : R E A D UNDERSCORE F I L E ;
|
||||
REGISTER : R E G I S T E R ;
|
||||
REPLICA : R E P L I C A ;
|
||||
REPLICAS : R E P L I C A S ;
|
||||
REPLICATION : R E P L I C A T I O N ;
|
||||
REVOKE : R E V O K E ;
|
||||
ROLE : R O L E ;
|
||||
ROLES : R O L E S ;
|
||||
QUOTE : Q U O T E ;
|
||||
SERVICE_URL : S E R V I C E UNDERSCORE U R L ;
|
||||
SESSION : S E S S I O N ;
|
||||
SETTING : S E T T I N G ;
|
||||
SETTINGS : S E T T I N G S ;
|
||||
SNAPSHOT : S N A P S H O T ;
|
||||
START : S T A R T ;
|
||||
STATISTICS : S T A T I S T I C S ;
|
||||
STATS : S T A T S ;
|
||||
STOP : S T O P ;
|
||||
STREAM : S T R E A M ;
|
||||
STREAMS : S T R E A M S ;
|
||||
SYNC : S Y N C ;
|
||||
TERMINATE : T E R M I N A T E ;
|
||||
TIMEOUT : T I M E O U T ;
|
||||
TO : T O ;
|
||||
TOPICS : T O P I C S;
|
||||
TRANSACTION : T R A N S A C T I O N ;
|
||||
TRANSACTION_MANAGEMENT : T R A N S A C T I O N UNDERSCORE M A N A G E M E N T ;
|
||||
TRANSACTIONS : T R A N S A C T I O N S ;
|
||||
TRANSFORM : T R A N S F O R M ;
|
||||
TRIGGER : T R I G G E R ;
|
||||
TRIGGERS : T R I G G E R S ;
|
||||
UNCOMMITTED : U N C O M M I T T E D ;
|
||||
UNLOCK : U N L O C K ;
|
||||
UPDATE : U P D A T E ;
|
||||
USER : U S E R ;
|
||||
USERS : U S E R S ;
|
||||
VERSION : V E R S I O N ;
|
||||
WEBSOCKET : W E B S O C K E T ;
|
||||
AFTER : A F T E R ;
|
||||
ALTER : A L T E R ;
|
||||
ANALYZE : A N A L Y Z E ;
|
||||
ASYNC : A S Y N C ;
|
||||
AUTH : A U T H ;
|
||||
BAD : B A D ;
|
||||
BATCH_INTERVAL : B A T C H UNDERSCORE I N T E R V A L ;
|
||||
BATCH_LIMIT : B A T C H UNDERSCORE L I M I T ;
|
||||
BATCH_SIZE : B A T C H UNDERSCORE S I Z E ;
|
||||
BEFORE : B E F O R E ;
|
||||
BOOTSTRAP_SERVERS : B O O T S T R A P UNDERSCORE S E R V E R S ;
|
||||
CALL : C A L L ;
|
||||
CHECK : C H E C K ;
|
||||
CLEAR : C L E A R ;
|
||||
COMMIT : C O M M I T ;
|
||||
COMMITTED : C O M M I T T E D ;
|
||||
CONFIG : C O N F I G ;
|
||||
CONFIGS : C O N F I G S;
|
||||
CONSUMER_GROUP : C O N S U M E R UNDERSCORE G R O U P ;
|
||||
CREATE_DELETE : C R E A T E UNDERSCORE D E L E T E ;
|
||||
CREDENTIALS : C R E D E N T I A L S ;
|
||||
CSV : C S V ;
|
||||
DATA : D A T A ;
|
||||
DELIMITER : D E L I M I T E R ;
|
||||
DATABASE : D A T A B A S E ;
|
||||
DENY : D E N Y ;
|
||||
DIRECTORY : D I R E C T O R Y ;
|
||||
DROP : D R O P ;
|
||||
DUMP : D U M P ;
|
||||
DURABILITY : D U R A B I L I T Y ;
|
||||
EDGE_TYPES : E D G E UNDERSCORE T Y P E S ;
|
||||
EXECUTE : E X E C U T E ;
|
||||
FOR : F O R ;
|
||||
FOREACH : F O R E A C H;
|
||||
FREE : F R E E ;
|
||||
FREE_MEMORY : F R E E UNDERSCORE M E M O R Y ;
|
||||
FROM : F R O M ;
|
||||
GLOBAL : G L O B A L ;
|
||||
GRANT : G R A N T ;
|
||||
GRAPH : G R A P H ;
|
||||
GRANTS : G R A N T S ;
|
||||
HEADER : H E A D E R ;
|
||||
IDENTIFIED : I D E N T I F I E D ;
|
||||
IGNORE : I G N O R E ;
|
||||
ISOLATION : I S O L A T I O N ;
|
||||
IN_MEMORY_ANALYTICAL : I N UNDERSCORE M E M O R Y UNDERSCORE A N A L Y T I C A L ;
|
||||
IN_MEMORY_TRANSACTIONAL : I N UNDERSCORE M E M O R Y UNDERSCORE T R A N S A C T I O N A L ;
|
||||
KAFKA : K A F K A ;
|
||||
LABELS : L A B E L S ;
|
||||
LEVEL : L E V E L ;
|
||||
LOAD : L O A D ;
|
||||
LOCK : L O C K ;
|
||||
MAIN : M A I N ;
|
||||
MODE : M O D E ;
|
||||
MODULE_READ : M O D U L E UNDERSCORE R E A D ;
|
||||
MODULE_WRITE : M O D U L E UNDERSCORE W R I T E ;
|
||||
NEXT : N E X T ;
|
||||
NO : N O ;
|
||||
NOTHING : N O T H I N G ;
|
||||
PASSWORD : P A S S W O R D ;
|
||||
PORT : P O R T ;
|
||||
PRIVILEGES : P R I V I L E G E S ;
|
||||
PULSAR : P U L S A R ;
|
||||
READ : R E A D ;
|
||||
READ_FILE : R E A D UNDERSCORE F I L E ;
|
||||
REGISTER : R E G I S T E R ;
|
||||
REPLICA : R E P L I C A ;
|
||||
REPLICAS : R E P L I C A S ;
|
||||
REPLICATION : R E P L I C A T I O N ;
|
||||
REVOKE : R E V O K E ;
|
||||
ROLE : R O L E ;
|
||||
ROLES : R O L E S ;
|
||||
QUOTE : Q U O T E ;
|
||||
SERVICE_URL : S E R V I C E UNDERSCORE U R L ;
|
||||
SESSION : S E S S I O N ;
|
||||
SETTING : S E T T I N G ;
|
||||
SETTINGS : S E T T I N G S ;
|
||||
SNAPSHOT : S N A P S H O T ;
|
||||
START : S T A R T ;
|
||||
STATISTICS : S T A T I S T I C S ;
|
||||
STATS : S T A T S ;
|
||||
STOP : S T O P ;
|
||||
STORAGE : S T O R A G E;
|
||||
STORAGE_MODE : S T O R A G E UNDERSCORE MODE;
|
||||
STREAM : S T R E A M ;
|
||||
STREAMS : S T R E A M S ;
|
||||
SYNC : S Y N C ;
|
||||
TERMINATE : T E R M I N A T E ;
|
||||
TIMEOUT : T I M E O U T ;
|
||||
TO : T O ;
|
||||
TOPICS : T O P I C S;
|
||||
TRANSACTION : T R A N S A C T I O N ;
|
||||
TRANSACTION_MANAGEMENT : T R A N S A C T I O N UNDERSCORE M A N A G E M E N T ;
|
||||
TRANSACTIONS : T R A N S A C T I O N S ;
|
||||
TRANSFORM : T R A N S F O R M ;
|
||||
TRIGGER : T R I G G E R ;
|
||||
TRIGGERS : T R I G G E R S ;
|
||||
UNCOMMITTED : U N C O M M I T T E D ;
|
||||
UNLOCK : U N L O C K ;
|
||||
UPDATE : U P D A T E ;
|
||||
USER : U S E R ;
|
||||
USERS : U S E R S ;
|
||||
VERSION : V E R S I O N ;
|
||||
WEBSOCKET : W E B S O C K E T ;
|
||||
|
||||
@@ -78,6 +78,8 @@ class PrivilegeExtractor : public QueryVisitor<void>, public HierarchicalTreeVis
|
||||
|
||||
void Visit(IsolationLevelQuery &isolation_level_query) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
|
||||
|
||||
void Visit(StorageModeQuery & /*storage_mode_query*/) override { AddPrivilege(AuthQuery::Privilege::STORAGE_MODE); }
|
||||
|
||||
void Visit(CreateSnapshotQuery &create_snapshot_query) override { AddPrivilege(AuthQuery::Privilege::DURABILITY); }
|
||||
|
||||
void Visit(SettingQuery & /*setting_query*/) override { AddPrivilege(AuthQuery::Privilege::CONFIG); }
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -208,7 +208,10 @@ const trie::Trie kKeywords = {"union",
|
||||
"websocket",
|
||||
"foreach",
|
||||
"labels",
|
||||
"edge_types"};
|
||||
"edge_types",
|
||||
"off",
|
||||
"in_memory_transactional",
|
||||
"in_memory_analytical"};
|
||||
|
||||
// Unicode codepoints that are allowed at the start of the unescaped name.
|
||||
const std::bitset<kBitsetSize> kUnescapedNameAllowedStarts(
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#include <functional>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
#include <thread>
|
||||
#include <unordered_map>
|
||||
@@ -62,11 +63,13 @@
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/on_scope_exit.hpp"
|
||||
#include "utils/readable_size.hpp"
|
||||
#include "utils/settings.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/tsc.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
namespace EventCounter {
|
||||
@@ -981,7 +984,8 @@ struct PullPlan {
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
|
||||
TriggerContextCollector *trigger_context_collector = nullptr,
|
||||
std::optional<size_t> memory_limit = {});
|
||||
std::optional<size_t> memory_limit = {}, bool use_monotonic_memory = true);
|
||||
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
@@ -1005,16 +1009,25 @@ struct PullPlan {
|
||||
// we have to keep track of any unsent results from previous `PullPlan::Pull`
|
||||
// manually by using this flag.
|
||||
bool has_unsent_results_ = false;
|
||||
|
||||
// In the case of LOAD CSV, we want to use only PoolResource without MonotonicMemoryResource
|
||||
// to reuse allocated memory. As LOAD CSV is processing row by row
|
||||
// it is possible to reduce memory usage significantly if MemoryResource deals with memory allocation
|
||||
// can reuse memory that was allocated on processing the first row on all subsequent rows.
|
||||
// This flag signals to `PullPlan::Pull` which MemoryResource to use
|
||||
bool use_monotonic_memory_;
|
||||
};
|
||||
|
||||
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
std::optional<std::string> username, std::atomic<TransactionStatus> *transaction_status,
|
||||
TriggerContextCollector *trigger_context_collector, const std::optional<size_t> memory_limit)
|
||||
TriggerContextCollector *trigger_context_collector, const std::optional<size_t> memory_limit,
|
||||
bool use_monotonic_memory)
|
||||
: plan_(plan),
|
||||
cursor_(plan->plan().MakeCursor(execution_memory)),
|
||||
frame_(plan->symbol_table().max_position(), execution_memory),
|
||||
memory_limit_(memory_limit) {
|
||||
memory_limit_(memory_limit),
|
||||
use_monotonic_memory_(use_monotonic_memory) {
|
||||
ctx_.db_accessor = dba;
|
||||
ctx_.symbol_table = plan->symbol_table();
|
||||
ctx_.evaluation_context.timestamp = QueryTimestamp();
|
||||
@@ -1023,7 +1036,14 @@ PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &par
|
||||
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, dba);
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (license::global_license_checker.IsEnterpriseValidFast() && username.has_value() && dba) {
|
||||
ctx_.auth_checker = interpreter_context->auth_checker->GetFineGrainedAuthChecker(*username, dba);
|
||||
auto auth_checker = interpreter_context->auth_checker->GetFineGrainedAuthChecker(*username, dba);
|
||||
|
||||
// if the user has global privileges to read, edit and write anything, we don't need to perform authorization
|
||||
// otherwise, we do assign the auth checker to check for label access control
|
||||
if (!auth_checker->HasGlobalPrivilegeOnVertices(AuthQuery::FineGrainedPrivilege::CREATE_DELETE) ||
|
||||
!auth_checker->HasGlobalPrivilegeOnEdges(AuthQuery::FineGrainedPrivilege::CREATE_DELETE)) {
|
||||
ctx_.auth_checker = std::move(auth_checker);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
if (interpreter_context->config.execution_timeout_sec > 0) {
|
||||
@@ -1043,21 +1063,30 @@ std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *strea
|
||||
// single `Pull`.
|
||||
static constexpr size_t stack_size = 256UL * 1024UL;
|
||||
char stack_data[stack_size];
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory, utils::NewDeleteResource());
|
||||
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
|
||||
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory{&stack_data[0], stack_size, &resource_with_exception};
|
||||
std::optional<utils::PoolResource> pool_memory;
|
||||
static constexpr auto kMaxBlockPerChunks = 128;
|
||||
|
||||
if (!use_monotonic_memory_) {
|
||||
pool_memory.emplace(kMaxBlockPerChunks, kExecutionPoolMaxBlockSize, &resource_with_exception,
|
||||
&resource_with_exception);
|
||||
} else {
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
pool_memory.emplace(kMaxBlockPerChunks, 1024, &monotonic_memory, &resource_with_exception);
|
||||
}
|
||||
|
||||
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
|
||||
if (memory_limit_) {
|
||||
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
|
||||
maybe_limited_resource.emplace(&*pool_memory, *memory_limit_);
|
||||
ctx_.evaluation_context.memory = &*maybe_limited_resource;
|
||||
} else {
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
ctx_.evaluation_context.memory = &*pool_memory;
|
||||
}
|
||||
|
||||
// Returns true if a result was pulled.
|
||||
@@ -1143,7 +1172,6 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
if (in_explicit_transaction_) {
|
||||
throw ExplicitTransactionUsageException("Nested transactions are not supported.");
|
||||
}
|
||||
|
||||
in_explicit_transaction_ = true;
|
||||
expect_rollback_ = false;
|
||||
|
||||
@@ -1216,16 +1244,19 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
if (memory_limit) {
|
||||
spdlog::info("Running query with memory limit of {}", utils::GetReadableSize(*memory_limit));
|
||||
}
|
||||
|
||||
if (const auto &clauses = cypher_query->single_query_->clauses_; std::any_of(
|
||||
clauses.begin(), clauses.end(), [](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) {
|
||||
auto clauses = cypher_query->single_query_->clauses_;
|
||||
bool contains_csv = false;
|
||||
if (std::any_of(clauses.begin(), clauses.end(),
|
||||
[](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) {
|
||||
notifications->emplace_back(
|
||||
SeverityLevel::INFO, NotificationCode::LOAD_CSV_TIP,
|
||||
"It's important to note that the parser parses the values as strings. It's up to the user to "
|
||||
"convert the parsed row values to the appropriate type. This can be done using the built-in "
|
||||
"conversion functions such as ToInteger, ToFloat, ToBoolean etc.");
|
||||
contains_csv = true;
|
||||
}
|
||||
|
||||
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
|
||||
auto use_monotonic_memory = !contains_csv;
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
|
||||
parsed_query.parameters,
|
||||
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
|
||||
@@ -1248,7 +1279,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
}
|
||||
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
|
||||
execution_memory, StringPointerToOptional(username), transaction_status,
|
||||
trigger_context_collector, memory_limit);
|
||||
trigger_context_collector, memory_limit, use_monotonic_memory);
|
||||
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
|
||||
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
@@ -1346,6 +1377,16 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
&interpreter_context->ast_cache, interpreter_context->config.query);
|
||||
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
|
||||
bool contains_csv = false;
|
||||
auto clauses = cypher_query->single_query_->clauses_;
|
||||
if (std::any_of(clauses.begin(), clauses.end(),
|
||||
[](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) {
|
||||
contains_csv = true;
|
||||
}
|
||||
// If this is LOAD CSV query, use PoolResource without MonotonicMemoryResource as we want to reuse allocated memory
|
||||
auto use_monotonic_memory = !contains_csv;
|
||||
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
|
||||
Frame frame(0);
|
||||
SymbolTable symbol_table;
|
||||
@@ -1370,14 +1411,14 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
// We want to execute the query we are profiling lazily, so we delay
|
||||
// the construction of the corresponding context.
|
||||
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
|
||||
pull_plan = std::shared_ptr<PullPlanVector>(nullptr), transaction_status](
|
||||
pull_plan = std::shared_ptr<PullPlanVector>(nullptr), transaction_status, use_monotonic_memory](
|
||||
AnyStream *stream, std::optional<int> n) mutable -> std::optional<QueryHandlerResult> {
|
||||
// No output symbols are given so that nothing is streamed.
|
||||
if (!stats_and_total_time) {
|
||||
stats_and_total_time =
|
||||
PullPlan(plan, parameters, true, dba, interpreter_context, execution_memory,
|
||||
optional_username, transaction_status, nullptr, memory_limit)
|
||||
.Pull(stream, {}, {}, summary);
|
||||
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
|
||||
execution_memory, optional_username, transaction_status,
|
||||
nullptr, memory_limit, use_monotonic_memory)
|
||||
.Pull(stream, {}, {}, summary);
|
||||
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
|
||||
}
|
||||
|
||||
@@ -1428,8 +1469,8 @@ std::vector<std::vector<TypedValue>> AnalyzeGraphQueryHandler::AnalyzeGraphCreat
|
||||
// Iterate over all indexed vertices
|
||||
std::for_each(indices_info.begin(), indices_info.end(), [execution_db_accessor, &counter](const LPIndex &index_info) {
|
||||
auto vertices = execution_db_accessor->Vertices(storage::View::OLD, index_info.first, index_info.second);
|
||||
std::for_each(vertices.begin(), vertices.end(), [&index_info, &counter](const auto &vertex) {
|
||||
counter[index_info][*vertex.GetProperty(storage::View::OLD, index_info.second)]++;
|
||||
std::for_each(vertices.begin(), vertices.end(), [&index_info, &counter](const auto *vertex) {
|
||||
counter[index_info][*vertex->GetProperty(storage::View::OLD, index_info.second)]++;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1970,7 +2011,16 @@ constexpr auto ToStorageIsolationLevel(const IsolationLevelQuery::IsolationLevel
|
||||
}
|
||||
}
|
||||
|
||||
PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
constexpr auto ToStorageMode(const StorageModeQuery::StorageMode storage_mode) noexcept {
|
||||
switch (storage_mode) {
|
||||
case StorageModeQuery::StorageMode::IN_MEMORY_TRANSACTIONAL:
|
||||
return storage::StorageMode::IN_MEMORY_TRANSACTIONAL;
|
||||
case StorageModeQuery::StorageMode::IN_MEMORY_ANALYTICAL:
|
||||
return storage::StorageMode::IN_MEMORY_ANALYTICAL;
|
||||
}
|
||||
}
|
||||
|
||||
PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
InterpreterContext *interpreter_context, Interpreter *interpreter) {
|
||||
if (in_explicit_transaction) {
|
||||
throw IsolationLevelModificationInMulticommandTxException();
|
||||
@@ -1985,7 +2035,15 @@ PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, bool in_expli
|
||||
interpreter]() -> std::function<void()> {
|
||||
switch (isolation_level_query->isolation_level_scope_) {
|
||||
case IsolationLevelQuery::IsolationLevelScope::GLOBAL:
|
||||
return [interpreter_context, isolation_level] { interpreter_context->db->SetIsolationLevel(isolation_level); };
|
||||
return [interpreter_context, isolation_level] {
|
||||
if (auto maybe_error = interpreter_context->db->SetIsolationLevel(isolation_level); maybe_error.HasError()) {
|
||||
switch (maybe_error.GetError()) {
|
||||
case storage::Storage::SetIsolationLevelError::DisabledForAnalyticalMode:
|
||||
throw IsolationLevelModificationInAnalyticsException();
|
||||
break;
|
||||
}
|
||||
}
|
||||
};
|
||||
case IsolationLevelQuery::IsolationLevelScope::SESSION:
|
||||
return [interpreter, isolation_level] { interpreter->SetSessionIsolationLevel(isolation_level); };
|
||||
case IsolationLevelQuery::IsolationLevelScope::NEXT:
|
||||
@@ -2003,6 +2061,41 @@ PreparedQuery PrepareIsolationLevelQuery(ParsedQuery parsed_query, bool in_expli
|
||||
RWType::NONE};
|
||||
}
|
||||
|
||||
PreparedQuery PrepareStorageModeQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
InterpreterContext *interpreter_context) {
|
||||
if (in_explicit_transaction) {
|
||||
throw StorageModeModificationInMulticommandTxException();
|
||||
}
|
||||
|
||||
auto *storage_mode_query = utils::Downcast<StorageModeQuery>(parsed_query.query);
|
||||
MG_ASSERT(storage_mode_query);
|
||||
const auto storage_mode = ToStorageMode(storage_mode_query->storage_mode_);
|
||||
|
||||
auto exists_active_transaction = interpreter_context->interpreters.WithLock([](const auto &interpreters_) {
|
||||
return std::any_of(interpreters_.begin(), interpreters_.end(), [](const auto &interpreter) {
|
||||
return interpreter->transaction_status_.load() != TransactionStatus::IDLE;
|
||||
});
|
||||
});
|
||||
if (exists_active_transaction) {
|
||||
spdlog::info(
|
||||
"Storage mode will be modified when there are no other active transactions. Check the status of the "
|
||||
"transactions using 'SHOW TRANSACTIONS' query and ensure no other transactions are active.");
|
||||
}
|
||||
|
||||
auto callback = [storage_mode, interpreter_context]() -> std::function<void()> {
|
||||
return [interpreter_context, storage_mode] { interpreter_context->db->SetStorageMode(storage_mode); };
|
||||
}();
|
||||
|
||||
return PreparedQuery{{},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[callback = std::move(callback)](AnyStream * /*stream*/,
|
||||
std::optional<int> /*n*/) -> std::optional<QueryHandlerResult> {
|
||||
callback();
|
||||
return QueryHandlerResult::COMMIT;
|
||||
},
|
||||
RWType::NONE};
|
||||
}
|
||||
|
||||
PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
InterpreterContext *interpreter_context) {
|
||||
if (in_explicit_transaction) {
|
||||
@@ -2013,11 +2106,18 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
|
||||
{},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[interpreter_context](AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
if (auto maybe_error = interpreter_context->db->CreateSnapshot(); maybe_error.HasError()) {
|
||||
if (auto maybe_error = interpreter_context->db->CreateSnapshot({}); maybe_error.HasError()) {
|
||||
switch (maybe_error.GetError()) {
|
||||
case storage::Storage::CreateSnapshotError::DisabledForReplica:
|
||||
throw utils::BasicException(
|
||||
"Failed to create a snapshot. Replica instances are not allowed to create them.");
|
||||
case storage::Storage::CreateSnapshotError::DisabledForAnalyticsPeriodicCommit:
|
||||
spdlog::warn(utils::MessageWithLink("Periodic snapshots are disabled for analytical mode.",
|
||||
"https://memgr.ph/replication"));
|
||||
break;
|
||||
case storage::Storage::CreateSnapshotError::ReachedMaxNumTries:
|
||||
spdlog::warn("Failed to create snapshot. Reached max number of tries. Please contact support");
|
||||
break;
|
||||
}
|
||||
}
|
||||
return QueryHandlerResult::COMMIT;
|
||||
@@ -2592,18 +2692,18 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
std::optional<std::string> user = StringPointerToOptional(username);
|
||||
username_ = user;
|
||||
|
||||
query_executions_.emplace_back(std::make_unique<QueryExecution>());
|
||||
auto &query_execution = query_executions_.back();
|
||||
|
||||
std::optional<int> qid =
|
||||
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
|
||||
|
||||
// Handle transaction control queries.
|
||||
|
||||
const auto upper_case_query = utils::ToUpperCase(query_string);
|
||||
const auto trimmed_query = utils::Trim(upper_case_query);
|
||||
|
||||
if (trimmed_query == "BEGIN" || trimmed_query == "COMMIT" || trimmed_query == "ROLLBACK") {
|
||||
query_executions_.emplace_back(
|
||||
std::make_unique<QueryExecution>(utils::MonotonicBufferResource(kExecutionMemoryBlockSize)));
|
||||
auto &query_execution = query_executions_.back();
|
||||
std::optional<int> qid =
|
||||
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
|
||||
|
||||
query_execution->prepared_query.emplace(PrepareTransactionQuery(trimmed_query));
|
||||
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
|
||||
}
|
||||
@@ -2619,18 +2719,50 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
// If we're not in an explicit transaction block and we have an open
|
||||
// transaction, abort it since we're about to prepare a new query.
|
||||
else if (db_accessor_) {
|
||||
AbortCommand(&query_execution);
|
||||
query_executions_.emplace_back(
|
||||
std::make_unique<QueryExecution>(utils::MonotonicBufferResource(kExecutionMemoryBlockSize)));
|
||||
AbortCommand(&query_executions_.back());
|
||||
}
|
||||
|
||||
std::unique_ptr<QueryExecution> *query_execution_ptr = nullptr;
|
||||
try {
|
||||
// Set a default cost estimate of 0. Individual queries can overwrite this
|
||||
// field with an improved estimate.
|
||||
query_execution->summary["cost_estimate"] = 0.0;
|
||||
|
||||
query_executions_.emplace_back(
|
||||
std::make_unique<QueryExecution>(utils::MonotonicBufferResource(kExecutionMemoryBlockSize)));
|
||||
query_execution_ptr = &query_executions_.back();
|
||||
utils::Timer parsing_timer;
|
||||
ParsedQuery parsed_query =
|
||||
ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query);
|
||||
query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count();
|
||||
TypedValue parsing_time{parsing_timer.Elapsed().count()};
|
||||
|
||||
if ((utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ProfileQuery>(parsed_query.query))) {
|
||||
CypherQuery *cypher_query = nullptr;
|
||||
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
|
||||
cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
|
||||
} else {
|
||||
auto *profile_query = utils::Downcast<ProfileQuery>(parsed_query.query);
|
||||
cypher_query = profile_query->cypher_query_;
|
||||
}
|
||||
|
||||
if (const auto &clauses = cypher_query->single_query_->clauses_;
|
||||
std::any_of(clauses.begin(), clauses.end(),
|
||||
[](const auto *clause) { return clause->GetTypeInfo() == LoadCsv::kType; })) {
|
||||
// Using PoolResource without MonotonicMemoryResouce for LOAD CSV reduces memory usage.
|
||||
// QueryExecution MemoryResource is mostly used for allocations done on Frame and storing `row`s
|
||||
query_executions_[query_executions_.size() - 1] = std::make_unique<QueryExecution>(
|
||||
utils::PoolResource(8, kExecutionPoolMaxBlockSize, utils::NewDeleteResource(), utils::NewDeleteResource()));
|
||||
query_execution_ptr = &query_executions_.back();
|
||||
}
|
||||
}
|
||||
|
||||
auto &query_execution = query_executions_.back();
|
||||
|
||||
std::optional<int> qid =
|
||||
in_explicit_transaction_ ? static_cast<int>(query_executions_.size() - 1) : std::optional<int>{};
|
||||
|
||||
query_execution->summary["parsing_time"] = std::move(parsing_time);
|
||||
|
||||
// Set a default cost estimate of 0. Individual queries can overwrite this
|
||||
// field with an improved estimate.
|
||||
query_execution->summary["cost_estimate"] = 0.0;
|
||||
|
||||
// Some queries require an active transaction in order to be prepared.
|
||||
if (!in_explicit_transaction_ &&
|
||||
@@ -2649,12 +2781,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
|
||||
utils::Timer planning_timer;
|
||||
PreparedQuery prepared_query;
|
||||
|
||||
utils::MemoryResource *memory_resource =
|
||||
std::visit([](auto &execution_memory) -> utils::MemoryResource * { return &execution_memory; },
|
||||
query_execution->execution_memory);
|
||||
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory,
|
||||
&query_execution->notifications, username, &transaction_status_,
|
||||
trigger_context_collector_ ? &*trigger_context_collector_ : nullptr);
|
||||
prepared_query =
|
||||
PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, memory_resource, &query_execution->notifications, username,
|
||||
&transaction_status_, trigger_context_collector_ ? &*trigger_context_collector_ : nullptr);
|
||||
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory_with_exception);
|
||||
@@ -2664,7 +2798,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory_with_exception, username, &transaction_status_);
|
||||
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory);
|
||||
memory_resource);
|
||||
} else if (utils::Downcast<IndexQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareIndexQuery(std::move(parsed_query), in_explicit_transaction_,
|
||||
&query_execution->notifications, interpreter_context_);
|
||||
@@ -2711,6 +2845,8 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_);
|
||||
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
|
||||
} else if (utils::Downcast<StorageModeQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareStorageModeQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
|
||||
} else if (utils::Downcast<TransactionQueueQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareTransactionQueueQuery(std::move(parsed_query), username_, in_explicit_transaction_,
|
||||
interpreter_context_, &*execution_db_accessor_);
|
||||
@@ -2736,7 +2872,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
return {query_execution->prepared_query->header, query_execution->prepared_query->privileges, qid};
|
||||
} catch (const utils::BasicException &) {
|
||||
EventCounter::IncrementCounter(EventCounter::FailedQuery);
|
||||
AbortCommand(&query_execution);
|
||||
AbortCommand(query_execution_ptr);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,7 @@ extern const Event FailedQuery;
|
||||
namespace memgraph::query {
|
||||
|
||||
inline constexpr size_t kExecutionMemoryBlockSize = 1UL * 1024UL * 1024UL;
|
||||
inline constexpr size_t kExecutionPoolMaxBlockSize = 2048UL; // 2 ^ 11
|
||||
|
||||
class AuthQueryHandler {
|
||||
public:
|
||||
@@ -340,13 +341,29 @@ class Interpreter final {
|
||||
private:
|
||||
struct QueryExecution {
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
|
||||
utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
|
||||
std::variant<utils::MonotonicBufferResource, utils::PoolResource> execution_memory;
|
||||
utils::ResourceWithOutOfMemoryException execution_memory_with_exception;
|
||||
|
||||
std::map<std::string, TypedValue> summary;
|
||||
std::vector<Notification> notifications;
|
||||
|
||||
explicit QueryExecution() = default;
|
||||
explicit QueryExecution(utils::MonotonicBufferResource monotonic_memory)
|
||||
: execution_memory(std::move(monotonic_memory)) {
|
||||
std::visit(
|
||||
[&](auto &memory_resource) {
|
||||
execution_memory_with_exception = utils::ResourceWithOutOfMemoryException(&memory_resource);
|
||||
},
|
||||
execution_memory);
|
||||
};
|
||||
|
||||
explicit QueryExecution(utils::PoolResource pool_resource) : execution_memory(std::move(pool_resource)) {
|
||||
std::visit(
|
||||
[&](auto &memory_resource) {
|
||||
execution_memory_with_exception = utils::ResourceWithOutOfMemoryException(&memory_resource);
|
||||
},
|
||||
execution_memory);
|
||||
};
|
||||
|
||||
QueryExecution(const QueryExecution &) = delete;
|
||||
QueryExecution(QueryExecution &&) = default;
|
||||
QueryExecution &operator=(const QueryExecution &) = delete;
|
||||
@@ -357,7 +374,7 @@ class Interpreter final {
|
||||
// destroy the prepared query which is using that instance
|
||||
// of execution memory.
|
||||
prepared_query.reset();
|
||||
execution_memory.Release();
|
||||
std::visit([](auto &memory_resource) { memory_resource.Release(); }, execution_memory);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -445,7 +462,9 @@ std::map<std::string, TypedValue> Interpreter::Pull(TStream *result_stream, std:
|
||||
try {
|
||||
// Wrap the (statically polymorphic) stream type into a common type which
|
||||
// the handler knows.
|
||||
AnyStream stream{result_stream, &query_execution->execution_memory};
|
||||
AnyStream stream{result_stream,
|
||||
std::visit([](auto &execution_memory) -> utils::MemoryResource * { return &execution_memory; },
|
||||
query_execution->execution_memory)};
|
||||
const auto maybe_res = query_execution->prepared_query->query_handler(&stream, n);
|
||||
// Stream is using execution memory of the query_execution which
|
||||
// can be deleted after its execution so the stream should be cleared
|
||||
|
||||
@@ -54,6 +54,7 @@
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/pmr/deque.hpp"
|
||||
#include "utils/pmr/list.hpp"
|
||||
#include "utils/pmr/unordered_map.hpp"
|
||||
#include "utils/pmr/unordered_set.hpp"
|
||||
@@ -439,7 +440,7 @@ class ScanAllCursor : public Cursor {
|
||||
// the roundabout assignment + emplace, instead of simple:
|
||||
// vertices _ = get_vertices_(frame, context);
|
||||
vertices_.emplace(std::move(next_vertices.value()));
|
||||
vertices_it_.emplace(vertices_.value().begin());
|
||||
vertices_it_ = vertices_.value().begin();
|
||||
}
|
||||
#ifdef MG_ENTERPRISE
|
||||
if (license::global_license_checker.IsEnterpriseValidFast() && context.auth_checker && !FindNextVertex(context)) {
|
||||
@@ -3249,7 +3250,7 @@ class AccumulateCursor : public Cursor {
|
||||
private:
|
||||
const Accumulate &self_;
|
||||
const UniqueCursorPtr input_cursor_;
|
||||
utils::pmr::vector<utils::pmr::vector<TypedValue>> cache_;
|
||||
utils::pmr::deque<utils::pmr::vector<TypedValue>> cache_;
|
||||
decltype(cache_.begin()) cache_it_ = cache_.begin();
|
||||
bool pulled_all_input_{false};
|
||||
};
|
||||
|
||||
@@ -564,7 +564,11 @@ class IndexLookupRewriter final : public HierarchicalLogicalOperatorVisitor {
|
||||
*/
|
||||
auto compare_indices = [](std::optional<LabelPropertyIndex> &found, std::optional<storage::IndexStats> &new_stats,
|
||||
int vertex_count) {
|
||||
if (!new_stats.has_value() || vertex_count / 10.0 > found->vertex_count) {
|
||||
if (!new_stats.has_value()) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (vertex_count / 10.0 > found->vertex_count) {
|
||||
return 1;
|
||||
}
|
||||
int cmp_avg_group = utils::CompareDecimal(new_stats->avg_group_size, found->index_stats->avg_group_size);
|
||||
|
||||
@@ -2589,13 +2589,15 @@ mgp_vertices_iterator::mgp_vertices_iterator(mgp_graph *graph, memgraph::utils::
|
||||
#endif
|
||||
|
||||
if (current_it != vertices.end()) {
|
||||
std::visit(
|
||||
memgraph::utils::Overloaded{
|
||||
[this, graph, memory](memgraph::query::DbAccessor *) { current_v.emplace(*current_it, graph, memory); },
|
||||
[this, graph, memory](memgraph::query::SubgraphDbAccessor *impl) {
|
||||
current_v.emplace(memgraph::query::SubgraphVertexAccessor(*current_it, impl->getGraph()), graph, memory);
|
||||
}},
|
||||
graph->impl);
|
||||
std::visit(memgraph::utils::Overloaded{[this, graph, memory](memgraph::query::DbAccessor *) {
|
||||
current_v.emplace(std::move(**current_it), graph, memory);
|
||||
},
|
||||
[this, graph, memory](memgraph::query::SubgraphDbAccessor *impl) {
|
||||
current_v.emplace(memgraph::query::SubgraphVertexAccessor(
|
||||
std::move(**current_it), impl->getGraph()),
|
||||
graph, memory);
|
||||
}},
|
||||
graph->impl);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2642,15 +2644,15 @@ mgp_error mgp_vertices_iterator_next(mgp_vertices_iterator *it, mgp_vertex **res
|
||||
}
|
||||
|
||||
memgraph::utils::OnScopeExit clean_up([it] { it->current_v = std::nullopt; });
|
||||
std::visit(memgraph::utils::Overloaded{[it](memgraph::query::DbAccessor *) {
|
||||
it->current_v.emplace(*it->current_it, it->graph,
|
||||
it->GetMemoryResource());
|
||||
},
|
||||
[it](memgraph::query::SubgraphDbAccessor *impl) {
|
||||
it->current_v.emplace(memgraph::query::SubgraphVertexAccessor(
|
||||
*it->current_it, impl->getGraph()),
|
||||
it->graph, it->GetMemoryResource());
|
||||
}},
|
||||
std::visit(memgraph::utils::Overloaded{
|
||||
[it](memgraph::query::DbAccessor *) {
|
||||
it->current_v.emplace(std::move(**it->current_it), it->graph, it->GetMemoryResource());
|
||||
},
|
||||
[it](memgraph::query::SubgraphDbAccessor *impl) {
|
||||
it->current_v.emplace(
|
||||
memgraph::query::SubgraphVertexAccessor(std::move(**it->current_it), impl->getGraph()),
|
||||
it->graph, it->GetMemoryResource());
|
||||
}},
|
||||
it->graph->impl);
|
||||
|
||||
clean_up.Disable();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,6 +19,7 @@
|
||||
#include <string_view>
|
||||
#include <utility>
|
||||
|
||||
#include "query/db_accessor.hpp"
|
||||
#include "storage/v2/temporal.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
@@ -462,6 +463,16 @@ TypedValue &TypedValue::operator=(const std::map<std::string, TypedValue> &other
|
||||
return *this;
|
||||
}
|
||||
|
||||
TypedValue &TypedValue::operator=(VertexAccessor *other) {
|
||||
if (this->type_ == TypedValue::Type::Vertex) {
|
||||
this->vertex_v = std::move(*other);
|
||||
} else {
|
||||
*this = TypedValue(std::move(*other), memory_);
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(const VertexAccessor &, Vertex, vertex_v)
|
||||
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(const EdgeAccessor &, Edge, edge_v)
|
||||
DEFINE_TYPED_VALUE_COPY_ASSIGNMENT(const Path &, Path, path_v)
|
||||
|
||||
@@ -445,6 +445,7 @@ class TypedValue {
|
||||
TypedValue &operator=(const TMap &);
|
||||
TypedValue &operator=(const std::map<std::string, TypedValue> &);
|
||||
TypedValue &operator=(const VertexAccessor &);
|
||||
TypedValue &operator=(VertexAccessor *);
|
||||
TypedValue &operator=(const EdgeAccessor &);
|
||||
TypedValue &operator=(const Path &);
|
||||
TypedValue &operator=(const utils::Date &);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -50,6 +50,11 @@ struct Config {
|
||||
|
||||
bool snapshot_on_exit{false};
|
||||
bool restore_replicas_on_startup{false};
|
||||
|
||||
uint64_t items_per_batch{1'000'000};
|
||||
uint64_t recovery_thread_count{8};
|
||||
|
||||
bool allow_parallel_index_creation{false};
|
||||
} durability;
|
||||
|
||||
struct Transaction {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -113,13 +113,15 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
|
||||
// to ensure that the indices and constraints are consistent at the end of the
|
||||
// recovery process.
|
||||
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
|
||||
Constraints *constraints, utils::SkipList<Vertex> *vertices) {
|
||||
Constraints *constraints, utils::SkipList<Vertex> *vertices,
|
||||
const std::optional<ParalellizedIndexCreationInfo> ¶lell_exec_info) {
|
||||
spdlog::info("Recreating indices from metadata.");
|
||||
// Recover label indices.
|
||||
spdlog::info("Recreating {} label indices from metadata.", indices_constraints.indices.label.size());
|
||||
for (const auto &item : indices_constraints.indices.label) {
|
||||
if (!indices->label_index.CreateIndex(item, vertices->access()))
|
||||
if (!indices->label_index.CreateIndex(item, vertices->access(), paralell_exec_info))
|
||||
throw RecoveryFailure("The label index must be created here!");
|
||||
|
||||
spdlog::info("A label index is recreated from metadata.");
|
||||
}
|
||||
spdlog::info("Label indices are recreated.");
|
||||
@@ -163,7 +165,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
|
||||
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
|
||||
std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
|
||||
Indices *indices, Constraints *constraints, Config::Items items,
|
||||
Indices *indices, Constraints *constraints, const Config &config,
|
||||
uint64_t *wal_seq_num) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
spdlog::info("Recovering persisted data using snapshot ({}) and WAL directory ({}).", snapshot_directory,
|
||||
@@ -195,7 +197,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
}
|
||||
spdlog::info("Starting snapshot recovery from {}.", path);
|
||||
try {
|
||||
recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history, name_id_mapper, edge_count, items);
|
||||
recovered_snapshot = LoadSnapshot(path, vertices, edges, epoch_history, name_id_mapper, edge_count, config);
|
||||
spdlog::info("Snapshot recovery successful!");
|
||||
break;
|
||||
} catch (const RecoveryFailure &e) {
|
||||
@@ -213,7 +215,11 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
*epoch_id = std::move(recovered_snapshot->snapshot_info.epoch_id);
|
||||
|
||||
if (!utils::DirExists(wal_directory)) {
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices);
|
||||
const auto par_exec_info = config.durability.allow_parallel_index_creation
|
||||
? std::make_optional(std::make_pair(recovery_info.vertex_batches,
|
||||
config.durability.recovery_thread_count))
|
||||
: std::nullopt;
|
||||
RecoverIndicesAndConstraints(indices_constraints, indices, constraints, vertices, par_exec_info);
|
||||
return recovered_snapshot->recovery_info;
|
||||
}
|
||||
} else {
|
||||
@@ -319,7 +325,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
}
|
||||
try {
|
||||
auto info = LoadWal(wal_file.path, &indices_constraints, last_loaded_timestamp, vertices, edges, name_id_mapper,
|
||||
edge_count, items);
|
||||
edge_count, config.items);
|
||||
recovery_info.next_vertex_id = std::max(recovery_info.next_vertex_id, info.next_vertex_id);
|
||||
recovery_info.next_edge_id = std::max(recovery_info.next_edge_id, info.next_edge_id);
|
||||
recovery_info.next_timestamp = std::max(recovery_info.next_timestamp, info.next_timestamp);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -91,13 +91,18 @@ std::optional<std::vector<WalDurabilityInfo>> GetWalFiles(const std::filesystem:
|
||||
std::string_view uuid = "",
|
||||
std::optional<size_t> current_seq_num = {});
|
||||
|
||||
using ParalellizedIndexCreationInfo =
|
||||
std::pair<std::vector<std::pair<Gid, uint64_t>> /*vertex_recovery_info*/, uint64_t /*thread_count*/>;
|
||||
|
||||
// Helper function used to recover all discovered indices and constraints. The
|
||||
// indices and constraints must be recovered after the data recovery is done
|
||||
// to ensure that the indices and constraints are consistent at the end of the
|
||||
// recovery process.
|
||||
/// @throw RecoveryFailure
|
||||
void RecoverIndicesAndConstraints(const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices,
|
||||
Constraints *constraints, utils::SkipList<Vertex> *vertices);
|
||||
void RecoverIndicesAndConstraints(
|
||||
const RecoveredIndicesAndConstraints &indices_constraints, Indices *indices, Constraints *constraints,
|
||||
utils::SkipList<Vertex> *vertices,
|
||||
const std::optional<ParalellizedIndexCreationInfo> ¶lell_exec_info = std::nullopt);
|
||||
|
||||
/// Recovers data either from a snapshot and/or WAL files.
|
||||
/// @throw RecoveryFailure
|
||||
@@ -108,7 +113,7 @@ std::optional<RecoveryInfo> RecoverData(const std::filesystem::path &snapshot_di
|
||||
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
|
||||
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges,
|
||||
std::atomic<uint64_t> *edge_count, NameIdMapper *name_id_mapper,
|
||||
Indices *indices, Constraints *constraints, Config::Items items,
|
||||
Indices *indices, Constraints *constraints, const Config &config,
|
||||
uint64_t *wal_seq_num);
|
||||
|
||||
} // namespace memgraph::storage::durability
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,6 +12,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <optional>
|
||||
#include <set>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
@@ -29,6 +30,8 @@ struct RecoveryInfo {
|
||||
|
||||
// last timestamp read from a WAL file
|
||||
std::optional<uint64_t> last_commit_timestamp;
|
||||
|
||||
std::vector<std::pair<Gid /*first vertex gid*/, uint64_t /*batch size*/>> vertex_batches;
|
||||
};
|
||||
|
||||
/// Structure used to track indices and constraints during recovery.
|
||||
|
||||
@@ -11,19 +11,27 @@
|
||||
|
||||
#include "storage/v2/durability/snapshot.hpp"
|
||||
|
||||
#include <thread>
|
||||
|
||||
#include "storage/v2/durability/exceptions.hpp"
|
||||
#include "storage/v2/durability/paths.hpp"
|
||||
#include "storage/v2/durability/serialization.hpp"
|
||||
#include "storage/v2/durability/version.hpp"
|
||||
#include "storage/v2/durability/wal.hpp"
|
||||
#include "storage/v2/edge.hpp"
|
||||
#include "storage/v2/edge_accessor.hpp"
|
||||
#include "storage/v2/edge_ref.hpp"
|
||||
#include "storage/v2/id_types.hpp"
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/storage.hpp"
|
||||
#include "storage/v2/vertex.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
#include "utils/file_locker.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::storage::durability {
|
||||
|
||||
@@ -41,6 +49,8 @@ namespace memgraph::storage::durability {
|
||||
// * offset to the constraints section
|
||||
// * offset to the mapper section
|
||||
// * offset to the metadata section
|
||||
// * offset to the offset-count pair of the first edge batch (`0` if properties on edges are disabled)
|
||||
// * offset to the offset-count pair of the first vertex batch
|
||||
//
|
||||
// 4) Encoded edges (if properties on edges are enabled); each edge is written
|
||||
// in the following format:
|
||||
@@ -88,9 +98,23 @@ namespace memgraph::storage::durability {
|
||||
// * number of edges
|
||||
// * number of vertices
|
||||
//
|
||||
// 10) Batch infos
|
||||
// * number of edge batch infos
|
||||
// * edge batch infos
|
||||
// * starting offset of the batch
|
||||
// * number of edges in the batch
|
||||
// * vertex batch infos
|
||||
// * starting offset of the batch
|
||||
// * number of vertices in the batch
|
||||
//
|
||||
// IMPORTANT: When changing snapshot encoding/decoding bump the snapshot/WAL
|
||||
// version in `version.hpp`.
|
||||
|
||||
struct BatchInfo {
|
||||
uint64_t offset;
|
||||
uint64_t count;
|
||||
};
|
||||
|
||||
// Function used to read information about the snapshot file.
|
||||
SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
|
||||
// Check magic and version.
|
||||
@@ -125,6 +149,13 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
|
||||
info.offset_mapper = read_offset();
|
||||
info.offset_epoch_history = read_offset();
|
||||
info.offset_metadata = read_offset();
|
||||
if (*version >= 15U) {
|
||||
info.offset_edge_batches = read_offset();
|
||||
info.offset_vertex_batches = read_offset();
|
||||
} else {
|
||||
info.offset_edge_batches = 0U;
|
||||
info.offset_vertex_batches = 0U;
|
||||
}
|
||||
}
|
||||
|
||||
// Read metadata.
|
||||
@@ -158,17 +189,385 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path) {
|
||||
return info;
|
||||
}
|
||||
|
||||
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
|
||||
utils::SkipList<Edge> *edges,
|
||||
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
|
||||
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, Config::Items items) {
|
||||
std::vector<BatchInfo> ReadBatchInfos(Decoder &snapshot) {
|
||||
std::vector<BatchInfo> infos;
|
||||
const auto infos_size = snapshot.ReadUint();
|
||||
if (!infos_size.has_value()) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
infos.reserve(*infos_size);
|
||||
|
||||
for (auto i{0U}; i < *infos_size; ++i) {
|
||||
const auto offset = snapshot.ReadUint();
|
||||
if (!offset.has_value()) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
const auto count = snapshot.ReadUint();
|
||||
if (!count.has_value()) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
infos.push_back(BatchInfo{*offset, *count});
|
||||
}
|
||||
return infos;
|
||||
}
|
||||
|
||||
template <typename TFunc>
|
||||
void LoadPartialEdges(const std::filesystem::path &path, utils::SkipList<Edge> &edges, const uint64_t from_offset,
|
||||
const uint64_t edges_count, const Config::Items items, TFunc get_property_from_id) {
|
||||
Decoder snapshot;
|
||||
snapshot.Initialize(path, kSnapshotMagic);
|
||||
|
||||
// Recover edges.
|
||||
auto edge_acc = edges.access();
|
||||
uint64_t last_edge_gid = 0;
|
||||
spdlog::info("Recovering {} edges.", edges_count);
|
||||
if (!snapshot.SetPosition(from_offset)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
std::vector<std::pair<PropertyId, PropertyValue>> read_properties;
|
||||
for (uint64_t i = 0; i < edges_count; ++i) {
|
||||
{
|
||||
const auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_EDGE) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
// Read edge GID.
|
||||
auto gid = snapshot.ReadUint();
|
||||
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
if (i > 0 && *gid <= last_edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
last_edge_gid = *gid;
|
||||
|
||||
if (items.properties_on_edges) {
|
||||
spdlog::debug("Recovering edge {} with properties.", *gid);
|
||||
auto [it, inserted] = edge_acc.insert(Edge{Gid::FromUint(*gid), nullptr});
|
||||
if (!inserted) throw RecoveryFailure("The edge must be inserted here!");
|
||||
|
||||
// Recover properties.
|
||||
{
|
||||
auto props_size = snapshot.ReadUint();
|
||||
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto &props = it->properties;
|
||||
read_properties.clear();
|
||||
read_properties.reserve(*props_size);
|
||||
for (uint64_t j = 0; j < *props_size; ++j) {
|
||||
auto key = snapshot.ReadUint();
|
||||
if (!key) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto value = snapshot.ReadPropertyValue();
|
||||
if (!value) throw RecoveryFailure("Invalid snapshot data!");
|
||||
read_properties.emplace_back(get_property_from_id(*key), std::move(*value));
|
||||
}
|
||||
props.InitProperties(std::move(read_properties));
|
||||
}
|
||||
} else {
|
||||
spdlog::debug("Ensuring edge {} doesn't have any properties.", *gid);
|
||||
// Read properties.
|
||||
{
|
||||
auto props_size = snapshot.ReadUint();
|
||||
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
if (*props_size != 0)
|
||||
throw RecoveryFailure(
|
||||
"The snapshot has properties on edges, but the storage is "
|
||||
"configured without properties on edges!");
|
||||
}
|
||||
}
|
||||
}
|
||||
spdlog::info("Partial edges are recovered.");
|
||||
}
|
||||
|
||||
// Returns the gid of the last recovered vertex
|
||||
template <typename TLabelFromIdFunc, typename TPropertyFromIdFunc>
|
||||
uint64_t LoadPartialVertices(const std::filesystem::path &path, utils::SkipList<Vertex> &vertices,
|
||||
const uint64_t from_offset, const uint64_t vertices_count,
|
||||
TLabelFromIdFunc get_label_from_id, TPropertyFromIdFunc get_property_from_id) {
|
||||
Decoder snapshot;
|
||||
snapshot.Initialize(path, kSnapshotMagic);
|
||||
if (!snapshot.SetPosition(from_offset)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto vertex_acc = vertices.access();
|
||||
uint64_t last_vertex_gid = 0;
|
||||
spdlog::info("Recovering {} vertices.", vertices_count);
|
||||
std::vector<std::pair<PropertyId, PropertyValue>> read_properties;
|
||||
for (uint64_t i = 0; i < vertices_count; ++i) {
|
||||
{
|
||||
auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
// Insert vertex.
|
||||
auto gid = snapshot.ReadUint();
|
||||
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
if (i > 0 && *gid <= last_vertex_gid) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
last_vertex_gid = *gid;
|
||||
spdlog::debug("Recovering vertex {}.", *gid);
|
||||
auto [it, inserted] = vertex_acc.insert(Vertex{Gid::FromUint(*gid), nullptr});
|
||||
if (!inserted) throw RecoveryFailure("The vertex must be inserted here!");
|
||||
|
||||
// Recover labels.
|
||||
spdlog::trace("Recovering labels for vertex {}.", *gid);
|
||||
{
|
||||
auto labels_size = snapshot.ReadUint();
|
||||
if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto &labels = it->labels;
|
||||
labels.reserve(*labels_size);
|
||||
for (uint64_t j = 0; j < *labels_size; ++j) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
labels.emplace_back(get_label_from_id(*label));
|
||||
}
|
||||
}
|
||||
|
||||
// Recover properties.
|
||||
spdlog::trace("Recovering properties for vertex {}.", *gid);
|
||||
{
|
||||
auto props_size = snapshot.ReadUint();
|
||||
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto &props = it->properties;
|
||||
read_properties.clear();
|
||||
read_properties.reserve(*props_size);
|
||||
for (uint64_t j = 0; j < *props_size; ++j) {
|
||||
auto key = snapshot.ReadUint();
|
||||
if (!key) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto value = snapshot.ReadPropertyValue();
|
||||
if (!value) throw RecoveryFailure("Invalid snapshot data!");
|
||||
read_properties.emplace_back(get_property_from_id(*key), std::move(*value));
|
||||
}
|
||||
props.InitProperties(std::move(read_properties));
|
||||
}
|
||||
|
||||
// Skip in edges.
|
||||
{
|
||||
auto in_size = snapshot.ReadUint();
|
||||
if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
for (uint64_t j = 0; j < *in_size; ++j) {
|
||||
auto edge_gid = snapshot.ReadUint();
|
||||
if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto from_gid = snapshot.ReadUint();
|
||||
if (!from_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto edge_type = snapshot.ReadUint();
|
||||
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
}
|
||||
|
||||
// Skip out edges.
|
||||
auto out_size = snapshot.ReadUint();
|
||||
if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
for (uint64_t j = 0; j < *out_size; ++j) {
|
||||
auto edge_gid = snapshot.ReadUint();
|
||||
if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto to_gid = snapshot.ReadUint();
|
||||
if (!to_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto edge_type = snapshot.ReadUint();
|
||||
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
}
|
||||
spdlog::info("Partial vertices are recovered.");
|
||||
|
||||
return last_vertex_gid;
|
||||
}
|
||||
|
||||
// Returns the number of edges recovered
|
||||
|
||||
struct LoadPartialConnectivityResult {
|
||||
uint64_t edge_count;
|
||||
uint64_t highest_edge_id;
|
||||
Gid first_vertex_gid;
|
||||
};
|
||||
|
||||
template <typename TEdgeTypeFromIdFunc>
|
||||
LoadPartialConnectivityResult LoadPartialConnectivity(const std::filesystem::path &path,
|
||||
utils::SkipList<Vertex> &vertices, utils::SkipList<Edge> &edges,
|
||||
const uint64_t from_offset, const uint64_t vertices_count,
|
||||
const Config::Items items, const bool snapshot_has_edges,
|
||||
TEdgeTypeFromIdFunc get_edge_type_from_id) {
|
||||
Decoder snapshot;
|
||||
snapshot.Initialize(path, kSnapshotMagic);
|
||||
if (!snapshot.SetPosition(from_offset)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto vertex_acc = vertices.access();
|
||||
auto edge_acc = edges.access();
|
||||
|
||||
// Read the first gid to find the necessary iterator in vertices
|
||||
const auto first_vertex_gid = std::invoke([&]() mutable {
|
||||
{
|
||||
auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
auto gid = snapshot.ReadUint();
|
||||
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
return Gid::FromUint(*gid);
|
||||
});
|
||||
|
||||
uint64_t edge_count{0};
|
||||
uint64_t highest_edge_gid{0};
|
||||
auto vertex_it = vertex_acc.find(first_vertex_gid);
|
||||
if (vertex_it == vertex_acc.end()) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
spdlog::info("Recovering connectivity for {} vertices.", vertices_count);
|
||||
|
||||
if (!snapshot.SetPosition(from_offset)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
for (uint64_t i = 0; i < vertices_count; ++i) {
|
||||
auto &vertex = *vertex_it;
|
||||
{
|
||||
auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_VERTEX) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
auto gid = snapshot.ReadUint();
|
||||
if (!gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
if (gid != vertex.gid.AsUint()) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
// Skip labels.
|
||||
{
|
||||
auto labels_size = snapshot.ReadUint();
|
||||
if (!labels_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
for (uint64_t j = 0; j < *labels_size; ++j) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
}
|
||||
|
||||
// Skip properties.
|
||||
{
|
||||
auto props_size = snapshot.ReadUint();
|
||||
if (!props_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
for (uint64_t j = 0; j < *props_size; ++j) {
|
||||
auto key = snapshot.ReadUint();
|
||||
if (!key) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto value = snapshot.SkipPropertyValue();
|
||||
if (!value) throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
}
|
||||
|
||||
// Recover in edges.
|
||||
{
|
||||
spdlog::trace("Recovering inbound edges for vertex {}.", vertex.gid.AsUint());
|
||||
auto in_size = snapshot.ReadUint();
|
||||
if (!in_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
vertex.in_edges.reserve(*in_size);
|
||||
for (uint64_t j = 0; j < *in_size; ++j) {
|
||||
auto edge_gid = snapshot.ReadUint();
|
||||
if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
highest_edge_gid = std::max(highest_edge_gid, *edge_gid);
|
||||
|
||||
auto from_gid = snapshot.ReadUint();
|
||||
if (!from_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto edge_type = snapshot.ReadUint();
|
||||
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
auto from_vertex = vertex_acc.find(Gid::FromUint(*from_gid));
|
||||
if (from_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid from vertex!");
|
||||
|
||||
EdgeRef edge_ref(Gid::FromUint(*edge_gid));
|
||||
if (items.properties_on_edges) {
|
||||
// The snapshot contains the individiual edges only if it was created with a config where properties are
|
||||
// allowed on edges. That means the snapshots that were created without edge properties will only contain the
|
||||
// edges in the in/out edges list of vertices, therefore the edges has to be created here.
|
||||
if (snapshot_has_edges) {
|
||||
auto edge = edge_acc.find(Gid::FromUint(*edge_gid));
|
||||
if (edge == edge_acc.end()) throw RecoveryFailure("Invalid edge!");
|
||||
edge_ref = EdgeRef(&*edge);
|
||||
} else {
|
||||
auto [edge, inserted] = edge_acc.insert(Edge{Gid::FromUint(*edge_gid), nullptr});
|
||||
edge_ref = EdgeRef(&*edge);
|
||||
}
|
||||
}
|
||||
vertex.in_edges.emplace_back(get_edge_type_from_id(*edge_type), &*from_vertex, edge_ref);
|
||||
}
|
||||
}
|
||||
|
||||
// Recover out edges.
|
||||
{
|
||||
spdlog::trace("Recovering outbound edges for vertex {}.", vertex.gid.AsUint());
|
||||
auto out_size = snapshot.ReadUint();
|
||||
if (!out_size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
vertex.out_edges.reserve(*out_size);
|
||||
for (uint64_t j = 0; j < *out_size; ++j) {
|
||||
auto edge_gid = snapshot.ReadUint();
|
||||
if (!edge_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
auto to_gid = snapshot.ReadUint();
|
||||
if (!to_gid) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto edge_type = snapshot.ReadUint();
|
||||
if (!edge_type) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
auto to_vertex = vertex_acc.find(Gid::FromUint(*to_gid));
|
||||
if (to_vertex == vertex_acc.end()) throw RecoveryFailure("Invalid to vertex!");
|
||||
|
||||
EdgeRef edge_ref(Gid::FromUint(*edge_gid));
|
||||
if (items.properties_on_edges) {
|
||||
// The snapshot contains the individiual edges only if it was created with a config where properties are
|
||||
// allowed on edges. That means the snapshots that were created without edge properties will only contain the
|
||||
// edges in the in/out edges list of vertices, therefore the edges has to be created here.
|
||||
if (snapshot_has_edges) {
|
||||
auto edge = edge_acc.find(Gid::FromUint(*edge_gid));
|
||||
if (edge == edge_acc.end()) throw RecoveryFailure("Invalid edge!");
|
||||
edge_ref = EdgeRef(&*edge);
|
||||
} else {
|
||||
auto [edge, inserted] = edge_acc.insert(Edge{Gid::FromUint(*edge_gid), nullptr});
|
||||
edge_ref = EdgeRef(&*edge);
|
||||
}
|
||||
}
|
||||
vertex.out_edges.emplace_back(get_edge_type_from_id(*edge_type), &*to_vertex, edge_ref);
|
||||
// Increment edge count. We only increment the count here because the
|
||||
// information is duplicated in in_edges.
|
||||
edge_count++;
|
||||
}
|
||||
}
|
||||
++vertex_it;
|
||||
}
|
||||
spdlog::info("Partial connectivities are recovered.");
|
||||
return {edge_count, highest_edge_gid, first_vertex_gid};
|
||||
}
|
||||
|
||||
template <typename TFunc>
|
||||
void RecoverOnMultipleThreads(size_t thread_count, const TFunc &func, const std::vector<BatchInfo> &batches) {
|
||||
utils::Synchronized<std::optional<RecoveryFailure>, utils::SpinLock> maybe_error{};
|
||||
{
|
||||
std::atomic<uint64_t> batch_counter = 0;
|
||||
thread_count = std::min(thread_count, batches.size());
|
||||
std::vector<std::jthread> threads;
|
||||
threads.reserve(thread_count);
|
||||
|
||||
for (auto i{0U}; i < thread_count; ++i) {
|
||||
threads.emplace_back([&func, &batches, &maybe_error, &batch_counter]() {
|
||||
while (!maybe_error.Lock()->has_value()) {
|
||||
const auto batch_index = batch_counter++;
|
||||
if (batch_index >= batches.size()) {
|
||||
return;
|
||||
}
|
||||
const auto &batch = batches[batch_index];
|
||||
try {
|
||||
func(batch_index, batch);
|
||||
} catch (RecoveryFailure &failure) {
|
||||
*maybe_error.Lock() = std::move(failure);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
if (maybe_error.Lock()->has_value()) {
|
||||
throw RecoveryFailure((*maybe_error.Lock())->what());
|
||||
}
|
||||
}
|
||||
|
||||
RecoveredSnapshot LoadSnapshotVersion14(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
|
||||
utils::SkipList<Edge> *edges,
|
||||
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
|
||||
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count,
|
||||
Config::Items items) {
|
||||
RecoveryInfo ret;
|
||||
RecoveredIndicesAndConstraints indices_constraints;
|
||||
|
||||
Decoder snapshot;
|
||||
auto version = snapshot.Initialize(path, kSnapshotMagic);
|
||||
if (!version) throw RecoveryFailure("Couldn't read snapshot magic and/or version!");
|
||||
if (!IsVersionSupported(*version)) throw RecoveryFailure(fmt::format("Invalid snapshot version {}", *version));
|
||||
if (*version != 14U) throw RecoveryFailure(fmt::format("Expected snapshot version is 14, but got {}", *version));
|
||||
|
||||
// Cleanup of loaded data in case of failure.
|
||||
bool success = false;
|
||||
@@ -626,10 +1025,297 @@ RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipLis
|
||||
return {info, ret, std::move(indices_constraints)};
|
||||
}
|
||||
|
||||
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
|
||||
utils::SkipList<Edge> *edges,
|
||||
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
|
||||
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, const Config &config) {
|
||||
RecoveryInfo recovery_info;
|
||||
RecoveredIndicesAndConstraints indices_constraints;
|
||||
|
||||
Decoder snapshot;
|
||||
const auto version = snapshot.Initialize(path, kSnapshotMagic);
|
||||
if (!version) throw RecoveryFailure("Couldn't read snapshot magic and/or version!");
|
||||
|
||||
if (!IsVersionSupported(*version)) throw RecoveryFailure(fmt::format("Invalid snapshot version {}", *version));
|
||||
if (*version == 14U) {
|
||||
return LoadSnapshotVersion14(path, vertices, edges, epoch_history, name_id_mapper, edge_count, config.items);
|
||||
}
|
||||
|
||||
// Cleanup of loaded data in case of failure.
|
||||
bool success = false;
|
||||
utils::OnScopeExit cleanup([&] {
|
||||
if (!success) {
|
||||
edges->clear();
|
||||
vertices->clear();
|
||||
epoch_history->clear();
|
||||
}
|
||||
});
|
||||
|
||||
// Read snapshot info.
|
||||
const auto info = ReadSnapshotInfo(path);
|
||||
spdlog::info("Recovering {} vertices and {} edges.", info.vertices_count, info.edges_count);
|
||||
// Check for edges.
|
||||
bool snapshot_has_edges = info.offset_edges != 0;
|
||||
|
||||
// Recover mapper.
|
||||
std::unordered_map<uint64_t, uint64_t> snapshot_id_map;
|
||||
{
|
||||
spdlog::info("Recovering mapper metadata.");
|
||||
if (!snapshot.SetPosition(info.offset_mapper)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_MAPPER) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto id = snapshot.ReadUint();
|
||||
if (!id) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto name = snapshot.ReadString();
|
||||
if (!name) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto my_id = name_id_mapper->NameToId(*name);
|
||||
snapshot_id_map.emplace(*id, my_id);
|
||||
SPDLOG_TRACE("Mapping \"{}\"from snapshot id {} to actual id {}.", *name, *id, my_id);
|
||||
}
|
||||
}
|
||||
auto get_label_from_id = [&snapshot_id_map](uint64_t snapshot_id) {
|
||||
auto it = snapshot_id_map.find(snapshot_id);
|
||||
if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!");
|
||||
return LabelId::FromUint(it->second);
|
||||
};
|
||||
auto get_property_from_id = [&snapshot_id_map](uint64_t snapshot_id) {
|
||||
auto it = snapshot_id_map.find(snapshot_id);
|
||||
if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!");
|
||||
return PropertyId::FromUint(it->second);
|
||||
};
|
||||
auto get_edge_type_from_id = [&snapshot_id_map](uint64_t snapshot_id) {
|
||||
auto it = snapshot_id_map.find(snapshot_id);
|
||||
if (it == snapshot_id_map.end()) throw RecoveryFailure("Invalid snapshot data!");
|
||||
return EdgeTypeId::FromUint(it->second);
|
||||
};
|
||||
|
||||
// Reset current edge count.
|
||||
edge_count->store(0, std::memory_order_release);
|
||||
|
||||
{
|
||||
spdlog::info("Recovering edges.");
|
||||
// Recover edges.
|
||||
if (snapshot_has_edges) {
|
||||
// We don't need to check whether we store properties on edge or not, because `LoadPartialEdges` will always
|
||||
// iterate over the edges in the snapshot (if they exist) and the current configuration of properties on edge only
|
||||
// affect what it does:
|
||||
// 1. If properties are allowed on edges, then it loads the edges.
|
||||
// 2. If properties are not allowed on edges, then it checks that none of the edges have any properties.
|
||||
if (!snapshot.SetPosition(info.offset_edge_batches)) {
|
||||
throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
}
|
||||
const auto edge_batches = ReadBatchInfos(snapshot);
|
||||
|
||||
RecoverOnMultipleThreads(
|
||||
config.durability.recovery_thread_count,
|
||||
[path, edges, items = config.items, &get_property_from_id](const size_t /*batch_index*/,
|
||||
const BatchInfo &batch) {
|
||||
LoadPartialEdges(path, *edges, batch.offset, batch.count, items, get_property_from_id);
|
||||
},
|
||||
edge_batches);
|
||||
}
|
||||
spdlog::info("Edges are recovered.");
|
||||
|
||||
// Recover vertices (labels and properties).
|
||||
spdlog::info("Recovering vertices.", info.vertices_count);
|
||||
uint64_t last_vertex_gid{0};
|
||||
|
||||
if (!snapshot.SetPosition(info.offset_vertex_batches)) {
|
||||
throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
}
|
||||
|
||||
const auto vertex_batches = ReadBatchInfos(snapshot);
|
||||
RecoverOnMultipleThreads(
|
||||
config.durability.recovery_thread_count,
|
||||
[path, vertices, &vertex_batches, &get_label_from_id, &get_property_from_id, &last_vertex_gid](
|
||||
const size_t batch_index, const BatchInfo &batch) {
|
||||
const auto last_vertex_gid_in_batch =
|
||||
LoadPartialVertices(path, *vertices, batch.offset, batch.count, get_label_from_id, get_property_from_id);
|
||||
if (batch_index == vertex_batches.size() - 1) {
|
||||
last_vertex_gid = last_vertex_gid_in_batch;
|
||||
}
|
||||
},
|
||||
vertex_batches);
|
||||
|
||||
spdlog::info("Vertices are recovered.");
|
||||
|
||||
// Recover vertices (in/out edges).
|
||||
spdlog::info("Recover connectivity.");
|
||||
recovery_info.vertex_batches.reserve(vertex_batches.size());
|
||||
for (const auto batch : vertex_batches) {
|
||||
recovery_info.vertex_batches.emplace_back(std::make_pair(Gid::FromUint(0), batch.count));
|
||||
}
|
||||
std::atomic<uint64_t> highest_edge_gid{0};
|
||||
|
||||
RecoverOnMultipleThreads(
|
||||
config.durability.recovery_thread_count,
|
||||
[path, vertices, edges, edge_count, items = config.items, snapshot_has_edges, &get_edge_type_from_id,
|
||||
&highest_edge_gid, &recovery_info](const size_t batch_index, const BatchInfo &batch) {
|
||||
const auto result = LoadPartialConnectivity(path, *vertices, *edges, batch.offset, batch.count, items,
|
||||
snapshot_has_edges, get_edge_type_from_id);
|
||||
edge_count->fetch_add(result.edge_count);
|
||||
auto known_highest_edge_gid = highest_edge_gid.load();
|
||||
while (known_highest_edge_gid < result.highest_edge_id) {
|
||||
highest_edge_gid.compare_exchange_weak(known_highest_edge_gid, result.highest_edge_id);
|
||||
}
|
||||
recovery_info.vertex_batches[batch_index].first = result.first_vertex_gid;
|
||||
},
|
||||
vertex_batches);
|
||||
|
||||
spdlog::info("Connectivity is recovered.");
|
||||
|
||||
// Set initial values for edge/vertex ID generators.
|
||||
recovery_info.next_edge_id = highest_edge_gid + 1;
|
||||
recovery_info.next_vertex_id = last_vertex_gid + 1;
|
||||
}
|
||||
|
||||
// Recover indices.
|
||||
{
|
||||
spdlog::info("Recovering metadata of indices.");
|
||||
if (!snapshot.SetPosition(info.offset_indices)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_INDICES) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
// Recover label indices.
|
||||
{
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} label indices.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
AddRecoveredIndexConstraint(&indices_constraints.indices.label, get_label_from_id(*label),
|
||||
"The label index already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of label index for :{}", name_id_mapper->IdToName(snapshot_id_map.at(*label)));
|
||||
}
|
||||
spdlog::info("Metadata of label indices are recovered.");
|
||||
}
|
||||
|
||||
// Recover label+property indices.
|
||||
{
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} label+property indices.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto property = snapshot.ReadUint();
|
||||
if (!property) throw RecoveryFailure("Invalid snapshot data!");
|
||||
AddRecoveredIndexConstraint(&indices_constraints.indices.label_property,
|
||||
{get_label_from_id(*label), get_property_from_id(*property)},
|
||||
"The label+property index already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of label+property index for :{}({})",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*label)),
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*property)));
|
||||
}
|
||||
spdlog::info("Metadata of label+property indices are recovered.");
|
||||
}
|
||||
spdlog::info("Metadata of indices are recovered.");
|
||||
}
|
||||
|
||||
// Recover constraints.
|
||||
{
|
||||
spdlog::info("Recovering metadata of constraints.");
|
||||
if (!snapshot.SetPosition(info.offset_constraints)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_CONSTRAINTS) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
// Recover existence constraints.
|
||||
{
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} existence constraints.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto property = snapshot.ReadUint();
|
||||
if (!property) throw RecoveryFailure("Invalid snapshot data!");
|
||||
AddRecoveredIndexConstraint(&indices_constraints.constraints.existence,
|
||||
{get_label_from_id(*label), get_property_from_id(*property)},
|
||||
"The existence constraint already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of existence constraint for :{}({})",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*label)),
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*property)));
|
||||
}
|
||||
spdlog::info("Metadata of existence constraints are recovered.");
|
||||
}
|
||||
|
||||
// Recover unique constraints.
|
||||
// Snapshot version should be checked since unique constraints were
|
||||
// implemented in later versions of snapshot.
|
||||
if (*version >= kUniqueConstraintVersion) {
|
||||
auto size = snapshot.ReadUint();
|
||||
if (!size) throw RecoveryFailure("Invalid snapshot data!");
|
||||
spdlog::info("Recovering metadata of {} unique constraints.", *size);
|
||||
for (uint64_t i = 0; i < *size; ++i) {
|
||||
auto label = snapshot.ReadUint();
|
||||
if (!label) throw RecoveryFailure("Invalid snapshot data!");
|
||||
auto properties_count = snapshot.ReadUint();
|
||||
if (!properties_count) throw RecoveryFailure("Invalid snapshot data!");
|
||||
std::set<PropertyId> properties;
|
||||
for (uint64_t j = 0; j < *properties_count; ++j) {
|
||||
auto property = snapshot.ReadUint();
|
||||
if (!property) throw RecoveryFailure("Invalid snapshot data!");
|
||||
properties.insert(get_property_from_id(*property));
|
||||
}
|
||||
AddRecoveredIndexConstraint(&indices_constraints.constraints.unique, {get_label_from_id(*label), properties},
|
||||
"The unique constraint already exists!");
|
||||
SPDLOG_TRACE("Recovered metadata of unique constraints for :{}",
|
||||
name_id_mapper->IdToName(snapshot_id_map.at(*label)));
|
||||
}
|
||||
spdlog::info("Metadata of unique constraints are recovered.");
|
||||
}
|
||||
spdlog::info("Metadata of constraints are recovered.");
|
||||
}
|
||||
|
||||
spdlog::info("Recovering metadata.");
|
||||
// Recover epoch history
|
||||
{
|
||||
if (!snapshot.SetPosition(info.offset_epoch_history)) throw RecoveryFailure("Couldn't read data from snapshot!");
|
||||
|
||||
const auto marker = snapshot.ReadMarker();
|
||||
if (!marker || *marker != Marker::SECTION_EPOCH_HISTORY) throw RecoveryFailure("Invalid snapshot data!");
|
||||
|
||||
const auto history_size = snapshot.ReadUint();
|
||||
if (!history_size) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
|
||||
for (int i = 0; i < *history_size; ++i) {
|
||||
auto maybe_epoch_id = snapshot.ReadString();
|
||||
if (!maybe_epoch_id) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
const auto maybe_last_commit_timestamp = snapshot.ReadUint();
|
||||
if (!maybe_last_commit_timestamp) {
|
||||
throw RecoveryFailure("Invalid snapshot data!");
|
||||
}
|
||||
epoch_history->emplace_back(std::move(*maybe_epoch_id), *maybe_last_commit_timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
spdlog::info("Metadata recovered.");
|
||||
// Recover timestamp.
|
||||
recovery_info.next_timestamp = info.start_timestamp + 1;
|
||||
|
||||
// Set success flag (to disable cleanup).
|
||||
success = true;
|
||||
|
||||
return {info, recovery_info, std::move(indices_constraints)};
|
||||
}
|
||||
|
||||
void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory,
|
||||
const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count,
|
||||
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
|
||||
Indices *indices, Constraints *constraints, Config::Items items, const std::string &uuid,
|
||||
Indices *indices, Constraints *constraints, const Config &config, const std::string &uuid,
|
||||
const std::string_view epoch_id, const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
|
||||
utils::FileRetainer *file_retainer) {
|
||||
// Ensure that the storage directory exists.
|
||||
@@ -650,6 +1336,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
uint64_t offset_mapper = 0;
|
||||
uint64_t offset_metadata = 0;
|
||||
uint64_t offset_epoch_history = 0;
|
||||
uint64_t offset_edge_batches = 0;
|
||||
uint64_t offset_vertex_batches = 0;
|
||||
{
|
||||
snapshot.WriteMarker(Marker::SECTION_OFFSETS);
|
||||
offset_offsets = snapshot.GetPosition();
|
||||
@@ -660,6 +1348,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
snapshot.WriteUint(offset_mapper);
|
||||
snapshot.WriteUint(offset_epoch_history);
|
||||
snapshot.WriteUint(offset_metadata);
|
||||
snapshot.WriteUint(offset_edge_batches);
|
||||
snapshot.WriteUint(offset_vertex_batches);
|
||||
}
|
||||
|
||||
// Object counters.
|
||||
@@ -673,9 +1363,13 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
snapshot.WriteUint(mapping.AsUint());
|
||||
};
|
||||
|
||||
std::vector<BatchInfo> edge_batch_infos;
|
||||
auto items_in_current_batch{0UL};
|
||||
auto batch_start_offset{0UL};
|
||||
// Store all edges.
|
||||
if (items.properties_on_edges) {
|
||||
if (config.items.properties_on_edges) {
|
||||
offset_edges = snapshot.GetPosition();
|
||||
batch_start_offset = offset_edges;
|
||||
auto acc = edges->access();
|
||||
for (auto &edge : acc) {
|
||||
// The edge visibility check must be done here manually because we don't
|
||||
@@ -715,7 +1409,7 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
// but that isn't an issue because we won't use that part of the API
|
||||
// here.
|
||||
auto ea = EdgeAccessor::Create(edge_ref, EdgeTypeId::FromUint(0UL), nullptr, nullptr, transaction, indices,
|
||||
constraints, items);
|
||||
constraints, config.items);
|
||||
|
||||
// Get edge data.
|
||||
auto maybe_props = ea->Properties(View::OLD);
|
||||
@@ -734,16 +1428,29 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
}
|
||||
|
||||
++edges_count;
|
||||
++items_in_current_batch;
|
||||
if (items_in_current_batch == config.durability.items_per_batch) {
|
||||
edge_batch_infos.push_back(BatchInfo{batch_start_offset, items_in_current_batch});
|
||||
batch_start_offset = snapshot.GetPosition();
|
||||
items_in_current_batch = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (items_in_current_batch > 0) {
|
||||
edge_batch_infos.push_back(BatchInfo{batch_start_offset, items_in_current_batch});
|
||||
}
|
||||
|
||||
std::vector<BatchInfo> vertex_batch_infos;
|
||||
// Store all vertices.
|
||||
{
|
||||
items_in_current_batch = 0;
|
||||
offset_vertices = snapshot.GetPosition();
|
||||
batch_start_offset = offset_vertices;
|
||||
auto acc = vertices->access();
|
||||
for (auto &vertex : acc) {
|
||||
// The visibility check is implemented for vertices so we use it here.
|
||||
auto va = VertexAccessor::Create(&vertex, transaction, indices, constraints, items, View::OLD);
|
||||
auto va = VertexAccessor::Create(&vertex, transaction, indices, constraints, config.items, View::OLD);
|
||||
if (!va) continue;
|
||||
|
||||
// Get vertex data.
|
||||
@@ -790,6 +1497,16 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
}
|
||||
|
||||
++vertices_count;
|
||||
++items_in_current_batch;
|
||||
if (items_in_current_batch == config.durability.items_per_batch) {
|
||||
vertex_batch_infos.push_back(BatchInfo{batch_start_offset, items_in_current_batch});
|
||||
batch_start_offset = snapshot.GetPosition();
|
||||
items_in_current_batch = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (items_in_current_batch > 0) {
|
||||
vertex_batch_infos.push_back(BatchInfo{batch_start_offset, items_in_current_batch});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -880,6 +1597,26 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
snapshot.WriteUint(vertices_count);
|
||||
}
|
||||
|
||||
auto write_batch_infos = [&snapshot](const std::vector<BatchInfo> &batch_infos) {
|
||||
snapshot.WriteUint(batch_infos.size());
|
||||
for (const auto &batch_info : batch_infos) {
|
||||
snapshot.WriteUint(batch_info.offset);
|
||||
snapshot.WriteUint(batch_info.count);
|
||||
}
|
||||
};
|
||||
|
||||
// Write edge batches
|
||||
{
|
||||
offset_edge_batches = snapshot.GetPosition();
|
||||
write_batch_infos(edge_batch_infos);
|
||||
}
|
||||
|
||||
// Write vertex batches
|
||||
{
|
||||
offset_vertex_batches = snapshot.GetPosition();
|
||||
write_batch_infos(vertex_batch_infos);
|
||||
}
|
||||
|
||||
// Write true offsets.
|
||||
{
|
||||
snapshot.SetPosition(offset_offsets);
|
||||
@@ -890,6 +1627,8 @@ void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snaps
|
||||
snapshot.WriteUint(offset_mapper);
|
||||
snapshot.WriteUint(offset_epoch_history);
|
||||
snapshot.WriteUint(offset_metadata);
|
||||
snapshot.WriteUint(offset_edge_batches);
|
||||
snapshot.WriteUint(offset_vertex_batches);
|
||||
}
|
||||
|
||||
// Finalize snapshot file.
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -37,6 +37,8 @@ struct SnapshotInfo {
|
||||
uint64_t offset_mapper;
|
||||
uint64_t offset_epoch_history;
|
||||
uint64_t offset_metadata;
|
||||
uint64_t offset_edge_batches;
|
||||
uint64_t offset_vertex_batches;
|
||||
|
||||
std::string uuid;
|
||||
std::string epoch_id;
|
||||
@@ -62,13 +64,13 @@ SnapshotInfo ReadSnapshotInfo(const std::filesystem::path &path);
|
||||
RecoveredSnapshot LoadSnapshot(const std::filesystem::path &path, utils::SkipList<Vertex> *vertices,
|
||||
utils::SkipList<Edge> *edges,
|
||||
std::deque<std::pair<std::string, uint64_t>> *epoch_history,
|
||||
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, Config::Items items);
|
||||
NameIdMapper *name_id_mapper, std::atomic<uint64_t> *edge_count, const Config &config);
|
||||
|
||||
/// Function used to create a snapshot using the given transaction.
|
||||
void CreateSnapshot(Transaction *transaction, const std::filesystem::path &snapshot_directory,
|
||||
const std::filesystem::path &wal_directory, uint64_t snapshot_retention_count,
|
||||
utils::SkipList<Vertex> *vertices, utils::SkipList<Edge> *edges, NameIdMapper *name_id_mapper,
|
||||
Indices *indices, Constraints *constraints, Config::Items items, const std::string &uuid,
|
||||
Indices *indices, Constraints *constraints, const Config &config, const std::string &uuid,
|
||||
std::string_view epoch_id, const std::deque<std::pair<std::string, uint64_t>> &epoch_history,
|
||||
utils::FileRetainer *file_retainer);
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -20,7 +20,7 @@ namespace memgraph::storage::durability {
|
||||
// The current version of snapshot and WAL encoding / decoding.
|
||||
// IMPORTANT: Please bump this version for every snapshot and/or WAL format
|
||||
// change!!!
|
||||
const uint64_t kVersion{14};
|
||||
const uint64_t kVersion{15};
|
||||
|
||||
const uint64_t kOldestSupportedVersion{14};
|
||||
const uint64_t kUniqueConstraintVersion{13};
|
||||
|
||||
@@ -13,12 +13,14 @@
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
#include <limits>
|
||||
#include <thread>
|
||||
|
||||
#include "storage/v2/mvcc.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "utils/bound.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
@@ -263,6 +265,95 @@ bool CurrentVersionHasLabelProperty(const Vertex &vertex, LabelId label, Propert
|
||||
return !deleted && has_label && current_value_equal_to_value;
|
||||
}
|
||||
|
||||
template <typename TIndexAccessor>
|
||||
void TryInsertLabelIndex(Vertex &vertex, LabelId label, TIndexAccessor &index_accessor) {
|
||||
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
|
||||
return;
|
||||
}
|
||||
|
||||
index_accessor.insert({&vertex, 0});
|
||||
}
|
||||
|
||||
template <typename TIndexAccessor>
|
||||
void TryInsertLabelPropertyIndex(Vertex &vertex, std::pair<LabelId, PropertyId> label_property_pair,
|
||||
TIndexAccessor &index_accessor) {
|
||||
if (vertex.deleted || !utils::Contains(vertex.labels, label_property_pair.first)) {
|
||||
return;
|
||||
}
|
||||
auto value = vertex.properties.GetProperty(label_property_pair.second);
|
||||
if (value.IsNull()) {
|
||||
return;
|
||||
}
|
||||
index_accessor.insert({std::move(value), &vertex, 0});
|
||||
}
|
||||
|
||||
template <typename TSkiplistIter, typename TIndex, typename TIndexKey, typename TFunc>
|
||||
void CreateIndexOnSingleThread(utils::SkipList<Vertex>::Accessor &vertices, TSkiplistIter it, TIndex &index,
|
||||
TIndexKey key, const TFunc &func) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
try {
|
||||
auto acc = it->second.access();
|
||||
for (Vertex &vertex : vertices) {
|
||||
func(vertex, key, acc);
|
||||
}
|
||||
} catch (const utils::OutOfMemoryException &) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
|
||||
index.erase(it);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
template <typename TIndex, typename TIndexKey, typename TSKiplistIter, typename TFunc>
|
||||
void CreateIndexOnMultipleThreads(utils::SkipList<Vertex>::Accessor &vertices, TSKiplistIter skiplist_iter,
|
||||
TIndex &index, TIndexKey key, const ParalellizedIndexCreationInfo ¶lell_exec_info,
|
||||
const TFunc &func) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
|
||||
const auto &vertex_batches = paralell_exec_info.first;
|
||||
const auto thread_count = std::min(paralell_exec_info.second, vertex_batches.size());
|
||||
|
||||
MG_ASSERT(!vertex_batches.empty(),
|
||||
"The size of batches should always be greater than zero if you want to use the parallel version of index "
|
||||
"creation!");
|
||||
|
||||
std::atomic<uint64_t> batch_counter = 0;
|
||||
|
||||
utils::Synchronized<std::optional<utils::OutOfMemoryException>, utils::SpinLock> maybe_error{};
|
||||
{
|
||||
std::vector<std::jthread> threads;
|
||||
threads.reserve(thread_count);
|
||||
|
||||
for (auto i{0U}; i < thread_count; ++i) {
|
||||
threads.emplace_back(
|
||||
[&skiplist_iter, &func, &index, &vertex_batches, &maybe_error, &batch_counter, &key, &vertices]() {
|
||||
while (!maybe_error.Lock()->has_value()) {
|
||||
const auto batch_index = batch_counter++;
|
||||
if (batch_index >= vertex_batches.size()) {
|
||||
return;
|
||||
}
|
||||
const auto &batch = vertex_batches[batch_index];
|
||||
auto index_accessor = index.at(key).access();
|
||||
auto it = vertices.find(batch.first);
|
||||
|
||||
try {
|
||||
for (auto i{0U}; i < batch.second; ++i, ++it) {
|
||||
func(*it, key, index_accessor);
|
||||
}
|
||||
|
||||
} catch (utils::OutOfMemoryException &failure) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
|
||||
index.erase(skiplist_iter);
|
||||
*maybe_error.Lock() = std::move(failure);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
if (maybe_error.Lock()->has_value()) {
|
||||
throw utils::OutOfMemoryException((*maybe_error.Lock())->what());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx) {
|
||||
@@ -272,27 +363,43 @@ void LabelIndex::UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transacti
|
||||
acc.insert(Entry{vertex, tx.start_timestamp});
|
||||
}
|
||||
|
||||
bool LabelIndex::CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
bool LabelIndex::CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices,
|
||||
const std::optional<ParalellizedIndexCreationInfo> ¶lell_exec_info) {
|
||||
auto create_index_seq = [this](LabelId label, utils::SkipList<Vertex>::Accessor &vertices,
|
||||
std::map<LabelId, utils::SkipList<Entry>>::iterator it) {
|
||||
using IndexAccessor = decltype(it->second.access());
|
||||
|
||||
CreateIndexOnSingleThread(vertices, it, index_, label,
|
||||
[](Vertex &vertex, LabelId label, IndexAccessor &index_accessor) {
|
||||
TryInsertLabelIndex(vertex, label, index_accessor);
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
auto create_index_par = [this](LabelId label, utils::SkipList<Vertex>::Accessor &vertices,
|
||||
std::map<LabelId, utils::SkipList<Entry>>::iterator label_it,
|
||||
const ParalellizedIndexCreationInfo ¶lell_exec_info) {
|
||||
using IndexAccessor = decltype(label_it->second.access());
|
||||
|
||||
CreateIndexOnMultipleThreads(vertices, label_it, index_, label, paralell_exec_info,
|
||||
[](Vertex &vertex, LabelId label, IndexAccessor &index_accessor) {
|
||||
TryInsertLabelIndex(vertex, label, index_accessor);
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
auto [it, emplaced] = index_.emplace(std::piecewise_construct, std::forward_as_tuple(label), std::forward_as_tuple());
|
||||
if (!emplaced) {
|
||||
// Index already exists.
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
auto acc = it->second.access();
|
||||
for (Vertex &vertex : vertices) {
|
||||
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
|
||||
continue;
|
||||
}
|
||||
acc.insert(Entry{&vertex, 0});
|
||||
}
|
||||
} catch (const utils::OutOfMemoryException &) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
|
||||
index_.erase(it);
|
||||
throw;
|
||||
|
||||
if (paralell_exec_info) {
|
||||
return create_index_par(label, vertices, it, *paralell_exec_info);
|
||||
}
|
||||
return true;
|
||||
return create_index_seq(label, vertices, it);
|
||||
}
|
||||
|
||||
std::vector<LabelId> LabelIndex::ListIndices() const {
|
||||
@@ -344,8 +451,13 @@ void LabelIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
}
|
||||
if (CurrentVersionHasLabel(*index_iterator_->vertex, self_->label_, self_->transaction_, self_->view_)) {
|
||||
current_vertex_ = index_iterator_->vertex;
|
||||
current_vertex_accessor_ = VertexAccessor::Create(current_vertex_, self_->transaction_, self_->indices_,
|
||||
self_->constraints_, self_->config_, self_->view_);
|
||||
if (current_vertex_accessor_) {
|
||||
current_vertex_accessor_->ReInit(current_vertex_, self_->transaction_, self_->indices_, self_->constraints_,
|
||||
self_->config_, self_->view_);
|
||||
} else {
|
||||
current_vertex_accessor_ = VertexAccessor::Create(current_vertex_, self_->transaction_, self_->indices_,
|
||||
self_->constraints_, self_->config_, self_->view_);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
@@ -415,32 +527,46 @@ void LabelPropertyIndex::UpdateOnSetProperty(PropertyId property, const Property
|
||||
}
|
||||
}
|
||||
|
||||
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionEnabler oom_exception;
|
||||
bool LabelPropertyIndex::CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices,
|
||||
const std::optional<ParalellizedIndexCreationInfo> ¶lell_exec_info) {
|
||||
auto create_index_seq = [this](LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor &vertices,
|
||||
std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>>::iterator it) {
|
||||
using IndexAccessor = decltype(it->second.access());
|
||||
|
||||
CreateIndexOnSingleThread(vertices, it, index_, std::make_pair(label, property),
|
||||
[](Vertex &vertex, std::pair<LabelId, PropertyId> key, IndexAccessor &index_accessor) {
|
||||
TryInsertLabelPropertyIndex(vertex, key, index_accessor);
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
auto create_index_par =
|
||||
[this](LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor &vertices,
|
||||
std::map<std::pair<LabelId, PropertyId>, utils::SkipList<Entry>>::iterator label_property_it,
|
||||
const ParalellizedIndexCreationInfo ¶lell_exec_info) {
|
||||
using IndexAccessor = decltype(label_property_it->second.access());
|
||||
|
||||
CreateIndexOnMultipleThreads(
|
||||
vertices, label_property_it, index_, std::make_pair(label, property), paralell_exec_info,
|
||||
[](Vertex &vertex, std::pair<LabelId, PropertyId> key, IndexAccessor &index_accessor) {
|
||||
TryInsertLabelPropertyIndex(vertex, key, index_accessor);
|
||||
});
|
||||
|
||||
return true;
|
||||
};
|
||||
|
||||
auto [it, emplaced] =
|
||||
index_.emplace(std::piecewise_construct, std::forward_as_tuple(label, property), std::forward_as_tuple());
|
||||
if (!emplaced) {
|
||||
// Index already exists.
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
auto acc = it->second.access();
|
||||
for (Vertex &vertex : vertices) {
|
||||
if (vertex.deleted || !utils::Contains(vertex.labels, label)) {
|
||||
continue;
|
||||
}
|
||||
auto value = vertex.properties.GetProperty(property);
|
||||
if (value.IsNull()) {
|
||||
continue;
|
||||
}
|
||||
acc.insert(Entry{std::move(value), &vertex, 0});
|
||||
}
|
||||
} catch (const utils::OutOfMemoryException &) {
|
||||
utils::MemoryTracker::OutOfMemoryExceptionBlocker oom_exception_blocker;
|
||||
index_.erase(it);
|
||||
throw;
|
||||
|
||||
if (paralell_exec_info) {
|
||||
return create_index_par(label, property, vertices, it, *paralell_exec_info);
|
||||
}
|
||||
return true;
|
||||
return create_index_seq(label, property, vertices, it);
|
||||
}
|
||||
|
||||
std::vector<std::pair<LabelId, PropertyId>> LabelPropertyIndex::ListIndices() const {
|
||||
@@ -513,8 +639,13 @@ void LabelPropertyIndex::Iterable::Iterator::AdvanceUntilValid() {
|
||||
if (CurrentVersionHasLabelProperty(*index_iterator_->vertex, self_->label_, self_->property_,
|
||||
index_iterator_->value, self_->transaction_, self_->view_)) {
|
||||
current_vertex_ = index_iterator_->vertex;
|
||||
current_vertex_accessor_ = VertexAccessor::Create(current_vertex_, self_->transaction_, self_->indices_,
|
||||
self_->constraints_, self_->config_, self_->view_);
|
||||
if (current_vertex_accessor_) {
|
||||
current_vertex_accessor_->ReInit(current_vertex_, self_->transaction_, self_->indices_, self_->constraints_,
|
||||
self_->config_, self_->view_);
|
||||
} else {
|
||||
current_vertex_accessor_ = VertexAccessor::Create(current_vertex_, self_->transaction_, self_->indices_,
|
||||
self_->constraints_, self_->config_, self_->view_);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,9 @@ namespace memgraph::storage {
|
||||
struct Indices;
|
||||
struct Constraints;
|
||||
|
||||
using ParalellizedIndexCreationInfo =
|
||||
std::pair<std::vector<std::pair<Gid, uint64_t>> /*vertex_recovery_info*/, uint64_t /*thread_count*/>;
|
||||
|
||||
class LabelIndex {
|
||||
private:
|
||||
struct Entry {
|
||||
@@ -58,7 +61,8 @@ class LabelIndex {
|
||||
void UpdateOnAddLabel(LabelId label, Vertex *vertex, const Transaction &tx);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
bool CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices);
|
||||
bool CreateIndex(LabelId label, utils::SkipList<Vertex>::Accessor vertices,
|
||||
const std::optional<ParalellizedIndexCreationInfo> ¶lell_exec_info = std::nullopt);
|
||||
|
||||
/// Returns false if there was no index to drop
|
||||
bool DropIndex(LabelId label) { return index_.erase(label) > 0; }
|
||||
@@ -168,7 +172,8 @@ class LabelPropertyIndex {
|
||||
void UpdateOnSetProperty(PropertyId property, const PropertyValue &value, Vertex *vertex, const Transaction &tx);
|
||||
|
||||
/// @throw std::bad_alloc
|
||||
bool CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices);
|
||||
bool CreateIndex(LabelId label, PropertyId property, utils::SkipList<Vertex>::Accessor vertices,
|
||||
const std::optional<ParalellizedIndexCreationInfo> ¶lell_exec_info = std::nullopt);
|
||||
|
||||
bool DropIndex(LabelId label, PropertyId property) { return index_.erase({label, property}) > 0; }
|
||||
|
||||
|
||||
@@ -118,6 +118,7 @@ Result<storage::PropertyValue> InMemoryEdgeAccessor::SetProperty(PropertyId prop
|
||||
// current code always follows the logical pattern of "create a delta" and
|
||||
// "modify in-place". Additionally, the created delta will make other
|
||||
// transactions get a SERIALIZATION_ERROR.
|
||||
|
||||
CreateAndLinkDelta(transaction_, edge_.ptr, Delta::SetPropertyTag(), property, current_value);
|
||||
edge_.ptr->properties.SetProperty(property, value);
|
||||
|
||||
|
||||
@@ -74,6 +74,7 @@ std::string RegisterReplicaErrorToString(InMemoryStorage::RegisterReplicaError e
|
||||
InMemoryStorage::InMemoryStorage(Config config)
|
||||
: indices_(&constraints_, config.items),
|
||||
isolation_level_(config.transaction.isolation_level),
|
||||
storage_mode_(StorageMode::IN_MEMORY_TRANSACTIONAL),
|
||||
config_(config),
|
||||
snapshot_directory_(config_.durability.storage_directory / durability::kSnapshotDirectory),
|
||||
wal_directory_(config_.durability.storage_directory / durability::kWalDirectory),
|
||||
@@ -116,7 +117,7 @@ InMemoryStorage::InMemoryStorage(Config config)
|
||||
if (config_.durability.recover_on_startup) {
|
||||
auto info = durability::RecoverData(snapshot_directory_, wal_directory_, &uuid_, &epoch_id_, &epoch_history_,
|
||||
&vertices_, &edges_, &edge_count_, &name_id_mapper_, &indices_, &constraints_,
|
||||
config_.items, &wal_seq_num_);
|
||||
config_, &wal_seq_num_);
|
||||
if (info) {
|
||||
vertex_id_ = info->next_vertex_id;
|
||||
edge_id_ = info->next_edge_id;
|
||||
@@ -156,12 +157,19 @@ InMemoryStorage::InMemoryStorage(Config config)
|
||||
}
|
||||
if (config_.durability.snapshot_wal_mode != Config::Durability::SnapshotWalMode::DISABLED) {
|
||||
snapshot_runner_.Run("Snapshot", config_.durability.snapshot_interval, [this] {
|
||||
if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) {
|
||||
if (auto maybe_error = this->CreateSnapshot({true}); maybe_error.HasError()) {
|
||||
switch (maybe_error.GetError()) {
|
||||
case CreateSnapshotError::DisabledForReplica:
|
||||
spdlog::warn(
|
||||
utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication"));
|
||||
break;
|
||||
case CreateSnapshotError::DisabledForAnalyticsPeriodicCommit:
|
||||
spdlog::warn(utils::MessageWithLink("Periodic snapshots are disabled for analytical mode.",
|
||||
"https://memgr.ph/durability"));
|
||||
break;
|
||||
case storage::Storage::CreateSnapshotError::ReachedMaxNumTries:
|
||||
spdlog::warn("Failed to create snapshot. Reached max number of tries. Please contact support");
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -204,23 +212,31 @@ InMemoryStorage::~InMemoryStorage() {
|
||||
snapshot_runner_.Stop();
|
||||
}
|
||||
if (config_.durability.snapshot_on_exit) {
|
||||
if (auto maybe_error = this->CreateSnapshot(); maybe_error.HasError()) {
|
||||
if (auto maybe_error = this->CreateSnapshot({false}); maybe_error.HasError()) {
|
||||
switch (maybe_error.GetError()) {
|
||||
case CreateSnapshotError::DisabledForReplica:
|
||||
spdlog::warn(utils::MessageWithLink("Snapshots are disabled for replicas.", "https://memgr.ph/replication"));
|
||||
break;
|
||||
case CreateSnapshotError::DisabledForAnalyticsPeriodicCommit:
|
||||
spdlog::warn(utils::MessageWithLink("Periodic snapshots are disabled for analytical mode.",
|
||||
"https://memgr.ph/replication"));
|
||||
break;
|
||||
case storage::Storage::CreateSnapshotError::ReachedMaxNumTries:
|
||||
spdlog::warn("Failed to create snapshot. Reached max number of tries. Please contact support");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(InMemoryStorage *storage, IsolationLevel isolation_level)
|
||||
InMemoryStorage::InMemoryAccessor::InMemoryAccessor(InMemoryStorage *storage, IsolationLevel isolation_level,
|
||||
StorageMode storage_mode)
|
||||
: storage_(storage),
|
||||
// The lock must be acquired before creating the transaction object to
|
||||
// prevent freshly created transactions from dangling in an active state
|
||||
// during exclusive operations.
|
||||
storage_guard_(storage_->main_lock_),
|
||||
transaction_(storage->CreateTransaction(isolation_level)),
|
||||
transaction_(storage->CreateTransaction(isolation_level, storage_mode)),
|
||||
is_transaction_active_(true),
|
||||
config_(storage->config_.items) {}
|
||||
|
||||
@@ -248,11 +264,15 @@ std::unique_ptr<VertexAccessor> InMemoryStorage::InMemoryAccessor::CreateVertex(
|
||||
OOMExceptionEnabler oom_exception;
|
||||
auto gid = storage_->vertex_id_.fetch_add(1, std::memory_order_acq_rel);
|
||||
auto acc = storage_->vertices_.access();
|
||||
auto delta = CreateDeleteObjectDelta(&transaction_);
|
||||
|
||||
auto *delta = CreateDeleteObjectDelta(&transaction_);
|
||||
auto [it, inserted] = acc.insert(Vertex{storage::Gid::FromUint(gid), delta});
|
||||
MG_ASSERT(inserted, "The vertex must be inserted here!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
|
||||
delta->prev.Set(&*it);
|
||||
|
||||
if (delta) {
|
||||
delta->prev.Set(&*it);
|
||||
}
|
||||
return std::make_unique<InMemoryVertexAccessor>(&*it, &transaction_, &storage_->indices_, &storage_->constraints_,
|
||||
config_);
|
||||
}
|
||||
@@ -268,11 +288,14 @@ std::unique_ptr<VertexAccessor> InMemoryStorage::InMemoryAccessor::CreateVertex(
|
||||
storage_->vertex_id_.store(std::max(storage_->vertex_id_.load(std::memory_order_acquire), gid.AsUint() + 1),
|
||||
std::memory_order_release);
|
||||
auto acc = storage_->vertices_.access();
|
||||
auto delta = CreateDeleteObjectDelta(&transaction_);
|
||||
|
||||
auto *delta = CreateDeleteObjectDelta(&transaction_);
|
||||
auto [it, inserted] = acc.insert(Vertex{gid, delta});
|
||||
MG_ASSERT(inserted, "The vertex must be inserted here!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Vertex accessor!");
|
||||
delta->prev.Set(&*it);
|
||||
if (delta) {
|
||||
delta->prev.Set(&*it);
|
||||
}
|
||||
return std::make_unique<InMemoryVertexAccessor>(&*it, &transaction_, &storage_->indices_, &storage_->constraints_,
|
||||
config_);
|
||||
}
|
||||
@@ -429,12 +452,15 @@ Result<std::unique_ptr<EdgeAccessor>> InMemoryStorage::InMemoryAccessor::CreateE
|
||||
EdgeRef edge(gid);
|
||||
if (config_.properties_on_edges) {
|
||||
auto acc = storage_->edges_.access();
|
||||
auto delta = CreateDeleteObjectDelta(&transaction_);
|
||||
|
||||
auto *delta = CreateDeleteObjectDelta(&transaction_);
|
||||
auto [it, inserted] = acc.insert(Edge(gid, delta));
|
||||
MG_ASSERT(inserted, "The edge must be inserted here!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Edge accessor!");
|
||||
edge = EdgeRef(&*it);
|
||||
delta->prev.Set(&*it);
|
||||
if (delta) {
|
||||
delta->prev.Set(&*it);
|
||||
}
|
||||
}
|
||||
|
||||
CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge);
|
||||
@@ -506,12 +532,15 @@ Result<std::unique_ptr<EdgeAccessor>> InMemoryStorage::InMemoryAccessor::CreateE
|
||||
EdgeRef edge(gid);
|
||||
if (config_.properties_on_edges) {
|
||||
auto acc = storage_->edges_.access();
|
||||
auto delta = CreateDeleteObjectDelta(&transaction_);
|
||||
|
||||
auto *delta = CreateDeleteObjectDelta(&transaction_);
|
||||
auto [it, inserted] = acc.insert(Edge(gid, delta));
|
||||
MG_ASSERT(inserted, "The edge must be inserted here!");
|
||||
MG_ASSERT(it != acc.end(), "Invalid Edge accessor!");
|
||||
edge = EdgeRef(&*it);
|
||||
delta->prev.Set(&*it);
|
||||
if (delta) {
|
||||
delta->prev.Set(&*it);
|
||||
}
|
||||
}
|
||||
|
||||
CreateAndLinkDelta(&transaction_, from_vertex, Delta::RemoveOutEdgeTag(), edge_type, to_vertex, edge);
|
||||
@@ -1177,7 +1206,7 @@ VerticesIterable InMemoryStorage::InMemoryAccessor::Vertices(
|
||||
storage_->indices_.label_property_index.Vertices(label, property, lower_bound, upper_bound, view, &transaction_));
|
||||
}
|
||||
|
||||
Transaction InMemoryStorage::CreateTransaction(IsolationLevel isolation_level) {
|
||||
Transaction InMemoryStorage::CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode) {
|
||||
// We acquire the transaction engine lock here because we access (and
|
||||
// modify) the transaction engine variables (`transaction_id` and
|
||||
// `timestamp`) below.
|
||||
@@ -1198,7 +1227,7 @@ Transaction InMemoryStorage::CreateTransaction(IsolationLevel isolation_level) {
|
||||
start_timestamp = timestamp_++;
|
||||
}
|
||||
}
|
||||
return {transaction_id, start_timestamp, isolation_level};
|
||||
return {transaction_id, start_timestamp, isolation_level, storage_mode};
|
||||
}
|
||||
|
||||
template <bool force>
|
||||
@@ -1705,28 +1734,48 @@ bool InMemoryStorage::AppendToWalDataDefinition(durability::StorageGlobalOperati
|
||||
return finalized_on_all_replicas;
|
||||
}
|
||||
|
||||
utils::BasicResult<InMemoryStorage::CreateSnapshotError> InMemoryStorage::CreateSnapshot() {
|
||||
utils::BasicResult<InMemoryStorage::CreateSnapshotError> InMemoryStorage::CreateSnapshot(
|
||||
std::optional<bool> is_periodic) {
|
||||
if (replication_role_.load() != ReplicationRole::MAIN) {
|
||||
return CreateSnapshotError::DisabledForReplica;
|
||||
}
|
||||
|
||||
auto snapshot_creator = [this]() {
|
||||
auto transaction = CreateTransaction(IsolationLevel::SNAPSHOT_ISOLATION, storage_mode_);
|
||||
// Create snapshot.
|
||||
durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_,
|
||||
config_.durability.snapshot_retention_count, &vertices_, &edges_, &name_id_mapper_,
|
||||
&indices_, &constraints_, config_, uuid_, epoch_id_, epoch_history_, &file_retainer_);
|
||||
// Finalize snapshot transaction.
|
||||
commit_log_->MarkFinished(transaction.start_timestamp);
|
||||
};
|
||||
|
||||
std::lock_guard snapshot_guard(snapshot_lock_);
|
||||
|
||||
// Take master RW lock (for reading).
|
||||
std::shared_lock<utils::RWLock> storage_guard(main_lock_);
|
||||
auto should_try_shared{true};
|
||||
auto max_num_tries{10};
|
||||
while (max_num_tries) {
|
||||
if (should_try_shared) {
|
||||
std::shared_lock<utils::RWLock> storage_guard(main_lock_);
|
||||
if (storage_mode_ == memgraph::storage::StorageMode::IN_MEMORY_TRANSACTIONAL) {
|
||||
snapshot_creator();
|
||||
return {};
|
||||
}
|
||||
} else {
|
||||
std::unique_lock main_guard{main_lock_};
|
||||
if (storage_mode_ == memgraph::storage::StorageMode::IN_MEMORY_ANALYTICAL) {
|
||||
if (is_periodic && *is_periodic) {
|
||||
return CreateSnapshotError::DisabledForAnalyticsPeriodicCommit;
|
||||
}
|
||||
snapshot_creator();
|
||||
return {};
|
||||
}
|
||||
}
|
||||
should_try_shared = !should_try_shared;
|
||||
max_num_tries--;
|
||||
}
|
||||
|
||||
// Create the transaction used to create the snapshot.
|
||||
auto transaction = CreateTransaction(IsolationLevel::SNAPSHOT_ISOLATION);
|
||||
|
||||
// Create snapshot.
|
||||
durability::CreateSnapshot(&transaction, snapshot_directory_, wal_directory_,
|
||||
config_.durability.snapshot_retention_count, &vertices_, &edges_, &name_id_mapper_,
|
||||
&indices_, &constraints_, config_.items, uuid_, epoch_id_, epoch_history_,
|
||||
&file_retainer_);
|
||||
|
||||
// Finalize snapshot transaction.
|
||||
commit_log_->MarkFinished(transaction.start_timestamp);
|
||||
return {};
|
||||
return CreateSnapshotError::ReachedMaxNumTries;
|
||||
}
|
||||
|
||||
bool InMemoryStorage::LockPath() {
|
||||
@@ -1913,11 +1962,24 @@ std::vector<InMemoryStorage::ReplicaInfo> InMemoryStorage::ReplicasInfo() {
|
||||
});
|
||||
}
|
||||
|
||||
void InMemoryStorage::SetIsolationLevel(IsolationLevel isolation_level) {
|
||||
utils::BasicResult<InMemoryStorage::SetIsolationLevelError> InMemoryStorage::SetIsolationLevel(
|
||||
IsolationLevel isolation_level) {
|
||||
std::unique_lock main_guard{main_lock_};
|
||||
if (storage_mode_ == storage::StorageMode::IN_MEMORY_ANALYTICAL) {
|
||||
return Storage::SetIsolationLevelError::DisabledForAnalyticalMode;
|
||||
}
|
||||
|
||||
isolation_level_ = isolation_level;
|
||||
return {};
|
||||
}
|
||||
|
||||
void InMemoryStorage::SetStorageMode(StorageMode storage_mode) {
|
||||
std::unique_lock main_guard{main_lock_};
|
||||
storage_mode_ = storage_mode;
|
||||
}
|
||||
|
||||
StorageMode InMemoryStorage::GetStorageMode() { return storage_mode_; }
|
||||
|
||||
void InMemoryStorage::RestoreReplicas() {
|
||||
MG_ASSERT(memgraph::storage::ReplicationRole::MAIN == GetReplicationRole());
|
||||
if (!ShouldStoreAndRestoreReplicas()) {
|
||||
|
||||
@@ -72,7 +72,7 @@ class InMemoryStorage final : public Storage {
|
||||
private:
|
||||
friend class InMemoryStorage;
|
||||
|
||||
explicit InMemoryAccessor(InMemoryStorage *storage, IsolationLevel isolation_level);
|
||||
explicit InMemoryAccessor(InMemoryStorage *storage, IsolationLevel isolation_level, StorageMode storage_mode);
|
||||
|
||||
public:
|
||||
InMemoryAccessor(const InMemoryAccessor &) = delete;
|
||||
@@ -245,7 +245,7 @@ class InMemoryStorage final : public Storage {
|
||||
|
||||
std::unique_ptr<Storage::Accessor> Access(std::optional<IsolationLevel> override_isolation_level) override {
|
||||
return std::unique_ptr<InMemoryAccessor>(
|
||||
new InMemoryAccessor{this, override_isolation_level.value_or(isolation_level_)});
|
||||
new InMemoryAccessor{this, override_isolation_level.value_or(isolation_level_), storage_mode_});
|
||||
}
|
||||
|
||||
const std::string &LabelToName(LabelId label) const override;
|
||||
@@ -366,12 +366,15 @@ class InMemoryStorage final : public Storage {
|
||||
|
||||
void FreeMemory() override;
|
||||
|
||||
void SetIsolationLevel(IsolationLevel isolation_level) override;
|
||||
utils::BasicResult<SetIsolationLevelError> SetIsolationLevel(IsolationLevel isolation_level) override;
|
||||
void SetStorageMode(StorageMode storage_mode) override;
|
||||
|
||||
utils::BasicResult<CreateSnapshotError> CreateSnapshot() override;
|
||||
StorageMode GetStorageMode() override;
|
||||
|
||||
utils::BasicResult<CreateSnapshotError> CreateSnapshot(std::optional<bool> is_periodic) override;
|
||||
|
||||
private:
|
||||
Transaction CreateTransaction(IsolationLevel isolation_level);
|
||||
Transaction CreateTransaction(IsolationLevel isolation_level, StorageMode storage_mode);
|
||||
|
||||
/// The force parameter determines the behaviour of the garbage collector.
|
||||
/// If it's set to true, it will behave as a global operation, i.e. it can't
|
||||
@@ -437,6 +440,7 @@ class InMemoryStorage final : public Storage {
|
||||
|
||||
utils::Synchronized<std::list<Transaction>, utils::SpinLock> committed_transactions_;
|
||||
IsolationLevel isolation_level_;
|
||||
StorageMode storage_mode_;
|
||||
|
||||
Config config_;
|
||||
utils::Scheduler gc_runner_;
|
||||
|
||||
@@ -71,6 +71,20 @@ std::unique_ptr<InMemoryVertexAccessor> InMemoryVertexAccessor::Create(Vertex *v
|
||||
return std::make_unique<InMemoryVertexAccessor>(vertex, transaction, indices, constraints, config);
|
||||
}
|
||||
|
||||
bool InMemoryVertexAccessor::ReInit(Vertex *vertex, Transaction *transaction, Indices *indices,
|
||||
Constraints *constraints, Config::Items config, View view) {
|
||||
if (const auto [exists, deleted] = detail::IsVisible(vertex, transaction, view); !exists || deleted) {
|
||||
return false;
|
||||
}
|
||||
|
||||
transaction_ = transaction;
|
||||
config_ = config;
|
||||
vertex_ = vertex;
|
||||
indices_ = indices;
|
||||
constraints_ = constraints;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool InMemoryVertexAccessor::IsVisible(View view) const {
|
||||
const auto [exists, deleted] = detail::IsVisible(vertex_, transaction_, view);
|
||||
return exists && (for_deleted_ || !deleted);
|
||||
@@ -224,6 +238,7 @@ Result<PropertyValue> InMemoryVertexAccessor::SetProperty(PropertyId property, c
|
||||
// current code always follows the logical pattern of "create a delta" and
|
||||
// "modify in-place". Additionally, the created delta will make other
|
||||
// transactions get a SERIALIZATION_ERROR.
|
||||
|
||||
CreateAndLinkDelta(transaction_, vertex_, Delta::SetPropertyTag(), property, current_value);
|
||||
vertex_->properties.SetProperty(property, value);
|
||||
|
||||
|
||||
@@ -43,6 +43,9 @@ class InMemoryVertexAccessor final : public VertexAccessor {
|
||||
static std::unique_ptr<InMemoryVertexAccessor> Create(Vertex *vertex, Transaction *transaction, Indices *indices,
|
||||
Constraints *constraints, Config::Items config, View view);
|
||||
|
||||
bool ReInit(Vertex *vertex, Transaction *transaction, Indices *indices, Constraints *constraints,
|
||||
Config::Items config, View view) override;
|
||||
|
||||
/// @return true if the object is visible from the current transaction
|
||||
bool IsVisible(View view) const override;
|
||||
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <atomic>
|
||||
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "storage/v2/transaction.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
@@ -95,6 +96,9 @@ inline bool PrepareForWrite(Transaction *transaction, TObj *object) {
|
||||
/// a `DELETE_OBJECT` delta).
|
||||
/// @throw std::bad_alloc
|
||||
inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
|
||||
if (transaction->storage_mode == StorageMode::IN_MEMORY_ANALYTICAL) {
|
||||
return nullptr;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
return &transaction->deltas.emplace_back(Delta::DeleteObjectTag(), transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
@@ -105,6 +109,9 @@ inline Delta *CreateDeleteObjectDelta(Transaction *transaction) {
|
||||
/// @throw std::bad_alloc
|
||||
template <typename TObj, class... Args>
|
||||
inline void CreateAndLinkDelta(Transaction *transaction, TObj *object, Args &&...args) {
|
||||
if (transaction->storage_mode == StorageMode::IN_MEMORY_ANALYTICAL) {
|
||||
return;
|
||||
}
|
||||
transaction->EnsureCommitTimestampExists();
|
||||
auto delta = &transaction->deltas.emplace_back(std::forward<Args>(args)..., transaction->commit_timestamp.get(),
|
||||
transaction->command_id);
|
||||
|
||||
@@ -1144,7 +1144,8 @@ bool PropertyStore::SetProperty(PropertyId property, const PropertyValue &value)
|
||||
return !existed;
|
||||
}
|
||||
|
||||
bool PropertyStore::InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
|
||||
template <typename TContainer>
|
||||
bool PropertyStore::DoInitProperties(const TContainer &properties) {
|
||||
uint64_t size = 0;
|
||||
uint8_t *data = nullptr;
|
||||
std::tie(size, data) = GetSizeData(buffer_);
|
||||
@@ -1201,6 +1202,20 @@ bool PropertyStore::InitProperties(const std::map<storage::PropertyId, storage::
|
||||
|
||||
return true;
|
||||
}
|
||||
template bool PropertyStore::DoInitProperties<std::map<PropertyId, PropertyValue>>(
|
||||
const std::map<PropertyId, PropertyValue> &);
|
||||
template bool PropertyStore::DoInitProperties<std::vector<std::pair<PropertyId, PropertyValue>>>(
|
||||
const std::vector<std::pair<PropertyId, PropertyValue>> &);
|
||||
|
||||
bool PropertyStore::InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties) {
|
||||
return DoInitProperties(properties);
|
||||
}
|
||||
|
||||
bool PropertyStore::InitProperties(std::vector<std::pair<storage::PropertyId, storage::PropertyValue>> properties) {
|
||||
std::sort(properties.begin(), properties.end());
|
||||
|
||||
return DoInitProperties(properties);
|
||||
}
|
||||
|
||||
bool PropertyStore::ClearProperties() {
|
||||
bool in_local_buffer = false;
|
||||
|
||||
@@ -60,11 +60,17 @@ class PropertyStore {
|
||||
bool SetProperty(PropertyId property, const PropertyValue &value);
|
||||
|
||||
/// Init property values and return `true` if insertion took place. `false` is
|
||||
/// returned if there exists property in property store and insertion couldn't take place. The time complexity of this
|
||||
/// function is O(n).
|
||||
/// returned if there is any existing property in property store and insertion couldn't take place. The time
|
||||
/// complexity of this function is O(n).
|
||||
/// @throw std::bad_alloc
|
||||
bool InitProperties(const std::map<storage::PropertyId, storage::PropertyValue> &properties);
|
||||
|
||||
/// Init property values and return `true` if insertion took place. `false` is
|
||||
/// returned if there is any existing property in property store and insertion couldn't take place. The time
|
||||
/// complexity of this function is O(n*log(n)):
|
||||
/// @throw std::bad_alloc
|
||||
bool InitProperties(std::vector<std::pair<storage::PropertyId, storage::PropertyValue>> properties);
|
||||
|
||||
/// Remove all properties and return `true` if any removal took place.
|
||||
/// `false` is returned if there were no properties to remove. The time
|
||||
/// complexity of this function is O(1).
|
||||
@@ -72,6 +78,9 @@ class PropertyStore {
|
||||
bool ClearProperties();
|
||||
|
||||
private:
|
||||
template <typename TContainer>
|
||||
bool DoInitProperties(const TContainer &properties);
|
||||
|
||||
uint8_t buffer_[sizeof(uint64_t) + sizeof(uint8_t *)];
|
||||
};
|
||||
|
||||
|
||||
@@ -174,7 +174,7 @@ void InMemoryStorage::ReplicationServer::SnapshotHandler(slk::Reader *req_reader
|
||||
spdlog::debug("Loading snapshot");
|
||||
auto recovered_snapshot = durability::LoadSnapshot(*maybe_snapshot_path, &storage_->vertices_, &storage_->edges_,
|
||||
&storage_->epoch_history_, &storage_->name_id_mapper_,
|
||||
&storage_->edge_count_, storage_->config_.items);
|
||||
&storage_->edge_count_, storage_->config_);
|
||||
spdlog::debug("Snapshot loaded successfully");
|
||||
// If this step is present it should always be the first step of
|
||||
// the recovery so we use the UUID we read from snasphost
|
||||
|
||||
@@ -19,8 +19,14 @@ auto AdvanceToVisibleVertex(utils::SkipList<Vertex>::Iterator it, utils::SkipLis
|
||||
std::unique_ptr<VertexAccessor> &vertex, Transaction *tx, View view, Indices *indices,
|
||||
Constraints *constraints, Config::Items config) {
|
||||
while (it != end) {
|
||||
vertex = VertexAccessor::Create(&*it, tx, indices, constraints, config, view);
|
||||
if (!vertex) {
|
||||
bool isVisible = true;
|
||||
if (vertex) {
|
||||
isVisible = vertex->ReInit(&*it, tx, indices, constraints, config, view);
|
||||
} else {
|
||||
vertex = VertexAccessor::Create(&*it, tx, indices, constraints, config, view);
|
||||
isVisible = (vertex != nullptr);
|
||||
}
|
||||
if (!isVisible) {
|
||||
++it;
|
||||
continue;
|
||||
}
|
||||
@@ -34,7 +40,9 @@ AllVerticesIterable::Iterator::Iterator(AllVerticesIterable *self, utils::SkipLi
|
||||
it_(AdvanceToVisibleVertex(it, self->vertices_accessor_.end(), self->vertex_, self->transaction_, self->view_,
|
||||
self->indices_, self_->constraints_, self->config_)) {}
|
||||
|
||||
VertexAccessor *AllVerticesIterable::Iterator::operator*() const { return self_->vertex_.get(); }
|
||||
VertexAccessor *AllVerticesIterable::Iterator::operator*() const {
|
||||
return (it_ != self_->vertices_accessor_.end() ? self_->vertex_.get() : nullptr);
|
||||
}
|
||||
|
||||
AllVerticesIterable::Iterator &AllVerticesIterable::Iterator::operator++() {
|
||||
++it_;
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include "storage/v2/indices.hpp"
|
||||
#include "storage/v2/result.hpp"
|
||||
#include "storage/v2/storage_error.hpp"
|
||||
#include "storage/v2/storage_mode.hpp"
|
||||
#include "storage/v2/vertex_accessor.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
@@ -481,11 +482,21 @@ class Storage {
|
||||
|
||||
virtual void FreeMemory() = 0;
|
||||
|
||||
virtual void SetIsolationLevel(IsolationLevel isolation_level) = 0;
|
||||
enum class SetIsolationLevelError : uint8_t { DisabledForAnalyticalMode };
|
||||
|
||||
enum class CreateSnapshotError : uint8_t { DisabledForReplica };
|
||||
virtual utils::BasicResult<SetIsolationLevelError> SetIsolationLevel(IsolationLevel isolation_level) = 0;
|
||||
|
||||
virtual utils::BasicResult<CreateSnapshotError> CreateSnapshot() = 0;
|
||||
virtual void SetStorageMode(StorageMode storage_mode) = 0;
|
||||
|
||||
virtual StorageMode GetStorageMode() = 0;
|
||||
|
||||
enum class CreateSnapshotError : uint8_t {
|
||||
DisabledForReplica,
|
||||
DisabledForAnalyticsPeriodicCommit,
|
||||
ReachedMaxNumTries
|
||||
};
|
||||
|
||||
virtual utils::BasicResult<CreateSnapshotError> CreateSnapshot(std::optional<bool> is_periodic) = 0;
|
||||
};
|
||||
|
||||
} // namespace memgraph::storage
|
||||
|
||||
20
src/storage/v2/storage_mode.hpp
Normal file
20
src/storage/v2/storage_mode.hpp
Normal file
@@ -0,0 +1,20 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace memgraph::storage {
|
||||
|
||||
enum class StorageMode : std::uint8_t { IN_MEMORY_ANALYTICAL, IN_MEMORY_TRANSACTIONAL };
|
||||
|
||||
} // namespace memgraph::storage
|
||||
@@ -22,6 +22,7 @@
|
||||
#include "storage/v2/edge.hpp"
|
||||
#include "storage/v2/isolation_level.hpp"
|
||||
#include "storage/v2/property_value.hpp"
|
||||
#include "storage/v2/storage_mode.hpp"
|
||||
#include "storage/v2/vertex.hpp"
|
||||
#include "storage/v2/view.hpp"
|
||||
|
||||
@@ -31,12 +32,14 @@ const uint64_t kTimestampInitialId = 0;
|
||||
const uint64_t kTransactionInitialId = 1ULL << 63U;
|
||||
|
||||
struct Transaction {
|
||||
Transaction(uint64_t transaction_id, uint64_t start_timestamp, IsolationLevel isolation_level)
|
||||
Transaction(uint64_t transaction_id, uint64_t start_timestamp, IsolationLevel isolation_level,
|
||||
StorageMode storage_mode)
|
||||
: transaction_id(transaction_id),
|
||||
start_timestamp(start_timestamp),
|
||||
command_id(0),
|
||||
must_abort(false),
|
||||
isolation_level(isolation_level) {}
|
||||
isolation_level(isolation_level),
|
||||
storage_mode(storage_mode) {}
|
||||
|
||||
Transaction(Transaction &&other) noexcept
|
||||
: transaction_id(other.transaction_id.load(std::memory_order_acquire)),
|
||||
@@ -45,7 +48,8 @@ struct Transaction {
|
||||
command_id(other.command_id),
|
||||
deltas(std::move(other.deltas)),
|
||||
must_abort(other.must_abort),
|
||||
isolation_level(other.isolation_level) {}
|
||||
isolation_level(other.isolation_level),
|
||||
storage_mode(other.storage_mode) {}
|
||||
|
||||
Transaction(const Transaction &) = delete;
|
||||
Transaction &operator=(const Transaction &) = delete;
|
||||
@@ -70,6 +74,7 @@ struct Transaction {
|
||||
std::list<Delta> deltas;
|
||||
bool must_abort;
|
||||
IsolationLevel isolation_level;
|
||||
StorageMode storage_mode;
|
||||
};
|
||||
|
||||
inline bool operator==(const Transaction &first, const Transaction &second) {
|
||||
|
||||
@@ -41,6 +41,10 @@ class VertexAccessor {
|
||||
static std::unique_ptr<VertexAccessor> Create(Vertex *vertex, Transaction *transaction, Indices *indices,
|
||||
Constraints *constraints, Config::Items config, View view);
|
||||
|
||||
/// Reinitialize the accessor with a new vertex to avoid additional allocations.
|
||||
virtual bool ReInit(Vertex *vertex, Transaction *transaction, Indices *indices, Constraints *constraints,
|
||||
Config::Items config, View view) = 0;
|
||||
|
||||
/// @return true if the object is visible from the current transaction
|
||||
virtual bool IsVisible(View view) const = 0;
|
||||
|
||||
|
||||
57
src/utils/enum.hpp
Normal file
57
src/utils/enum.hpp
Normal file
@@ -0,0 +1,57 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include <string>
|
||||
|
||||
#include "utils/result.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::utils {
|
||||
enum class ValidationError : uint8_t { EmptyValue, InvalidValue };
|
||||
|
||||
// Returns joined string representations for every enum in the mapping.
|
||||
std::string GetAllowedEnumValuesString(const auto &mappings) {
|
||||
std::vector<std::string> allowed_values;
|
||||
allowed_values.reserve(mappings.size());
|
||||
std::transform(mappings.begin(), mappings.end(), std::back_inserter(allowed_values),
|
||||
[](const auto &mapping) { return std::string(mapping.first); });
|
||||
return memgraph::utils::Join(allowed_values, ", ");
|
||||
}
|
||||
|
||||
// Checks if the string value can be represented as an enum.
|
||||
// If not, the BasicResult will contain an error.
|
||||
memgraph::utils::BasicResult<ValidationError> IsValidEnumValueString(const auto &value, const auto &mappings) {
|
||||
if (value.empty()) {
|
||||
return ValidationError::EmptyValue;
|
||||
}
|
||||
|
||||
if (std::find_if(mappings.begin(), mappings.end(), [&](const auto &mapping) { return mapping.first == value; }) ==
|
||||
mappings.cend()) {
|
||||
return ValidationError::InvalidValue;
|
||||
}
|
||||
|
||||
return {};
|
||||
}
|
||||
// Tries to convert a string into enum, which would then contain a value if the conversion
|
||||
// has been successful.
|
||||
template <typename Enum>
|
||||
std::optional<Enum> StringToEnum(const auto &value, const auto &mappings) {
|
||||
const auto mapping_iter =
|
||||
std::find_if(mappings.begin(), mappings.end(), [&](const auto &mapping) { return mapping.first == value; });
|
||||
if (mapping_iter == mappings.cend()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return mapping_iter->second;
|
||||
}
|
||||
} // namespace memgraph::utils
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -187,14 +187,19 @@ void *Pool::Allocate() {
|
||||
for (unsigned char i = 0U; i < blocks_per_chunk_; ++i) {
|
||||
*(data + (i * block_size_)) = i + 1U;
|
||||
}
|
||||
Chunk chunk{data, 0, blocks_per_chunk_};
|
||||
// Insert the big block in the sorted position.
|
||||
auto it = std::lower_bound(chunks_.begin(), chunks_.end(), chunk,
|
||||
[](const auto &a, const auto &b) { return a.data < b.data; });
|
||||
try {
|
||||
chunks_.push_back(Chunk{data, 0, blocks_per_chunk_});
|
||||
it = chunks_.insert(it, chunk);
|
||||
} catch (...) {
|
||||
GetUpstreamResource()->Deallocate(data, data_size, alignment);
|
||||
throw;
|
||||
}
|
||||
last_alloc_chunk_ = &chunks_.back();
|
||||
last_dealloc_chunk_ = &chunks_.back();
|
||||
|
||||
last_alloc_chunk_ = &*it;
|
||||
last_dealloc_chunk_ = &*it;
|
||||
return allocate_block_from_chunk(last_alloc_chunk_);
|
||||
}
|
||||
|
||||
@@ -223,18 +228,20 @@ void Pool::Deallocate(void *p) {
|
||||
deallocate_block_from_chunk(last_dealloc_chunk_);
|
||||
return;
|
||||
}
|
||||
|
||||
// Find the chunk which served this allocation
|
||||
for (auto &chunk : chunks_) {
|
||||
if (is_in_chunk(chunk)) {
|
||||
// Update last_alloc_chunk_ as well because it now has a free block.
|
||||
// Additionally this corresponds with C++ pattern of allocations and
|
||||
// deallocations being done in reverse order.
|
||||
last_alloc_chunk_ = &chunk;
|
||||
last_dealloc_chunk_ = &chunk;
|
||||
deallocate_block_from_chunk(&chunk);
|
||||
return;
|
||||
}
|
||||
}
|
||||
Chunk chunk{reinterpret_cast<unsigned char *>(p) - blocks_per_chunk_ * block_size_, 0, 0};
|
||||
auto it = std::lower_bound(chunks_.begin(), chunks_.end(), chunk,
|
||||
[](const auto &a, const auto &b) { return a.data <= b.data; });
|
||||
MG_ASSERT(it != chunks_.end(), "Failed deallocation in utils::Pool");
|
||||
MG_ASSERT(is_in_chunk(*it), "Failed deallocation in utils::Pool");
|
||||
|
||||
// Update last_alloc_chunk_ as well because it now has a free block.
|
||||
// Additionally this corresponds with C++ pattern of allocations and
|
||||
// deallocations being done in reverse order.
|
||||
last_alloc_chunk_ = &*it;
|
||||
last_dealloc_chunk_ = &*it;
|
||||
deallocate_block_from_chunk(last_dealloc_chunk_);
|
||||
// TODO: We could release the Chunk to upstream memory
|
||||
}
|
||||
|
||||
|
||||
23
src/utils/pmr/deque.hpp
Normal file
23
src/utils/pmr/deque.hpp
Normal file
@@ -0,0 +1,23 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <deque>
|
||||
|
||||
#include "utils/memory.hpp"
|
||||
|
||||
namespace memgraph::utils::pmr {
|
||||
|
||||
template <class T>
|
||||
using deque = std::deque<T, utils::Allocator<T>>;
|
||||
|
||||
} // namespace memgraph::utils::pmr
|
||||
@@ -171,6 +171,7 @@ enum class TypeId : uint64_t {
|
||||
AST_FREE_MEMORY_QUERY,
|
||||
AST_TRIGGER_QUERY,
|
||||
AST_ISOLATION_LEVEL_QUERY,
|
||||
AST_STORAGE_MODE_QUERY,
|
||||
AST_CREATE_SNAPSHOT_QUERY,
|
||||
AST_STREAM_QUERY,
|
||||
AST_SETTING_QUERY,
|
||||
|
||||
@@ -39,7 +39,7 @@ static void MapLiteral(benchmark::State &state) {
|
||||
memgraph::query::SymbolTable symbol_table;
|
||||
TMemory memory;
|
||||
memgraph::query::Frame frame(symbol_table.max_position(), memory.get());
|
||||
std::unique_ptr<memgraph::storage::Storage> db(new memgraph::storage::InMemoryStorage());
|
||||
std::unique_ptr<memgraph::storage::Storage> db = std::make_unique<memgraph::storage::InMemoryStorage>();
|
||||
auto storage_dba = db->Access();
|
||||
memgraph::query::DbAccessor dba(storage_dba.get());
|
||||
std::unordered_map<memgraph::query::PropertyIx, memgraph::query::Expression *> elements;
|
||||
@@ -68,7 +68,7 @@ static void AdditionOperator(benchmark::State &state) {
|
||||
memgraph::query::SymbolTable symbol_table;
|
||||
TMemory memory;
|
||||
memgraph::query::Frame frame(symbol_table.max_position(), memory.get());
|
||||
std::unique_ptr<memgraph::storage::Storage> db(new memgraph::storage::InMemoryStorage());
|
||||
std::unique_ptr<memgraph::storage::Storage> db = std::make_unique<memgraph::storage::InMemoryStorage>();
|
||||
auto storage_dba = db->Access();
|
||||
memgraph::query::DbAccessor dba(storage_dba.get());
|
||||
memgraph::query::Expression *expr = ast.Create<memgraph::query::PrimitiveLiteral>(0);
|
||||
|
||||
@@ -124,7 +124,7 @@ template <class TMemory>
|
||||
static void Distinct(benchmark::State &state) {
|
||||
memgraph::query::AstStorage ast;
|
||||
memgraph::query::Parameters parameters;
|
||||
std::unique_ptr<memgraph::storage::Storage> db(new memgraph::storage::InMemoryStorage());
|
||||
std::unique_ptr<memgraph::storage::Storage> db = std::make_unique<memgraph::storage::InMemoryStorage>();
|
||||
AddVertices(db.get(), state.range(0));
|
||||
auto storage_dba = db->Access();
|
||||
memgraph::query::DbAccessor dba(storage_dba.get());
|
||||
@@ -188,8 +188,9 @@ static void ExpandVariable(benchmark::State &state) {
|
||||
TMemory memory;
|
||||
memgraph::query::Frame frame(symbol_table.max_position(), memory.get());
|
||||
auto cursor = expand_variable.MakeCursor(memory.get());
|
||||
for (const auto &v : dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel))) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(memgraph::query::VertexAccessor(v));
|
||||
auto vertices = dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel));
|
||||
for (auto vIt = vertices.begin(); vIt != vertices.end(); ++vIt) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(*vIt);
|
||||
while (cursor->Pull(frame, execution_context)) per_pull_memory.Reset();
|
||||
}
|
||||
}
|
||||
@@ -227,8 +228,9 @@ static void ExpandBfs(benchmark::State &state) {
|
||||
TMemory memory;
|
||||
memgraph::query::Frame frame(symbol_table.max_position(), memory.get());
|
||||
auto cursor = expand_variable.MakeCursor(memory.get());
|
||||
for (const auto &v : dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel))) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(memgraph::query::VertexAccessor(v));
|
||||
auto vertices = dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel));
|
||||
for (auto vIt = vertices.begin(); vIt != vertices.end(); ++vIt) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(*vIt);
|
||||
while (cursor->Pull(frame, execution_context)) per_pull_memory.Reset();
|
||||
}
|
||||
}
|
||||
@@ -262,10 +264,11 @@ static void ExpandShortest(benchmark::State &state) {
|
||||
TMemory memory;
|
||||
memgraph::query::Frame frame(symbol_table.max_position(), memory.get());
|
||||
auto cursor = expand_variable.MakeCursor(memory.get());
|
||||
for (const auto &v : dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel))) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(memgraph::query::VertexAccessor(v));
|
||||
for (const auto &dest : dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel))) {
|
||||
frame[dest_symbol] = memgraph::query::TypedValue(memgraph::query::VertexAccessor(dest));
|
||||
auto vertices = dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel));
|
||||
for (auto vIt = vertices.begin(); vIt != vertices.end(); ++vIt) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(*vIt);
|
||||
for (auto destIt = vertices.begin(); destIt != vertices.end(); ++destIt) {
|
||||
frame[dest_symbol] = memgraph::query::TypedValue(*destIt);
|
||||
while (cursor->Pull(frame, execution_context)) per_pull_memory.Reset();
|
||||
}
|
||||
}
|
||||
@@ -303,10 +306,11 @@ static void ExpandWeightedShortest(benchmark::State &state) {
|
||||
TMemory memory;
|
||||
memgraph::query::Frame frame(symbol_table.max_position(), memory.get());
|
||||
auto cursor = expand_variable.MakeCursor(memory.get());
|
||||
for (const auto &v : dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel))) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(memgraph::query::VertexAccessor(v));
|
||||
for (const auto &dest : dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel))) {
|
||||
frame[dest_symbol] = memgraph::query::TypedValue(memgraph::query::VertexAccessor(dest));
|
||||
auto vertices = dba.Vertices(memgraph::storage::View::OLD, dba.NameToLabel(kStartLabel));
|
||||
for (auto vIt = vertices.begin(); vIt != vertices.end(); ++vIt) {
|
||||
frame[expand_variable.input_symbol_] = memgraph::query::TypedValue(*vIt);
|
||||
for (auto destIt = vertices.begin(); destIt != vertices.end(); ++destIt) {
|
||||
frame[dest_symbol] = memgraph::query::TypedValue(*destIt);
|
||||
while (cursor->Pull(frame, execution_context)) per_pull_memory.Reset();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -58,7 +58,8 @@ int main(int argc, char *argv[]) {
|
||||
gflags::ParseCommandLineFlags(&argc, &argv, true);
|
||||
|
||||
for (const auto &config : TestConfigurations) {
|
||||
std::unique_ptr<memgraph::storage::Storage> storage(new memgraph::storage::InMemoryStorage(config.second));
|
||||
std::unique_ptr<memgraph::storage::Storage> storage =
|
||||
std::make_unique<memgraph::storage::InMemoryStorage>(config.second);
|
||||
std::vector<memgraph::storage::Gid> vertices;
|
||||
{
|
||||
auto acc = storage->Access();
|
||||
|
||||
@@ -38,7 +38,12 @@ def test_does_default_config_match():
|
||||
flag_name = flag[0]
|
||||
|
||||
# The default value of these is dependent on the given machine.
|
||||
machine_dependent_configurations = ["bolt_num_workers", "data_directory", "log_file"]
|
||||
machine_dependent_configurations = [
|
||||
"bolt_num_workers",
|
||||
"data_directory",
|
||||
"log_file",
|
||||
"storage_recovery_thread_count",
|
||||
]
|
||||
if flag_name in machine_dependent_configurations:
|
||||
continue
|
||||
|
||||
|
||||
@@ -98,6 +98,12 @@ startup_config_dict = {
|
||||
"IP address on which the websocket server for Memgraph monitoring should listen.",
|
||||
),
|
||||
"monitoring_port": ("7444", "7444", "Port on which the websocket server for Memgraph monitoring should listen."),
|
||||
"storage_parallel_index_recovery": (
|
||||
"false",
|
||||
"false",
|
||||
"Controls whether the index creation can be done in a multithreaded fashion.",
|
||||
),
|
||||
"password_encryption_algorithm": ("bcrypt", "bcrypt", "The password encryption algorithm used for authentication."),
|
||||
"pulsar_service_url": ("", "", "Default URL used while connecting to Pulsar brokers."),
|
||||
"query_execution_timeout_sec": (
|
||||
"600",
|
||||
@@ -115,12 +121,18 @@ startup_config_dict = {
|
||||
"The time duration between two replica checks/pings. If < 1, replicas will NOT be checked at all. NOTE: The MAIN instance allocates a new thread for each REPLICA.",
|
||||
),
|
||||
"storage_gc_cycle_sec": ("30", "30", "Storage garbage collector interval (in seconds)."),
|
||||
"storage_items_per_batch": (
|
||||
"1000000",
|
||||
"1000000",
|
||||
"The number of edges and vertices stored in a batch in a snapshot file.",
|
||||
),
|
||||
"storage_properties_on_edges": ("false", "true", "Controls whether edges have properties."),
|
||||
"storage_recover_on_startup": (
|
||||
"false",
|
||||
"false",
|
||||
"Controls whether the storage recovers persisted data on startup.",
|
||||
),
|
||||
"storage_recovery_thread_count": ("12", "12", "The number of threads used to recover persisted data from disk."),
|
||||
"storage_snapshot_interval_sec": (
|
||||
"0",
|
||||
"300",
|
||||
|
||||
@@ -37,6 +37,7 @@ BASIC_PRIVILEGES = [
|
||||
"WEBSOCKET",
|
||||
"MODULE_WRITE",
|
||||
"TRANSACTION_MANAGEMENT",
|
||||
"STORAGE_MODE",
|
||||
]
|
||||
|
||||
|
||||
@@ -60,7 +61,7 @@ def test_lba_procedures_show_privileges_first_user():
|
||||
cursor = connect(username="Josip", password="").cursor()
|
||||
result = execute_and_fetch_all(cursor, "SHOW PRIVILEGES FOR Josip;")
|
||||
|
||||
assert len(result) == 31
|
||||
assert len(result) == 32
|
||||
|
||||
fine_privilege_results = [res for res in result if res[0] not in BASIC_PRIVILEGES]
|
||||
|
||||
|
||||
@@ -30,3 +30,6 @@ add_subdirectory(env_variable_check)
|
||||
|
||||
#flag check binaries
|
||||
add_subdirectory(flag_check)
|
||||
|
||||
#flag check binaries
|
||||
add_subdirectory(storage_mode)
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
// --storage-items-per-batch is set to 10
|
||||
CREATE INDEX ON :`label2`(`prop2`);
|
||||
CREATE INDEX ON :`label2`(`prop`);
|
||||
CREATE INDEX ON :`label`;
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `ext`: 2, `prop`: "joj"});
|
||||
CREATE (:__mg_vertex__:`label2`:`label` {__mg_id__: 1, `ext`: 2, `prop`: "joj"});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2});
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE;
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE INDEX ON :`label`;
|
||||
CREATE INDEX ON :`label2`(`prop2`);
|
||||
CREATE INDEX ON :`label2`(`prop`);
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `ext`: 2, `prop`: "joj"});
|
||||
CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `ext`: 2, `prop`: "joj"});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2});
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`ext`: [false, {`k`: "l"}], `prop`: -1}]->(v);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE;
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE INDEX ON :`label`;
|
||||
CREATE INDEX ON :`label2`(`prop2`);
|
||||
CREATE INDEX ON :`label2`(`prop`);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`prop2`, u.`prop` IS UNIQUE;
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 0, `prop2`: ["kaj", 2, Null, {`prop4`: -1.341}], `prop`: "joj", `ext`: 2});
|
||||
CREATE (:__mg_vertex__:`label`:`label2` {__mg_id__: 1, `prop`: "joj", `ext`: 2});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 2, `prop2`: 2, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 3, `prop2`: 2, `prop`: 2});
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 0 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 1 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 2 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 1 AND v.__mg_id__ = 3 CREATE (u)-[:`link` {`prop`: -1, `ext`: [false, {`k`: "l"}]}]->(v);
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
BIN
tests/integration/durability/tests/v15/test_all/snapshot.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_all/snapshot.bin
Normal file
Binary file not shown.
BIN
tests/integration/durability/tests/v15/test_all/wal.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_all/wal.bin
Normal file
Binary file not shown.
@@ -0,0 +1,6 @@
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`a`, u.`b` IS UNIQUE;
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`b`, u.`a` IS UNIQUE;
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT EXISTS (u.`ext2`);
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT EXISTS (u.`ext`);
|
||||
CREATE CONSTRAINT ON (u:`label2`) ASSERT u.`a`, u.`b` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`a` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`b` IS UNIQUE;
|
||||
CREATE CONSTRAINT ON (u:`label`) ASSERT u.`c` IS UNIQUE;
|
||||
Binary file not shown.
BIN
tests/integration/durability/tests/v15/test_constraints/wal.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_constraints/wal.bin
Normal file
Binary file not shown.
@@ -0,0 +1,59 @@
|
||||
// --storage-items-per-batch is set to 7
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 0});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 1});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 2});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 3});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 4});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 5});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 6});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 7});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 8});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 9});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 10});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 11});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 12});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 13});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 14});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 15});
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
@@ -0,0 +1,58 @@
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 0});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 1});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 2});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 3});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 4});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 5});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 6});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 7});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 8});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 9});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 10});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 11});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 12});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 13});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 14});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 15});
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
@@ -0,0 +1,58 @@
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 0});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 1});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 2});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 3});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 4});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 5});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 6});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 7});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 8});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 9});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 10});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 11});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 12});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 13});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 14});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 15});
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 0 AND v.__mg_id__ = 1 CREATE (u)-[:`edge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 2 AND v.__mg_id__ = 3 CREATE (u)-[:`edge` {`prop`: 11}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 4 AND v.__mg_id__ = 5 CREATE (u)-[:`edge` {`prop`: true}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 6 AND v.__mg_id__ = 7 CREATE (u)-[:`edge2`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 8 AND v.__mg_id__ = 9 CREATE (u)-[:`edge2` {`prop`: -3.141}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 10 AND v.__mg_id__ = 11 CREATE (u)-[:`edgelink` {`prop`: {`prop`: 1, `prop2`: {`prop4`: 9}}}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 12 AND v.__mg_id__ = 13 CREATE (u)-[:`edgelink` {`prop`: [1, Null, false, "\n\n\n\n\\\"\"\n\t"]}]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 14 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 0 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 1 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 2 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 3 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 4 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 5 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 6 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 7 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 8 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 9 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 10 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 11 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 12 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 13 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 14 CREATE (u)-[:`testedge`]->(v);
|
||||
MATCH (u:__mg_vertex__), (v:__mg_vertex__) WHERE u.__mg_id__ = 15 AND v.__mg_id__ = 15 CREATE (u)-[:`testedge`]->(v);
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
BIN
tests/integration/durability/tests/v15/test_edges/snapshot.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_edges/snapshot.bin
Normal file
Binary file not shown.
BIN
tests/integration/durability/tests/v15/test_edges/wal.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_edges/wal.bin
Normal file
Binary file not shown.
@@ -0,0 +1,4 @@
|
||||
CREATE INDEX ON :`label2`;
|
||||
CREATE INDEX ON :`label2`(`prop2`);
|
||||
CREATE INDEX ON :`label`(`prop2`);
|
||||
CREATE INDEX ON :`label`(`prop`);
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE INDEX ON :`label2`;
|
||||
CREATE INDEX ON :`label`(`prop`);
|
||||
CREATE INDEX ON :`label`(`prop2`);
|
||||
CREATE INDEX ON :`label2`(`prop2`);
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE INDEX ON :`label2`;
|
||||
CREATE INDEX ON :`label2`(`prop2`);
|
||||
CREATE INDEX ON :`label`(`prop2`);
|
||||
CREATE INDEX ON :`label`(`prop`);
|
||||
BIN
tests/integration/durability/tests/v15/test_indices/snapshot.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_indices/snapshot.bin
Normal file
Binary file not shown.
BIN
tests/integration/durability/tests/v15/test_indices/wal.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_indices/wal.bin
Normal file
Binary file not shown.
@@ -0,0 +1,17 @@
|
||||
// --storage-items-per-batch is set to 5
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 0});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 1});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141});
|
||||
CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop3`: true, `prop2`: -314000000});
|
||||
CREATE (:__mg_vertex__:`label3`:`label1`:`label2` {__mg_id__: 7});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop3`: "str", `prop2`: 2, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label2`:`label1` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]});
|
||||
CREATE (:__mg_vertex__:`label3`:`label` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 12, `prop`: " \n\"\'\t\\%"});
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 0});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 1});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141});
|
||||
CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop3`: true, `prop2`: -314000000});
|
||||
CREATE (:__mg_vertex__:`label2`:`label3`:`label1` {__mg_id__: 7});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop3`: "str", `prop2`: 2, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label1`:`label2` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]});
|
||||
CREATE (:__mg_vertex__:`label`:`label3` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 12, `prop`: " \n\"\'\t\\%"});
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
@@ -0,0 +1,16 @@
|
||||
CREATE INDEX ON :__mg_vertex__(__mg_id__);
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 0});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 1});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 2, `prop`: false});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 3, `prop`: true});
|
||||
CREATE (:__mg_vertex__:`label2` {__mg_id__: 4, `prop`: 1});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 5, `prop2`: 3.141});
|
||||
CREATE (:__mg_vertex__:`label6` {__mg_id__: 6, `prop2`: -314000000, `prop3`: true});
|
||||
CREATE (:__mg_vertex__:`label2`:`label3`:`label1` {__mg_id__: 7});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 8, `prop`: 1, `prop2`: 2, `prop3`: "str"});
|
||||
CREATE (:__mg_vertex__:`label1`:`label2` {__mg_id__: 9, `prop`: {`prop_nes`: "kaj je"}});
|
||||
CREATE (:__mg_vertex__:`label` {__mg_id__: 10, `prop_array`: [1, false, Null, "str", {`prop2`: 2}]});
|
||||
CREATE (:__mg_vertex__:`label`:`label3` {__mg_id__: 11, `prop`: {`prop`: [1, false], `prop2`: {}, `prop3`: "test2", `prop4`: "test"}});
|
||||
CREATE (:__mg_vertex__ {__mg_id__: 12, `prop`: " \n\"\'\t\\%"});
|
||||
DROP INDEX ON :__mg_vertex__(__mg_id__);
|
||||
MATCH (u) REMOVE u:__mg_vertex__, u.__mg_id__;
|
||||
Binary file not shown.
BIN
tests/integration/durability/tests/v15/test_vertices/wal.bin
Normal file
BIN
tests/integration/durability/tests/v15/test_vertices/wal.bin
Normal file
Binary file not shown.
6
tests/integration/storage_mode/CMakeLists.txt
Normal file
6
tests/integration/storage_mode/CMakeLists.txt
Normal file
@@ -0,0 +1,6 @@
|
||||
set(target_name memgraph__integration__storage_mode)
|
||||
set(tester_target_name ${target_name}__tester)
|
||||
|
||||
add_executable(${tester_target_name} tester.cpp)
|
||||
set_target_properties(${tester_target_name} PROPERTIES OUTPUT_NAME tester)
|
||||
target_link_libraries(${tester_target_name} mg-communication)
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user