Compare commits

..

5 Commits

Author SHA1 Message Date
Tyler Neely
18c8473a6b Check-in high-level skeleton of some things for the protobuf transport 2022-10-05 12:41:32 +00:00
Tyler Neely
bc602bb93c Check-in protobuf transport spaghetti 2022-09-23 15:38:16 +00:00
Tyler Neely
a7125894cf Start to fill out UberMessage 2022-09-23 15:14:20 +00:00
Tyler Neely
d46e6e36a2 Start to use UberMessage protobuf 2022-09-23 14:41:54 +00:00
Tyler Neely
63871d0b03 Check-in initial scaffolding for ProtobufTransport 2022-09-23 13:57:17 +00:00
302 changed files with 40901 additions and 23506 deletions

View File

@@ -16,7 +16,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -26,7 +26,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -64,7 +64,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -75,39 +75,14 @@ jobs:
- name: Fetch all history for all tags and branches
run: git fetch
- name: Initialize deps
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
- name: Set base branch
if: ${{ github.event_name == 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
- name: Set base branch # if we manually dispatch or push to master
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
- name: Python code analysis
run: |
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
for file in ${CHANGED_FILES}; do
echo ${file}
if [[ ${file} == *.py ]]; then
python3 -m black --check --diff ${file}
python3 -m isort --check-only --profile "black" --diff ${file}
fi
done
- name: Build combined ASAN, UBSAN and coverage binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
cd build
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
make -j$THREADS memgraph__unit
@@ -135,7 +110,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -170,7 +145,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -189,23 +164,52 @@ jobs:
cmake ..
make -j$THREADS
- name: Run simulation tests
- name: Run leftover CTest tests
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run simulation tests.
# Run leftover CTest tests (all except unit and benchmark tests).
cd build
ctest -R memgraph__simulation --output-on-failure -j$THREADS
ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
- name: Run single benchmark test
- name: Run drivers tests
run: |
./tests/drivers/run.sh
- name: Run integration tests
run: |
cd tests/integration
for name in *; do
if [ ! -d $name ]; then continue; fi
pushd $name >/dev/null
echo "Running: $name"
if [ -x prepare.sh ]; then
./prepare.sh
fi
if [ -x runner.py ]; then
./runner.py
elif [ -x runner.sh ]; then
./runner.sh
fi
echo
popd >/dev/null
done
- name: Run cppcheck and clang-format
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run simulation tests.
cd tests/mgbench
./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
# Run cppcheck and clang-format.
cd tools/github
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
release_build:
name: "Release build"
@@ -217,7 +221,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -236,6 +240,19 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run GQL Behave tests
run: |
cd tests/gql_behave
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
tests/gql_behave/gql_behave_status.csv
tests/gql_behave/gql_behave_status.html
- name: Run unit tests
run: |
# Activate toolchain.
@@ -250,19 +267,10 @@ jobs:
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run simulation tests.
# Run unit tests.
cd build
ctest -R memgraph__simulation --output-on-failure -j$THREADS
- name: Run single benchmark test
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Run simulation tests.
cd tests/mgbench
./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
- name: Run e2e tests
run: |
# TODO(gitbuda): Setup mgclient and pymgclient properly.
@@ -270,12 +278,131 @@ jobs:
./setup.sh
source ve3/bin/activate
cd e2e
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory ./distributed_queries
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
- name: Run query performance tests
- name: Run stress test (plain)
run: |
cd tests/manual
./query_performance_runner.py
cd tests/stress
./continuous_integration
- name: Run stress test (SSL)
run: |
cd tests/stress
./continuous_integration --use-ssl
- name: Run durability test
run: |
cd tests/stress
source ve3/bin/activate
python3 durability --num-steps 5
- name: Create enterprise DEB package
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
cd build
# create mgconsole
# we use the -B to force the build
make -j$THREADS -B mgconsole
# Create enterprise DEB package.
mkdir output && cd output
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
- name: Save test data
uses: actions/upload-artifact@v2
if: always()
with:
name: "Test data"
path: |
# multiple paths could be defined
build/logs
release_jepsen_test:
name: "Release Jepsen Test"
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
#continue-on-error: true
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build only memgraph release binarie.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS memgraph
- name: Run Jepsen tests
run: |
cd tests/jepsen
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: "Jepsen Report"
path: tests/jepsen/Jepsen.tar.gz
release_benchmarks:
name: "Release benchmarks"
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
env:
THREADS: 24
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
steps:
- name: Set up repository
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
fetch-depth: 0
- name: Build release binaries
run: |
# Activate toolchain.
source /opt/toolchain-v4/activate
# Initialize dependencies.
./init
# Build only memgraph release binaries.
cd build
cmake -DCMAKE_BUILD_TYPE=release ..
make -j$THREADS
- name: Run macro benchmarks
run: |
cd tests/macro_benchmark
./harness QuerySuite MemgraphRunner \
--groups aggregation 1000_create unwind_create dense_expand match \
--no-strict
- name: Get branch name (merge)
if: github.event_name != 'pull_request'
@@ -293,8 +420,25 @@ jobs:
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "query_performance" \
--benchmark-results-path "../../build/tests/manual/query_performance_benchmark/summary.json" \
./main.py --benchmark-name "macro_benchmark" \
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"
- name: Run mgbench
run: |
cd tests/mgbench
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
- name: Upload mgbench results
run: |
cd tools/bench-graph-client
virtualenv -p python3 ve3
source ve3/bin/activate
pip install -r requirements.txt
./main.py --benchmark-name "mgbench" \
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
--github-run-id "${{ github.run_id }}" \
--github-run-number "${{ github.run_number }}" \
--head-branch-name "${{ env.BRANCH_NAME }}"

View File

@@ -14,7 +14,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)

View File

@@ -17,7 +17,7 @@ jobs:
run: |
./release/package/run.sh package centos-7
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: centos-7
path: build/output/centos-7/memgraph*.rpm
@@ -34,7 +34,7 @@ jobs:
run: |
./release/package/run.sh package centos-9
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: centos-9
path: build/output/centos-9/memgraph*.rpm
@@ -51,7 +51,7 @@ jobs:
run: |
./release/package/run.sh package debian-10
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-10
path: build/output/debian-10/memgraph*.deb
@@ -68,7 +68,7 @@ jobs:
run: |
./release/package/run.sh package debian-11
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11
path: build/output/debian-11/memgraph*.deb
@@ -87,7 +87,7 @@ jobs:
./run.sh package debian-11 --for-docker
./run.sh docker
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: docker
path: build/output/docker/memgraph*.tar.gz
@@ -104,7 +104,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-18.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-1804
path: build/output/ubuntu-18.04/memgraph*.deb
@@ -121,7 +121,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-20.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-2004
path: build/output/ubuntu-20.04/memgraph*.deb
@@ -138,7 +138,7 @@ jobs:
run: |
./release/package/run.sh package ubuntu-22.04
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: ubuntu-2204
path: build/output/ubuntu-22.04/memgraph*.deb
@@ -155,7 +155,7 @@ jobs:
run: |
./release/package/run.sh package debian-11 --for-platform
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11-platform
path: build/output/debian-11/memgraph*.deb
@@ -172,7 +172,7 @@ jobs:
run: |
./release/package/run.sh package debian-11-arm
- name: "Upload package"
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: debian-11-arm
path: build/output/debian-11-arm/memgraph*.deb

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -225,7 +225,7 @@ jobs:
rpmlint memgraph*.rpm
- name: Save enterprise RPM package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise RPM package"
path: build/output/memgraph*.rpm
@@ -262,7 +262,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |
@@ -324,7 +324,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -349,7 +349,7 @@ jobs:
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
- name: Save Jepsen report
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
if: ${{ always() }}
with:
name: "Jepsen Report"

View File

@@ -19,7 +19,7 @@ jobs:
DOCKER_REPOSITORY_NAME: memgraph
steps:
- name: Checkout
uses: actions/checkout@v3
uses: actions/checkout@v2
- name: Set up QEMU
uses: docker/setup-qemu-action@v1

View File

@@ -17,7 +17,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -55,7 +55,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -97,7 +97,7 @@ jobs:
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
- name: Save code coverage
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/generated/code_coverage.tar.gz
@@ -112,7 +112,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -173,7 +173,7 @@ jobs:
./cppcheck_and_clang_format diff
- name: Save cppcheck and clang-format errors
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Code coverage"
path: tools/github/cppcheck_and_clang_format.txt
@@ -189,7 +189,7 @@ jobs:
steps:
- name: Set up repository
uses: actions/checkout@v3
uses: actions/checkout@v2
with:
# Number of commits to fetch. `0` indicates all history for all
# branches and tags. (default: 1)
@@ -224,7 +224,7 @@ jobs:
cpack -G DEB --config ../CPackConfig.cmake
- name: Save enterprise DEB package
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "Enterprise DEB package"
path: build/output/memgraph*.deb
@@ -261,7 +261,7 @@ jobs:
./continuous_integration
- name: Save quality assurance status
uses: actions/upload-artifact@v3
uses: actions/upload-artifact@v2
with:
name: "GQL Behave Status"
path: |

View File

@@ -6,15 +6,18 @@ repos:
- id: end-of-file-fixer
- id: trailing-whitespace
- repo: https://github.com/psf/black
rev: 22.10.0
rev: 22.3.0
hooks:
- id: black
- repo: https://github.com/pycqa/isort
rev: 5.10.1
hooks:
- id: isort
name: isort (python)
args: ["--profile", "black"]
args: # arguments to configure black
- --line-length=120
- --include='\.pyi?$'
# these folders wont be formatted by black
- --exclude="""\.git |
\.__pycache__|
build|
libs|
.cache"""
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: v13.0.0
hooks:

View File

@@ -182,8 +182,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
# c99-designator is disabled because of required mixture of designated and
# non-designated initializers in Python Query Module code (`py_module.cpp`).
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
-Werror=switch -Werror=switch-bool -Werror=implicit-fallthrough \
-Werror=return-type \
-Werror=switch -Werror=switch-bool -Werror=return-type \
-Werror=return-stack-address \
-Wno-c99-designator \
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")

View File

@@ -83,9 +83,13 @@ modifications:
value: "true"
override: true
# - name: "query_modules_directory"
# value: "/usr/lib/memgraph/query_modules"
# override: true
- name: "query_modules_directory"
value: "/usr/lib/memgraph/query_modules"
override: true
- name: "auth_module_executable"
value: "/usr/lib/memgraph/auth_module/example.py"
override: false
- name: "memory_limit"
value: "0"

7
init
View File

@@ -140,12 +140,5 @@ done;
python3 -m pip install pre-commit
python3 -m pre_commit install
# Install py format tools
echo "Install black formatter"
python3 -m pip install black==22.10.*
echo "Install isort"
python3 -m pip install isort==5.10.*
# Link `include/mgp.py` with `release/mgp/mgp.py`
ln -v -f include/mgp.py release/mgp/mgp.py

View File

@@ -116,7 +116,7 @@ declare -A primary_urls=(
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
["spdlog"]="http://$local_cache_host/git/spdlog"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
["protobuf"]="http://$local_cache_host/git/protobuf.git"
@@ -141,7 +141,7 @@ declare -A secondary_urls=(
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
["spdlog"]="https://github.com/gabime/spdlog"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
@@ -171,7 +171,7 @@ benchmark_tag="v1.6.0"
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
# google test
googletest_tag="release-1.12.1"
googletest_tag="release-1.8.0"
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
# libbcrypt

View File

@@ -1,12 +0,0 @@
[tool.black]
line-length = 120
include = '\.pyi?$'
extend-exclude = '''
/(
| .git
| .__pycache__
| build
| libs
| .cache
)/
'''

View File

@@ -21,7 +21,6 @@ add_subdirectory(auth)
add_subdirectory(parser)
add_subdirectory(expr)
add_subdirectory(coordinator)
add_subdirectory(functions)
if (MG_ENTERPRISE)
add_subdirectory(audit)
@@ -38,13 +37,13 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
# Memgraph Single Node v2 Executable
# ----------------------------------------------------------------------------
set(mg_single_node_v2_sources
glue/v2/communication.cpp
memgraph.cpp
glue/v2/auth.cpp
glue/communication.cpp
memgraph.cpp
glue/auth.cpp
)
set(mg_single_node_v2_libs stdc++fs Threads::Threads
telemetry_lib mg-query-v2 mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-io mg-coordinator)
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
if (MG_ENTERPRISE)
# These are enterprise subsystems
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
@@ -127,3 +126,19 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
# ----------------------------------------------------------------------------
# Memgraph CSV Import Tool Executable
# ----------------------------------------------------------------------------
add_executable(mg_import_csv mg_import_csv.cpp)
target_link_libraries(mg_import_csv mg-storage-v2)
# Strip the executable in release build.
if (lower_build_type STREQUAL "release")
add_custom_command(TARGET mg_import_csv POST_BUILD
COMMAND strip -s mg_import_csv
COMMENT "Stripping symbols and sections from mg_import_csv")
endif()
install(TARGETS mg_import_csv RUNTIME DESTINATION bin)
# ----------------------------------------------------------------------------
# Memgraph CSV Import Tool Executable
# ----------------------------------------------------------------------------

View File

@@ -1,68 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <cstdint>
#include <string_view>
namespace memgraph::common {
enum class ErrorCode : uint8_t {
SERIALIZATION_ERROR,
NONEXISTENT_OBJECT,
DELETED_OBJECT,
VERTEX_HAS_EDGES,
PROPERTIES_DISABLED,
VERTEX_ALREADY_INSERTED,
// Schema Violations
SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL,
SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
OBJECT_NOT_FOUND,
};
constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
switch (code) {
case ErrorCode::SERIALIZATION_ERROR:
return "SERIALIZATION_ERROR";
case ErrorCode::NONEXISTENT_OBJECT:
return "NONEXISTENT_OBJECT";
case ErrorCode::DELETED_OBJECT:
return "DELETED_OBJECT";
case ErrorCode::VERTEX_HAS_EDGES:
return "VERTEX_HAS_EDGES";
case ErrorCode::PROPERTIES_DISABLED:
return "PROPERTIES_DISABLED";
case ErrorCode::VERTEX_ALREADY_INSERTED:
return "VERTEX_ALREADY_INSERTED";
case ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
return "SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL";
case ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
return "SCHEMA_VERTEX_PROPERTY_WRONG_TYPE";
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
return "SCHEMA_VERTEX_UPDATE_PRIMARY_KEY";
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
return "SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL";
case ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
return "SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY";
case ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
case ErrorCode::OBJECT_NOT_FOUND:
return "OBJECT_NOT_FOUND";
}
}
} // namespace memgraph::common

View File

@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
}
case Signature::Route: {
if constexpr (bolt_minor >= 3) {
return HandleRoute<TSession>(session);
if (signature == Signature::Route) return HandleRoute<TSession>(session);
} else {
spdlog::trace("Supported only in bolt v4.3");
return State::Close;

View File

@@ -30,7 +30,6 @@
#include "communication/context.hpp"
#include "communication/v2/pool.hpp"
#include "communication/v2/session.hpp"
#include "utils/message.hpp"
#include "utils/spin_lock.hpp"
#include "utils/synchronized.hpp"
@@ -59,10 +58,10 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
bool IsRunning() const noexcept { return alive_.load(std::memory_order_relaxed); }
private:
Listener(boost::asio::io_context &io_context, TSessionData &data, ServerContext *server_context,
Listener(boost::asio::io_context &io_context, TSessionData *data, ServerContext *server_context,
tcp::endpoint &endpoint, const std::string_view service_name, const uint64_t inactivity_timeout_sec)
: io_context_(io_context),
data_(&data),
data_(data),
server_context_(server_context),
acceptor_(io_context_),
endpoint_{endpoint},
@@ -111,7 +110,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
return OnError(ec, "accept");
}
auto session = SessionHandler::Create(std::move(socket), *data_, *server_context_, endpoint_, inactivity_timeout_,
auto session = SessionHandler::Create(std::move(socket), data_, *server_context_, endpoint_, inactivity_timeout_,
service_name_);
session->Start();
DoAccept();

View File

@@ -72,7 +72,7 @@ class Server final {
* Constructs and binds server to endpoint, operates on session data and
* invokes workers_count workers
*/
Server(ServerEndpoint &endpoint, TSessionData &session_data, ServerContext *server_context,
Server(ServerEndpoint &endpoint, TSessionData *session_data, ServerContext *server_context,
const int inactivity_timeout_sec, const std::string_view service_name,
size_t workers_count = std::thread::hardware_concurrency())
: endpoint_{endpoint},

View File

@@ -41,7 +41,6 @@
#include <boost/beast/websocket/rfc6455.hpp>
#include <boost/system/detail/error_code.hpp>
#include "communication/buffer.hpp"
#include "communication/context.hpp"
#include "communication/exceptions.hpp"
#include "utils/logging.hpp"
@@ -140,7 +139,7 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
private:
// Take ownership of the socket
explicit WebsocketSession(tcp::socket &&socket, TSessionData &data, tcp::endpoint endpoint,
explicit WebsocketSession(tcp::socket &&socket, TSessionData *data, tcp::endpoint endpoint,
std::string_view service_name)
: ws_(std::move(socket)),
strand_{boost::asio::make_strand(ws_.get_executor())},
@@ -312,13 +311,13 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
}
private:
explicit Session(tcp::socket &&socket, TSessionData &data, ServerContext &server_context, tcp::endpoint endpoint,
explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &server_context, tcp::endpoint endpoint,
const std::chrono::seconds inactivity_timeout_sec, std::string_view service_name)
: socket_(CreateSocket(std::move(socket), server_context)),
strand_{boost::asio::make_strand(GetExecutor())},
output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
data_{&data},
data_{data},
endpoint_{endpoint},
remote_endpoint_{GetRemoteEndpoint()},
service_name_{service_name},
@@ -374,7 +373,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
spdlog::info("Switching {} to websocket connection", remote_endpoint_);
if (std::holds_alternative<TCPSocket>(socket_)) {
auto sock = std::get<TCPSocket>(std::move(socket_));
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), *data_, endpoint_, service_name_)
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), data_, endpoint_, service_name_)
->DoAccept(parser.release());
execution_active_ = false;
return;
@@ -466,7 +465,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
// The deadline has passed. Stop the session. The other actors will
// terminate as soon as possible.
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_.count());
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
DoShutdown();
} else {
// Put the actor back to sleep.

View File

@@ -18,7 +18,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_r
// add this storage engine to any under-replicated shards that it is not already a part of
return shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
auto initializing_rsms_for_shard_manager =
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
return HeartbeatResponse{
.shards_to_initialize = initializing_rsms_for_shard_manager,
};
}
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
@@ -62,8 +67,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&a
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
SplitShardResponse res{};
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
split_shard_request.split_key);
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
res.success = false;
} else {
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
split_shard_request.split_key);
}
return res;
}

View File

@@ -121,6 +121,15 @@ struct InitializeLabelResponse {
std::optional<ShardMap> fresher_shard_map;
};
struct HeartbeatRequest {
Address from_storage_manager;
std::set<boost::uuids::uuid> initialized_rsms;
};
struct HeartbeatResponse {
std::vector<ShardToInitialize> shards_to_initialize;
};
using CoordinatorWriteRequests =
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
@@ -149,10 +158,10 @@ class Coordinator {
private:
ShardMap shard_map_;
uint64_t highest_allocated_timestamp_{0};
uint64_t highest_allocated_timestamp_;
/// Query engines need to periodically request batches of unique edge IDs.
uint64_t highest_allocated_edge_id_{0};
uint64_t highest_allocated_edge_id_;
CoordinatorReadResponses HandleRead(GetShardMapRequest && /* get_shard_map_request */) {
GetShardMapResponse res;

View File

@@ -1,168 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <deque>
#include <memory>
#include <queue>
#include <variant>
#include "coordinator/coordinator.hpp"
#include "coordinator/coordinator_rsm.hpp"
#include "coordinator/shard_map.hpp"
#include "io/address.hpp"
#include "io/future.hpp"
#include "io/messages.hpp"
#include "io/rsm/raft.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
#include "query/v2/requests.hpp"
namespace memgraph::coordinator::coordinator_worker {
/// Obligations:
/// * ShutDown
/// * Cron
/// * RouteMessage
using coordinator::Coordinator;
using coordinator::CoordinatorRsm;
using io::Address;
using io::RequestId;
using io::Time;
using io::messages::CoordinatorMessages;
using msgs::ReadRequests;
using msgs::ReadResponses;
using msgs::WriteRequests;
using msgs::WriteResponses;
struct ShutDown {};
struct Cron {};
struct RouteMessage {
CoordinatorMessages message;
RequestId request_id;
Address to;
Address from;
};
using Message = std::variant<RouteMessage, Cron, ShutDown>;
struct QueueInner {
std::mutex mu{};
std::condition_variable cv;
// TODO(tyler) handle simulator communication std::shared_ptr<std::atomic<int>> blocked;
// TODO(tyler) investigate using a priority queue that prioritizes messages in a way that
// improves overall QoS. For example, maybe we want to schedule raft Append messages
// ahead of Read messages or generally writes before reads for lowering the load on the
// overall system faster etc... When we do this, we need to make sure to avoid
// starvation by sometimes randomizing priorities, rather than following a strict
// prioritization.
std::deque<Message> queue;
uint64_t submitted = 0;
uint64_t calls_to_pop = 0;
};
/// There are two reasons to implement our own Queue instead of using
/// one off-the-shelf:
/// 1. we will need to know in the simulator when all threads are waiting
/// 2. we will want to implement our own priority queue within this for QoS
class Queue {
std::shared_ptr<QueueInner> inner_ = std::make_shared<QueueInner>();
public:
void Push(Message &&message) {
{
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
inner_->submitted++;
inner_->queue.emplace_back(std::move(message));
} // lock dropped before notifying condition variable
inner_->cv.notify_all();
}
Message Pop() {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
inner_->calls_to_pop++;
inner_->cv.notify_all();
while (inner_->queue.empty()) {
inner_->cv.wait(lock);
}
Message message = std::move(inner_->queue.front());
inner_->queue.pop_front();
return message;
}
void BlockOnQuiescence() const {
MG_ASSERT(inner_.use_count() > 0);
std::unique_lock<std::mutex> lock(inner_->mu);
while (inner_->calls_to_pop <= inner_->submitted) {
inner_->cv.wait(lock);
}
}
};
/// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
template <typename IoImpl>
class CoordinatorWorker {
io::Io<IoImpl> io_;
Queue queue_;
CoordinatorRsm<IoImpl> coordinator_;
bool Process(ShutDown && /*shut_down*/) { return false; }
bool Process(Cron && /* cron */) {
coordinator_.Cron();
return true;
}
bool Process(RouteMessage &&route_message) {
coordinator_.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
return true;
}
public:
CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
: io_(std::move(io)), queue_(std::move(queue)), coordinator_{std::move(io_), {}, std::move(coordinator)} {}
CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
CoordinatorWorker(const CoordinatorWorker &) = delete;
CoordinatorWorker &operator=(const CoordinatorWorker &) = delete;
~CoordinatorWorker() = default;
void Run() {
bool should_continue = true;
while (should_continue) {
Message message = queue_.Pop();
should_continue = std::visit([this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); },
std::move(message));
}
}
};
} // namespace memgraph::coordinator::coordinator_worker

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,10 +11,7 @@
#pragma once
#include <chrono>
#include <compare>
#include <ctime>
#include <iomanip>
#include "io/time.hpp"
@@ -23,26 +20,17 @@ namespace memgraph::coordinator {
using Time = memgraph::io::Time;
/// Hybrid-logical clock
struct Hlc final {
uint64_t logical_id{0};
Time coordinator_wall_clock = Time::min();
struct Hlc {
uint64_t logical_id;
Time coordinator_wall_clock;
auto operator<=>(const Hlc &other) const noexcept { return logical_id <=> other.logical_id; }
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
bool operator==(const Hlc &other) const noexcept = default;
bool operator<(const Hlc &other) const noexcept = default;
bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);
in << "Hlc { logical_id: " << hlc.logical_id;
in << ", coordinator_wall_clock: " << std::put_time(std::localtime(&wall_clock), "%F %T");
in << " }";
return in;
}
bool operator==(const Hlc &other) const = default;
bool operator<(const Hlc &other) const = default;
bool operator==(const uint64_t other) const { return logical_id == other; }
bool operator<(const uint64_t other) const { return logical_id < other; }
bool operator>=(const uint64_t other) const { return logical_id >= other; }
};
} // namespace memgraph::coordinator

View File

@@ -9,19 +9,8 @@
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include <map>
#include <optional>
#include <unordered_map>
#include <vector>
#include "common/types.hpp"
#include "coordinator/shard_map.hpp"
#include "spdlog/spdlog.h"
#include "storage/v3/schemas.hpp"
#include "storage/v3/temporal.hpp"
#include "utils/cast.hpp"
#include "utils/exceptions.hpp"
#include "utils/string.hpp"
namespace memgraph::coordinator {
@@ -68,352 +57,6 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
return ret;
}
ShardMap ShardMap::Parse(std::istream &input_stream) {
ShardMap shard_map;
const auto read_size = [&input_stream] {
size_t size{0};
input_stream >> size;
return size;
};
// Reads a string until the next whitespace
const auto read_word = [&input_stream] {
std::string word;
input_stream >> word;
return word;
};
const auto read_names = [&read_size, &read_word] {
const auto number_of_names = read_size();
spdlog::trace("Reading {} names", number_of_names);
std::vector<std::string> names;
names.reserve(number_of_names);
for (auto name_index = 0; name_index < number_of_names; ++name_index) {
names.push_back(read_word());
spdlog::trace("Read '{}'", names.back());
}
return names;
};
const auto read_line = [&input_stream] {
std::string line;
std::getline(input_stream, line);
return line;
};
const auto parse_type = [](const std::string &type) {
static const auto type_map = std::unordered_map<std::string, common::SchemaType>{
{"string", common::SchemaType::STRING}, {"int", common::SchemaType::INT}, {"bool", common::SchemaType::BOOL}};
const auto lower_case_type = utils::ToLowerCase(type);
auto it = type_map.find(lower_case_type);
MG_ASSERT(it != type_map.end(), "Invalid type in split files: {}", type);
return it->second;
};
const auto parse_property_value = [](std::string text, const common::SchemaType type) {
if (type == common::SchemaType::STRING) {
return storage::v3::PropertyValue{std::move(text)};
}
if (type == common::SchemaType::INT) {
size_t processed{0};
int64_t value = std::stoll(text, &processed);
MG_ASSERT(processed == text.size() || text[processed] == ' ', "Invalid integer format: '{}'", text);
return storage::v3::PropertyValue{value};
}
LOG_FATAL("Not supported type: {}", utils::UnderlyingCast(type));
};
spdlog::debug("Reading properties");
const auto properties = read_names();
MG_ASSERT(shard_map.AllocatePropertyIds(properties).size() == properties.size(),
"Unexpected number of properties created!");
spdlog::debug("Reading edge types");
const auto edge_types = read_names();
MG_ASSERT(shard_map.AllocateEdgeTypeIds(edge_types).size() == edge_types.size(),
"Unexpected number of properties created!");
spdlog::debug("Reading primary labels");
const auto number_of_primary_labels = read_size();
spdlog::debug("Reading {} primary labels", number_of_primary_labels);
for (auto label_index = 0; label_index < number_of_primary_labels; ++label_index) {
const auto primary_label = read_word();
spdlog::debug("Reading primary label named '{}'", primary_label);
const auto number_of_primary_properties = read_size();
spdlog::debug("Reading {} primary properties", number_of_primary_properties);
std::vector<std::string> pp_names;
std::vector<common::SchemaType> pp_types;
pp_names.reserve(number_of_primary_properties);
pp_types.reserve(number_of_primary_properties);
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
pp_names.push_back(read_word());
spdlog::debug("Reading primary property named '{}'", pp_names.back());
pp_types.push_back(parse_type(read_word()));
}
auto pp_mapping = shard_map.AllocatePropertyIds(pp_names);
std::vector<SchemaProperty> schema;
schema.reserve(number_of_primary_properties);
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
schema.push_back(storage::v3::SchemaProperty{pp_mapping.at(pp_names[property_index]), pp_types[property_index]});
}
const auto hlc = shard_map.GetHlc();
MG_ASSERT(shard_map.InitializeNewLabel(primary_label, schema, 1, hlc).has_value(),
"Cannot initialize new label: {}", primary_label);
const auto number_of_split_points = read_size();
spdlog::debug("Reading {} split points", number_of_split_points);
[[maybe_unused]] const auto remainder_from_last_line = read_line();
for (auto split_point_index = 0; split_point_index < number_of_split_points; ++split_point_index) {
const auto line = read_line();
spdlog::debug("Read split point '{}'", line);
MG_ASSERT(line.front() == '[', "Invalid split file format!");
MG_ASSERT(line.back() == ']', "Invalid split file format!");
std::string_view line_view{line};
line_view.remove_prefix(1);
line_view.remove_suffix(1);
static constexpr std::string_view kDelimiter{","};
auto pk_values_as_text = utils::Split(line_view, kDelimiter);
std::vector<PropertyValue> pk;
pk.reserve(number_of_primary_properties);
MG_ASSERT(pk_values_as_text.size() == number_of_primary_properties,
"Split point contains invalid number of values '{}'", line);
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
pk.push_back(parse_property_value(std::move(pk_values_as_text[property_index]), schema[property_index].type));
}
shard_map.SplitShard(shard_map.GetHlc(), shard_map.labels.at(primary_label), pk);
}
}
return shard_map;
}
std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map) {
using utils::print_helpers::operator<<;
in << "ShardMap { shard_map_version: " << shard_map.shard_map_version;
in << ", max_property_id: " << shard_map.max_property_id;
in << ", max_edge_type_id: " << shard_map.max_edge_type_id;
in << ", properties: " << shard_map.properties;
in << ", edge_types: " << shard_map.edge_types;
in << ", max_label_id: " << shard_map.max_label_id;
in << ", labels: " << shard_map.labels;
in << ", label_spaces: " << shard_map.label_spaces;
in << ", schemas: " << shard_map.schemas;
in << "}";
return in;
}
Shards ShardMap::GetShardsForLabel(const LabelName &label) const {
const auto id = labels.at(label);
const auto &shards = label_spaces.at(id).shards;
return shards;
}
std::vector<Shards> ShardMap::GetAllShards() const {
std::vector<Shards> all_shards;
all_shards.reserve(label_spaces.size());
std::transform(label_spaces.begin(), label_spaces.end(), std::back_inserter(all_shards),
[](const auto &label_space) { return label_space.second.shards; });
return all_shards;
}
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well
Hlc ShardMap::IncrementShardMapVersion() noexcept {
++shard_map_version.logical_id;
return shard_map_version;
}
// TODO(antaljanosbenjamin) use a single map for all name id
// mapping and a single counter to maintain the next id
std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
std::unordered_map<uint64_t, std::string> id_to_names;
const auto map_type_ids = [&id_to_names](const auto &name_to_id_type) {
for (const auto &[name, id] : name_to_id_type) {
id_to_names.emplace(id.AsUint(), name);
}
};
map_type_ids(edge_types);
map_type_ids(labels);
map_type_ids(properties);
return id_to_names;
}
Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
return boost::uuids::uuid{0,
0,
0,
0,
0,
0,
0,
0,
static_cast<unsigned char>(shard_id >> 56U),
static_cast<unsigned char>(shard_id >> 48U),
static_cast<unsigned char>(shard_id >> 40U),
static_cast<unsigned char>(shard_id >> 32U),
static_cast<unsigned char>(shard_id >> 24U),
static_cast<unsigned char>(shard_id >> 16U),
static_cast<unsigned char>(shard_id >> 8U),
static_cast<unsigned char>(shard_id)};
}
HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
HeartbeatResponse ret{};
bool mutated = false;
for (auto &[label_id, label_space] : label_spaces) {
for (auto it = label_space.shards.begin(); it != label_space.shards.end(); it++) {
auto &[low_key, shard] = *it;
std::optional<PrimaryKey> high_key;
if (const auto next_it = std::next(it); next_it != label_space.shards.end()) {
high_key = next_it->first;
}
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
bool shard_assigned_to_machine = false;
for (auto &peer_metadata : shard.peers) {
const bool same_machine = peer_metadata.address.last_known_ip == storage_manager.last_known_ip &&
peer_metadata.address.last_known_port == storage_manager.last_known_port;
if (initialized.contains(peer_metadata.address.unique_id)) {
shard_assigned_to_machine = true;
if (!same_machine) {
// set the last known ip and port to the storage manager that has heartbeated it just now
mutated = true;
peer_metadata.address.last_known_ip = storage_manager.last_known_ip;
peer_metadata.address.last_known_port = storage_manager.last_known_port;
}
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
mutated = true;
spdlog::info("marking shard as full consensus participant: {}", peer_metadata.address.unique_id);
peer_metadata.status = Status::CONSENSUS_PARTICIPANT;
}
} else if (same_machine && peer_metadata.status == Status::INITIALIZING) {
// we are expecting this shard to be initialized (rather than split) on this machine,
// so send it an initialization request
shard_assigned_to_machine = true;
spdlog::info("reminding shard manager that they should begin participating in shard");
ret.shards_to_initialize.push_back(ShardToInitialize{
.uuid = peer_metadata.address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = high_key,
.schema = schemas[label_id],
.config = Config{},
.id_to_names = IdToNames(),
});
} else if (same_machine && peer_metadata.status == Status::PENDING_SPLIT) {
// we are expecting this shard to be split, so send it a split request
ret.shards_to_split.push_back(ShardToSplit{
.shard_to_split_uuid = peer_metadata.split_from,
.new_right_side_uuid = peer_metadata.address.unique_id,
.split_requested_at = shard.version,
.label_id = label_id,
.split_key = low_key,
.schema = schemas[label_id],
.config = Config{},
.id_to_names = IdToNames(),
});
} else {
MG_ASSERT(
!same_machine,
"failed to properly handle a new Status type in the heartbeat management and shard assignment code");
}
}
if (!shard_assigned_to_machine && shard.peers.size() < label_space.replication_factor) {
Address address = storage_manager;
// NB: increment version for each new uuid for deterministic creation
IncrementShardMapVersion();
address.unique_id = NewShardUuid(shard_map_version.logical_id);
spdlog::info("assigning shard manager to shard");
ret.shards_to_initialize.push_back(ShardToInitialize{
.uuid = address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = high_key,
.schema = schemas[label_id],
.config = Config{},
.id_to_names = IdToNames(),
});
PeerMetadata peer_metadata = {
.address = address,
.status = Status::INITIALIZING,
};
shard.peers.emplace_back(peer_metadata);
}
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
if (previous_shard_map_version != shard_map_version || !label_spaces.contains(label_id)) {
return false;
}
auto &label_space = label_spaces.at(label_id);
auto &shards_in_map = label_space.shards;
MG_ASSERT(!shards_in_map.empty());
MG_ASSERT(!shards_in_map.contains(key));
// Finding the ShardMetadata that the new PrimaryKey should map to.
ShardMetadata duplicated_shard = GetShardForKey(label_id, key);
std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
for (auto &peer_metadata : duplicated_shard.peers) {
peer_metadata.status = Status::PENDING_SPLIT;
peer_metadata.split_from = peer_metadata.address.unique_id;
// NB: increment version for each new uuid for deterministic creation
IncrementShardMapVersion();
auto new_uuid = NewShardUuid(shard_map_version.logical_id);
// store new uuid for the right side of each shard
split_mapping.emplace(peer_metadata.address.unique_id, new_uuid);
peer_metadata.address.unique_id = new_uuid;
}
// Apply the split
shards_in_map[key] = duplicated_shard;
IncrementShardMapVersion();
return true;
}
std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
size_t replication_factor, Hlc last_shard_map_version) {
if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
@@ -425,18 +68,17 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
labels.emplace(std::move(label_name), label_id);
PrimaryKey initial_key = SchemaToMinKey(schema);
ShardMetadata empty_shard = {};
Shard empty_shard = {};
Shards shards = {
{initial_key, empty_shard},
};
LabelSpace label_space{
.schema = schema,
.schema = std::move(schema),
.shards = shards,
.replication_factor = replication_factor,
};
schemas[label_id] = std::move(schema);
label_spaces.emplace(label_id, label_space);
@@ -445,174 +87,4 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
return label_id;
}
void ShardMap::AddServer(Address server_address) {
// Find a random place for the server to plug in
}
std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
if (const auto it = labels.find(label); it != labels.end()) {
return it->second;
}
return std::nullopt;
}
const std::string &ShardMap::GetLabelName(const LabelId label) const {
if (const auto it =
std::ranges::find_if(labels, [label](const auto &name_id_pair) { return name_id_pair.second == label; });
it != labels.end()) {
return it->first;
}
throw utils::BasicException("GetLabelName fails on the given label id!");
}
std::optional<PropertyId> ShardMap::GetPropertyId(const std::string &property_name) const {
if (const auto it = properties.find(property_name); it != properties.end()) {
return it->second;
}
return std::nullopt;
}
const std::string &ShardMap::GetPropertyName(const PropertyId property) const {
if (const auto it = std::ranges::find_if(
properties, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
it != properties.end()) {
return it->first;
}
throw utils::BasicException("PropertyId not found!");
}
std::optional<EdgeTypeId> ShardMap::GetEdgeTypeId(const std::string &edge_type) const {
if (const auto it = edge_types.find(edge_type); it != edge_types.end()) {
return it->second;
}
return std::nullopt;
}
const std::string &ShardMap::GetEdgeTypeName(const EdgeTypeId property) const {
if (const auto it = std::ranges::find_if(
edge_types, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
it != edge_types.end()) {
return it->first;
}
throw utils::BasicException("EdgeTypeId not found!");
}
Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key,
const PrimaryKey &end_key) const {
MG_ASSERT(start_key <= end_key);
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
const auto &shards_for_label = label_space.shards;
MG_ASSERT(shards_for_label.begin()->first <= start_key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
auto it = std::prev(shards_for_label.upper_bound(start_key));
const auto end_it = shards_for_label.upper_bound(end_key);
Shards shards{};
std::copy(it, end_it, std::inserter(shards, shards.end()));
return shards;
}
ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
MG_ASSERT(label_spaces.contains(label_id));
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
PropertyMap ShardMap::AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
PropertyMap ret{};
bool mutated = false;
for (const auto &property_name : new_properties) {
if (properties.contains(property_name)) {
auto property_id = properties.at(property_name);
ret.emplace(property_name, property_id);
} else {
mutated = true;
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
ret.emplace(property_name, property_id);
properties.emplace(property_name, property_id);
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types) {
EdgeTypeIdMap ret;
bool mutated = false;
for (const auto &edge_type_name : new_edge_types) {
if (edge_types.contains(edge_type_name)) {
auto edge_type_id = edge_types.at(edge_type_name);
ret.emplace(edge_type_name, edge_type_id);
} else {
mutated = true;
const EdgeTypeId edge_type_id = EdgeTypeId::FromUint(++max_edge_type_id);
ret.emplace(edge_type_name, edge_type_id);
edge_types.emplace(edge_type_name, edge_type_id);
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
bool ShardMap::ClusterInitialized() const {
for (const auto &[label_id, label_space] : label_spaces) {
for (const auto &[low_key, shard] : label_space.shards) {
if (shard.peers.size() < label_space.replication_factor) {
spdlog::info("label_space below desired replication factor");
return false;
}
for (const auto &peer_metadata : shard.peers) {
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
return false;
}
}
}
}
return true;
}
} // namespace memgraph::coordinator

View File

@@ -11,7 +11,6 @@
#pragma once
#include <algorithm>
#include <limits>
#include <map>
#include <set>
@@ -25,20 +24,14 @@
#include "io/address.hpp"
#include "storage/v3/config.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/name_id_mapper.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
#include "storage/v3/temporal.hpp"
#include "utils/exceptions.hpp"
#include "utils/print_helpers.hpp"
namespace memgraph::coordinator {
constexpr int64_t kNotExistingId{0};
using memgraph::io::Address;
using memgraph::storage::v3::Config;
using memgraph::storage::v3::EdgeTypeId;
using memgraph::storage::v3::LabelId;
using memgraph::storage::v3::PropertyId;
using memgraph::storage::v3::PropertyValue;
@@ -47,186 +40,237 @@ using memgraph::storage::v3::SchemaProperty;
enum class Status : uint8_t {
CONSENSUS_PARTICIPANT,
INITIALIZING,
PENDING_SPLIT,
// TODO(tyler) this will possibly have more states,
// depending on the reconfiguration protocol that we
// implement.
};
struct PeerMetadata {
struct AddressAndStatus {
memgraph::io::Address address;
Status status;
boost::uuids::uuid split_from;
friend bool operator<(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address < rhs.address; }
friend std::ostream &operator<<(std::ostream &in, const PeerMetadata &peer_metadata) {
in << "PeerMetadata { address: ";
in << peer_metadata.address;
if (peer_metadata.status == Status::CONSENSUS_PARTICIPANT) {
in << ", status: CONSENSUS_PARTICIPANT";
} else if (peer_metadata.status == Status::INITIALIZING) {
in << ", status: INITIALIZING";
} else if (peer_metadata.status == Status::PENDING_SPLIT) {
in << ", status: PENDING_SPLIT";
} else {
MG_ASSERT(false, "failed to update the operator<< implementation for Status");
}
in << ", split_from: " << peer_metadata.split_from << " }";
return in;
}
friend bool operator==(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address == rhs.address; }
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
};
using PrimaryKey = std::vector<PropertyValue>;
struct ShardMetadata {
std::vector<PeerMetadata> peers;
uint64_t version;
friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
using utils::print_helpers::operator<<;
in << "ShardMetadata { peers: ";
in << shard.peers;
in << " version: ";
in << shard.version;
in << " }";
return in;
}
friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
if (lhs.peers != rhs.peers) {
return lhs.peers < rhs.peers;
}
return lhs.version < rhs.version;
}
};
using Shards = std::map<PrimaryKey, ShardMetadata>;
using Shard = std::vector<AddressAndStatus>;
using Shards = std::map<PrimaryKey, Shard>;
using LabelName = std::string;
using PropertyName = std::string;
using EdgeTypeName = std::string;
using PropertyMap = std::map<PropertyName, PropertyId>;
using EdgeTypeIdMap = std::map<EdgeTypeName, EdgeTypeId>;
struct ShardToInitialize {
boost::uuids::uuid uuid;
LabelId label_id;
PrimaryKey min_key;
std::optional<PrimaryKey> max_key;
std::vector<SchemaProperty> schema;
Config config;
std::unordered_map<uint64_t, std::string> id_to_names;
};
struct ShardToSplit {
boost::uuids::uuid shard_to_split_uuid;
boost::uuids::uuid new_right_side_uuid;
Hlc split_requested_at;
LabelId label_id;
PrimaryKey split_key;
std::vector<SchemaProperty> schema;
Config config;
std::unordered_map<uint64_t, std::string> id_to_names;
};
struct HeartbeatRequest {
Address from_storage_manager;
std::set<boost::uuids::uuid> initialized_rsms;
};
struct HeartbeatResponse {
std::vector<ShardToInitialize> shards_to_initialize;
std::vector<ShardToSplit> shards_to_split;
};
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
struct LabelSpace {
std::vector<SchemaProperty> schema;
// Maps between the smallest primary key stored in the shard and the shard
std::map<PrimaryKey, ShardMetadata> shards;
std::map<PrimaryKey, Shard> shards;
size_t replication_factor;
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
using utils::print_helpers::operator<<;
in << "LabelSpace { schema: ";
in << label_space.schema;
in << ", shards: ";
in << label_space.shards;
in << ", replication_factor: " << label_space.replication_factor << "}";
return in;
}
};
struct ShardMap {
Hlc shard_map_version;
uint64_t max_property_id{kNotExistingId};
uint64_t max_edge_type_id{kNotExistingId};
uint64_t max_property_id;
std::map<PropertyName, PropertyId> properties;
std::map<EdgeTypeName, EdgeTypeId> edge_types;
uint64_t max_label_id{kNotExistingId};
uint64_t max_label_id;
std::map<LabelName, LabelId> labels;
std::map<LabelId, LabelSpace> label_spaces;
std::map<LabelId, std::vector<SchemaProperty>> schemas;
std::map<LabelName, LabelId> labels;
[[nodiscard]] static ShardMap Parse(std::istream &input_stream);
friend std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map);
Shards GetShardsForLabel(const LabelName &label) const;
std::vector<Shards> GetAllShards() const;
Shards GetShards(const LabelName &label) {
const auto id = labels.at(label);
auto &shards = label_spaces.at(id).shards;
return shards;
}
// TODO(gabor) later we will want to update the wallclock time with
// the given Io<impl>'s time as well
Hlc IncrementShardMapVersion() noexcept;
Hlc GetHlc() const noexcept;
Hlc IncrementShardMapVersion() noexcept {
++shard_map_version.logical_id;
return shard_map_version;
}
std::unordered_map<uint64_t, std::string> IdToNames();
Hlc GetHlc() const noexcept { return shard_map_version; }
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
HeartbeatResponse AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
std::vector<ShardToInitialize> ret{};
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
bool mutated = false;
for (auto &[label_id, label_space] : label_spaces) {
for (auto &[low_key, shard] : label_space.shards) {
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
bool machine_contains_shard = false;
for (auto &aas : shard) {
if (initialized.contains(aas.address.unique_id)) {
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
aas.status = Status::CONSENSUS_PARTICIPANT;
machine_contains_shard = true;
} else {
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
aas.address.last_known_port == storage_manager.last_known_port;
if (same_machine) {
machine_contains_shard = true;
ret.push_back(ShardToInitialize{
.uuid = aas.address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = std::nullopt,
.config = Config{},
});
}
}
}
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
Address address = storage_manager;
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
ret.push_back(ShardToInitialize{
.uuid = address.unique_id,
.label_id = label_id,
.min_key = low_key,
.max_key = std::nullopt,
.config = Config{},
});
AddressAndStatus aas = {
.address = address,
.status = Status::INITIALIZING,
};
shard.emplace_back(aas);
}
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
if (previous_shard_map_version != shard_map_version) {
return false;
}
auto &label_space = label_spaces.at(label_id);
auto &shards_in_map = label_space.shards;
MG_ASSERT(!shards_in_map.empty());
MG_ASSERT(!shards_in_map.contains(key));
MG_ASSERT(label_spaces.contains(label_id));
// Finding the Shard that the new PrimaryKey should map to.
auto prev = std::prev(shards_in_map.upper_bound(key));
Shard duplicated_shard = prev->second;
// Apply the split
shards_in_map[key] = duplicated_shard;
return true;
}
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
size_t replication_factor, Hlc last_shard_map_version);
void AddServer(Address server_address);
void AddServer(Address server_address) {
// Find a random place for the server to plug in
}
std::optional<LabelId> GetLabelId(const std::string &label) const;
// TODO(antaljanosbenjamin): Remove this and instead use NameIdMapper
const std::string &GetLabelName(LabelId label) const;
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const;
const std::string &GetPropertyName(PropertyId property) const;
std::optional<EdgeTypeId> GetEdgeTypeId(const std::string &edge_type) const;
const std::string &GetEdgeTypeName(EdgeTypeId property) const;
LabelId GetLabelId(const std::string &label) const { return labels.at(label); }
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const {
MG_ASSERT(start_key <= end_key);
MG_ASSERT(labels.contains(label_name));
ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
LabelId label_id = labels.at(label_name);
ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
const auto &label_space = label_spaces.at(label_id);
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
const auto &shards_for_label = label_space.shards;
EdgeTypeIdMap AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types);
MG_ASSERT(shards_for_label.begin()->first <= start_key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
/// Returns true if all shards have the desired number of replicas and they are in
/// the CONSENSUS_PARTICIPANT state. Note that this does not necessarily mean that
/// there is also an active leader for each shard.
bool ClusterInitialized() const;
auto it = std::prev(shards_for_label.upper_bound(start_key));
const auto end_it = shards_for_label.upper_bound(end_key);
Shards shards{};
std::copy(it, end_it, std::inserter(shards, shards.end()));
return shards;
}
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
MG_ASSERT(labels.contains(label_name));
LabelId label_id = labels.at(label_name);
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
MG_ASSERT(label_spaces.contains(label_id));
const auto &label_space = label_spaces.at(label_id);
MG_ASSERT(label_space.shards.begin()->first <= key,
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
return std::prev(label_space.shards.upper_bound(key))->second;
}
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
PropertyMap ret{};
bool mutated = false;
for (const auto &property_name : new_properties) {
if (properties.contains(property_name)) {
auto property_id = properties.at(property_name);
ret.emplace(property_name, property_id);
} else {
mutated = true;
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
ret.emplace(property_name, property_id);
properties.emplace(property_name, property_id);
}
}
if (mutated) {
IncrementShardMapVersion();
}
return ret;
}
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const {
if (properties.contains(property_name)) {
return properties.at(property_name);
}
return std::nullopt;
}
};
} // namespace memgraph::coordinator

View File

@@ -17,4 +17,4 @@ target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser mg-functions)
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)

View File

@@ -13,7 +13,7 @@
#ifndef MG_AST_INCLUDE_PATH
#ifdef MG_CLANG_TIDY_CHECK
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#include "query/v2/bindings/bindings.hpp"
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp"
#else
#error Missing AST include path
#endif
@@ -21,6 +21,8 @@
#ifndef MG_INJECTED_NAMESPACE_NAME
#ifdef MG_CLANG_TIDY_CHECK
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2
#else
#error Missing AST namespace
#endif

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -51,7 +51,7 @@ constexpr char kId[] = "ID";
namespace MG_INJECTED_NAMESPACE_NAME {
namespace detail {
using antlropencypher::v2::MemgraphCypher;
using antlropencypher::MemgraphCypher;
template <typename TVisitor>
std::optional<std::pair<Expression *, size_t>> VisitMemoryLimit(MemgraphCypher::MemoryLimitContext *memory_limit_ctx,
@@ -211,13 +211,13 @@ inline std::string_view ToString(const PulsarConfigKey key) {
}
} // namespace detail
using antlropencypher::v2::MemgraphCypher;
using antlropencypher::MemgraphCypher;
struct ParsingContext {
bool is_query_cached = false;
};
class CypherMainVisitor : public antlropencypher::v2::MemgraphCypherBaseVisitor {
class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
public:
explicit CypherMainVisitor(ParsingContext context, AstStorage *storage) : context_(context), storage_(storage) {}

View File

@@ -1,279 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iostream>
#include <type_traits>
#include "expr/ast.hpp"
#include "expr/typed_value.hpp"
#include "utils/algorithm.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
namespace memgraph::expr {
inline constexpr const char *identifier_node_symbol = "MG_SYMBOL_NODE";
inline constexpr const char *identifier_edge_symbol = "MG_SYMBOL_EDGE";
namespace detail {
template <typename T>
void PrintObject(std::ostream *out, const T &arg) {
static_assert(!std::is_convertible<T, Expression *>::value,
"This overload shouldn't be called with pointers convertible "
"to Expression *. This means your other PrintObject overloads aren't "
"being called for certain AST nodes when they should (or perhaps such "
"overloads don't exist yet).");
*out << arg;
}
inline void PrintObject(std::ostream *out, const std::string &str) { *out << str; }
inline void PrintObject(std::ostream * /*out*/, Aggregation::Op /*op*/) {
throw utils::NotYetImplemented("PrintObject: Aggregation::Op");
}
inline void PrintObject(std::ostream * /*out*/, Expression * /*expr*/);
inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
template <typename T>
void PrintObject(std::ostream * /*out*/, const std::vector<T> & /*vec*/) {
throw utils::NotYetImplemented("PrintObject: vector<T>");
}
template <typename T>
void PrintObject(std::ostream * /*out*/, const std::vector<T, utils::Allocator<T>> & /*vec*/) {
throw utils::NotYetImplemented("PrintObject: vector<T, utils::Allocator<T>>");
}
template <typename K, typename V>
void PrintObject(std::ostream * /*out*/, const std::map<K, V> & /*map*/) {
throw utils::NotYetImplemented("PrintObject: map<K, V>");
}
template <typename T>
void PrintObject(std::ostream * /*out*/, const utils::pmr::map<utils::pmr::string, T> & /*map*/) {
throw utils::NotYetImplemented("PrintObject: map<utils::pmr::string, V>");
}
template <typename T1, typename T2, typename T3>
inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
using TypedValue = TypedValueT<T1, T2, T3>;
switch (value.type()) {
case TypedValue::Type::Null:
*out << "null";
break;
case TypedValue::Type::String:
PrintObject(out, value.ValueString());
break;
case TypedValue::Type::Bool:
*out << (value.ValueBool() ? "true" : "false");
break;
case TypedValue::Type::Int:
PrintObject(out, value.ValueInt());
break;
case TypedValue::Type::Double:
PrintObject(out, value.ValueDouble());
break;
case TypedValue::Type::List:
PrintObject(out, value.ValueList());
break;
case TypedValue::Type::Map:
PrintObject(out, value.ValueMap());
break;
case TypedValue::Type::Date:
PrintObject(out, value.ValueDate());
break;
case TypedValue::Type::Duration:
PrintObject(out, value.ValueDuration());
break;
case TypedValue::Type::LocalTime:
PrintObject(out, value.ValueLocalTime());
break;
case TypedValue::Type::LocalDateTime:
PrintObject(out, value.ValueLocalDateTime());
break;
default:
MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
}
}
template <typename T>
void PrintOperatorArgs(const std::string & /*name*/, std::ostream *out, bool with_parenthesis, const T &arg) {
PrintObject(out, arg);
if (with_parenthesis) {
*out << ")";
}
}
template <typename T, typename... Ts>
void PrintOperatorArgs(const std::string &name, std::ostream *out, bool with_parenthesis, const T &arg,
const Ts &...args) {
PrintObject(out, arg);
*out << " " << name << " ";
PrintOperatorArgs(name, out, with_parenthesis, args...);
}
template <typename... Ts>
void PrintOperator(const std::string &name, std::ostream *out, bool with_parenthesis, const Ts &...args) {
if (with_parenthesis) {
*out << "(";
}
PrintOperatorArgs(name, out, with_parenthesis, args...);
}
// new
template <typename T>
void PrintOperatorArgs(std::ostream *out, const T &arg) {
PrintObject(out, arg);
}
template <typename T, typename... Ts>
void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
PrintObject(out, arg);
PrintOperatorArgs(out, args...);
}
template <typename... Ts>
void PrintOperator(std::ostream *out, const Ts &...args) {
PrintOperatorArgs(out, args...);
}
} // namespace detail
class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
public:
explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
// Unary operators
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE &op) override { detail::PrintOperator(OP_STR, out_, false /*with_parenthesis*/, op.expression_); }
UNARY_OPERATOR_VISIT(NotOperator, "Not");
UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
#undef UNARY_OPERATOR_VISIT
// Binary operators
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE &op) override { \
detail::PrintOperator(OP_STR, out_, true /*with_parenthesis*/, op.expression1_, op.expression2_); \
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define BINARY_OPERATOR_VISIT_NOT_IMPL(OP_NODE, OP_STR) \
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
void Visit(OP_NODE & /*op*/) override { throw utils::NotYetImplemented("OP_NODE"); }
BINARY_OPERATOR_VISIT(OrOperator, "Or");
BINARY_OPERATOR_VISIT(XorOperator, "Xor");
BINARY_OPERATOR_VISIT(AndOperator, "And");
BINARY_OPERATOR_VISIT(AdditionOperator, "+");
BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
BINARY_OPERATOR_VISIT(DivisionOperator, "/");
BINARY_OPERATOR_VISIT(ModOperator, "%");
BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
BINARY_OPERATOR_VISIT(EqualOperator, "=");
BINARY_OPERATOR_VISIT(LessOperator, "<");
BINARY_OPERATOR_VISIT(GreaterOperator, ">");
BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
BINARY_OPERATOR_VISIT_NOT_IMPL(InListOperator, "In");
BINARY_OPERATOR_VISIT_NOT_IMPL(SubscriptOperator, "Subscript");
#undef BINARY_OPERATOR_VISIT
#undef BINARY_OPERATOR_VISIT_NOT_IMPL
// Other
void Visit(ListSlicingOperator & /*op*/) override { throw utils::NotYetImplemented("ListSlicingOperator"); }
void Visit(IfOperator & /*op*/) override { throw utils::NotYetImplemented("IfOperator"); }
void Visit(ListLiteral & /*op*/) override { throw utils::NotYetImplemented("ListLiteral"); }
void Visit(MapLiteral & /*op*/) override { throw utils::NotYetImplemented("MapLiteral"); }
void Visit(LabelsTest & /*op*/) override { throw utils::NotYetImplemented("LabelsTest"); }
void Visit(Aggregation & /*op*/) override { throw utils::NotYetImplemented("Aggregation"); }
void Visit(Function & /*op*/) override { throw utils::NotYetImplemented("Function"); }
void Visit(Reduce & /*op*/) override { throw utils::NotYetImplemented("Reduce"); }
void Visit(Coalesce & /*op*/) override { throw utils::NotYetImplemented("Coalesce"); }
void Visit(Extract & /*op*/) override { throw utils::NotYetImplemented("Extract"); }
void Visit(All & /*op*/) override { throw utils::NotYetImplemented("All"); }
void Visit(Single & /*op*/) override { throw utils::NotYetImplemented("Single"); }
void Visit(Any & /*op*/) override { throw utils::NotYetImplemented("Any"); }
void Visit(None & /*op*/) override { throw utils::NotYetImplemented("None"); }
void Visit(Identifier &op) override {
auto is_node = true;
auto is_edge = false;
auto is_other = false;
if (is_node) {
detail::PrintOperator(out_, identifier_node_symbol);
} else if (is_edge) {
detail::PrintOperator(out_, identifier_edge_symbol);
} else {
MG_ASSERT(is_other);
detail::PrintOperator(out_, op.name_);
}
}
void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
void Visit(PropertyLookup &op) override { detail::PrintOperator(out_, op.expression_, ".", op.property_.name); }
void Visit(ParameterLookup & /*op*/) override { throw utils::NotYetImplemented("ParameterLookup"); }
void Visit(NamedExpression & /*op*/) override { throw utils::NotYetImplemented("NamedExpression"); }
void Visit(RegexMatch & /*op*/) override { throw utils::NotYetImplemented("RegexMatch"); }
private:
std::ostream *out_;
};
namespace detail {
inline void PrintObject(std::ostream *out, Expression *expr) {
if (expr) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
} else {
*out << "<null>";
}
}
} // namespace detail
inline void PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(Expression *expr, std::ostream *out) {
ExpressionPrettyPrinter printer{out};
expr->Accept(printer);
}
inline std::string ExpressiontoStringWhileReplacingNodeAndEdgeSymbols(Expression *expr) {
std::ostringstream ss;
expr::PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(expr, &ss);
return ss.str();
}
} // namespace memgraph::expr

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -24,7 +24,6 @@
#include "expr/exceptions.hpp"
#include "expr/interpret/frame.hpp"
#include "expr/semantic/symbol_table.hpp"
#include "functions/awesome_memgraph_functions.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::expr {
@@ -36,8 +35,8 @@ template <typename TypedValue, typename EvaluationContext, typename DbAccessor,
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
public:
ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
StorageView view)
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
DbAccessor *dba, StorageView view)
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
using ExpressionVisitor<TypedValue>::Visit;
@@ -47,12 +46,12 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
TypedValue Visit(NamedExpression &named_expression) override {
const auto &symbol = symbol_table_->at(named_expression);
auto value = named_expression.expression_->Accept(*this);
frame_->At(symbol) = value;
frame_->at(symbol) = value;
return value;
}
TypedValue Visit(Identifier &ident) override {
return TypedValue(frame_->At(symbol_table_->at(ident)), ctx_->memory);
return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
}
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
@@ -101,28 +100,6 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
#undef BINARY_OPERATOR_VISITOR
#undef UNARY_OPERATOR_VISITOR
void HandleObjectAccessError(Error &shard_error, const std::string_view accessed_object) {
switch (shard_error) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to access {} on a deleted object.", accessed_object);
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to access {} from a node object doesn't exist.", accessed_object);
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
case Error::VERTEX_ALREADY_INSERTED:
case Error::OBJECT_NOT_FOUND:
throw ExpressionRuntimeException("Unexpected error when accessing {}.", accessed_object);
case Error::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
case Error::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
}
}
TypedValue Visit(AndOperator &op) override {
auto value1 = op.expression1_->Accept(*this);
if (value1.IsBool() && !value1.ValueBool()) {
@@ -419,7 +396,16 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
}
if (has_label.HasError()) {
HandleObjectAccessError(has_label.GetError().code, "labels");
switch (has_label.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
}
}
return *has_label;
}
@@ -428,7 +414,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
return vertex.HasLabel(label);
auto has_label = vertex.HasLabel(label);
return !has_label;
}
TypedValue Visit(LabelsTest &labels_test) override {
@@ -470,7 +457,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(Aggregation &aggregation) override {
return TypedValue(frame_->At(symbol_table_->at(aggregation)), ctx_->memory);
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
}
TypedValue Visit(Coalesce &coalesce) override {
@@ -491,7 +478,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
}
TypedValue Visit(Function &function) override {
functions::FunctionContext<DbAccessor> function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
// Stack allocate evaluated arguments when there's a small number of them.
if (function.arguments_.size() <= 8) {
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
@@ -528,8 +515,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
auto accumulator = reduce.initializer_->Accept(*this);
for (const auto &element : list) {
frame_->At(accumulator_symbol) = accumulator;
frame_->At(element_symbol) = element;
frame_->at(accumulator_symbol) = accumulator;
frame_->at(element_symbol) = element;
accumulator = reduce.expression_->Accept(*this);
}
return accumulator;
@@ -551,7 +538,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
if (element.IsNull()) {
result.emplace_back();
} else {
frame_->At(element_symbol) = element;
frame_->at(element_symbol) = element;
result.emplace_back(extract.expression_->Accept(*this));
}
}
@@ -571,7 +558,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
bool has_null_elements = false;
bool has_value = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = all.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
@@ -608,7 +595,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
bool has_value = false;
bool predicate_satisfied = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = single.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
@@ -645,7 +632,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
const auto &symbol = symbol_table_->at(*any.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = any.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
@@ -677,7 +664,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
const auto &symbol = symbol_table_->at(*none.identifier_);
bool has_value = false;
for (const auto &element : list) {
frame_->At(symbol) = element;
frame_->at(symbol) = element;
auto result = none.where_->expression_->Accept(*this);
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
@@ -731,7 +718,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
TReturnType GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
auto maybe_prop = record_accessor.GetProperty(prop.name);
// Handler non existent property
return conv_(maybe_prop, dba_);
return conv_(maybe_prop);
}
template <class TRecordAccessor, class TTag = Tag,
@@ -739,7 +726,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
TReturnType GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
auto maybe_prop = record_accessor.GetProperty(std::string(name));
// Handler non existent property
return conv_(maybe_prop, dba_);
return conv_(maybe_prop);
}
template <class TRecordAccessor, class TTag = Tag,
@@ -756,7 +743,16 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
}
if (maybe_prop.HasError()) {
HandleObjectAccessError(maybe_prop.GetError().code, "property");
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
@@ -775,14 +771,23 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
}
if (maybe_prop.HasError()) {
HandleObjectAccessError(maybe_prop.GetError().code, "property");
switch (maybe_prop.GetError()) {
case Error::DELETED_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
case Error::NONEXISTENT_OBJECT:
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
case Error::SERIALIZATION_ERROR:
case Error::VERTEX_HAS_EDGES:
case Error::PROPERTIES_DISABLED:
throw ExpressionRuntimeException("Unexpected error when getting a property.");
}
}
return conv_(*maybe_prop, ctx_->memory);
}
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
Frame *frame_;
Frame<TypedValue> *frame_;
const SymbolTable *symbol_table_;
const EvaluationContext *ctx_;
DbAccessor *dba_;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -20,43 +20,26 @@
namespace memgraph::expr {
template <typename TypedValue>
class Frame {
public:
/// Create a Frame of given size backed by a utils::NewDeleteResource()
explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
TypedValue &At(const Symbol &symbol) { return elems_.at(symbol.position()); }
const TypedValue &At(const Symbol &symbol) const { return elems_.at(symbol.position()); }
TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
uint64_t Id() const { return id_; }
void SetId(const uint64_t id) { id_ = id; }
const utils::pmr::vector<TypedValue> &Elems() const { return elems_; }
auto &elems() { return elems_; }
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
private:
uint64_t id_{0U};
utils::pmr::vector<TypedValue> elems_;
};
class FrameWithValidity final : public Frame {
public:
explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {}
FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
bool IsValid() const noexcept { return is_valid_; }
void MakeValid() noexcept { is_valid_ = true; }
void MakeInvalid() noexcept { is_valid_ = false; }
private:
bool is_valid_;
};
} // namespace memgraph::expr

View File

@@ -21,7 +21,6 @@
#include <utility>
#include <vector>
#include "expr/typed_value_exception.hpp"
#include "utils/algorithm.hpp"
#include "utils/exceptions.hpp"
#include "utils/fnv.hpp"
@@ -33,6 +32,16 @@
namespace memgraph::expr {
/**
* An exception raised by the TypedValue system. Typically when
* trying to perform operations (such as addition) on TypedValues
* of incompatible Types.
*/
class TypedValueException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
};
// TODO: Neo4j does overflow checking. Should we also implement it?
/**
* Stores a query runtime value and its type.

View File

@@ -1 +0,0 @@
add_library(mg-functions INTERFACE)

File diff suppressed because it is too large Load Diff

View File

@@ -15,17 +15,9 @@
#include <string>
#include <vector>
#include "common/errors.hpp"
#include "coordinator/shard_map.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/edge_accessor.hpp"
#include "storage/v3/id_types.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/vertex_accessor.hpp"
#include "storage/v3/view.hpp"
#include "utils/exceptions.hpp"
#include "utils/temporal.hpp"
using memgraph::communication::bolt::Value;
@@ -71,139 +63,127 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
}
}
communication::bolt::Vertex ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex,
const query::v2::RequestRouterInterface *request_router,
storage::v3::View /*view*/) {
auto id = communication::bolt::Id::FromUint(0);
auto labels = vertex.Labels();
std::vector<std::string> new_labels;
new_labels.reserve(labels.size());
for (const auto &label : labels) {
new_labels.push_back(request_router->LabelToName(label.id));
}
auto properties = vertex.Properties();
std::map<std::string, Value> new_properties;
for (const auto &[prop, property_value] : properties) {
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
}
return communication::bolt::Vertex{id, new_labels, new_properties};
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltVertex(vertex.impl_, db, view);
}
communication::bolt::Edge ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge,
const query::v2::RequestRouterInterface *request_router,
storage::v3::View /*view*/) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto from = communication::bolt::Id::FromUint(0);
auto to = communication::bolt::Id::FromUint(0);
const auto &type = request_router->EdgeTypeToName(edge.EdgeType());
auto properties = edge.Properties();
std::map<std::string, Value> new_properties;
for (const auto &[prop, property_value] : properties) {
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
}
return communication::bolt::Edge{id, from, to, type, new_properties};
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view) {
return ToBoltEdge(edge.impl_, db, view);
}
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path & /*edge*/,
const query::v2::RequestRouterInterface * /*request_router*/,
storage::v3::View /*view*/) {
// TODO(jbajic) Fix bolt communication
MG_ASSERT(false, "Path is unimplemented!");
return {};
}
Value ToBoltValue(const query::v2::TypedValue &value, const query::v2::RequestRouterInterface *request_router,
storage::v3::View view) {
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db,
storage::v3::View view) {
switch (value.type()) {
case query::v2::TypedValue::Type::Null:
return {};
return Value();
case query::v2::TypedValue::Type::Bool:
return {value.ValueBool()};
return Value(value.ValueBool());
case query::v2::TypedValue::Type::Int:
return {value.ValueInt()};
return Value(value.ValueInt());
case query::v2::TypedValue::Type::Double:
return {value.ValueDouble()};
return Value(value.ValueDouble());
case query::v2::TypedValue::Type::String:
return {std::string(value.ValueString())};
return Value(std::string(value.ValueString()));
case query::v2::TypedValue::Type::List: {
std::vector<Value> values;
values.reserve(value.ValueList().size());
for (const auto &v : value.ValueList()) {
auto value = ToBoltValue(v, request_router, view);
values.emplace_back(std::move(value));
auto maybe_value = ToBoltValue(v, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
values.emplace_back(std::move(*maybe_value));
}
return {std::move(values)};
return Value(std::move(values));
}
case query::v2::TypedValue::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.ValueMap()) {
auto value = ToBoltValue(kv.second, request_router, view);
map.emplace(kv.first, std::move(value));
auto maybe_value = ToBoltValue(kv.second, db, view);
if (maybe_value.HasError()) return maybe_value.GetError();
map.emplace(kv.first, std::move(*maybe_value));
}
return {std::move(map)};
return Value(std::move(map));
}
case query::v2::TypedValue::Type::Vertex: {
auto vertex = ToBoltVertex(value.ValueVertex(), request_router, view);
return {std::move(vertex)};
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
return Value(std::move(*maybe_vertex));
}
case query::v2::TypedValue::Type::Edge: {
auto edge = ToBoltEdge(value.ValueEdge(), request_router, view);
return {std::move(edge)};
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
return Value(std::move(*maybe_edge));
}
case query::v2::TypedValue::Type::Path: {
auto path = ToBoltPath(value.ValuePath(), request_router, view);
return {std::move(path)};
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
if (maybe_path.HasError()) return maybe_path.GetError();
return Value(std::move(*maybe_path));
}
case query::v2::TypedValue::Type::Date:
return {value.ValueDate()};
return Value(value.ValueDate());
case query::v2::TypedValue::Type::LocalTime:
return {value.ValueLocalTime()};
return Value(value.ValueLocalTime());
case query::v2::TypedValue::Type::LocalDateTime:
return {value.ValueLocalDateTime()};
return Value(value.ValueLocalDateTime());
case query::v2::TypedValue::Type::Duration:
return {value.ValueDuration()};
return Value(value.ValueDuration());
}
}
Value ToBoltValue(msgs::Value value) {
switch (value.type) {
case msgs::Value::Type::Null:
return {};
case msgs::Value::Type::Bool:
return {value.bool_v};
case msgs::Value::Type::Int64:
return {value.int_v};
case msgs::Value::Type::Double:
return {value.double_v};
case msgs::Value::Type::String:
return {std::string(value.string_v)};
case msgs::Value::Type::List: {
std::vector<Value> values;
values.reserve(value.list_v.size());
for (const auto &v : value.list_v) {
auto maybe_value = ToBoltValue(v);
values.emplace_back(std::move(maybe_value));
}
return Value{std::move(values)};
}
case msgs::Value::Type::Map: {
std::map<std::string, Value> map;
for (const auto &kv : value.map_v) {
auto maybe_value = ToBoltValue(kv.second);
map.emplace(kv.first, std::move(maybe_value));
}
return Value{std::move(map)};
}
case msgs::Value::Type::Vertex:
case msgs::Value::Type::Edge: {
throw utils::BasicException("Vertex and Edge not supported!");
}
// TODO Value to Date types not supported
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto maybe_labels = vertex.Labels(view);
if (maybe_labels.HasError()) return maybe_labels.GetError();
std::vector<std::string> labels;
labels.reserve(maybe_labels->size());
for (const auto &label : *maybe_labels) {
labels.push_back(db.LabelToName(label));
}
auto maybe_properties = vertex.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Vertex{id, labels, properties};
}
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view) {
// TODO(jbajic) Fix bolt communication
auto id = communication::bolt::Id::FromUint(0);
auto from = communication::bolt::Id::FromUint(0);
auto to = communication::bolt::Id::FromUint(0);
const auto &type = db.EdgeTypeToName(edge.EdgeType());
auto maybe_properties = edge.Properties(view);
if (maybe_properties.HasError()) return maybe_properties.GetError();
std::map<std::string, Value> properties;
for (const auto &prop : *maybe_properties) {
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
}
return communication::bolt::Edge{id, from, to, type, properties};
}
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view) {
std::vector<communication::bolt::Vertex> vertices;
vertices.reserve(path.vertices().size());
for (const auto &v : path.vertices()) {
auto maybe_vertex = ToBoltVertex(v, db, view);
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
vertices.emplace_back(std::move(*maybe_vertex));
}
std::vector<communication::bolt::Edge> edges;
edges.reserve(path.edges().size());
for (const auto &e : path.edges()) {
auto maybe_edge = ToBoltEdge(e, db, view);
if (maybe_edge.HasError()) return maybe_edge.GetError();
edges.emplace_back(std::move(*maybe_edge));
}
return communication::bolt::Path(vertices, edges);
}
storage::v3::PropertyValue ToPropertyValue(const Value &value) {

View File

@@ -13,14 +13,10 @@
#pragma once
#include "communication/bolt/v1/value.hpp"
#include "coordinator/shard_map.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/request_router.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/view.hpp"
#include "utils/result.hpp"
namespace memgraph::storage::v3 {
class EdgeAccessor;
@@ -32,37 +28,36 @@ namespace memgraph::glue::v2 {
/// @param storage::v3::VertexAccessor for converting to
/// communication::bolt::Vertex.
/// @param query::v2::RequestRouterInterface *request_router getting label and property names.
/// @param storage::v3::Shard for getting label and property names.
/// @param storage::v3::View for deciding which vertex attributes are visible.
///
/// @throw std::bad_alloc
communication::bolt::Vertex ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const query::v2::RequestRouterInterface *request_router,
storage::v3::View view);
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
const storage::v3::Shard &db, storage::v3::View view);
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
/// @param query::v2::RequestRouterInterface *request_router getting edge type and property names.
/// @param storage::v3::Shard for getting edge type and property names.
/// @param storage::v3::View for deciding which edge attributes are visible.
///
/// @throw std::bad_alloc
communication::bolt::Edge ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
const storage::v3::Shard &db, storage::v3::View view);
/// @param query::v2::Path for converting to communication::bolt::Path.
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path &path,
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
storage::v3::View view);
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
///
/// @throw std::bad_alloc
communication::bolt::Value ToBoltValue(const query::v2::TypedValue &value,
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
const storage::v3::Shard &db, storage::v3::View view);
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
@@ -70,9 +65,4 @@ communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
communication::bolt::Value ToBoltValue(msgs::Value value);
communication::bolt::Value ToBoltValue(msgs::Value value, const query::v2::RequestRouterInterface *request_router,
storage::v3::View view);
} // namespace memgraph::glue::v2

View File

@@ -15,38 +15,11 @@
#include <fmt/format.h>
#include <boost/asio/ip/tcp.hpp>
#include <boost/functional/hash.hpp>
#include <boost/uuid/uuid.hpp>
#include <boost/uuid/uuid_generators.hpp>
#include <boost/uuid/uuid_io.hpp>
#include "utils/logging.hpp"
namespace memgraph::io {
struct PartialAddress {
boost::asio::ip::address ip;
uint16_t port;
friend bool operator==(const PartialAddress &lhs, const PartialAddress &rhs) = default;
/// unique_id is most dominant for ordering, then ip, then port
friend bool operator<(const PartialAddress &lhs, const PartialAddress &rhs) {
if (lhs.ip != rhs.ip) {
return lhs.ip < rhs.ip;
}
return lhs.port < rhs.port;
}
std::string ToString() const { return fmt::format("PartialAddress {{ ip: {}, port: {} }}", ip.to_string(), port); }
friend std::ostream &operator<<(std::ostream &in, const PartialAddress &partial_address) {
in << partial_address.ToString();
return in;
}
};
struct Address {
// It's important for all participants to have a
// unique identifier - IP and port alone are not
@@ -60,39 +33,18 @@ struct Address {
uint16_t last_known_port;
static Address TestAddress(uint16_t port) {
MG_ASSERT(port <= 255);
return Address{
.unique_id = boost::uuids::uuid{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static_cast<unsigned char>(port)},
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
.last_known_port = port,
};
}
// NB: don't use this in test code because it is non-deterministic
static Address UniqueLocalAddress() {
return Address{
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
};
}
/// `Coordinator`s have constant UUIDs because there is at most one per ip/port pair.
Address ForkLocalCoordinator() {
return Address{
.unique_id = boost::uuids::uuid{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
.last_known_ip = last_known_ip,
.last_known_port = last_known_port,
};
}
/// `ShardManager`s have constant UUIDs because there is at most one per ip/port pair.
Address ForkLocalShardManager() {
return Address{
.unique_id = boost::uuids::uuid{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
.last_known_ip = last_known_ip,
.last_known_port = last_known_port,
};
}
/// Returns a new ID with the same IP and port but a unique UUID.
Address ForkUniqueAddress() {
return Address{
@@ -102,13 +54,6 @@ struct Address {
};
}
PartialAddress ToPartialAddress() const {
return PartialAddress{
.ip = last_known_ip,
.port = last_known_port,
};
}
friend bool operator==(const Address &lhs, const Address &rhs) = default;
/// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
@@ -128,36 +73,5 @@ struct Address {
return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
}
friend std::ostream &operator<<(std::ostream &in, const Address &address) {
in << address.ToString();
return in;
}
};
}; // namespace memgraph::io
namespace std {
template <>
struct hash<memgraph::io::PartialAddress> {
size_t operator()(const memgraph::io::PartialAddress &pa) const {
using boost::hash_combine;
using boost::hash_value;
// Start with a hash value of 0 .
std::size_t seed = 0;
if (pa.ip.is_v4()) {
auto h = std::hash<boost::asio::ip::address_v4>()(pa.ip.to_v4());
hash_combine(seed, h);
} else {
auto h = std::hash<boost::asio::ip::address_v6>()(pa.ip.to_v6());
hash_combine(seed, h);
}
hash_combine(seed, hash_value(pa.port));
// Return the result.
return seed;
}
};
} // namespace std

View File

@@ -11,16 +11,11 @@
#pragma once
#include "utils/exceptions.hpp"
namespace memgraph::io {
/// Protocol:
/// crc32: 4 bytes
/// len: 8 bytes
/// buffer: <len bytes>
namespace memgraph::expr {
/**
* An exception raised by the TypedValue system. Typically when
* trying to perform operations (such as addition) on TypedValues
* of incompatible Types.
*/
class TypedValueException : public utils::BasicException {
public:
using utils::BasicException::BasicException;
};
} // namespace memgraph::expr
std::optional<std::pair<size_t, size_t>> make_frame(char *ptr, size_t len) { return std::nullopt; }
} // namespace memgraph::io

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -35,13 +35,10 @@ class Shared {
std::optional<T> item_;
bool consumed_ = false;
bool waiting_ = false;
bool filled_ = false;
std::function<bool()> wait_notifier_ = nullptr;
std::function<void()> fill_notifier_ = nullptr;
std::function<bool()> simulator_notifier_ = nullptr;
public:
explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
: wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
Shared() = default;
Shared(Shared &&) = delete;
Shared &operator=(Shared &&) = delete;
@@ -67,7 +64,8 @@ class Shared {
waiting_ = true;
while (!item_) {
if (wait_notifier_) [[unlikely]] {
bool simulator_progressed = false;
if (simulator_notifier_) [[unlikely]] {
// We can't hold our own lock while notifying
// the simulator because notifying the simulator
// involves acquiring the simulator's mutex
@@ -79,7 +77,7 @@ class Shared {
// so we have to get out of its way to avoid
// a cyclical deadlock.
lock.unlock();
std::invoke(wait_notifier_);
simulator_progressed = std::invoke(simulator_notifier_);
lock.lock();
if (item_) {
// item may have been filled while we
@@ -87,7 +85,8 @@ class Shared {
// the simulator of our waiting_ status.
break;
}
} else {
}
if (!simulator_progressed) [[likely]] {
cv_.wait(lock);
}
MG_ASSERT(!consumed_, "Future consumed twice!");
@@ -118,19 +117,11 @@ class Shared {
std::unique_lock<std::mutex> lock(mu_);
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
MG_ASSERT(!filled_, "Promise filled twice!");
MG_ASSERT(!item_, "Promise filled twice!");
item_ = std::move(item);
filled_ = true;
item_ = item;
} // lock released before condition variable notification
if (fill_notifier_) {
spdlog::trace("calling fill notifier");
std::invoke(fill_notifier_);
} else {
spdlog::trace("not calling fill notifier");
}
cv_.notify_all();
}
@@ -157,14 +148,6 @@ class Future {
old.consumed_or_moved_ = true;
}
Future &operator=(Future &&old) noexcept {
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
shared_ = std::move(old.shared_);
consumed_or_moved_ = old.consumed_or_moved_;
old.consumed_or_moved_ = true;
return *this;
}
Future(const Future &) = delete;
Future &operator=(const Future &) = delete;
~Future() = default;
@@ -209,7 +192,7 @@ class Future {
template <typename T>
class Promise {
std::shared_ptr<details::Shared<T>> shared_;
bool filled_or_moved_{false};
bool filled_or_moved_ = false;
public:
explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
@@ -229,13 +212,12 @@ class Promise {
Promise(const Promise &) = delete;
Promise &operator=(const Promise &) = delete;
// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Branch)
~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
// Fill the expected item into the Future.
void Fill(T item) {
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
shared_->Fill(std::move(item));
shared_->Fill(item);
filled_or_moved_ = true;
}
@@ -262,9 +244,8 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
}
template <typename T>
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
std::function<void()> fill_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
Future<T> future = Future<T>(shared);
Promise<T> promise = Promise<T>(shared);

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -30,11 +30,15 @@ class LocalTransport {
explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
: local_transport_handle_(std::move(local_transport_handle)) {}
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
std::function<void()> fill_notifier, Duration timeout) {
return local_transport_handle_->template SubmitRequest<ResponseT, RequestT>(
to_address, from_address, std::move(request), timeout, fill_notifier);
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
Duration timeout) {
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
local_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
@@ -43,8 +47,8 @@ class LocalTransport {
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
}
Time Now() const { return local_transport_handle_->Now(); }
@@ -56,7 +60,5 @@ class LocalTransport {
std::random_device rng;
return distrib(rng);
}
LatencyHistogramSummaries ResponseLatencies() { return local_transport_handle_->ResponseLatencies(); }
};
}; // namespace memgraph::io::local_transport

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,7 +19,6 @@
#include "io/errors.hpp"
#include "io/message_conversion.hpp"
#include "io/message_histogram_collector.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
@@ -29,8 +28,6 @@ class LocalTransportHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
bool should_shut_down_ = false;
MessageHistogramCollector histograms_;
RequestId request_id_counter_ = 0;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
@@ -57,11 +54,6 @@ class LocalTransportHandle {
return should_shut_down_;
}
LatencyHistogramSummaries ResponseLatencies() {
std::unique_lock<std::mutex> lock(mu_);
return histograms_.ResponseLatencies();
}
static Time Now() {
auto nano_time = std::chrono::system_clock::now();
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
@@ -104,17 +96,15 @@ class LocalTransportHandle {
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
auto type_info = TypeInfoFor(message);
std::any message_any(std::move(message));
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
std::any message_any(std::forward<M>(message));
OpaqueMessage opaque_message{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any),
.type_info = type_info};
.message = std::move(message_any)};
PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
PromiseKey promise_key{
.requester_address = to_address, .request_id = opaque_message.request_id, .replier_address = from_address};
{
std::unique_lock<std::mutex> lock(mu_);
@@ -125,10 +115,7 @@ class LocalTransportHandle {
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
Duration response_latency = Now() - dop.requested_at;
dop.promise.Fill(std::move(opaque_message), response_latency);
histograms_.Measure(type_info, response_latency);
dop.promise.Fill(std::move(opaque_message));
} else {
spdlog::info("placing message in can_receive_");
can_receive_.emplace_back(std::move(opaque_message));
@@ -138,39 +125,26 @@ class LocalTransportHandle {
cv_.notify_all();
}
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
Duration timeout, std::function<void()> fill_notifier) {
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
// set null notifier for when the Future::Wait is called
nullptr,
// set notifier for when Promise::Fill is called
std::move(fill_notifier));
template <Message RequestT, Message ResponseT>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
Duration timeout, ResponsePromise<ResponseT> promise) {
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
MG_ASSERT(port_matches && ip_matches);
const Time deadline = Now() + timeout;
const auto now = Now();
const Time deadline = now + timeout;
RequestId request_id = 0;
{
std::unique_lock<std::mutex> lock(mu_);
request_id = ++request_id_counter_;
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
PromiseKey promise_key{
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.requested_at = now, .deadline = deadline, .promise = std::move(opaque_promise)};
MG_ASSERT(!promises_.contains(promise_key));
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
} // lock dropped
Send<RequestT>(to_address, from_address, request_id, std::move(request));
return std::move(future);
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
}
};

View File

@@ -11,17 +11,20 @@
#pragma once
#include <boost/core/demangle.hpp>
#include "io/time.hpp"
#include "io/transport.hpp"
#include "utils/type_info_ref.hpp"
namespace memgraph::io {
using memgraph::io::Duration;
using memgraph::io::Message;
using memgraph::io::Time;
struct PromiseKey {
Address requester_address;
uint64_t request_id;
// TODO(tyler) possibly remove replier_address from promise key
// once we want to support DSR.
Address replier_address;
public:
friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
@@ -29,7 +32,11 @@ struct PromiseKey {
return lhs.requester_address < rhs.requester_address;
}
return lhs.request_id < rhs.request_id;
if (lhs.request_id != rhs.request_id) {
return lhs.request_id < rhs.request_id;
}
return lhs.replier_address < rhs.replier_address;
}
};
@@ -38,8 +45,6 @@ struct OpaqueMessage {
Address from_address;
uint64_t request_id;
std::any message;
utils::TypeInfoRef type_info;
Time deliverable_at;
/// Recursively tries to match a specific type from the outer
/// variant's parameter pack against the type of the std::any,
@@ -87,10 +92,6 @@ struct OpaqueMessage {
};
}
std::string demangled_name = "\"" + boost::core::demangle(message.type().name()) + "\"";
spdlog::error("failed to cast message of type {} to expected request type (probably in Receive argument types)",
demangled_name);
return std::nullopt;
}
};
@@ -99,7 +100,7 @@ class OpaquePromiseTraitBase {
public:
virtual const std::type_info *TypeInfo() const = 0;
virtual bool IsAwaited(void *ptr) const = 0;
virtual void Fill(void *ptr, OpaqueMessage &&, Duration) const = 0;
virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
virtual void TimeOut(void *ptr) const = 0;
virtual ~OpaquePromiseTraitBase() = default;
@@ -117,13 +118,12 @@ class OpaquePromiseTrait : public OpaquePromiseTraitBase {
bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
void Fill(void *ptr, OpaqueMessage &&opaque_message, Duration response_latency) const override {
void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
T message = std::any_cast<T>(std::move(opaque_message.message));
auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
.request_id = opaque_message.request_id,
.to_address = opaque_message.to_address,
.from_address = opaque_message.from_address,
.response_latency = response_latency};
.from_address = opaque_message.from_address};
auto promise = static_cast<ResponsePromise<T> *>(ptr);
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
unique_promise->Fill(std::move(response_envelope));
@@ -187,9 +187,9 @@ class OpaquePromise {
ptr_ = nullptr;
}
void Fill(OpaqueMessage &&opaque_message, Duration response_latency) {
void Fill(OpaqueMessage &&opaque_message) {
MG_ASSERT(ptr_ != nullptr);
trait_->Fill(ptr_, std::move(opaque_message), response_latency);
trait_->Fill(ptr_, std::move(opaque_message));
ptr_ = nullptr;
}
@@ -199,24 +199,18 @@ class OpaquePromise {
};
struct DeadlineAndOpaquePromise {
Time requested_at;
Time deadline;
OpaquePromise promise;
};
template <class From>
std::type_info const &TypeInfoForVariant(From const &from) {
std::type_info const &type_info_for_variant(From const &from) {
return std::visit([](auto &&x) -> decltype(auto) { return typeid(x); }, from);
}
template <class T>
utils::TypeInfoRef TypeInfoFor(const T & /* t */) {
return typeid(T);
}
template <typename From, typename Return, typename Head, typename... Rest>
std::optional<Return> ConvertVariantInner(From &&a) {
if (typeid(Head) == TypeInfoForVariant(a)) {
if (typeid(Head) == type_info_for_variant(a)) {
Head concrete = std::get<Head>(std::forward<From>(a));
return concrete;
}

View File

@@ -1,132 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <cmath>
#include <compare>
#include <unordered_map>
#include <boost/core/demangle.hpp>
#include "io/time.hpp"
#include "utils/histogram.hpp"
#include "utils/logging.hpp"
#include "utils/print_helpers.hpp"
#include "utils/type_info_ref.hpp"
namespace memgraph::io {
struct LatencyHistogramSummary {
uint64_t count;
Duration p0;
Duration p50;
Duration p75;
Duration p90;
Duration p95;
Duration p975;
Duration p99;
Duration p999;
Duration p9999;
Duration p100;
Duration sum;
friend bool operator==(const LatencyHistogramSummary &lhs, const LatencyHistogramSummary &rhs) = default;
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummary &histo) {
in << "{ \"count\": " << histo.count;
in << ", \"p0\": " << histo.p0.count();
in << ", \"p50\": " << histo.p50.count();
in << ", \"p75\": " << histo.p75.count();
in << ", \"p90\": " << histo.p90.count();
in << ", \"p95\": " << histo.p95.count();
in << ", \"p975\": " << histo.p975.count();
in << ", \"p99\": " << histo.p99.count();
in << ", \"p999\": " << histo.p999.count();
in << ", \"p9999\": " << histo.p9999.count();
in << ", \"p100\": " << histo.p100.count();
in << ", \"sum\": " << histo.sum.count();
in << " }";
return in;
}
};
struct LatencyHistogramSummaries {
std::unordered_map<std::string, LatencyHistogramSummary> latencies;
std::string SummaryTable() {
std::string output;
const auto row = [&output](const auto &c1, const auto &c2, const auto &c3, const auto &c4, const auto &c5,
const auto &c6, const auto &c7) {
output +=
fmt::format("{: >50} | {: >8} | {: >8} | {: >8} | {: >8} | {: >8} | {: >8}\n", c1, c2, c3, c4, c5, c6, c7);
};
row("name", "count", "min (μs)", "med (μs)", "p99 (μs)", "max (μs)", "sum (ms)");
for (const auto &[name, histo] : latencies) {
row(name, histo.count, histo.p0.count(), histo.p50.count(), histo.p99.count(), histo.p100.count(),
histo.sum.count() / 1000);
}
output += "\n";
return output;
}
friend bool operator==(const LatencyHistogramSummaries &lhs, const LatencyHistogramSummaries &rhs) = default;
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummaries &histo) {
using memgraph::utils::print_helpers::operator<<;
in << histo.latencies;
return in;
}
};
class MessageHistogramCollector {
std::unordered_map<utils::TypeInfoRef, utils::Histogram, utils::TypeInfoHasher, utils::TypeInfoEqualTo> histograms_;
public:
void Measure(const std::type_info &type_info, const Duration &duration) {
auto &histo = histograms_[type_info];
histo.Measure(duration.count());
}
LatencyHistogramSummaries ResponseLatencies() {
std::unordered_map<std::string, LatencyHistogramSummary> ret{};
for (const auto &[type_id, histo] : histograms_) {
std::string demangled_name = "\"" + boost::core::demangle(type_id.get().name()) + "\"";
LatencyHistogramSummary latency_histogram_summary{
.count = histo.Count(),
.p0 = Duration(static_cast<int64_t>(histo.Percentile(0.0))),
.p50 = Duration(static_cast<int64_t>(histo.Percentile(50.0))),
.p75 = Duration(static_cast<int64_t>(histo.Percentile(75.0))),
.p90 = Duration(static_cast<int64_t>(histo.Percentile(90.0))),
.p95 = Duration(static_cast<int64_t>(histo.Percentile(95.0))),
.p975 = Duration(static_cast<int64_t>(histo.Percentile(97.5))),
.p99 = Duration(static_cast<int64_t>(histo.Percentile(99.0))),
.p999 = Duration(static_cast<int64_t>(histo.Percentile(99.9))),
.p9999 = Duration(static_cast<int64_t>(histo.Percentile(99.99))),
.p100 = Duration(static_cast<int64_t>(histo.Percentile(100.0))),
.sum = Duration(histo.Sum()),
};
ret.emplace(demangled_name, latency_histogram_summary);
}
return LatencyHistogramSummaries{.latencies = ret};
}
};
} // namespace memgraph::io

View File

@@ -1,93 +0,0 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <condition_variable>
#include <functional>
#include <mutex>
#include <optional>
#include <vector>
namespace memgraph::io {
class ReadinessToken {
size_t id_;
public:
explicit ReadinessToken(size_t id) : id_(id) {}
size_t GetId() const { return id_; }
};
class Inner {
std::condition_variable cv_;
std::mutex mu_;
std::vector<ReadinessToken> ready_;
std::optional<std::function<bool()>> tick_simulator_;
public:
void Notify(ReadinessToken readiness_token) {
{
std::unique_lock<std::mutex> lock(mu_);
ready_.emplace_back(readiness_token);
} // mutex dropped
cv_.notify_all();
}
ReadinessToken Await() {
std::unique_lock<std::mutex> lock(mu_);
while (ready_.empty()) {
if (tick_simulator_) [[unlikely]] {
// This avoids a deadlock in a similar way that
// Future::Wait will release its mutex while
// interacting with the simulator, due to
// the fact that the simulator may cause
// notifications that we are interested in.
lock.unlock();
std::invoke(tick_simulator_.value());
lock.lock();
} else {
cv_.wait(lock);
}
}
ReadinessToken ret = ready_.back();
ready_.pop_back();
return ret;
}
void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
std::unique_lock<std::mutex> lock(mu_);
tick_simulator_ = tick_simulator;
}
};
class Notifier {
std::shared_ptr<Inner> inner_;
public:
Notifier() : inner_(std::make_shared<Inner>()) {}
Notifier(const Notifier &) = default;
Notifier &operator=(const Notifier &) = default;
Notifier(Notifier &&old) = default;
Notifier &operator=(Notifier &&old) = default;
~Notifier() = default;
void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
ReadinessToken Await() const { return inner_->Await(); }
void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
};
} // namespace memgraph::io

View File

@@ -0,0 +1,61 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include "io/address.hpp"
#include "io/message_conversion.hpp"
#include "io/protobuf_transport/protobuf_transport_handle.hpp"
#include "io/transport.hpp"
namespace memgraph::io::protobuf_transport {
class ProtobufTransport {
std::shared_ptr<ProtobufTransportHandle> protobuf_transport_handle_;
uint16_t listen_port_;
public:
explicit ProtobufTransport(uint16_t listen_port)
: protobuf_transport_handle_(std::make_unique<ProtobufTransportHandle>()), listen_port_(listen_port) {}
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
Duration timeout) {
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
protobuf_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
return protobuf_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
return protobuf_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
}
Time Now() const { return protobuf_transport_handle_->Now(); }
bool ShouldShutDown() const { return protobuf_transport_handle_->ShouldShutDown(); }
template <class D = std::poisson_distribution<>, class Return = uint64_t>
Return Rand(D distrib) {
std::random_device rng;
return distrib(rng);
}
};
}; // namespace memgraph::io::protobuf_transport

View File

@@ -0,0 +1,161 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <map>
#include <mutex>
#include "google/protobuf/message.h"
#include "io/address.hpp"
#include "io/transport.hpp"
//#include "protobuf/messages.pb.cc"
#include "protobuf/messages.pb.h"
namespace memgraph::io::protobuf_transport {
using PbAddress = memgraph::protobuf::Address;
using memgraph::protobuf::UberMessage;
class ProtobufTransportHandle {
mutable std::mutex mu_{};
mutable std::condition_variable cv_;
bool should_shut_down_ = false;
// the responses to requests that are being waited on
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::vector<std::string> can_receive_;
// serialized outbound messages
std::vector<std::string> outbox_;
public:
~ProtobufTransportHandle() {
for (auto &&[pk, promise] : promises_) {
std::move(promise.promise).TimeOut();
}
promises_.clear();
}
void ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
cv_.notify_all();
}
bool ShouldShutDown() const {
std::unique_lock<std::mutex> lock(mu_);
return should_shut_down_;
}
static Time Now() {
auto nano_time = std::chrono::system_clock::now();
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
std::unique_lock lock(mu_);
Time before = Now();
spdlog::info("can_receive_ size: {}", can_receive_.size());
while (can_receive_.empty()) {
Time now = Now();
// protection against non-monotonic timesources
auto maxed_now = std::max(now, before);
auto elapsed = maxed_now - before;
if (timeout < elapsed) {
return TimedOut{};
}
Duration relative_timeout = timeout - elapsed;
std::cv_status cv_status_value = cv_.wait_for(lock, relative_timeout);
if (cv_status_value == std::cv_status::timeout) {
return TimedOut{};
}
}
auto current_message = std::move(can_receive_.back());
can_receive_.pop_back();
auto m_opt = std::move(current_message).Take<Ms...>();
return std::move(m_opt).value();
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
PromiseKey promise_key{.requester_address = to_address, .request_id = request_id, .replier_address = from_address};
{
std::unique_lock<std::mutex> lock(mu_);
if (promises_.contains(promise_key)) {
// hair-pin local message optimization
spdlog::info("using message to fill local promise");
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
promises_.erase(promise_key);
std::any message_any(std::forward<M>(message));
OpaqueMessage opaque_message{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any)};
dop.promise.Fill(std::move(opaque_message));
} else {
spdlog::info("placing message in outbox");
// serialize protobuf message and place it in the outbox
std::string bytes;
bool success = message.SerializeToString(&bytes);
MG_ASSERT(success);
outbox_.emplace_back(std::move(bytes));
}
} // lock dropped
cv_.notify_all();
}
template <Message RequestT, Message ResponseT>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
Duration timeout, ResponsePromise<ResponseT> promise) {
const Time deadline = Now() + timeout;
{
std::unique_lock<std::mutex> lock(mu_);
PromiseKey promise_key{
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
} // lock dropped
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
}
};
} // namespace memgraph::io::protobuf_transport

View File

@@ -11,19 +11,15 @@
#pragma once
#include <functional>
#include <typeinfo>
#include <chrono>
#include <condition_variable>
#include <iostream>
#include <map>
#include <mutex>
namespace memgraph::utils {
#include "google/protobuf/message.h"
using TypeInfoRef = std::reference_wrapper<const std::type_info>;
#include "io/address.hpp"
#include "io/transport.hpp"
struct TypeInfoHasher {
std::size_t operator()(TypeInfoRef code) const { return code.get().hash_code(); }
};
struct TypeInfoEqualTo {
bool operator()(TypeInfoRef lhs, TypeInfoRef rhs) const { return lhs.get() == rhs.get(); }
};
} // namespace memgraph::utils
namespace memgraph::io::protobuf_transport {}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,12 +19,9 @@
#include <map>
#include <set>
#include <thread>
#include <type_traits>
#include <unordered_map>
#include <vector>
#include <boost/core/demangle.hpp>
#include "io/message_conversion.hpp"
#include "io/simulator/simulator.hpp"
#include "io/transport.hpp"
#include "utils/concepts.hpp"
@@ -91,46 +88,6 @@ struct ReadResponse {
std::optional<Address> retry_leader;
};
template <class... ReadReturn>
utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
return TypeInfoForVariant(response.read_return);
}
template <class ReadReturn>
utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
return typeid(ReadReturn);
}
template <class ReadOperation>
utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
return typeid(ReadOperation);
}
template <class... ReadOperations>
utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
return TypeInfoForVariant(request.operation);
}
template <class... WriteReturn>
utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
return TypeInfoForVariant(response.write_return);
}
template <class WriteReturn>
utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
return typeid(WriteReturn);
}
template <class WriteOperation>
utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
return typeid(WriteOperation);
}
template <class... WriteOperations>
utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
return TypeInfoForVariant(request.operation);
}
/// AppendRequest is a raft-level message that the Leader
/// periodically broadcasts to all Follower peers. This
/// serves three main roles:
@@ -192,7 +149,7 @@ struct PendingClientRequest {
struct Leader {
std::map<Address, FollowerTracker> followers;
std::map<LogIndex, PendingClientRequest> pending_client_requests;
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
Time last_broadcast = Time::min();
std::string static ToString() { return "\tLeader \t"; }
@@ -247,7 +204,7 @@ to a CAS operation.
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
typename ReadResponseValue>
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
{ state.Read(std::move(r)) } -> std::same_as<ReadResponseValue>;
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
};
@@ -403,7 +360,7 @@ class Raft {
const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
leader.pending_client_requests.erase(apply_index);
WriteResponse<WriteResponseValue> resp{
const WriteResponse<WriteResponseValue> resp{
.success = true,
.write_return = std::move(write_return),
.raft_index = apply_index,
@@ -555,7 +512,7 @@ class Raft {
for (const auto &peer : peers_) {
// request_id not necessary to set because it's not a Future-backed Request.
static constexpr auto request_id = 0;
io_.template Send(peer, request_id, VoteRequest{request});
io_.template Send<VoteRequest>(peer, request_id, request);
outstanding_votes.insert(peer);
}
@@ -591,7 +548,7 @@ class Raft {
const Time now = io_.Now();
const Duration broadcast_timeout = RandomTimeout(kMinimumBroadcastTimeout, kMaximumBroadcastTimeout);
if (now > leader.last_broadcast + broadcast_timeout) {
if (now - leader.last_broadcast > broadcast_timeout) {
BroadcastAppendEntries(leader.followers);
leader.last_broadcast = now;
}
@@ -625,12 +582,13 @@ class Raft {
MG_ASSERT(std::max(req.term, state_.term) == req.term);
}
io_.Send(from_address, request_id,
VoteResponse{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
});
const VoteResponse res{
.term = std::max(req.term, state_.term),
.committed_log_size = state_.committed_log_size,
.vote_granted = new_leader,
};
io_.Send(from_address, request_id, res);
if (new_leader) {
// become a follower
@@ -692,7 +650,7 @@ class Raft {
return Leader{
.followers = std::move(followers),
.pending_client_requests = std::map<LogIndex, PendingClientRequest>(),
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
};
}
@@ -718,10 +676,6 @@ class Raft {
.log_size = state_.log.size(),
};
static_assert(std::is_trivially_copyable_v<AppendResponse>,
"This function copies this message, therefore it is important to be trivially copyable. Otherwise it "
"should be moved");
if constexpr (std::is_same<ALL, Leader>()) {
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
}
@@ -740,7 +694,7 @@ class Raft {
// become follower of this leader, reply with our log status
state_.term = req.term;
io_.Send(from_address, request_id, AppendResponse{res});
io_.Send(from_address, request_id, res);
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
return Follower{
@@ -751,7 +705,7 @@ class Raft {
if (req.term < state_.term) {
// nack this request from an old leader
io_.Send(from_address, request_id, AppendResponse{res});
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -812,7 +766,7 @@ class Raft {
Log("returning log_size of ", res.log_size);
io_.Send(from_address, request_id, AppendResponse{res});
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -860,20 +814,18 @@ class Raft {
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
Address from_address) {
auto type_info = TypeInfoFor(req);
std::string demangled_name = boost::core::demangle(type_info.get().name());
Log("handling ReadOperation<" + demangled_name + ">");
ReadOperation &read_operation = req.operation;
Log("handling ReadOperation");
ReadOperation read_operation = req.operation;
ReadResponseValue read_return = replicated_state_.Read(std::move(read_operation));
ReadResponseValue read_return = replicated_state_.Read(read_operation);
ReadResponse<ReadResponseValue> resp{
const ReadResponse<ReadResponseValue> resp{
.success = true,
.read_return = std::move(read_return),
.retry_leader = std::nullopt,
};
io_.Send(from_address, request_id, std::move(resp));
io_.Send(from_address, request_id, resp);
return std::nullopt;
}
@@ -882,11 +834,11 @@ class Raft {
std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
RequestId request_id, Address from_address) {
Log("received ReadOperation - not redirecting because no Leader is known");
ReadResponse<ReadResponseValue> res{
const ReadResponse<ReadResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
Cron();
@@ -898,12 +850,12 @@ class Raft {
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
ReadResponse<ReadResponseValue> res{
const ReadResponse<ReadResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -917,12 +869,12 @@ class Raft {
Address from_address) {
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
WriteResponse<WriteResponseValue> res{
const WriteResponse<WriteResponseValue> res{
.success = false,
.retry_leader = follower.leader_address,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
return std::nullopt;
}
@@ -931,11 +883,11 @@ class Raft {
RequestId request_id, Address from_address) {
Log("received WriteRequest - not redirecting because no Leader is known");
WriteResponse<WriteResponseValue> res{
const WriteResponse<WriteResponseValue> res{
.success = false,
};
io_.Send(from_address, request_id, std::move(res));
io_.Send(from_address, request_id, res);
Cron();
@@ -945,9 +897,7 @@ class Raft {
// only leaders actually handle replication requests from clients
std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
Address from_address) {
auto type_info = TypeInfoFor(req);
std::string demangled_name = boost::core::demangle(type_info.get().name());
Log("handling WriteRequest<" + demangled_name + ">");
Log("handling WriteRequest");
// we are the leader. add item to log and send Append to peers
MG_ASSERT(state_.term >= LastLogTerm());

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,13 +13,9 @@
#include <iostream>
#include <optional>
#include <type_traits>
#include <unordered_map>
#include <vector>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/notifier.hpp"
#include "io/rsm/raft.hpp"
#include "utils/result.hpp"
@@ -38,14 +34,6 @@ using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::utils::BasicResult;
template <typename RequestT, typename ResponseT>
struct AsyncRequest {
Time start_time;
RequestT request;
Notifier notifier;
ResponseFuture<ResponseT> future;
};
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
typename ReadResponseT>
class RsmClient {
@@ -55,28 +43,21 @@ class RsmClient {
Address leader_;
ServerPool server_addrs_;
/// State for single async read/write operations. In the future this could become a map
/// of async operations that can be accessed via an ID etc...
std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
void SelectRandomLeader() {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = io_.Rand(addr_distrib);
leader_ = server_addrs_[addr_index];
spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
}
template <typename ResponseT>
void PossiblyRedirectLeader(const ResponseT &response) {
if (response.retry_leader) {
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
leader_ = response.retry_leader.value();
spdlog::debug("client redirected to leader server {}", leader_.ToString());
}
if (!response.success) {
SelectRandomLeader();
} else if (!response.success) {
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
size_t addr_index = io_.Rand(addr_distrib);
leader_ = server_addrs_[addr_index];
spdlog::debug(
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
"a RSM Candidate) trying a random one at index {} with address {}",
addr_index, leader_.ToString());
}
}
@@ -84,173 +65,71 @@ class RsmClient {
RsmClient(Io<IoImpl> io, Address leader, ServerPool server_addrs)
: io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
RsmClient(const RsmClient &) = delete;
RsmClient &operator=(const RsmClient &) = delete;
RsmClient(RsmClient &&) noexcept = default;
RsmClient &operator=(RsmClient &&) noexcept = default;
RsmClient() = delete;
~RsmClient() = default;
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
Notifier notifier;
const ReadinessToken readiness_token{0};
SendAsyncWriteRequest(std::move(req), notifier, readiness_token);
auto poll_result = AwaitAsyncWriteRequest(readiness_token);
while (!poll_result) {
poll_result = AwaitAsyncWriteRequest(readiness_token);
}
return poll_result.value();
WriteRequest<WriteRequestT> client_req;
client_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const Time before = io_.Now();
do {
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
if (response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return response_result.GetError();
}
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
if (write_response.success) {
return std::move(write_response.write_return);
}
PossiblyRedirectLeader(write_response);
} while (io_.Now() < before + overall_timeout);
return TimedOut{};
}
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
Notifier notifier;
const ReadinessToken readiness_token{0};
SendAsyncReadRequest(std::move(req), notifier, readiness_token);
auto poll_result = AwaitAsyncReadRequest(readiness_token);
while (!poll_result) {
poll_result = AwaitAsyncReadRequest(readiness_token);
}
return poll_result.value();
}
/// AsyncRead methods
void SendAsyncReadRequest(ReadRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
ReadRequest<ReadRequestT> read_req = {.operation = req};
AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
.start_time = io_.Now(),
.request = std::move(req),
.notifier = notifier,
.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
leader_, std::move(read_req), notifier, readiness_token),
};
async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
}
void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_reads_.at(readiness_token.GetId());
ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
async_request.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
leader_, std::move(read_req), async_request.notifier, readiness_token);
}
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_reads_.at(readiness_token.GetId());
if (!async_request.future.IsReady()) {
return std::nullopt;
}
return AwaitAsyncReadRequest(readiness_token);
}
std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_reads_.at(readiness_token.GetId());
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
ReadRequest<ReadRequestT> read_req;
read_req.operation = req;
const Duration overall_timeout = io_.GetDefaultTimeout();
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
const bool result_has_error = get_response_result.HasError();
const Time before = io_.Now();
if (result_has_error && past_time_out) {
// TODO static assert the exact type of error.
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
async_reads_.erase(readiness_token.GetId());
return TimedOut{};
}
do {
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
// receive response
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
if (get_response_result.HasError()) {
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
return get_response_result.GetError();
}
if (!result_has_error) {
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
PossiblyRedirectLeader(read_get_response);
if (read_get_response.success) {
async_reads_.erase(readiness_token.GetId());
spdlog::debug("returning read_return for RSM request");
return std::move(read_get_response.read_return);
}
} else {
SelectRandomLeader();
}
ResendAsyncReadRequest(readiness_token);
PossiblyRedirectLeader(read_get_response);
} while (io_.Now() < before + overall_timeout);
return std::nullopt;
}
/// AsyncWrite methods
void SendAsyncWriteRequest(WriteRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
WriteRequest<WriteRequestT> write_req = {.operation = req};
AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
.start_time = io_.Now(),
.request = std::move(req),
.notifier = notifier,
.future = io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
leader_, std::move(write_req), notifier, readiness_token),
};
async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
}
void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_writes_.at(readiness_token.GetId());
WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
async_request.future =
io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
leader_, std::move(write_req), async_request.notifier, readiness_token);
}
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_writes_.at(readiness_token.GetId());
if (!async_request.future.IsReady()) {
return std::nullopt;
}
return AwaitAsyncWriteRequest(readiness_token);
}
std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
auto &async_request = async_writes_.at(readiness_token.GetId());
ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
const Duration overall_timeout = io_.GetDefaultTimeout();
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
const bool result_has_error = get_response_result.HasError();
if (result_has_error && past_time_out) {
// TODO static assert the exact type of error.
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
async_writes_.erase(readiness_token.GetId());
return TimedOut{};
}
if (!result_has_error) {
ResponseEnvelope<WriteResponse<WriteResponseT>> &&get_response_envelope =
std::move(get_response_result.GetValue());
WriteResponse<WriteResponseT> &&write_get_response = std::move(get_response_envelope.message);
PossiblyRedirectLeader(write_get_response);
if (write_get_response.success) {
async_writes_.erase(readiness_token.GetId());
return std::move(write_get_response.write_return);
}
} else {
SelectRandomLeader();
}
ResendAsyncWriteRequest(readiness_token);
return std::nullopt;
return TimedOut{};
}
};

View File

@@ -29,8 +29,6 @@ class Simulator {
explicit Simulator(SimulatorConfig config)
: rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
~Simulator() { ShutDown(); }
void ShutDown() { simulator_handle_->ShutDown(); }
Io<SimulatorTransport> RegisterNew() {
@@ -41,7 +39,7 @@ class Simulator {
Io<SimulatorTransport> Register(Address address) {
std::uniform_int_distribution<uint64_t> seed_distrib;
uint64_t seed = seed_distrib(rng_);
return Io{SimulatorTransport(simulator_handle_, address, seed), address};
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
}
void IncrementServerCountAndWaitForQuiescentState(Address address) {
@@ -49,14 +47,5 @@ class Simulator {
}
SimulatorStats Stats() { return simulator_handle_->Stats(); }
std::shared_ptr<SimulatorHandle> GetSimulatorHandle() const { return simulator_handle_; }
std::function<bool()> GetSimulatorTickClosure() {
std::function<bool()> tick_closure = [handle_copy = simulator_handle_] {
return handle_copy->MaybeTickSimulator();
};
return tick_closure;
}
};
}; // namespace memgraph::io::simulator

View File

@@ -26,6 +26,5 @@ struct SimulatorConfig {
uint64_t rng_seed = 0;
Time start_time = Time::min();
Time abort_time = Time::max();
Duration message_delay = std::chrono::microseconds(100);
};
}; // namespace memgraph::io::simulator

View File

@@ -16,19 +16,15 @@
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
#include "io/transport.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::io::simulator {
using memgraph::io::Duration;
using memgraph::io::Time;
void SimulatorHandle::ShutDown() {
std::unique_lock<std::mutex> lock(mu_);
should_shut_down_ = true;
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
std::move(dop).promise.TimeOut();
it = promises_.erase(it);
}
can_receive_.clear();
cv_.notify_all();
}
@@ -37,11 +33,6 @@ bool SimulatorHandle::ShouldShutDown() const {
return should_shut_down_;
}
LatencyHistogramSummaries SimulatorHandle::ResponseLatencies() {
std::unique_lock<std::mutex> lock(mu_);
return histograms_.ResponseLatencies();
}
void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
std::unique_lock<std::mutex> lock(mu_);
server_addresses_.insert(address);
@@ -52,84 +43,47 @@ void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address addre
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
if (all_servers_blocked) {
spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
server_addresses_.size());
return;
}
spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
server_addresses_.size());
cv_.wait(lock);
}
}
bool SortInFlight(const std::pair<Address, OpaqueMessage> &lhs, const std::pair<Address, OpaqueMessage> &rhs) {
// NB: never sort based on the request ID etc...
// This should only be used from std::stable_sort
// because by comparing on the from_address alone,
// we expect the sender ordering to remain
// deterministic.
const auto &[addr_1, om_1] = lhs;
const auto &[addr_2, om_2] = rhs;
return om_1.from_address < om_2.from_address;
}
bool SimulatorHandle::MaybeTickSimulator() {
std::unique_lock<std::mutex> lock(mu_);
const size_t blocked_servers = blocked_on_receive_.size();
if (should_shut_down_ || blocked_servers < server_addresses_.size()) {
if (blocked_servers < server_addresses_.size()) {
// we only need to advance the simulator when all
// servers have reached a quiescent state, blocked
// on their own futures or receive methods.
return false;
}
// We allow the simulator to progress the state of the system only
// after all servers are blocked on receive.
spdlog::trace("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ simulator tick ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
stats_.simulator_ticks++;
blocked_on_receive_.clear();
cv_.notify_all();
bool timed_anything_out = TimeoutPromisesPastDeadline();
if (timed_anything_out) {
spdlog::trace("simulator progressing: timed out a request");
}
const Duration clock_advance = std::chrono::microseconds{time_distrib_(rng_)};
// We don't always want to advance the clock with every message that we deliver because
// when we advance it for every message, it causes timeouts to occur for any "request path"
// over a certain length. Alternatively, we don't want to simply deliver multiple messages
// in a single simulator tick because that would reduce the amount of concurrent message
// mixing that may naturally occur in production. This approach is to mod the random clock
// advance by a prime number (hopefully avoiding most harmonic effects that would be introduced
// by only advancing the clock by an even amount etc...) and only advancing the clock close to
// half of the time.
if (clock_advance.count() % 97 > 49) {
spdlog::trace("simulator progressing: clock advanced by {}", clock_advance.count());
cluster_wide_time_microseconds_ += clock_advance;
stats_.elapsed_time = cluster_wide_time_microseconds_ - config_.start_time;
}
if (cluster_wide_time_microseconds_ >= config_.abort_time) {
spdlog::error(
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time. The SimulatorConfig.rng_seed for this run is {}",
config_.rng_seed);
throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
}
TimeoutPromisesPastDeadline();
if (in_flight_.empty()) {
// return early here because there are no messages to schedule
// We tick the clock forward when all servers are blocked but
// there are no in-flight messages to schedule delivery of.
std::poisson_distribution<> time_distrib(50);
Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
cluster_wide_time_microseconds_ += clock_advance;
MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
"in an expected amount of time.");
return true;
}
std::stable_sort(in_flight_.begin(), in_flight_.end(), SortInFlight);
if (config_.scramble_messages && in_flight_.size() > 1) {
if (config_.scramble_messages) {
// scramble messages
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
const size_t swap_index = swap_distrib(rng_);
@@ -139,14 +93,17 @@ bool SimulatorHandle::MaybeTickSimulator() {
auto [to_address, opaque_message] = std::move(in_flight_.back());
in_flight_.pop_back();
const int drop_threshold = drop_distrib_(rng_);
std::uniform_int_distribution<int> drop_distrib(0, 99);
const int drop_threshold = drop_distrib(rng_);
const bool should_drop = drop_threshold < config_.drop_percent;
if (should_drop) {
stats_.dropped_messages++;
}
PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
PromiseKey promise_key{.requester_address = to_address,
.request_id = opaque_message.request_id,
.replier_address = opaque_message.from_address};
if (promises_.contains(promise_key)) {
// complete waiting promise if it's there
@@ -158,25 +115,16 @@ bool SimulatorHandle::MaybeTickSimulator() {
if (should_drop || normal_timeout) {
stats_.timed_out_requests++;
dop.promise.TimeOut();
spdlog::trace("simulator timing out request ");
} else {
stats_.total_responses++;
Duration response_latency = cluster_wide_time_microseconds_ - dop.requested_at;
auto type_info = opaque_message.type_info;
dop.promise.Fill(std::move(opaque_message), response_latency);
histograms_.Measure(type_info, response_latency);
spdlog::trace("simulator replying to request");
dop.promise.Fill(std::move(opaque_message));
}
} else if (should_drop) {
// don't add it anywhere, let it drop
spdlog::trace("simulator silently dropping request");
} else {
// add to can_receive_ if not
spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
opaque_message.to_address.last_known_port);
const auto &[om_vec, inserted] =
can_receive_.try_emplace(to_address.ToPartialAddress(), std::deque<OpaqueMessage>());
om_vec->second.emplace_front(std::move(opaque_message));
const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
om_vec->second.emplace_back(std::move(opaque_message));
}
return true;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,12 +22,9 @@
#include <variant>
#include <vector>
#include <boost/core/demangle.hpp>
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/message_conversion.hpp"
#include "io/message_histogram_collector.hpp"
#include "io/simulator/simulator_config.hpp"
#include "io/simulator/simulator_stats.hpp"
#include "io/time.hpp"
@@ -46,7 +43,7 @@ class SimulatorHandle {
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
// messages that are sent to servers that may later receive them
std::map<PartialAddress, std::deque<OpaqueMessage>> can_receive_;
std::map<Address, std::vector<OpaqueMessage>> can_receive_;
Time cluster_wide_time_microseconds_;
bool should_shut_down_ = false;
@@ -54,45 +51,29 @@ class SimulatorHandle {
std::set<Address> blocked_on_receive_;
std::set<Address> server_addresses_;
std::mt19937 rng_;
std::uniform_int_distribution<int> time_distrib_{0, 30000};
std::uniform_int_distribution<int> drop_distrib_{0, 99};
SimulatorConfig config_;
MessageHistogramCollector histograms_;
RequestId request_id_counter_{0};
bool TimeoutPromisesPastDeadline() {
bool timed_anything_out = false;
void TimeoutPromisesPastDeadline() {
const Time now = cluster_wide_time_microseconds_;
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
if (dop.deadline < now && config_.perform_timeouts) {
spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
if (dop.deadline < now) {
spdlog::info("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
promise_key.replier_address.ToString());
std::move(dop).promise.TimeOut();
it = promises_.erase(it);
stats_.timed_out_requests++;
timed_anything_out = true;
} else {
++it;
}
}
return timed_anything_out;
}
public:
explicit SimulatorHandle(SimulatorConfig config)
: cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
LatencyHistogramSummaries ResponseLatencies();
~SimulatorHandle() {
for (auto it = promises_.begin(); it != promises_.end();) {
auto &[promise_key, dop] = *it;
std::move(dop).promise.TimeOut();
it = promises_.erase(it);
}
}
void IncrementServerCountAndWaitForQuiescentState(Address address);
/// This method causes most of the interesting simulation logic to happen, wrt network behavior.
@@ -105,122 +86,80 @@ class SimulatorHandle {
bool ShouldShutDown() const;
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
Duration timeout, std::function<bool()> &&maybe_tick_simulator,
std::function<void()> &&fill_notifier) {
auto type_info = TypeInfoFor(request);
std::string demangled_name = boost::core::demangle(type_info.get().name());
spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
template <Message Request, Message Response>
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, Request &&request,
Duration timeout, ResponsePromise<Response> &&promise) {
std::unique_lock<std::mutex> lock(mu_);
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
// set notifier for when the Future::Wait is called
std::move(maybe_tick_simulator),
// set notifier for when Promise::Fill is called
std::move(fill_notifier));
const Time deadline = cluster_wide_time_microseconds_ + timeout;
{
std::unique_lock<std::mutex> lock(mu_);
std::any message(request);
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message)};
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
RequestId request_id = ++request_id_counter_;
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
promises_.emplace(std::move(promise_key), std::move(dop));
const Time deadline = cluster_wide_time_microseconds_ + timeout;
std::any message(std::move(request));
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message),
.type_info = type_info,
.deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
OpaquePromise opaque_promise(std::move(promise).ToUnique());
DeadlineAndOpaquePromise dop{
.requested_at = cluster_wide_time_microseconds_,
.deadline = deadline,
.promise = std::move(opaque_promise),
};
MG_ASSERT(!promises_.contains(promise_key));
promises_.emplace(std::move(promise_key), std::move(dop));
stats_.total_messages++;
stats_.total_requests++;
} // lock dropped here
stats_.total_messages++;
stats_.total_requests++;
cv_.notify_all();
return std::move(future);
}
template <Message... Ms>
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
std::unique_lock<std::mutex> lock(mu_);
blocked_on_receive_.emplace(receiver);
const Time deadline = cluster_wide_time_microseconds_ + timeout;
auto partial_address = receiver.ToPartialAddress();
while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
if (can_receive_.contains(partial_address)) {
std::deque<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
bool contains_items = !can_rx.empty();
bool can_receive = contains_items && can_rx.back().deliverable_at <= cluster_wide_time_microseconds_;
if (can_receive) {
if (can_receive_.contains(receiver)) {
std::vector<OpaqueMessage> &can_rx = can_receive_.at(receiver);
if (!can_rx.empty()) {
OpaqueMessage message = std::move(can_rx.back());
can_rx.pop_back();
// TODO(tyler) search for item in can_receive_ that matches the desired types, rather
// than asserting that the last item in can_rx matches.
auto m_opt = std::move(message).Take<Ms...>();
MG_ASSERT(m_opt.has_value(), "Wrong message type received compared to the expected type");
blocked_on_receive_.erase(receiver);
return std::move(m_opt).value();
}
if (contains_items) {
auto count = can_rx.back().deliverable_at.time_since_epoch().count();
auto now_count = cluster_wide_time_microseconds_.time_since_epoch().count();
spdlog::trace("can't receive message yet due to artificial latency. deliverable_at: {}, now: {}", count,
now_count);
}
}
if (!should_shut_down_) {
if (!blocked_on_receive_.contains(receiver)) {
blocked_on_receive_.emplace(receiver);
spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
cv_.notify_all();
}
lock.unlock();
bool made_progress = MaybeTickSimulator();
lock.lock();
if (!should_shut_down_ && !made_progress) {
cv_.wait(lock);
}
}
spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
blocked_on_receive_.erase(receiver);
return TimedOut{};
}
template <Message M>
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
auto type_info = TypeInfoFor(message);
{
std::unique_lock<std::mutex> lock(mu_);
std::any message_any(std::move(message));
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any),
.type_info = type_info,
.deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
std::unique_lock<std::mutex> lock(mu_);
std::any message_any(std::move(message));
OpaqueMessage om{.to_address = to_address,
.from_address = from_address,
.request_id = request_id,
.message = std::move(message_any)};
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
stats_.total_messages++;
} // lock dropped before cv notification
stats_.total_messages++;
cv_.notify_all();
}

View File

@@ -13,10 +13,6 @@
#include <cstdint>
#include <fmt/format.h>
#include "io/time.hpp"
namespace memgraph::io::simulator {
struct SimulatorStats {
uint64_t total_messages = 0;
@@ -25,22 +21,5 @@ struct SimulatorStats {
uint64_t total_requests = 0;
uint64_t total_responses = 0;
uint64_t simulator_ticks = 0;
Duration elapsed_time;
friend bool operator==(const SimulatorStats & /* lhs */, const SimulatorStats & /* rhs */) = default;
friend std::ostream &operator<<(std::ostream &in, const SimulatorStats &stats) {
auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(stats.elapsed_time).count();
std::string formated = fmt::format(
"SimulatorStats {{ total_messages: {}, dropped_messages: {}, timed_out_requests: {}, total_requests: {}, "
"total_responses: {}, simulator_ticks: {}, elapsed_time: {}ms }}",
stats.total_messages, stats.dropped_messages, stats.timed_out_requests, stats.total_requests,
stats.total_responses, stats.simulator_ticks, elapsed_ms);
in << formated;
return in;
}
};
}; // namespace memgraph::io::simulator

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -15,7 +15,6 @@
#include <utility>
#include "io/address.hpp"
#include "io/notifier.hpp"
#include "io/simulator/simulator_handle.hpp"
#include "io/time.hpp"
@@ -26,22 +25,24 @@ using memgraph::io::Time;
class SimulatorTransport {
std::shared_ptr<SimulatorHandle> simulator_handle_;
Address address_;
const Address address_;
std::mt19937 rng_;
public:
SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
std::function<void()> notification, Duration timeout) {
std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
return handle_copy->MaybeTickSimulator();
};
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, uint64_t request_id, RequestT request,
Duration timeout) {
std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
auto [future, promise] =
memgraph::io::FuturePromisePairWithNotifier<ResponseResult<ResponseT>>(maybe_tick_simulator);
return simulator_handle_->template SubmitRequest<ResponseT, RequestT>(
to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
simulator_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
std::move(promise));
return std::move(future);
}
template <Message... Ms>
@@ -50,8 +51,8 @@ class SimulatorTransport {
}
template <Message M>
void Send(Address to_address, Address from_address, uint64_t request_id, RValueRef<M> message) {
return simulator_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
}
Time Now() const { return simulator_handle_->Now(); }
@@ -62,7 +63,5 @@ class SimulatorTransport {
Return Rand(D distrib) {
return distrib(rng_);
}
LatencyHistogramSummaries ResponseLatencies() { return simulator_handle_->ResponseLatencies(); }
};
}; // namespace memgraph::io::simulator

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,10 +19,7 @@
#include "io/address.hpp"
#include "io/errors.hpp"
#include "io/future.hpp"
#include "io/message_histogram_collector.hpp"
#include "io/notifier.hpp"
#include "io/time.hpp"
#include "utils/concepts.hpp"
#include "utils/result.hpp"
namespace memgraph::io {
@@ -33,15 +30,7 @@ using memgraph::utils::BasicResult;
// reasonable constraints around message types over time,
// as we adapt things to use Thrift-generated message types.
template <typename T>
concept Message = std::movable<T> && std::copyable<T>;
template <utils::Object T>
struct RValueRefEnforcer {
using Type = T &&;
};
template <typename T>
using RValueRef = typename RValueRefEnforcer<T>::Type;
concept Message = std::same_as<T, std::decay_t<T>>;
using RequestId = uint64_t;
@@ -51,7 +40,6 @@ struct ResponseEnvelope {
RequestId request_id;
Address to_address;
Address from_address;
Duration response_latency;
};
template <Message M>
@@ -78,6 +66,7 @@ template <typename I>
class Io {
I implementation_;
Address address_;
RequestId request_id_counter_ = 0;
Duration default_timeout_ = std::chrono::microseconds{100000};
public:
@@ -91,44 +80,22 @@ class Io {
Duration GetDefaultTimeout() { return default_timeout_; }
/// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RValueRef<RequestT> request, Duration timeout) {
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
const RequestId request_id = ++request_id_counter_;
const Address from_address = address_;
std::function<void()> fill_notifier = nullptr;
return implementation_.template Request<ResponseT, RequestT>(address, from_address, std::move(request),
fill_notifier, timeout);
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request_id, request, timeout);
}
/// Issue a request that times out after the default timeout. This tends
/// to be used by clients.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> Request(Address to_address, RValueRef<RequestT> request) {
template <Message RequestT, Message ResponseT>
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
const RequestId request_id = ++request_id_counter_;
const Duration timeout = default_timeout_;
const Address from_address = address_;
std::function<void()> fill_notifier = nullptr;
return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Issue a request that will notify a Notifier when it is filled or times out.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RValueRef<RequestT> request, Notifier notifier,
ReadinessToken readiness_token) {
const Duration timeout = default_timeout_;
const Address from_address = address_;
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
fill_notifier, timeout);
}
/// Issue a request that will notify a Notifier when it is filled or times out.
template <Message ResponseT, Message RequestT>
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT &&request, Notifier notifier,
ReadinessToken readiness_token, Duration timeout) {
const Address from_address = address_;
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
return implementation_.template Request<ResponseT>(to_address, from_address, std::forward<RequestT>(request),
fill_notifier, timeout);
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, request_id,
std::move(request), timeout);
}
/// Wait for an explicit number of microseconds for a request of one of the
@@ -150,9 +117,9 @@ class Io {
/// responses are not necessarily expected, and for servers to respond to requests.
/// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
template <Message M>
void Send(Address to_address, RequestId request_id, M &&message) {
void Send(Address to_address, RequestId request_id, M message) {
Address from_address = address_;
return implementation_.template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
}
/// The current system time. This time source should be preferred over any other,
@@ -172,15 +139,6 @@ class Io {
Address GetAddress() { return address_; }
void SetAddress(Address address) { address_ = address; }
Io<I> ForkLocal(boost::uuids::uuid uuid) {
Address new_address{
.unique_id = uuid,
.last_known_ip = address_.last_known_ip,
.last_known_port = address_.last_known_port,
};
return Io(implementation_, new_address);
}
LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
Io<I> ForkLocal() { return Io(implementation_, address_.ForkUniqueAddress()); }
};
}; // namespace memgraph::io

View File

@@ -11,11 +11,7 @@
#pragma once
#include <algorithm>
#include <thread>
#include <boost/asio/ip/tcp.hpp>
#include "io/address.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/schemas.hpp"
@@ -41,8 +37,6 @@ struct MachineConfig {
bool is_query_engine;
boost::asio::ip::address listen_ip;
uint16_t listen_port;
size_t shard_worker_threads = std::max(static_cast<unsigned int>(1), std::thread::hardware_concurrency());
bool sync_message_handling = false;
};
} // namespace memgraph::machine_manager

View File

@@ -11,43 +11,39 @@
#pragma once
#include "coordinator/coordinator_rsm.hpp"
#include "coordinator/coordinator_worker.hpp"
#include "io/message_conversion.hpp"
#include "io/messages.hpp"
#include "io/rsm/rsm_client.hpp"
#include "io/time.hpp"
#include "machine_manager/machine_config.hpp"
#include "storage/v3/shard_manager.hpp"
#include <coordinator/coordinator_rsm.hpp>
#include <io/message_conversion.hpp>
#include <io/messages.hpp>
#include <io/rsm/rsm_client.hpp>
#include <io/time.hpp>
#include <machine_manager/machine_config.hpp>
#include <storage/v3/shard_manager.hpp>
namespace memgraph::machine_manager {
using coordinator::Coordinator;
using coordinator::CoordinatorReadRequests;
using coordinator::CoordinatorReadResponses;
using coordinator::CoordinatorRsm;
using coordinator::CoordinatorWriteRequests;
using coordinator::CoordinatorWriteResponses;
using coordinator::coordinator_worker::CoordinatorWorker;
using CoordinatorRouteMessage = coordinator::coordinator_worker::RouteMessage;
using CoordinatorQueue = coordinator::coordinator_worker::Queue;
using io::ConvertVariant;
using io::Duration;
using io::RequestId;
using io::Time;
using io::messages::CoordinatorMessages;
using io::messages::ShardManagerMessages;
using io::messages::ShardMessages;
using io::messages::StorageReadRequest;
using io::messages::StorageWriteRequest;
using io::rsm::AppendRequest;
using io::rsm::AppendResponse;
using io::rsm::ReadRequest;
using io::rsm::VoteRequest;
using io::rsm::VoteResponse;
using io::rsm::WriteRequest;
using io::rsm::WriteResponse;
using storage::v3::ShardManager;
using memgraph::coordinator::Coordinator;
using memgraph::coordinator::CoordinatorReadRequests;
using memgraph::coordinator::CoordinatorReadResponses;
using memgraph::coordinator::CoordinatorRsm;
using memgraph::coordinator::CoordinatorWriteRequests;
using memgraph::coordinator::CoordinatorWriteResponses;
using memgraph::io::ConvertVariant;
using memgraph::io::Duration;
using memgraph::io::RequestId;
using memgraph::io::Time;
using memgraph::io::messages::CoordinatorMessages;
using memgraph::io::messages::ShardManagerMessages;
using memgraph::io::messages::ShardMessages;
using memgraph::io::messages::StorageReadRequest;
using memgraph::io::messages::StorageWriteRequest;
using memgraph::io::rsm::AppendRequest;
using memgraph::io::rsm::AppendResponse;
using memgraph::io::rsm::ReadRequest;
using memgraph::io::rsm::VoteRequest;
using memgraph::io::rsm::VoteResponse;
using memgraph::io::rsm::WriteRequest;
using memgraph::io::rsm::WriteResponse;
using memgraph::storage::v3::ShardManager;
/// The MachineManager is responsible for:
/// * starting the entire system and ensuring that high-level
@@ -66,11 +62,9 @@ template <typename IoImpl>
class MachineManager {
io::Io<IoImpl> io_;
MachineConfig config_;
Address coordinator_address_;
CoordinatorQueue coordinator_queue_;
std::jthread coordinator_handle_;
CoordinatorRsm<IoImpl> coordinator_;
ShardManager<IoImpl> shard_manager_;
Time next_cron_ = Time::min();
Time next_cron_;
public:
// TODO initialize ShardManager with "real" coordinator addresses instead of io.GetAddress
@@ -78,49 +72,20 @@ class MachineManager {
MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
: io_(io),
config_(config),
coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
coordinator_address_} {
auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
}
coordinator_{std::move(io.ForkLocal()), {}, std::move(coordinator)},
shard_manager_(ShardManager{io.ForkLocal(), coordinator_.GetAddress()}) {}
MachineManager(MachineManager &&) noexcept = default;
MachineManager &operator=(MachineManager &&) noexcept = default;
MachineManager(const MachineManager &) = delete;
MachineManager &operator=(const MachineManager &) = delete;
~MachineManager() {
if (coordinator_handle_.joinable()) {
coordinator_queue_.Push(coordinator::coordinator_worker::ShutDown{});
coordinator_handle_.join();
}
}
Address CoordinatorAddress() { return coordinator_address_; }
Address CoordinatorAddress() { return coordinator_.GetAddress(); }
void Run() {
while (true) {
MaybeBlockOnSyncHandling();
if (io_.ShouldShutDown()) {
break;
}
while (!io_.ShouldShutDown()) {
const auto now = io_.Now();
uint64_t now_us = now.time_since_epoch().count();
uint64_t next_us = next_cron_.time_since_epoch().count();
if (now >= next_cron_) {
spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
next_cron_ = Cron();
} else {
spdlog::info("now {} < next_cron_ {}", now_us, next_us);
}
Duration receive_timeout = std::max(next_cron_, now) - now;
Duration receive_timeout = next_cron_ - now;
// Note: this parameter pack must be kept in-sync with the ReceiveWithTimeout parameter pack below
using AllMessages =
@@ -129,7 +94,7 @@ class MachineManager {
WriteResponse<CoordinatorWriteResponses>, ReadRequest<StorageReadRequest>,
AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>;
spdlog::info("MM waiting on Receive on address {}", io_.GetAddress().ToString());
spdlog::info("MM waiting on Receive");
// Note: this parameter pack must be kept in-sync with the AllMessages parameter pack above
auto request_result = io_.template ReceiveWithTimeout<
@@ -140,6 +105,7 @@ class MachineManager {
if (request_result.HasError()) {
// time to do Cron
spdlog::info("MM got timeout");
continue;
}
@@ -148,7 +114,8 @@ class MachineManager {
spdlog::info("MM got message to {}", request_envelope.to_address.ToString());
// If message is for the coordinator, cast it to subset and pass it to the coordinator
bool to_coordinator = coordinator_address_ == request_envelope.to_address;
bool to_coordinator = coordinator_.GetAddress() == request_envelope.to_address;
spdlog::info("coordinator: {}", coordinator_.GetAddress().ToString());
if (to_coordinator) {
std::optional<CoordinatorMessages> conversion_attempt =
ConvertVariant<AllMessages, ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>,
@@ -161,13 +128,8 @@ class MachineManager {
CoordinatorMessages &&cm = std::move(conversion_attempt.value());
CoordinatorRouteMessage route_message{
.message = std::move(cm),
.request_id = request_envelope.request_id,
.to = request_envelope.to_address,
.from = request_envelope.from_address,
};
coordinator_queue_.Push(std::move(route_message));
coordinator_.Handle(std::forward<CoordinatorMessages>(cm), request_envelope.request_id,
request_envelope.from_address);
continue;
}
@@ -194,7 +156,7 @@ class MachineManager {
AppendResponse, WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>(
std::move(request_envelope.message));
MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {} - incorrect message type",
MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {}",
request_envelope.to_address.ToString());
spdlog::info("got shard rsm message");
@@ -206,27 +168,9 @@ class MachineManager {
}
private:
// This method exists for controlling concurrency
// during deterministic simulation testing.
void MaybeBlockOnSyncHandling() {
if (!config_.sync_message_handling) {
return;
}
// block on coordinator
coordinator_queue_.BlockOnQuiescence();
// block on shards
shard_manager_.BlockOnQuiescence();
}
Time Cron() {
spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
MaybeBlockOnSyncHandling();
Time ret = shard_manager_.Cron();
MaybeBlockOnSyncHandling();
return ret;
return shard_manager_.Cron();
}
};

File diff suppressed because it is too large Load Diff

View File

@@ -23,7 +23,7 @@ add_custom_command(
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher::v2
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,10 +14,10 @@
#include <string>
#include "antlr4-runtime.h"
#include "utils/exceptions.hpp"
#include "parser/opencypher/generated/MemgraphCypher.h"
#include "parser/opencypher/generated/MemgraphCypherLexer.h"
#include "utils/concepts.hpp"
#include "utils/exceptions.hpp"
namespace memgraph::frontend::opencypher {
@@ -32,9 +32,11 @@ class SyntaxException : public utils::BasicException {
* This thing must me a class since parser.cypher() returns pointer and there is
* no way for us to get ownership over the object.
*/
enum class ParserOpTag : uint8_t { CYPHER, EXPRESSION };
enum class ParserOpTag : uint8_t {
CYPHER, EXPRESSION
};
template <ParserOpTag Tag = ParserOpTag::CYPHER>
template<ParserOpTag Tag = ParserOpTag::CYPHER>
class Parser {
public:
/**
@@ -44,9 +46,10 @@ class Parser {
Parser(const std::string query) : query_(std::move(query)) {
parser_.removeErrorListeners();
parser_.addErrorListener(&error_listener_);
if constexpr (Tag == ParserOpTag::CYPHER) {
if constexpr(Tag == ParserOpTag::CYPHER) {
tree_ = parser_.cypher();
} else {
}
else {
tree_ = parser_.expression();
}
if (parser_.getNumberOfSyntaxErrors()) {
@@ -72,11 +75,11 @@ class Parser {
FirstMessageErrorListener error_listener_;
std::string query_;
antlr4::ANTLRInputStream input_{query_};
antlropencypher::v2::MemgraphCypherLexer lexer_{&input_};
antlropencypher::MemgraphCypherLexer lexer_{&input_};
antlr4::CommonTokenStream tokens_{&lexer_};
// generate ast
antlropencypher::v2::MemgraphCypher parser_{&tokens_};
antlropencypher::MemgraphCypher parser_{&tokens_};
antlr4::tree::ParseTree *tree_ = nullptr;
};
} // namespace memgraph::frontend::opencypher

1053
src/protobuf/messages.pb.cc Normal file

File diff suppressed because it is too large Load Diff

1231
src/protobuf/messages.pb.h Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,24 @@
syntax = "proto3";
package memgraph.protobuf;
message Address {
bytes unique_id = 1;
// this is either 4 bytes for ipv4 or 16 bytes for ipv6
bytes last_known_ip = 2;
uint32 last_known_port = 3;
}
message TestRequest {
uint64 request_id = 1;
string content = 2;
}
message UberMessage {
uint64 request_id = 1;
Address to_address = 2;
Address from_address = 3;
oneof specific_message {
TestRequest test_request = 4;
}
}

View File

@@ -48,20 +48,18 @@ add_dependencies(mg-query generate_lcp_query)
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_link_libraries(mg-query dl cppitertools Boost::headers)
target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
endif()
if("${MG_PYTHON_VERSION}" STREQUAL "")
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
else()
find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
endif()
target_link_libraries(mg-query Python3::Python)
# Generate Antlr openCypher parser
set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher)
set(opencypher_generated ${opencypher_frontend}/generated)
set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
@@ -84,15 +82,15 @@ add_custom_command(
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
COMMAND
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
-Dlanguage=Cpp -visitor -package antlropencypher
-o ${opencypher_generated}
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
DEPENDS
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
${opencypher_frontend}/grammar/CypherLexer.g4
${opencypher_frontend}/grammar/Cypher.g4)
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
${opencypher_frontend}/grammar/CypherLexer.g4
${opencypher_frontend}/grammar/Cypher.g4)
add_custom_target(generate_opencypher_parser
DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})

View File

@@ -9,8 +9,10 @@ set(mg_query_v2_sources
${lcp_query_v2_cpp_files}
common.cpp
cypher_query_interpreter.cpp
dump.cpp
frontend/semantic/required_privileges.cpp
frontend/stripped.cpp
interpret/awesome_memgraph_functions.cpp
interpreter.cpp
metadata.cpp
plan/operator.cpp
@@ -21,10 +23,18 @@ set(mg_query_v2_sources
plan/rewrite/index_lookup.cpp
plan/rule_based_planner.cpp
plan/variable_start_planner.cpp
# procedure/mg_procedure_impl.cpp
# procedure/mg_procedure_helpers.cpp
# procedure/module.cpp
# procedure/py_module.cpp
serialization/property_value.cpp
# stream/streams.cpp
# stream/sources.cpp
# stream/common.cpp
# trigger.cpp
# trigger_context.cpp
bindings/typed_value.cpp
accessors.cpp
multiframe.cpp)
accessors.cpp)
find_package(Boost REQUIRED)
@@ -33,8 +43,8 @@ add_dependencies(mg-query-v2 generate_lcp_query_v2)
target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory mg-coordinator)
target_link_libraries(mg-query-v2 mg-expr mg-functions)
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory)
target_link_libraries(mg-query-v2 mg-expr)
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")

View File

@@ -10,67 +10,37 @@
// licenses/APL.txt.
#include "query/v2/accessors.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/requests.hpp"
#include "storage/v3/id_types.hpp"
namespace memgraph::query::v2::accessors {
EdgeAccessor::EdgeAccessor(Edge edge, const RequestRouterInterface *request_router)
: edge(std::move(edge)), request_router_(request_router) {}
EdgeAccessor::EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props)
: edge(std::move(edge)), properties(std::move(props)) {}
EdgeTypeId EdgeAccessor::EdgeType() const { return edge.type.id; }
uint64_t EdgeAccessor::EdgeType() const { return edge.type.id; }
const std::vector<std::pair<PropertyId, Value>> &EdgeAccessor::Properties() const { return edge.properties; }
Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
if (!maybe_prop) {
return {};
}
const auto prop_id = *maybe_prop;
auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
return it->second;
std::vector<std::pair<PropertyId, Value>> EdgeAccessor::Properties() const {
return properties;
// std::map<std::string, TypedValue> res;
// for (const auto &[name, value] : *properties) {
// res[name] = ValueToTypedValue(value);
// }
// return res;
}
const Edge &EdgeAccessor::GetEdge() const { return edge; }
bool EdgeAccessor::IsCycle() const { return edge.src == edge.dst; };
size_t EdgeAccessor::CypherId() const { return edge.id.gid; }
VertexAccessor EdgeAccessor::To() const {
return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value EdgeAccessor::GetProperty(const std::string & /*prop_name*/) const {
// TODO(kostasrim) fix this
return {};
}
VertexAccessor EdgeAccessor::From() const {
return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
}
Edge EdgeAccessor::GetEdge() const { return edge; }
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
const RequestRouterInterface *request_router)
: vertex(std::move(v)), properties(std::move(props)), request_router_(request_router) {}
VertexAccessor EdgeAccessor::To() const { return VertexAccessor(Vertex{edge.dst}, {}); }
VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
const RequestRouterInterface *request_router)
: vertex(std::move(v)), request_router_(request_router) {
properties.reserve(props.size());
for (auto &[id, value] : props) {
properties.emplace_back(std::make_pair(id, std::move(value)));
}
}
VertexAccessor EdgeAccessor::From() const { return VertexAccessor(Vertex{edge.src}, {}); }
VertexAccessor::VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props,
const RequestRouterInterface *request_router)
: vertex(std::move(v)), request_router_(request_router) {
properties.reserve(props.size());
for (const auto &[id, value] : props) {
properties.emplace_back(std::make_pair(id, value));
}
}
Label VertexAccessor::PrimaryLabel() const { return vertex.id.first; }
const msgs::VertexId &VertexAccessor::Id() const { return vertex.id; }
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props)
: vertex(std::move(v)), properties(std::move(props)) {}
std::vector<Label> VertexAccessor::Labels() const { return vertex.labels; }
@@ -79,23 +49,25 @@ bool VertexAccessor::HasLabel(Label &label) const {
[label](const auto &l) { return l.id == label.id; }) != vertex.labels.end();
}
const std::vector<std::pair<PropertyId, Value>> &VertexAccessor::Properties() const { return properties; }
std::vector<std::pair<PropertyId, Value>> VertexAccessor::Properties() const {
// std::map<std::string, TypedValue> res;
// for (const auto &[name, value] : *properties) {
// res[name] = ValueToTypedValue(value);
// }
// return res;
return properties;
}
Value VertexAccessor::GetProperty(PropertyId prop_id) const {
auto it = std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; });
if (it == properties.end()) {
return {};
}
return it->second;
return std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; })->second;
// return ValueToTypedValue(properties[prop_name]);
}
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
Value VertexAccessor::GetProperty(const std::string &prop_name) const {
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
if (!maybe_prop) {
return {};
}
return GetProperty(*maybe_prop);
Value VertexAccessor::GetProperty(const std::string & /*prop_name*/) const {
// TODO(kostasrim) Add string mapping
return {};
// return ValueToTypedValue(properties[prop_name]);
}
msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }

View File

@@ -11,7 +11,6 @@
#pragma once
#include <map>
#include <optional>
#include <utility>
#include <vector>
@@ -24,80 +23,108 @@
#include "utils/memory.hpp"
#include "utils/memory_tracker.hpp"
namespace memgraph::query::v2 {
class RequestRouterInterface;
} // namespace memgraph::query::v2
namespace memgraph::query::v2::accessors {
using Value = msgs::Value;
using Edge = msgs::Edge;
using Vertex = msgs::Vertex;
using Label = msgs::Label;
using PropertyId = msgs::PropertyId;
using EdgeTypeId = msgs::EdgeTypeId;
using Value = memgraph::msgs::Value;
using Edge = memgraph::msgs::Edge;
using Vertex = memgraph::msgs::Vertex;
using Label = memgraph::msgs::Label;
using PropertyId = memgraph::msgs::PropertyId;
class VertexAccessor;
class EdgeAccessor final {
public:
explicit EdgeAccessor(Edge edge, const RequestRouterInterface *request_router);
EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props);
[[nodiscard]] EdgeTypeId EdgeType() const;
uint64_t EdgeType() const;
[[nodiscard]] const std::vector<std::pair<PropertyId, Value>> &Properties() const;
std::vector<std::pair<PropertyId, Value>> Properties() const;
[[nodiscard]] Value GetProperty(const std::string &prop_name) const;
Value GetProperty(const std::string &prop_name) const;
[[nodiscard]] const Edge &GetEdge() const;
Edge GetEdge() const;
[[nodiscard]] bool IsCycle() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
inline size_t CypherId() const { return 10; }
[[nodiscard]] size_t CypherId() const;
// bool HasSrcAccessor const { return src == nullptr; }
// bool HasDstAccessor const { return dst == nullptr; }
[[nodiscard]] VertexAccessor To() const;
[[nodiscard]] VertexAccessor From() const;
VertexAccessor To() const;
VertexAccessor From() const;
friend bool operator==(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return lhs.edge == rhs.edge; }
friend bool operator==(const EdgeAccessor &lhs, const EdgeAccessor &rhs) {
return lhs.edge == rhs.edge && lhs.properties == rhs.properties;
}
friend bool operator!=(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return !(lhs == rhs); }
private:
Edge edge;
const RequestRouterInterface *request_router_;
std::vector<std::pair<PropertyId, Value>> properties;
};
class VertexAccessor final {
public:
using PropertyId = msgs::PropertyId;
using Label = msgs::Label;
using VertexId = msgs::VertexId;
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
const RequestRouterInterface *request_router);
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props);
VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const RequestRouterInterface *request_router);
VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const RequestRouterInterface *request_router);
std::vector<Label> Labels() const;
[[nodiscard]] Label PrimaryLabel() const;
bool HasLabel(Label &label) const;
[[nodiscard]] const msgs::VertexId &Id() const;
std::vector<std::pair<PropertyId, Value>> Properties() const;
[[nodiscard]] std::vector<Label> Labels() const;
Value GetProperty(PropertyId prop_id) const;
Value GetProperty(const std::string &prop_name) const;
[[nodiscard]] bool HasLabel(Label &label) const;
[[nodiscard]] const std::vector<std::pair<PropertyId, Value>> &Properties() const;
[[nodiscard]] Value GetProperty(PropertyId prop_id) const;
[[nodiscard]] Value GetProperty(const std::string &prop_name) const;
[[nodiscard]] msgs::Vertex GetVertex() const;
msgs::Vertex GetVertex() const;
// Dummy function
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
[[nodiscard]] size_t InDegree() const { throw utils::NotYetImplemented("InDegree() not yet implemented"); }
inline size_t CypherId() const { return 10; }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
[[nodiscard]] size_t OutDegree() const { throw utils::NotYetImplemented("OutDegree() not yet implemented"); }
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
// auto maybe_edges = impl_.InEdges(view, edge_types);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto InEdges(storage::View view) const { return InEdges(view, {}); }
//
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest)
// const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
// auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
// auto maybe_edges = impl_.OutEdges(view, edge_types);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
//
// auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
//
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
// const VertexAccessor &dest) const
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
// auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
// if (maybe_edges.HasError()) return maybe_edges.GetError();
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
// }
// storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
//
// storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
//
friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
@@ -108,9 +135,12 @@ class VertexAccessor final {
private:
Vertex vertex;
std::vector<std::pair<PropertyId, Value>> properties;
const RequestRouterInterface *request_router_;
};
// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
// inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
// Highly mocked interface. Won't work if used.
class Path {
public:
@@ -151,14 +181,7 @@ class Path {
friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
utils::MemoryResource *GetMemoryResource() { return mem; }
auto &vertices() { return vertices_; }
auto &edges() { return edges_; }
const auto &vertices() const { return vertices_; }
const auto &edges() const { return edges_; }
private:
std::vector<VertexAccessor> vertices_;
std::vector<EdgeAccessor> edges_;
utils::MemoryResource *mem = utils::NewDeleteResource();
};
} // namespace memgraph::query::v2::accessors

View File

@@ -11,9 +11,5 @@
#pragma once
#ifdef MG_AST_INCLUDE_PATH
#error You are probably trying to include files from expr from both the storage and query engines! You will have a rough time kid!
#endif
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp" // NOLINT(cppcoreguidelines-macro-usage)
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2 // NOLINT(cppcoreguidelines-macro-usage)

View File

@@ -21,27 +21,24 @@
#include "query/v2/requests.hpp"
#include "storage/v3/conversions.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/view.hpp"
namespace memgraph::query::v2 {
class RequestRouterInterface;
inline const auto lam = [](const auto &val) { return ValueToTypedValue(val); };
namespace detail {
class Callable {
public:
auto operator()(const storage::v3::PropertyValue &val) const {
return storage::v3::PropertyToTypedValue<TypedValue>(val);
};
auto operator()(const msgs::Value &val, RequestRouterInterface *request_router) const {
return ValueToTypedValue(val, request_router);
auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
};
auto operator()(const msgs::Value &val) const { return ValueToTypedValue(val); };
};
} // namespace detail
using ExpressionEvaluator = expr::ExpressionEvaluator<TypedValue, query::v2::EvaluationContext, RequestRouterInterface,
storage::v3::View, storage::v3::LabelId, msgs::Value,
detail::Callable, common::ErrorCode, expr::QueryEngineTag>;
using ExpressionEvaluator =
memgraph::expr::ExpressionEvaluator<TypedValue, memgraph::query::v2::EvaluationContext, DbAccessor,
storage::v3::View, storage::v3::LabelId, msgs::Value, detail::Callable,
memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
} // namespace memgraph::query::v2

View File

@@ -13,10 +13,9 @@
#include "query/v2/bindings/bindings.hpp"
#include "expr/interpret/frame.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "expr/interpret/frame.hpp"
namespace memgraph::query::v2 {
using Frame = memgraph::expr::Frame;
using FrameWithValidity = memgraph::expr::FrameWithValidity;
} // namespace memgraph::query::v2
using Frame = memgraph::expr::Frame<TypedValue>;
} // namespace memgraph::query::v2

View File

@@ -20,6 +20,7 @@
#include "query/v2/bindings/symbol.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/path.hpp"
@@ -27,6 +28,7 @@
#include "storage/v3/id_types.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/result.hpp"
#include "storage/v3/schema_validator.hpp"
#include "storage/v3/view.hpp"
#include "utils/exceptions.hpp"
#include "utils/logging.hpp"
@@ -80,5 +82,102 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
}
template <typename T>
concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
};
template <typename T>
concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
const storage::v3::PropertyValue new_value) {
{
accessor.SetPropertyAndValidate(key, new_value)
} -> std::same_as<storage::v3::ResultSchema<storage::v3::PropertyValue>>;
};
template <typename TRecordAccessor>
concept RecordAccessor =
AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) {
switch (schema_violation.status) {
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: {
throw SchemaViolationException(
fmt::format("Primary key {} not defined on label :{}",
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type),
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL: {
throw SchemaViolationException(
fmt::format("Label :{} is not a primary label", dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE: {
throw SchemaViolationException(
fmt::format("Wrong type of property {} in schema :{}, should be of type {}",
*schema_violation.violated_property_value, dba.LabelToName(schema_violation.label),
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY: {
throw SchemaViolationException(fmt::format("Updating of primary key {} on schema :{} not supported",
*schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL: {
throw SchemaViolationException(fmt::format(
"Adding primary label as secondary or removing primary label:", *schema_violation.violated_property_value,
dba.LabelToName(schema_violation.label)));
}
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: {
throw SchemaViolationException(fmt::format("Cannot create vertex where primary label is secondary:{}",
dba.LabelToName(schema_violation.label)));
}
}
}
inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
switch (error) {
case storage::v3::Error::SERIALIZATION_ERROR:
throw TransactionSerializationException();
case storage::v3::Error::DELETED_OBJECT:
throw QueryRuntimeException("Trying to set properties on a deleted object.");
case storage::v3::Error::PROPERTIES_DISABLED:
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::NONEXISTENT_OBJECT:
throw QueryRuntimeException("Unexpected error when setting a property.");
}
}
/// Set a property `value` mapped with given `key` on a `record`.
///
/// @throw QueryRuntimeException if value cannot be set as a property value
template <RecordAccessor T>
storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key,
const TypedValue &value) {
try {
if constexpr (std::is_same_v<T, VertexAccessor>) {
const auto maybe_old_value = record->SetPropertyAndValidate(key, storage::v3::TypedToPropertyValue(value));
if (maybe_old_value.HasError()) {
std::visit(utils::Overloaded{[](const storage::v3::Error error) { HandleErrorOnPropertyUpdate(error); },
[&dba](const storage::v3::SchemaViolation &schema_violation) {
HandleSchemaViolation(schema_violation, dba);
}},
maybe_old_value.GetError());
}
return std::move(*maybe_old_value);
} else {
// No validation on edge properties
const auto maybe_old_value = record->SetProperty(key, storage::v3::TypedToPropertyValue(value));
if (maybe_old_value.HasError()) {
HandleErrorOnPropertyUpdate(maybe_old_value.GetError());
}
return std::move(*maybe_old_value);
}
} catch (const expr::TypedValueException &) {
throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
}
}
int64_t QueryTimestamp();
} // namespace memgraph::query::v2

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -13,35 +13,17 @@
#include <type_traits>
#include "io/local_transport/local_transport.hpp"
#include "query/v2/bindings/symbol_table.hpp"
#include "query/v2/common.hpp"
#include "query/v2/metadata.hpp"
#include "query/v2/parameters.hpp"
#include "query/v2/plan/profile.hpp"
//#include "query/v2/trigger.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/shard_request_manager.hpp"
#include "utils/async_timer.hpp"
namespace memgraph::query::v2 {
// Used to store range of ids that are available
// Used for edge id assignment
class IdAllocator {
public:
IdAllocator() = default;
IdAllocator(uint64_t low, uint64_t high) : current_edge_id_{low}, max_id_{high} {};
uint64_t AllocateId() {
MG_ASSERT(current_edge_id_ < max_id_, "Current Edge Id went above max id");
return current_edge_id_++;
}
private:
uint64_t current_edge_id_;
uint64_t max_id_;
};
struct EvaluationContext {
/// Memory for allocations during evaluation of a *single* Pull call.
///
@@ -61,27 +43,20 @@ struct EvaluationContext {
};
inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
RequestRouterInterface *request_router) {
DbAccessor *dba) {
std::vector<storage::v3::PropertyId> properties;
// TODO Fix by using reference
properties.reserve(property_names.size());
if (request_router != nullptr) {
for (const auto &name : property_names) {
properties.push_back(request_router->NameToProperty(name));
}
for (const auto &name : property_names) {
properties.push_back(dba->NameToProperty(name));
}
return properties;
}
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names,
RequestRouterInterface *request_router) {
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names, DbAccessor *dba) {
std::vector<storage::v3::LabelId> labels;
labels.reserve(label_names.size());
// TODO Fix by using reference
if (request_router != nullptr) {
for (const auto &name : label_names) {
labels.push_back(request_router->NameToLabel(name));
}
for (const auto &name : label_names) {
labels.push_back(dba->NameToLabel(name));
}
return labels;
}
@@ -96,9 +71,9 @@ struct ExecutionContext {
plan::ProfilingStats stats;
plan::ProfilingStats *stats_root{nullptr};
ExecutionStats execution_stats;
// TriggerContextCollector *trigger_context_collector{nullptr};
utils::AsyncTimer timer;
RequestRouterInterface *request_router{nullptr};
IdAllocator *edge_ids_alloc;
std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager{nullptr};
};
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");

View File

@@ -12,12 +12,11 @@
#pragma once
#include "bindings/typed_value.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/request_router.hpp"
#include "query/v2/requests.hpp"
namespace memgraph::query::v2 {
inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInterface *request_router) {
inline TypedValue ValueToTypedValue(const msgs::Value &value) {
using Value = msgs::Value;
switch (value.type) {
case Value::Type::Null:
@@ -35,7 +34,7 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
std::vector<TypedValue> dst;
dst.reserve(lst.size());
for (const auto &elem : lst) {
dst.push_back(ValueToTypedValue(elem, request_router));
dst.push_back(ValueToTypedValue(elem));
}
return TypedValue(std::move(dst));
}
@@ -43,23 +42,20 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
const auto &value_map = value.map_v;
std::map<std::string, TypedValue> dst;
for (const auto &[key, val] : value_map) {
dst[key] = ValueToTypedValue(val, request_router);
dst[key] = ValueToTypedValue(val);
}
return TypedValue(std::move(dst));
}
case Value::Type::Vertex:
return TypedValue(accessors::VertexAccessor(
value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, request_router));
return TypedValue(accessors::VertexAccessor(value.vertex_v, {}));
case Value::Type::Edge:
return TypedValue(accessors::EdgeAccessor(value.edge_v, request_router));
return TypedValue(accessors::EdgeAccessor(value.edge_v, {}));
case Value::Type::Path:
break;
}
throw std::runtime_error("Incorrect type in conversion");
}
inline const auto ValueToTypedValueFunctor = [](const msgs::Value &value, RequestRouterInterface *request_router) {
return ValueToTypedValue(value, request_router);
};
inline msgs::Value TypedValueToValue(const TypedValue &value) {
using Value = msgs::Value;
switch (value.type()) {
@@ -95,10 +91,7 @@ inline msgs::Value TypedValueToValue(const TypedValue &value) {
case TypedValue::Type::Edge:
return Value(value.ValueEdge().GetEdge());
case TypedValue::Type::Path:
case TypedValue::Type::LocalTime:
case TypedValue::Type::LocalDateTime:
case TypedValue::Type::Date:
case TypedValue::Type::Duration:
default:
break;
}
throw std::runtime_error("Incorrect type in conversion");

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,12 +11,11 @@
#include "query/v2/cypher_query_interpreter.hpp"
#include "query/v2/bindings/symbol_generator.hpp"
#include "query/v2/request_router.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_HIDDEN_bool(query_v2_cost_planner, true, "Use the cost-estimating query planner.");
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_VALIDATED_int32(query_v2_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
namespace memgraph::query::v2 {
@@ -118,19 +117,19 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
}
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
RequestRouterInterface *request_router,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
auto vertex_counts = plan::MakeVertexCountCache(request_router);
auto vertex_counts = plan::MakeVertexCountCache(db_accessor);
auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_v2_cost_planner);
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
std::move(symbol_table));
}
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
RequestRouterInterface *request_router,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers) {
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
if (plan_cache) {
@@ -146,7 +145,7 @@ std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_stor
}
auto plan = std::make_shared<CachedPlan>(
MakeLogicalPlan(std::move(ast_storage), query, parameters, request_router, predefined_identifiers));
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
if (plan_cache_access) {
plan_cache_access->insert({hash, plan});
}

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -22,9 +22,9 @@
#include "utils/timer.hpp"
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_bool(query_v2_cost_planner);
DECLARE_bool(query_cost_planner);
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_int32(query_v2_plan_cache_ttl);
DECLARE_int32(query_plan_cache_ttl);
namespace memgraph::query::v2 {
@@ -58,7 +58,7 @@ class CachedPlan {
bool IsExpired() const {
// NOLINTNEXTLINE (modernize-use-nullptr)
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_v2_plan_cache_ttl);
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
};
private:
@@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
};
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters &parameters,
RequestRouterInterface *request_router,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers);
/**
@@ -145,7 +145,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
*/
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
const Parameters &parameters, utils::SkipList<PlanCacheEntry> *plan_cache,
RequestRouterInterface *request_router,
DbAccessor *db_accessor,
const std::vector<Identifier *> &predefined_identifiers = {});
} // namespace memgraph::query::v2

View File

@@ -36,7 +36,7 @@
// This cannot be avoided by simple include orderings so we
// simply undefine those macros as we're sure that libkrb5
// won't and can't be used anywhere in the query engine.
#include "storage/v3/shard.hpp"
#include "storage/v3/storage.hpp"
#include "utils/logging.hpp"
#include "utils/result.hpp"
@@ -64,11 +64,24 @@ class EdgeAccessor final {
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetProperty(key, value);
}
storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
return SetProperty(key, storage::v3::PropertyValue());
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
VertexAccessor To() const;
VertexAccessor From() const;
@@ -100,19 +113,51 @@ class VertexAccessor final {
auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
storage::v3::ShardResult<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
storage::v3::ResultSchema<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabelAndValidate(label); }
storage::v3::ResultSchema<bool> AddLabelAndValidate(storage::v3::LabelId label) {
return impl_.AddLabelAndValidate(label);
}
storage::v3::ResultSchema<bool> RemoveLabel(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::ResultSchema<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
return impl_.RemoveLabelAndValidate(label);
}
storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
return impl_.HasLabel(label, view);
}
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
storage::v3::PropertyId key) const {
return impl_.GetProperty(key, view);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> SetPropertyAndValidate(
storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
return impl_.SetPropertyAndValidate(key, value);
}
storage::v3::ResultSchema<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
}
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
return impl_.ClearProperties();
}
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
auto maybe_edges = impl_.InEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -122,7 +167,7 @@ class VertexAccessor final {
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
@@ -130,7 +175,7 @@ class VertexAccessor final {
}
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
auto maybe_edges = impl_.OutEdges(view, edge_types);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
@@ -140,16 +185,16 @@ class VertexAccessor final {
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
const VertexAccessor &dest) const
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
const auto dest_id = dest.impl_.Id(view).GetValue();
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
if (maybe_edges.HasError()) return maybe_edges.GetError();
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
}
storage::v3::ShardResult<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
storage::v3::ShardResult<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
// TODO(jbajic) Fix Remove Gid
static int64_t CypherId() { return 1; }
@@ -174,7 +219,195 @@ inline VertexAccessor EdgeAccessor::From() const { return *static_cast<VertexAcc
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
class DbAccessor final {
storage::v3::Shard::Accessor *accessor_;
class VerticesIterable final {
storage::v3::VerticesIterable iterable_;
public:
class Iterator final {
storage::v3::VerticesIterable::Iterator it_;
public:
explicit Iterator(storage::v3::VerticesIterable::Iterator it) : it_(it) {}
VertexAccessor operator*() const { return VertexAccessor(*it_); }
Iterator &operator++() {
++it_;
return *this;
}
bool operator==(const Iterator &other) const { return it_ == other.it_; }
bool operator!=(const Iterator &other) const { return !(other == *this); }
};
explicit VerticesIterable(storage::v3::VerticesIterable iterable) : iterable_(std::move(iterable)) {}
Iterator begin() { return Iterator(iterable_.begin()); }
Iterator end() { return Iterator(iterable_.end()); }
};
public:
explicit DbAccessor(storage::v3::Shard::Accessor *accessor) : accessor_(accessor) {}
// TODO(jbajic) Fix Remove Gid
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
std::optional<VertexAccessor> FindVertex(uint64_t /*unused*/) { return std::nullopt; }
std::optional<VertexAccessor> FindVertex(storage::v3::PrimaryKey &primary_key, storage::v3::View view) {
auto maybe_vertex = accessor_->FindVertex(primary_key, view);
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
return std::nullopt;
}
VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); }
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) {
return VerticesIterable(accessor_->Vertices(label, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) {
return VerticesIterable(accessor_->Vertices(label, property, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
const storage::v3::PropertyValue &value) {
return VerticesIterable(accessor_->Vertices(label, property, value, view));
}
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) {
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
}
storage::v3::ResultSchema<VertexAccessor> InsertVertexAndValidate(
const storage::v3::LabelId primary_label, const std::vector<storage::v3::LabelId> &labels,
const std::vector<std::pair<storage::v3::PropertyId, storage::v3::PropertyValue>> &properties) {
auto maybe_vertex_acc = accessor_->CreateVertexAndValidate(primary_label, labels, properties);
if (maybe_vertex_acc.HasError()) {
return {std::move(maybe_vertex_acc.GetError())};
}
return VertexAccessor{maybe_vertex_acc.GetValue()};
}
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
const storage::v3::EdgeTypeId &edge_type) {
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
auto maybe_edge = accessor_->CreateEdge(from->impl_.Id(storage::v3::View::NEW).GetValue(),
to->impl_.Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
if (maybe_edge.HasError()) return storage::v3::Result<EdgeAccessor>(maybe_edge.GetError());
return EdgeAccessor(*maybe_edge);
}
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
auto res = accessor_->DeleteEdge(edge->impl_.FromVertex(), edge->impl_.ToVertex(), edge->impl_.Gid());
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<EdgeAccessor>{};
}
return std::make_optional<EdgeAccessor>(*value);
}
storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
VertexAccessor *vertex_accessor) {
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<ReturnType>{};
}
const auto &[vertex, edges] = *value;
std::vector<EdgeAccessor> deleted_edges;
deleted_edges.reserve(edges.size());
std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
}
storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
if (res.HasError()) {
return res.GetError();
}
const auto &value = res.GetValue();
if (!value) {
return std::optional<VertexAccessor>{};
}
return {std::make_optional<VertexAccessor>(*value)};
}
// TODO(jbajic) Query engine should have a map of labels, properties and edge
// types
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); }
const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); }
const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); }
void AdvanceCommand() { accessor_->AdvanceCommand(); }
void Commit() { return accessor_->Commit(coordinator::Hlc{}); }
void Abort() { accessor_->Abort(); }
bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); }
bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const {
return accessor_->LabelPropertyIndexExists(label, prop);
}
int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); }
int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); }
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const {
return accessor_->ApproximateVertexCount(label, property);
}
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
const storage::v3::PropertyValue &value) const {
return accessor_->ApproximateVertexCount(label, property, value);
}
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) const {
return accessor_->ApproximateVertexCount(label, property, lower, upper);
}
storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); }
const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); }
storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); }
};
} // namespace memgraph::query::v2

483
src/query/v2/dump.cpp Normal file
View File

@@ -0,0 +1,483 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/dump.hpp"
#include <iomanip>
#include <limits>
#include <map>
#include <optional>
#include <ostream>
#include <utility>
#include <vector>
#include <fmt/format.h>
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/stream.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/storage.hpp"
#include "utils/algorithm.hpp"
#include "utils/logging.hpp"
#include "utils/string.hpp"
#include "utils/temporal.hpp"
namespace memgraph::query::v2 {
namespace {
// Property that is used to make a difference among vertices. It is added to
// property set of vertices to match edges and removed after the entire graph
// is built.
const char *kInternalPropertyId = "__mg_id__";
// Label that is attached to each vertex and is used for easier creation of
// index on internal property id.
const char *kInternalVertexLabel = "__mg_vertex__";
/// A helper function that escapes label, edge type and property names.
std::string EscapeName(const std::string_view value) {
std::string out;
out.reserve(value.size() + 2);
out.append(1, '`');
for (auto c : value) {
if (c == '`') {
out.append("``");
} else {
out.append(1, c);
}
}
out.append(1, '`');
return out;
}
void DumpPreciseDouble(std::ostream *os, double value) {
// A temporary stream is used to keep precision of the original output
// stream unchanged.
std::ostringstream temp_oss;
temp_oss << std::setprecision(std::numeric_limits<double>::max_digits10) << value;
*os << temp_oss.str();
}
namespace {
void DumpDate(std::ostream &os, const storage::v3::TemporalData &value) {
utils::Date date(value.microseconds);
os << "DATE(\"" << date << "\")";
}
void DumpLocalTime(std::ostream &os, const storage::v3::TemporalData &value) {
utils::LocalTime lt(value.microseconds);
os << "LOCALTIME(\"" << lt << "\")";
}
void DumpLocalDateTime(std::ostream &os, const storage::v3::TemporalData &value) {
utils::LocalDateTime ldt(value.microseconds);
os << "LOCALDATETIME(\"" << ldt << "\")";
}
void DumpDuration(std::ostream &os, const storage::v3::TemporalData &value) {
utils::Duration dur(value.microseconds);
os << "DURATION(\"" << dur << "\")";
}
void DumpTemporalData(std::ostream &os, const storage::v3::TemporalData &value) {
switch (value.type) {
case storage::v3::TemporalType::Date: {
DumpDate(os, value);
return;
}
case storage::v3::TemporalType::LocalTime: {
DumpLocalTime(os, value);
return;
}
case storage::v3::TemporalType::LocalDateTime: {
DumpLocalDateTime(os, value);
return;
}
case storage::v3::TemporalType::Duration: {
DumpDuration(os, value);
return;
}
}
}
} // namespace
void DumpPropertyValue(std::ostream *os, const storage::v3::PropertyValue &value) {
switch (value.type()) {
case storage::v3::PropertyValue::Type::Null:
*os << "Null";
return;
case storage::v3::PropertyValue::Type::Bool:
*os << (value.ValueBool() ? "true" : "false");
return;
case storage::v3::PropertyValue::Type::String:
*os << utils::Escape(value.ValueString());
return;
case storage::v3::PropertyValue::Type::Int:
*os << value.ValueInt();
return;
case storage::v3::PropertyValue::Type::Double:
DumpPreciseDouble(os, value.ValueDouble());
return;
case storage::v3::PropertyValue::Type::List: {
*os << "[";
const auto &list = value.ValueList();
utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); });
*os << "]";
return;
}
case storage::v3::PropertyValue::Type::Map: {
*os << "{";
const auto &map = value.ValueMap();
utils::PrintIterable(*os, map, ", ", [](auto &os, const auto &kv) {
os << EscapeName(kv.first) << ": ";
DumpPropertyValue(&os, kv.second);
});
*os << "}";
return;
}
case storage::v3::PropertyValue::Type::TemporalData: {
DumpTemporalData(*os, value.ValueTemporalData());
return;
}
}
}
void DumpProperties(std::ostream *os, query::v2::DbAccessor *dba,
const std::map<storage::v3::PropertyId, storage::v3::PropertyValue> &store,
std::optional<int64_t> property_id = std::nullopt) {
*os << "{";
if (property_id) {
*os << kInternalPropertyId << ": " << *property_id;
if (store.size() > 0) *os << ", ";
}
utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) {
os << EscapeName(dba->PropertyToName(kv.first)) << ": ";
DumpPropertyValue(&os, kv.second);
});
*os << "}";
}
void DumpVertex(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::VertexAccessor &vertex) {
*os << "CREATE (";
*os << ":" << kInternalVertexLabel;
auto maybe_labels = vertex.Labels(storage::v3::View::OLD);
if (maybe_labels.HasError()) {
switch (maybe_labels.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get labels from a deleted node.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw query::v2::QueryRuntimeException("Unexpected error when getting labels.");
}
}
for (const auto &label : *maybe_labels) {
*os << ":" << EscapeName(dba->LabelToName(label));
}
*os << " ";
auto maybe_props = vertex.Properties(storage::v3::View::OLD);
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from a node that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
}
}
DumpProperties(os, dba, *maybe_props, vertex.CypherId());
*os << ");";
}
void DumpEdge(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::EdgeAccessor &edge) {
*os << "MATCH ";
*os << "(u:" << kInternalVertexLabel << "), ";
*os << "(v:" << kInternalVertexLabel << ")";
*os << " WHERE ";
*os << "u." << kInternalPropertyId << " = " << edge.From().CypherId();
*os << " AND ";
*os << "v." << kInternalPropertyId << " = " << edge.To().CypherId() << " ";
*os << "CREATE (u)-[";
*os << ":" << EscapeName(dba->EdgeTypeToName(edge.EdgeType()));
auto maybe_props = edge.Properties(storage::v3::View::OLD);
if (maybe_props.HasError()) {
switch (maybe_props.GetError()) {
case storage::v3::Error::DELETED_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
case storage::v3::Error::NONEXISTENT_OBJECT:
throw query::v2::QueryRuntimeException("Trying to get properties from an edge that doesn't exist.");
case storage::v3::Error::SERIALIZATION_ERROR:
case storage::v3::Error::VERTEX_HAS_EDGES:
case storage::v3::Error::PROPERTIES_DISABLED:
throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
}
}
if (maybe_props->size() > 0) {
*os << " ";
DumpProperties(os, dba, *maybe_props);
}
*os << "]->(v);";
}
void DumpLabelIndex(std::ostream *os, query::v2::DbAccessor *dba, const storage::v3::LabelId label) {
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";";
}
void DumpLabelPropertyIndex(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
storage::v3::PropertyId property) {
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property))
<< ");";
}
void DumpExistenceConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
storage::v3::PropertyId property) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u."
<< EscapeName(dba->PropertyToName(property)) << ");";
}
void DumpUniqueConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
const std::set<storage::v3::PropertyId> &properties) {
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT ";
utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) {
stream << "u." << EscapeName(dba->PropertyToName(property));
});
*os << " IS UNIQUE;";
}
} // namespace
PullPlanDump::PullPlanDump(DbAccessor *dba)
: dba_(dba),
vertices_iterable_(dba->Vertices(storage::v3::View::OLD)),
pull_chunks_{// Dump all label indices
CreateLabelIndicesPullChunk(),
// Dump all label property indices
CreateLabelPropertyIndicesPullChunk(),
// Create internal index for faster edge creation
CreateInternalIndexPullChunk(),
// Dump all vertices
CreateVertexPullChunk(),
// Dump all edges
CreateEdgePullChunk(),
// Drop the internal index
CreateDropInternalIndexPullChunk(),
// Internal index cleanup
CreateInternalIndexCleanupPullChunk()} {}
bool PullPlanDump::Pull(AnyStream *stream, std::optional<int> n) {
// Iterate all functions that stream some results.
// Each function should return number of results it streamed after it
// finishes. If the function did not finish streaming all the results,
// std::nullopt should be returned because n results have already been sent.
while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) {
const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n);
if (!maybe_streamed_count) {
// n wasn't large enough to stream all the results from the current chunk
break;
}
if (n) {
// chunk finished streaming its results
// subtract number of results streamed in current pull
// so we know how many results we need to stream from future
// chunks.
*n -= *maybe_streamed_count;
}
++current_chunk_index_;
}
return current_chunk_index_ == pull_chunks_.size();
}
PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() {
// Dump all label indices
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of indices vectors
if (!indices_info_) {
indices_info_.emplace(dba_->ListAllIndices());
}
const auto &label = indices_info_->label;
size_t local_counter = 0;
while (global_index < label.size() && (!n || local_counter < *n)) {
std::ostringstream os;
DumpLabelIndex(&os, dba_, label[global_index]);
stream->Result({TypedValue(os.str())});
++global_index;
++local_counter;
}
if (global_index == label.size()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the construction of indices vectors
if (!indices_info_) {
indices_info_.emplace(dba_->ListAllIndices());
}
const auto &label_property = indices_info_->label_property;
size_t local_counter = 0;
while (global_index < label_property.size() && (!n || local_counter < *n)) {
std::ostringstream os;
const auto &label_property_index = label_property[global_index];
DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second);
stream->Result({TypedValue(os.str())});
++global_index;
++local_counter;
}
if (global_index == label_property.size()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() {
return [this](AnyStream *stream, std::optional<int>) mutable -> std::optional<size_t> {
if (vertices_iterable_.begin() != vertices_iterable_.end()) {
std::ostringstream os;
os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
stream->Result({TypedValue(os.str())});
internal_index_created_ = true;
return 1;
}
return 0;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
return [this, maybe_current_iter = std::optional<VertexAccessorIterableIterator>{}](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
// states.
if (!maybe_current_iter) {
maybe_current_iter.emplace(vertices_iterable_.begin());
}
auto &current_iter{*maybe_current_iter};
size_t local_counter = 0;
while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) {
std::ostringstream os;
DumpVertex(&os, dba_, *current_iter);
stream->Result({TypedValue(os.str())});
++local_counter;
++current_iter;
}
if (current_iter == vertices_iterable_.end()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
return [this, maybe_current_vertex_iter = std::optional<VertexAccessorIterableIterator>{},
// we need to save the iterable which contains list of accessor so
// our saved iterator is valid in the next run
maybe_edge_iterable = std::shared_ptr<EdgeAccessorIterable>{nullptr},
maybe_current_edge_iter = std::optional<EdgeAccessorIterableIterator>{}](
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
// Delay the call of begin() function
// If multiple begins are called before an iteration,
// one iteration will make the rest of iterators be in undefined
// states.
if (!maybe_current_vertex_iter) {
maybe_current_vertex_iter.emplace(vertices_iterable_.begin());
}
auto &current_vertex_iter{*maybe_current_vertex_iter};
size_t local_counter = 0U;
for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) {
const auto &vertex = *current_vertex_iter;
// If we have a saved iterable from a previous pull
// we need to use the same iterable
if (!maybe_edge_iterable) {
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::v3::View::OLD));
}
auto &maybe_edges = *maybe_edge_iterable;
MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin();
for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) {
std::ostringstream os;
DumpEdge(&os, dba_, *current_edge_iter);
stream->Result({TypedValue(os.str())});
++local_counter;
}
if (current_edge_iter != maybe_edges->end()) {
maybe_current_edge_iter.emplace(current_edge_iter);
return std::nullopt;
}
maybe_current_edge_iter = std::nullopt;
maybe_edge_iterable = nullptr;
}
if (current_vertex_iter == vertices_iterable_.end()) {
return local_counter;
}
return std::nullopt;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() {
return [this](AnyStream *stream, std::optional<int>) {
if (internal_index_created_) {
std::ostringstream os;
os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
stream->Result({TypedValue(os.str())});
return 1;
}
return 0;
};
}
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() {
return [this](AnyStream *stream, std::optional<int>) {
if (internal_index_created_) {
std::ostringstream os;
os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";";
stream->Result({TypedValue(os.str())});
return 1;
}
return 0;
};
}
void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); }
} // namespace memgraph::query::v2

63
src/query/v2/dump.hpp Normal file
View File

@@ -0,0 +1,63 @@
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <ostream>
#include "query/v2/db_accessor.hpp"
#include "query/v2/stream.hpp"
#include "storage/v3/storage.hpp"
namespace memgraph::query::v2 {
void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream);
struct PullPlanDump {
explicit PullPlanDump(query::v2::DbAccessor *dba);
/// Pull the dump results lazily
/// @return true if all results were returned, false otherwise
bool Pull(AnyStream *stream, std::optional<int> n);
private:
query::v2::DbAccessor *dba_ = nullptr;
std::optional<storage::v3::IndicesInfo> indices_info_ = std::nullopt;
using VertexAccessorIterable = decltype(std::declval<query::v2::DbAccessor>().Vertices(storage::v3::View::OLD));
using VertexAccessorIterableIterator = decltype(std::declval<VertexAccessorIterable>().begin());
using EdgeAccessorIterable = decltype(std::declval<VertexAccessor>().OutEdges(storage::v3::View::OLD));
using EdgeAccessorIterableIterator = decltype(std::declval<EdgeAccessorIterable>().GetValue().begin());
VertexAccessorIterable vertices_iterable_;
bool internal_index_created_ = false;
size_t current_chunk_index_ = 0;
using PullChunk = std::function<std::optional<size_t>(AnyStream *stream, std::optional<int> n)>;
// We define every part of the dump query in a self contained function.
// Each functions is responsible of keeping track of its execution status.
// If a function did finish its execution, it should return number of results
// it streamed so we know how many rows should be pulled from the next
// function, otherwise std::nullopt is returned.
std::vector<PullChunk> pull_chunks_;
PullChunk CreateLabelIndicesPullChunk();
PullChunk CreateLabelPropertyIndicesPullChunk();
PullChunk CreateInternalIndexPullChunk();
PullChunk CreateVertexPullChunk();
PullChunk CreateEdgePullChunk();
PullChunk CreateDropInternalIndexPullChunk();
PullChunk CreateInternalIndexCleanupPullChunk();
};
} // namespace memgraph::query::v2

View File

@@ -224,4 +224,12 @@ class VersionInfoInMulticommandTxException : public QueryException {
: QueryException("Version info query not allowed in multicommand transactions.") {}
};
/**
* An exception for an illegal operation that violates schema
*/
class SchemaViolationException : public QueryRuntimeException {
public:
using QueryRuntimeException::QueryRuntimeException;
};
} // namespace memgraph::query::v2

View File

@@ -18,15 +18,12 @@
#include <vector>
#include "query/v2/bindings/ast_visitor.hpp"
#include "common/types.hpp"
#include "query/v2/bindings/symbol.hpp"
#include "functions/awesome_memgraph_functions.hpp"
#include "query/v2/interpret/awesome_memgraph_functions.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/path.hpp"
#include "query/v2/request_router.hpp"
#include "utils/typeinfo.hpp"
#include "query/v2/conversions.hpp"
cpp<#
@@ -838,15 +835,13 @@ cpp<#
:slk-load (slk-load-ast-vector "Expression"))
(function-name "std::string" :scope :public)
(function "std::function<TypedValue(const TypedValue *, int64_t,
const functions::FunctionContext<RequestRouterInterface> &)>"
const FunctionContext &)>"
:scope :public
:dont-save t
:clone :copy
:slk-load (lambda (member)
#>cpp
self->${member} = functions::NameToFunction<TypedValue,
functions::FunctionContext<RequestRouterInterface>,
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(self->function_name_);
self->${member} = query::v2::NameToFunction(self->function_name_);
cpp<#)))
(:public
#>cpp
@@ -869,8 +864,7 @@ cpp<#
const std::vector<Expression *> &arguments)
: arguments_(arguments),
function_name_(function_name),
function_(functions::NameToFunction<TypedValue, functions::FunctionContext<RequestRouterInterface>,
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(function_name_)) {
function_(NameToFunction(function_name_)) {
if (!function_) {
throw SemanticException("Function '{}' doesn't exist.", function_name);
}

File diff suppressed because it is too large Load Diff

View File

@@ -16,13 +16,12 @@
#include <unordered_map>
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/db_accessor.hpp"
#include "storage/v3/view.hpp"
#include "utils/memory.hpp"
namespace memgraph::query::v2 {
class RequestRouterInterface;
class DbAccessor;
namespace {
const char kStartsWith[] = "STARTSWITH";
@@ -32,9 +31,7 @@ const char kId[] = "ID";
} // namespace
struct FunctionContext {
// TODO(kostasrim) consider optional here. RequestRouter does not exist on the storage.
// DbAccessor *db_accessor;
RequestRouterInterface *request_router;
DbAccessor *db_accessor;
utils::MemoryResource *memory;
int64_t timestamp;
std::unordered_map<std::string, int64_t> *counters;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -19,13 +19,9 @@
#include <cstdint>
#include <functional>
#include <limits>
#include <memory>
#include <optional>
#include "coordinator/coordinator_client.hpp"
#include "expr/ast/ast_visitor.hpp"
#include "io/local_transport/local_system.hpp"
#include "io/local_transport/local_transport.hpp"
#include "memory/memory_control.hpp"
#include "parser/opencypher/parser.hpp"
#include "query/v2/bindings/eval.hpp"
@@ -37,17 +33,17 @@
#include "query/v2/context.hpp"
#include "query/v2/cypher_query_interpreter.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/dump.hpp"
#include "query/v2/exceptions.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/frontend/semantic/required_privileges.hpp"
#include "query/v2/metadata.hpp"
#include "query/v2/multiframe.hpp"
#include "query/v2/plan/planner.hpp"
#include "query/v2/plan/profile.hpp"
#include "query/v2/plan/vertex_count_cache.hpp"
#include "query/v2/request_router.hpp"
#include "storage/v3/property_value.hpp"
#include "storage/v3/shard.hpp"
#include "storage/v3/storage.hpp"
#include "utils/algorithm.hpp"
#include "utils/csv_parsing.hpp"
#include "utils/event_counter.hpp"
@@ -64,11 +60,7 @@
#include "utils/tsc.hpp"
#include "utils/variant_helpers.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
namespace EventCounter {
extern Event ReadQuery;
extern Event WriteQuery;
extern Event ReadWriteQuery;
@@ -78,7 +70,6 @@ extern const Event LabelPropertyIndexCreated;
extern const Event StreamsCreated;
extern const Event TriggersCreated;
} // namespace EventCounter
namespace memgraph::query::v2 {
@@ -123,6 +114,17 @@ std::optional<TResult> GetOptionalValue(query::v2::Expression *expression, Expre
return {};
};
std::optional<std::string> GetOptionalStringValue(query::v2::Expression *expression, ExpressionEvaluator &evaluator) {
if (expression != nullptr) {
auto value = expression->Accept(evaluator);
MG_ASSERT(value.IsNull() || value.IsString());
if (value.IsString()) {
return {std::string(value.ValueString().begin(), value.ValueString().end())};
}
}
return {};
};
class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
public:
explicit ReplQueryHandler(storage::v3::Shard * /*db*/) {}
@@ -149,18 +151,18 @@ class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
/// @throw QueryRuntimeException if an error ocurred.
Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Parameters &parameters,
RequestRouterInterface *request_router) {
DbAccessor *db_accessor) {
// Empty frame for evaluation of password expression. This is OK since
// password should be either null or string literal and it's evaluation
// should not depend on frame.
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
// the argument to Callback.
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD);
std::string username = auth_query->user_;
std::string rolename = auth_query->role_;
@@ -318,16 +320,16 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
}
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &parameters,
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
InterpreterContext *interpreter_context, DbAccessor *db_accessor,
std::vector<Notification> *notifications) {
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
// the argument to Callback.
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD);
Callback callback;
switch (repl_query->action_) {
@@ -453,9 +455,23 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
}
}
Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &parameters,
RequestRouterInterface *request_router) {
expr::Frame frame(0);
std::optional<std::string> StringPointerToOptional(const std::string *str) {
return str == nullptr ? std::nullopt : std::make_optional(*str);
}
std::vector<std::string> EvaluateTopicNames(ExpressionEvaluator &evaluator,
std::variant<Expression *, std::vector<std::string>> topic_variant) {
return std::visit(utils::Overloaded{[&](Expression *expression) {
auto topic_names = expression->Accept(evaluator);
MG_ASSERT(topic_names.IsString());
return utils::Split(topic_names.ValueString(), ",");
},
[&](std::vector<std::string> topic_names) { return topic_names; }},
std::move(topic_variant));
}
Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &parameters, DbAccessor *db_accessor) {
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
@@ -464,7 +480,7 @@ Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters &param
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
.count();
evaluation_context.parameters = parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD);
Callback callback;
switch (setting_query->action_) {
@@ -655,21 +671,16 @@ struct PullPlanVector {
struct PullPlan {
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters &parameters, bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router = nullptr,
// TriggerContextCollector *trigger_context_collector = nullptr,
std::optional<size_t> memory_limit = {});
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary);
std::optional<plan::ProfilingStatsWithTotalTime> PullMultiple(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary);
private:
std::shared_ptr<CachedPlan> plan_ = nullptr;
plan::UniqueCursorPtr cursor_ = nullptr;
expr::FrameWithValidity frame_;
MultiFrame multi_frame_;
expr::Frame<TypedValue> frame_;
ExecutionContext ctx_;
std::optional<size_t> memory_limit_;
@@ -689,137 +700,29 @@ struct PullPlan {
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters &parameters, const bool is_profile_query,
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router, const std::optional<size_t> memory_limit)
const std::optional<size_t> memory_limit)
// TriggerContextCollector *trigger_context_collector, const std::optional<size_t> memory_limit)
: plan_(plan),
cursor_(plan->plan().MakeCursor(execution_memory)),
frame_(plan->symbol_table().max_position(), execution_memory),
multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
memory_limit_(memory_limit) {
ctx_.db_accessor = dba;
ctx_.symbol_table = plan->symbol_table();
ctx_.evaluation_context.timestamp = QueryTimestamp();
ctx_.evaluation_context.parameters = parameters;
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, request_router);
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, request_router);
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, dba);
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, dba);
if (interpreter_context->config.execution_timeout_sec > 0) {
ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec};
}
ctx_.is_shutting_down = &interpreter_context->is_shutting_down;
ctx_.is_profile_query = is_profile_query;
ctx_.request_router = request_router;
ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc;
}
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary) {
// Set up temporary memory for a single Pull. Initial memory comes from the
// stack. 256 KiB should fit on the stack and should be more than enough for a
// single `Pull`.
MG_ASSERT(!n.has_value(), "should pull all!");
static constexpr size_t stack_size = 256UL * 1024UL;
char stack_data[stack_size];
utils::ResourceWithOutOfMemoryException resource_with_exception;
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
// We can throw on every query because a simple queries for deleting will use only
// the stack allocated buffer.
// Also, we want to throw only when the query engine requests more memory and not the storage
// so we add the exception to the allocator.
// TODO (mferencevic): Tune the parameters accordingly.
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
if (memory_limit_) {
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
ctx_.evaluation_context.memory = &*maybe_limited_resource;
} else {
ctx_.evaluation_context.memory = &pool_memory;
}
// Returns true if a result was pulled.
const auto pull_result = [&]() -> bool { return cursor_->PullMultiple(multi_frame_, ctx_); };
const auto stream_values = [&output_symbols, &stream](const Frame &frame) {
// TODO: The streamed values should also probably use the above memory.
std::vector<TypedValue> values;
values.reserve(output_symbols.size());
for (const auto &symbol : output_symbols) {
values.emplace_back(frame[symbol]);
}
stream->Result(values);
};
// Get the execution time of all possible result pulls and streams.
utils::Timer timer;
int i = 0;
if (has_unsent_results_ && !output_symbols.empty()) {
// stream unsent results from previous pull
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
stream_values(frame);
frame.MakeInvalid();
++i;
if (i == n) {
break;
}
}
}
for (; !n || i < n;) {
if (!pull_result()) {
break;
}
if (!output_symbols.empty()) {
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
stream_values(frame);
frame.MakeInvalid();
++i;
if (i == n) {
break;
}
}
} else {
multi_frame_.MakeAllFramesInvalid();
}
}
// If we finished because we streamed the requested n results,
// we try to pull the next result to see if there is more.
// If there is additional result, we leave the pulled result in the frame
// and set the flag to true.
has_unsent_results_ = i == n && pull_result();
execution_time_ += timer.Elapsed();
if (has_unsent_results_) {
return std::nullopt;
}
summary->insert_or_assign("plan_execution_time", execution_time_.count());
// We are finished with pulling all the data, therefore we can send any
// metadata about the results i.e. notifications and statistics
const bool is_any_counter_set =
std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(),
[](const auto &counter) { return counter > 0; });
if (is_any_counter_set) {
std::map<std::string, TypedValue> stats;
for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) {
stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]);
}
summary->insert_or_assign("stats", std::move(stats));
}
cursor_->Shutdown();
ctx_.profile_execution_time = execution_time_;
return GetStatsWithTotalTime(ctx_);
// ctx_.trigger_context_collector = trigger_context_collector;
}
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
const std::vector<Symbol> &output_symbols,
std::map<std::string, TypedValue> *summary) {
if (FLAGS_use_multi_frame) {
return PullMultiple(stream, n, output_symbols, summary);
}
// Set up temporary memory for a single Pull. Initial memory comes from the
// stack. 256 KiB should fit on the stack and should be more than enough for a
// single `Pull`.
@@ -910,26 +813,13 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
} // namespace
InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config,
const std::filesystem::path & /*data_directory*/,
std::unique_ptr<RequestRouterFactory> request_router_factory,
coordinator::Address coordinator_addr)
: db(db),
config(config),
coordinator_address{coordinator_addr},
request_router_factory_{std::move(request_router_factory)} {}
const std::filesystem::path &data_directory)
// : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory /
// "streams"} {}
: db(db), config(config) {}
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
request_router_ =
interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address);
// Get edge ids
const auto edge_ids_alloc_min_max_pair =
request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address);
if (edge_ids_alloc_min_max_pair) {
interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second};
}
}
PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) {
@@ -942,6 +832,14 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
}
in_explicit_transaction_ = true;
expect_rollback_ = false;
db_accessor_ = std::make_unique<storage::v3::Shard::Accessor>(
interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride()));
execution_db_accessor_.emplace(db_accessor_.get());
// if (interpreter_context_->trigger_store.HasTriggers()) {
// trigger_context_collector_.emplace(interpreter_context_->trigger_store.GetEventTypes());
// }
};
} else if (query_upper == "COMMIT") {
handler = [this] {
@@ -988,17 +886,16 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
RequestRouterInterface *request_router) {
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications) {
// TriggerContextCollector *trigger_context_collector = nullptr) {
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parsed_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::v3::View::OLD);
const auto memory_limit =
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
if (memory_limit) {
@@ -1013,9 +910,10 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
"convert the parsed row values to the appropriate type. This can be done using the built-in "
"conversion functions such as ToInteger, ToFloat, ToBoolean etc.");
}
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
parsed_query.parameters,
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
summary->insert_or_assign("cost_estimate", plan->cost());
auto rw_type_checker = plan::ReadWriteTypeChecker();
@@ -1034,7 +932,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
}
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
execution_memory, request_router, memory_limit);
execution_memory, memory_limit);
// execution_memory, trigger_context_collector, memory_limit);
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
@@ -1048,7 +946,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
}
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
InterpreterContext *interpreter_context, DbAccessor *dba,
utils::MemoryResource *execution_memory) {
const std::string kExplainQueryStart = "explain ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
@@ -1067,20 +965,19 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
auto cypher_query_plan =
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
cypher_query, parsed_inner_query.parameters,
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
std::stringstream printed_plan;
plan::PrettyPrint(*request_router, &cypher_query_plan->plan(), &printed_plan);
plan::PrettyPrint(*dba, &cypher_query_plan->plan(), &printed_plan);
std::vector<std::vector<TypedValue>> printed_plan_rows;
for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) {
printed_plan_rows.push_back(std::vector<TypedValue>{TypedValue(row)});
}
summary->insert_or_assign("explain", plan::PlanToJson(*request_router, &cypher_query_plan->plan()).dump());
summary->insert_or_assign("explain", plan::PlanToJson(*dba, &cypher_query_plan->plan()).dump());
return PreparedQuery{{"QUERY PLAN"},
std::move(parsed_query.required_privileges),
@@ -1096,8 +993,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router = nullptr) {
DbAccessor *dba, utils::MemoryResource *execution_memory) {
const std::string kProfileQueryStart = "profile ";
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
@@ -1135,26 +1031,25 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
expr::Frame frame(0);
expr::Frame<TypedValue> frame(0);
SymbolTable symbol_table;
EvaluationContext evaluation_context;
evaluation_context.timestamp = QueryTimestamp();
evaluation_context.parameters = parsed_inner_query.parameters;
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::v3::View::OLD);
const auto memory_limit =
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
auto cypher_query_plan =
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
cypher_query, parsed_inner_query.parameters,
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
auto cypher_query_plan = CypherQueryToPlan(
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
auto rw_type_checker = plan::ReadWriteTypeChecker();
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME", "CUSTOM DATA"},
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
std::move(parsed_query.required_privileges),
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
summary, dba, interpreter_context, execution_memory, memory_limit, request_router,
summary, dba, interpreter_context, execution_memory, memory_limit,
// We want to execute the query we are profiling lazily, so we delay
// the construction of the corresponding context.
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
@@ -1163,7 +1058,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
// No output symbols are given so that nothing is streamed.
if (!stats_and_total_time) {
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
execution_memory, request_router, memory_limit)
execution_memory, memory_limit)
.Pull(stream, {}, {}, summary);
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
}
@@ -1182,7 +1077,16 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
utils::MemoryResource *execution_memory) {
throw QueryRuntimeException("Dump query is not supported!");
return PreparedQuery{{"QUERY"},
std::move(parsed_query.required_privileges),
[pull_plan = std::make_shared<PullPlanDump>(dba)](
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
if (pull_plan->Pull(stream, n)) {
return QueryHandlerResult::COMMIT;
}
return std::nullopt;
},
RWType::R};
}
PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
@@ -1289,15 +1193,14 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
DbAccessor *dba, utils::MemoryResource *execution_memory,
RequestRouterInterface *request_router) {
DbAccessor *dba, utils::MemoryResource *execution_memory) {
if (in_explicit_transaction) {
throw UserModificationInMulticommandTxException();
}
auto *auth_query = utils::Downcast<AuthQuery>(parsed_query.query);
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, request_router);
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, dba);
SymbolTable symbol_table;
std::vector<Symbol> output_symbols;
@@ -1326,14 +1229,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
std::vector<Notification> *notifications, InterpreterContext *interpreter_context,
RequestRouterInterface *request_router) {
DbAccessor *dba) {
if (in_explicit_transaction) {
throw ReplicationModificationInMulticommandTxException();
}
auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context,
request_router, notifications);
auto callback =
HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context, dba, notifications);
return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -1421,15 +1324,14 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
throw SemanticException("CreateSnapshot query is not supported!");
}
PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
RequestRouterInterface *request_router) {
PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, DbAccessor *dba) {
if (in_explicit_transaction) {
throw SettingConfigInMulticommandTxException{};
}
auto *setting_query = utils::Downcast<SettingQuery>(parsed_query.query);
MG_ASSERT(setting_query);
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, request_router);
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, dba);
return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges),
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
@@ -1623,10 +1525,15 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
ParsedQuery parsed_query =
ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query);
query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count();
// Some queries require an active transaction in order to be prepared.
if (!in_explicit_transaction_ &&
(utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ExplainQuery>(parsed_query.query) ||
utils::Downcast<ProfileQuery>(parsed_query.query))) {
request_router_->StartTransaction();
utils::Downcast<ProfileQuery>(parsed_query.query) || utils::Downcast<DumpQuery>(parsed_query.query) ||
utils::Downcast<TriggerQuery>(parsed_query.query))) {
db_accessor_ = std::make_unique<storage::v3::Shard::Accessor>(
interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride()));
execution_db_accessor_.emplace(db_accessor_.get());
}
utils::Timer planning_timer;
@@ -1635,14 +1542,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*execution_db_accessor_, &query_execution->execution_memory,
&query_execution->notifications, request_router_.get());
&query_execution->notifications);
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
&*request_router_, &query_execution->execution_memory_with_exception);
&*execution_db_accessor_, &query_execution->execution_memory_with_exception);
} else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, request_router_.get());
&query_execution->execution_memory_with_exception);
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
&query_execution->execution_memory);
@@ -1652,7 +1559,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<AuthQuery>(parsed_query.query)) {
prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, &*execution_db_accessor_,
&query_execution->execution_memory_with_exception, request_router_.get());
&query_execution->execution_memory_with_exception);
} else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
interpreter_context_, interpreter_context_->db,
@@ -1663,7 +1570,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
} else if (utils::Downcast<ReplicationQuery>(parsed_query.query)) {
prepared_query =
PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
interpreter_context_, request_router_.get());
interpreter_context_, &*execution_db_accessor_);
} else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_,
&*execution_db_accessor_);
@@ -1680,7 +1587,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
prepared_query =
PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
} else if (utils::Downcast<SettingQuery>(parsed_query.query)) {
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, request_router_.get());
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_);
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
} else if (utils::Downcast<SchemaQuery>(parsed_query.query)) {
@@ -1721,7 +1628,6 @@ void Interpreter::Commit() {
// For now, we will not check if there are some unfinished queries.
// We should document clearly that all results should be pulled to complete
// a query.
request_router_->Commit();
if (!db_accessor_) return;
const auto reset_necessary_members = [this]() {

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -12,11 +12,7 @@
#pragma once
#include <gflags/gflags.h>
#include <cstdint>
#include "coordinator/coordinator.hpp"
#include "coordinator/coordinator_client.hpp"
#include "io/transport.hpp"
#include "query/v2/auth_checker.hpp"
#include "query/v2/bindings/cypher_main_visitor.hpp"
#include "query/v2/bindings/typed_value.hpp"
@@ -30,7 +26,6 @@
#include "query/v2/metadata.hpp"
#include "query/v2/plan/operator.hpp"
#include "query/v2/plan/read_write_type_checker.hpp"
#include "query/v2/requests.hpp"
#include "query/v2/stream.hpp"
#include "storage/v3/isolation_level.hpp"
#include "storage/v3/name_id_mapper.hpp"
@@ -170,9 +165,7 @@ struct PreparedQuery {
*/
struct InterpreterContext {
explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
const std::filesystem::path &data_directory,
std::unique_ptr<RequestRouterFactory> request_router_factory,
coordinator::Address coordinator_addr);
const std::filesystem::path &data_directory);
storage::v3::Shard *db;
@@ -186,26 +179,22 @@ struct InterpreterContext {
utils::SkipList<PlanCacheEntry> plan_cache;
const InterpreterConfig config;
IdAllocator edge_ids_alloc;
coordinator::Address coordinator_address;
std::unique_ptr<RequestRouterFactory> request_router_factory_;
storage::v3::LabelId NameToLabelId(std::string_view label_name) {
return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name));
return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
}
storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name));
return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
}
storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name));
return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
}
private:
// TODO Replace with local map of labels, properties and edge type ids
storage::v3::NameIdMapper query_id_mapper_;
storage::v3::NameIdMapper query_id_mapper;
};
/// Function that is used to tell all active interpreters that they should stop
@@ -294,16 +283,11 @@ class Interpreter final {
*/
void Abort();
const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
request_router_->InstallSimulatorTicker(tick_simulator);
}
private:
struct QueryExecution {
std::optional<PreparedQuery> prepared_query;
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
std::optional<PreparedQuery> prepared_query;
std::map<std::string, TypedValue> summary;
std::vector<Notification> notifications;
@@ -340,10 +324,9 @@ class Interpreter final {
// This cannot be std::optional because we need to move this accessor later on into a lambda capture
// which is assigned to std::function. std::function requires every object to be copyable, so we
// move this unique_ptr into a shared_ptr.
// move this unique_ptr into a shrared_ptr.
std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
std::optional<DbAccessor> execution_db_accessor_;
std::unique_ptr<RequestRouterInterface> request_router_;
bool in_explicit_transaction_{false};
bool expect_rollback_{false};

View File

@@ -1,153 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#include "query/v2/multiframe.hpp"
#include <algorithm>
#include <iterator>
#include "query/v2/bindings/frame.hpp"
#include "utils/pmr/vector.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
namespace memgraph::query::v2 {
static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
: frames_(utils::pmr::vector<FrameWithValidity>(
number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
MG_ASSERT(number_of_frames > 0);
}
MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
// NOLINTNEXTLINE (bugprone-exception-escape)
MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
FrameWithValidity &MultiFrame::GetFirstFrame() {
MG_ASSERT(!frames_.empty());
return frames_.front();
}
void MultiFrame::MakeAllFramesInvalid() noexcept {
std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
}
bool MultiFrame::HasValidFrame() const noexcept {
return std::any_of(frames_.begin(), frames_.end(), [](const auto &frame) { return frame.IsValid(); });
}
bool MultiFrame::HasInvalidFrame() const noexcept {
return std::any_of(frames_.rbegin(), frames_.rend(), [](const auto &frame) { return !frame.IsValid(); });
}
// NOLINTNEXTLINE (bugprone-exception-escape)
void MultiFrame::DefragmentValidFrames() noexcept {
static constexpr auto kIsValid = [](const FrameWithValidity &frame) { return frame.IsValid(); };
static constexpr auto kIsInvalid = [](const FrameWithValidity &frame) { return !frame.IsValid(); };
auto first_invalid_frame = std::find_if(frames_.begin(), frames_.end(), kIsInvalid);
auto following_first_valid = std::find_if(first_invalid_frame, frames_.end(), kIsValid);
while (first_invalid_frame != frames_.end() && following_first_valid != frames_.end()) {
std::swap(*first_invalid_frame, *following_first_valid);
first_invalid_frame++;
first_invalid_frame = std::find_if(first_invalid_frame, frames_.end(), kIsInvalid);
following_first_valid++;
following_first_valid = std::find_if(following_first_valid, frames_.end(), kIsValid);
}
}
ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
/*
From: https://en.cppreference.com/w/cpp/algorithm/find
Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
find_if searches for an element for which predicate p returns true
Return value
Iterator to the first element satisfying the condition or last if no such element is found.
-> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
*/
auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
[](const auto &frame) { return !frame.IsValid(); });
after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
}
ValidFramesReader::Iterator ValidFramesReader::begin() {
if (multiframe_->frames_[0].IsValid()) {
return Iterator{&multiframe_->frames_[0]};
}
return end();
}
ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
ValidFramesModifier::Iterator ValidFramesModifier::begin() {
if (multiframe_->frames_[0].IsValid()) {
return Iterator{&multiframe_->frames_[0], *this};
}
return end();
}
ValidFramesModifier::Iterator ValidFramesModifier::end() {
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
}
ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
// NOLINTNEXTLINE (bugprone-exception-escape)
ValidFramesConsumer::~ValidFramesConsumer() noexcept {
// TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
// cost too much to store it
multiframe_->DefragmentValidFrames();
}
ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
if (multiframe_->frames_[0].IsValid()) {
return Iterator{&multiframe_->frames_[0], *this};
}
return end();
}
ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
}
InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
for (auto &frame : multiframe_->frames_) {
if (!frame.IsValid()) {
return Iterator{&frame};
}
}
return end();
}
InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
}
} // namespace memgraph::query::v2

View File

@@ -1,308 +0,0 @@
// Copyright 2023 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
// License, and you may not use this file except in compliance with the Business Source License.
//
// As of the Change Date specified in that file, in accordance with
// the Business Source License, use of this software will be governed
// by the Apache License, Version 2.0, included in the file
// licenses/APL.txt.
#pragma once
#include <iterator>
#include <gflags/gflags.h>
#include "query/v2/bindings/frame.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
DECLARE_uint64(default_multi_frame_size);
namespace memgraph::query::v2 {
class ValidFramesConsumer;
class ValidFramesModifier;
class ValidFramesReader;
class InvalidFramesPopulator;
class MultiFrame {
public:
friend class ValidFramesConsumer;
friend class ValidFramesModifier;
friend class ValidFramesReader;
friend class InvalidFramesPopulator;
MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
~MultiFrame() = default;
// Assigning and moving the MultiFrame is not allowed if any accessor from the above ones are alive.
MultiFrame(const MultiFrame &other);
MultiFrame(MultiFrame &&other) noexcept;
MultiFrame &operator=(const MultiFrame &other) = delete;
MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
* state in the MultiFrame.
* Iteration goes in a deterministic order.
* One can't modify the validity of the Frame nor its content with this implementation.
*/
ValidFramesReader GetValidFramesReader();
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
* state in the MultiFrame.
* Iteration goes in a deterministic order.
* One can't modify the validity of the Frame with this implementation. One can modify its content.
*/
ValidFramesModifier GetValidFramesModifier();
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
* state in the MultiFrame.
* Iteration goes in a deterministic order.
* One can modify the validity of the Frame with this implementation.
* If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
* this is faster.
*/
ValidFramesConsumer GetValidFramesConsumer();
/*
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
* invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
* the Frame with this implementation.
*/
InvalidFramesPopulator GetInvalidFramesPopulator();
/**
* Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
* using the iterators instead.
* The Frame can be valid or invalid.
*/
FrameWithValidity &GetFirstFrame();
void MakeAllFramesInvalid() noexcept;
bool HasValidFrame() const noexcept;
bool HasInvalidFrame() const noexcept;
inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
private:
void DefragmentValidFrames() noexcept;
utils::pmr::vector<FrameWithValidity> frames_;
};
class ValidFramesReader {
public:
explicit ValidFramesReader(MultiFrame &multiframe);
~ValidFramesReader() = default;
ValidFramesReader(const ValidFramesReader &other) = delete;
ValidFramesReader(ValidFramesReader &&other) noexcept = default;
ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = const Frame;
using pointer = value_type *;
using reference = const Frame &;
Iterator() = default;
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
Iterator &operator++() {
ptr_++;
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
ptr_++;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
};
Iterator begin();
Iterator end();
private:
FrameWithValidity *after_last_valid_frame_;
MultiFrame *multiframe_;
};
class ValidFramesModifier {
public:
explicit ValidFramesModifier(MultiFrame &multiframe);
~ValidFramesModifier() = default;
ValidFramesModifier(const ValidFramesModifier &other) = delete;
ValidFramesModifier(ValidFramesModifier &&other) noexcept = default;
ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = Frame;
using pointer = value_type *;
using reference = Frame &;
Iterator() = default;
Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
// Prefix increment
Iterator &operator++() {
do {
ptr_++;
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
++*this;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
ValidFramesModifier *iterator_wrapper_{nullptr};
};
Iterator begin();
Iterator end();
private:
MultiFrame *multiframe_;
};
class ValidFramesConsumer {
public:
explicit ValidFramesConsumer(MultiFrame &multiframe);
~ValidFramesConsumer() noexcept;
ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default;
ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = FrameWithValidity;
using pointer = value_type *;
using reference = FrameWithValidity &;
Iterator() = default;
Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
Iterator &operator++() {
do {
ptr_++;
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
++*this;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
ValidFramesConsumer *iterator_wrapper_{nullptr};
};
Iterator begin();
Iterator end();
private:
MultiFrame *multiframe_;
};
class InvalidFramesPopulator {
public:
explicit InvalidFramesPopulator(MultiFrame &multiframe);
~InvalidFramesPopulator() = default;
InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = default;
InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = default;
struct Iterator {
using iterator_category = std::forward_iterator_tag;
using difference_type = std::ptrdiff_t;
using value_type = FrameWithValidity;
using pointer = value_type *;
using reference = FrameWithValidity &;
Iterator() = default;
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
reference operator*() const { return *ptr_; }
pointer operator->() { return ptr_; }
Iterator &operator++() {
ptr_->MakeValid();
ptr_++;
return *this;
}
// NOLINTNEXTLINE(cert-dcl21-cpp)
Iterator operator++(int) {
auto old = *this;
++ptr_;
return old;
}
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
private:
FrameWithValidity *ptr_{nullptr};
};
Iterator begin();
Iterator end();
private:
MultiFrame *multiframe_;
};
} // namespace memgraph::query::v2

View File

@@ -12,8 +12,8 @@
#pragma once
#include <algorithm>
#include <unordered_map>
#include <utility>
#include <vector>
#include "storage/v3/property_value.hpp"
#include "utils/logging.hpp"
@@ -32,7 +32,7 @@ struct Parameters {
* @param position Token position in query of value.
* @param value
*/
void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace(position, value); }
void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace_back(position, value); }
/**
* Returns the value found for the given token position.
@@ -41,11 +41,23 @@ struct Parameters {
* @return Value for the given token position.
*/
const storage::v3::PropertyValue &AtTokenPosition(int position) const {
auto found = storage_.find(position);
auto found = std::find_if(storage_.begin(), storage_.end(), [&](const auto &a) { return a.first == position; });
MG_ASSERT(found != storage_.end(), "Token position must be present in container");
return found->second;
}
/**
* Returns the position-th stripped value. Asserts that this
* container has at least (position + 1) elements.
*
* @param position Which stripped param is sought.
* @return Token position and value for sought param.
*/
const std::pair<int, storage::v3::PropertyValue> &At(int position) const {
MG_ASSERT(position < static_cast<int>(storage_.size()), "Invalid position");
return storage_[position];
}
/** Returns the number of arguments in this container */
auto size() const { return storage_.size(); }
@@ -53,7 +65,7 @@ struct Parameters {
auto end() const { return storage_.end(); }
private:
std::unordered_map<int, storage::v3::PropertyValue> storage_;
std::vector<std::pair<int, storage::v3::PropertyValue>> storage_;
};
} // namespace memgraph::query::v2

View File

@@ -14,8 +14,6 @@
#include <functional>
#include <utility>
#include "query/db_accessor.hpp"
#include "query/v2/accessors.hpp"
#include "query/v2/db_accessor.hpp"
#include "utils/logging.hpp"
#include "utils/memory.hpp"
@@ -30,8 +28,6 @@ namespace memgraph::query::v2 {
*/
class Path {
public:
using VertexAccessor = accessors::VertexAccessor;
using EdgeAccessor = accessors::EdgeAccessor;
/** Allocator type so that STL containers are aware that we need one */
using allocator_type = utils::Allocator<char>;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -149,6 +149,8 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
return true;
}
// TODO: Cost estimate ScanAllById?
// For the given op first increments the cardinality and then cost.
#define POST_VISIT_CARD_FIRST(NAME) \
bool PostVisit(NAME &) override { \

File diff suppressed because it is too large Load Diff

View File

@@ -22,13 +22,12 @@
#include <variant>
#include <vector>
#include "expr/semantic/symbol.hpp"
#include "query/v2/common.hpp"
#include "query/v2/frontend/ast/ast.hpp"
#include "expr/semantic/symbol.hpp"
#include "query/v2/bindings/typed_value.hpp"
#include "query/v2/bindings/frame.hpp"
#include "query/v2/bindings/symbol_table.hpp"
#include "query/v2/multiframe.hpp"
#include "storage/v3/id_types.hpp"
#include "utils/bound.hpp"
#include "utils/fnv.hpp"
@@ -72,24 +71,6 @@ class Cursor {
/// @throws QueryRuntimeException if something went wrong with execution
virtual bool Pull(Frame &, ExecutionContext &) = 0;
/// Run an iteration of a @c LogicalOperator with MultiFrame.
///
/// Since operators may be chained, the iteration may pull results from
/// multiple operators.
///
/// @param MultiFrame May be read from or written to while performing the
/// iteration.
/// @param ExecutionContext Used to get the position of symbols in frame and
/// other information.
/// @return True if the operator was able to populate at least one Frame on the MultiFrame,
/// thus if an operator returns true, that means there is at least one valid Frame in the
/// MultiFrame.
///
/// @throws QueryRuntimeException if something went wrong with execution
virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
virtual void PushDown(const MultiFrame&) { MG_ASSERT(false, "PushDownIsNotImplemented"); }
/// Resets the Cursor to its initial state.
virtual void Reset() = 0;
@@ -129,7 +110,7 @@ class ScanAllByLabel;
class ScanAllByLabelPropertyRange;
class ScanAllByLabelPropertyValue;
class ScanAllByLabelProperty;
class ScanByPrimaryKey;
class ScanAllById;
class Expand;
class ExpandVariable;
class ConstructNamedPath;
@@ -160,7 +141,7 @@ class Foreach;
using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue,
ScanAllByLabelProperty, ScanByPrimaryKey,
ScanAllByLabelProperty, ScanAllById,
Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
@@ -351,14 +332,11 @@ and false on every following Pull.")
class OnceCursor : public Cursor {
public:
OnceCursor() {}
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void PushDown(const MultiFrame&) override;
bool Pull(Frame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
private:
std::optional<MultiFrame> pushed_down_multi_frame_;
bool did_pull_{false};
};
cpp<#)
@@ -863,21 +841,19 @@ given label and property.
(:serialize (:slk))
(:clone))
(lcp:define-class scan-by-primary-key (scan-all)
((label "::storage::v3::LabelId" :scope :public)
(primary-key "std::vector<Expression*>" :scope :public)
(expression "Expression *" :scope :public
(lcp:define-class scan-all-by-id (scan-all)
((expression "Expression *" :scope :public
:slk-save #'slk-save-ast-pointer
:slk-load (slk-load-ast-pointer "Expression")))
(:documentation
"ScanAll producing a single node with specified by the label and primary key")
"ScanAll producing a single node with ID equal to evaluated expression")
(:public
#>cpp
ScanByPrimaryKey() {}
ScanByPrimaryKey(const std::shared_ptr<LogicalOperator> &input,
Symbol output_symbol,
storage::v3::LabelId label,
std::vector<query::v2::Expression*> primary_key,
ScanAllById() {}
ScanAllById(const std::shared_ptr<LogicalOperator> &input,
Symbol output_symbol, Expression *expression,
storage::v3::View view = storage::v3::View::OLD);
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
@@ -1180,7 +1156,6 @@ a boolean value.")
public:
FilterCursor(const Filter &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1232,7 +1207,6 @@ RETURN clause) the Produce's pull succeeds exactly once.")
public:
ProduceCursor(const Produce &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1280,7 +1254,6 @@ Has a flag for using DETACH DELETE when deleting vertices.")
public:
DeleteCursor(const Delete &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1574,7 +1547,6 @@ edge lists).")
EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
@@ -1971,6 +1943,27 @@ and returns true, once.")
input_ = input;
}
cpp<#)
(:private
#>cpp
class OptionalCursor : public Cursor {
public:
OptionalCursor(const Optional &, utils::MemoryResource *);
bool Pull(Frame &, ExecutionContext &) override;
void Shutdown() override;
void Reset() override;
private:
const Optional &self_;
const UniqueCursorPtr input_cursor_;
const UniqueCursorPtr optional_cursor_;
// indicates if the next Pull from this cursor should
// perform a Pull from the input_cursor_
// this is true when:
// - first pulling from this Cursor
// - previous Pull from this cursor exhausted the optional_cursor_
bool pull_input_{true};
};
cpp<#)
(:serialize (:slk))
(:clone))

View File

@@ -398,7 +398,7 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
auto add_id_equal = [&](auto *maybe_id_fun, auto *val_expr) -> bool {
auto *id_fun = utils::Downcast<Function>(maybe_id_fun);
if (!id_fun) return false;
if (id_fun->function_name_ != functions::kId) return false;
if (id_fun->function_name_ != kId) return false;
if (id_fun->arguments_.size() != 1U) return false;
auto *ident = utils::Downcast<Identifier>(id_fun->arguments_.front());
if (!ident) return false;

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -14,13 +14,11 @@
#include "query/v2/bindings/pretty_print.hpp"
#include "query/v2/db_accessor.hpp"
#include "query/v2/request_router.hpp"
#include "utils/string.hpp"
namespace memgraph::query::v2::plan {
PlanPrinter::PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out)
: request_router_(request_router), out_(out) {}
PlanPrinter::PlanPrinter(const DbAccessor *dba, std::ostream *out) : dba_(dba), out_(out) {}
#define PRE_VISIT(TOp) \
bool PlanPrinter::PreVisit(TOp &) { \
@@ -34,7 +32,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) {
WithPrintLn([&](auto &out) {
out << "* CreateExpand (" << op.input_symbol_.name() << ")"
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
<< op.edge_info_.symbol.name() << ":" << request_router_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
<< op.edge_info_.symbol.name() << ":" << dba_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
<< op.node_info_.symbol.name() << ")";
});
@@ -54,7 +52,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabel"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << ")";
});
return true;
}
@@ -62,8 +60,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabelPropertyValue"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
<< request_router_->PropertyToName(op.property_) << "})";
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
<< dba_->PropertyToName(op.property_) << "})";
});
return true;
}
@@ -71,8 +69,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabelPropertyRange"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
<< request_router_->PropertyToName(op.property_) << "})";
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
<< dba_->PropertyToName(op.property_) << "})";
});
return true;
}
@@ -80,16 +78,16 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) {
WithPrintLn([&](auto &out) {
out << "* ScanAllByLabelProperty"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
<< request_router_->PropertyToName(op.property_) << "})";
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
<< dba_->PropertyToName(op.property_) << "})";
});
return true;
}
bool PlanPrinter::PreVisit(query::v2::plan::ScanByPrimaryKey &op) {
bool PlanPrinter::PreVisit(ScanAllById &op) {
WithPrintLn([&](auto &out) {
out << "* ScanByPrimaryKey"
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
out << "* ScanAllById"
<< " (" << op.output_symbol_.name() << ")";
});
return true;
}
@@ -100,7 +98,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) {
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
<< op.common_.edge_symbol.name();
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
stream << ":" << request_router_->EdgeTypeToName(edge_type);
stream << ":" << dba_->EdgeTypeToName(edge_type);
});
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
<< op.common_.node_symbol.name() << ")";
@@ -129,7 +127,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) {
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
<< op.common_.edge_symbol.name();
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
stream << ":" << request_router_->EdgeTypeToName(edge_type);
stream << ":" << dba_->EdgeTypeToName(edge_type);
});
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
<< op.common_.node_symbol.name() << ")";
@@ -263,14 +261,14 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string
--depth_;
}
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out) {
PlanPrinter printer(&request_router, out);
void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out) {
PlanPrinter printer(&dba, out);
// FIXME(mtomic): We should make visitors that take const arguments.
const_cast<LogicalOperator *>(plan_root)->Accept(printer);
}
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
impl::PlanToJsonVisitor visitor(&request_router);
nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root) {
impl::PlanToJsonVisitor visitor(&dba);
// FIXME(mtomic): We should make visitors that take const arguments.
const_cast<LogicalOperator *>(plan_root)->Accept(visitor);
return visitor.output();
@@ -348,17 +346,11 @@ json ToJson(const utils::Bound<Expression *> &bound) {
json ToJson(const Symbol &symbol) { return symbol.name(); }
json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router) {
return request_router.EdgeTypeToName(edge_type);
}
json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba) { return dba.EdgeTypeToName(edge_type); }
json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router) {
return request_router.LabelToName(label);
}
json ToJson(storage::v3::LabelId label, const DbAccessor &dba) { return dba.LabelToName(label); }
json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router) {
return request_router.PropertyToName(property);
}
json ToJson(storage::v3::PropertyId property, const DbAccessor &dba) { return dba.PropertyToName(property); }
json ToJson(NamedExpression *nexpr) {
json json;
@@ -367,30 +359,29 @@ json ToJson(NamedExpression *nexpr) {
return json;
}
json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
const RequestRouterInterface &request_router) {
json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties, const DbAccessor &dba) {
json json;
for (const auto &prop_pair : properties) {
json.emplace(ToJson(prop_pair.first, request_router), ToJson(prop_pair.second));
json.emplace(ToJson(prop_pair.first, dba), ToJson(prop_pair.second));
}
return json;
}
json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router) {
json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba) {
json self;
self["symbol"] = ToJson(node_info.symbol);
self["labels"] = ToJson(node_info.labels, request_router);
self["labels"] = ToJson(node_info.labels, dba);
const auto *props = std::get_if<PropertiesMapList>(&node_info.properties);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba);
return self;
}
json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router) {
json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba) {
json self;
self["symbol"] = ToJson(edge_info.symbol);
const auto *props = std::get_if<PropertiesMapList>(&edge_info.properties);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
self["edge_type"] = ToJson(edge_info.edge_type, request_router);
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba);
self["edge_type"] = ToJson(edge_info.edge_type, dba);
self["direction"] = ToString(edge_info.direction);
return self;
}
@@ -432,7 +423,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
json self;
self["name"] = "ScanAllByLabel";
self["label"] = ToJson(op.label_, *request_router_);
self["label"] = ToJson(op.label_, *dba_);
self["output_symbol"] = ToJson(op.output_symbol_);
op.input_->Accept(*this);
@@ -445,8 +436,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
json self;
self["name"] = "ScanAllByLabelPropertyRange";
self["label"] = ToJson(op.label_, *request_router_);
self["property"] = ToJson(op.property_, *request_router_);
self["label"] = ToJson(op.label_, *dba_);
self["property"] = ToJson(op.property_, *dba_);
self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json();
self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json();
self["output_symbol"] = ToJson(op.output_symbol_);
@@ -461,8 +452,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
json self;
self["name"] = "ScanAllByLabelPropertyValue";
self["label"] = ToJson(op.label_, *request_router_);
self["property"] = ToJson(op.property_, *request_router_);
self["label"] = ToJson(op.label_, *dba_);
self["property"] = ToJson(op.property_, *dba_);
self["expression"] = ToJson(op.expression_);
self["output_symbol"] = ToJson(op.output_symbol_);
@@ -476,8 +467,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
json self;
self["name"] = "ScanAllByLabelProperty";
self["label"] = ToJson(op.label_, *request_router_);
self["property"] = ToJson(op.property_, *request_router_);
self["label"] = ToJson(op.label_, *dba_);
self["property"] = ToJson(op.property_, *dba_);
self["output_symbol"] = ToJson(op.output_symbol_);
op.input_->Accept(*this);
@@ -487,15 +478,12 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
return false;
}
bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
json self;
self["name"] = "ScanByPrimaryKey";
self["label"] = ToJson(op.label_, *request_router_);
self["name"] = "ScanAllById";
self["output_symbol"] = ToJson(op.output_symbol_);
op.input_->Accept(*this);
self["input"] = PopOutput();
output_ = std::move(self);
return false;
}
@@ -503,7 +491,7 @@ bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
bool PlanToJsonVisitor::PreVisit(CreateNode &op) {
json self;
self["name"] = "CreateNode";
self["node_info"] = ToJson(op.node_info_, *request_router_);
self["node_info"] = ToJson(op.node_info_, *dba_);
op.input_->Accept(*this);
self["input"] = PopOutput();
@@ -516,8 +504,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) {
json self;
self["name"] = "CreateExpand";
self["input_symbol"] = ToJson(op.input_symbol_);
self["node_info"] = ToJson(op.node_info_, *request_router_);
self["edge_info"] = ToJson(op.edge_info_, *request_router_);
self["node_info"] = ToJson(op.node_info_, *dba_);
self["edge_info"] = ToJson(op.edge_info_, *dba_);
self["existing_node"] = op.existing_node_;
op.input_->Accept(*this);
@@ -533,7 +521,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) {
self["input_symbol"] = ToJson(op.input_symbol_);
self["node_symbol"] = ToJson(op.common_.node_symbol);
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
self["edge_types"] = ToJson(op.common_.edge_types, *dba_);
self["direction"] = ToString(op.common_.direction);
self["existing_node"] = op.common_.existing_node;
@@ -550,7 +538,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) {
self["input_symbol"] = ToJson(op.input_symbol_);
self["node_symbol"] = ToJson(op.common_.node_symbol);
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
self["edge_types"] = ToJson(op.common_.edge_types, *dba_);
self["direction"] = ToString(op.common_.direction);
self["type"] = ToString(op.type_);
self["is_reverse"] = op.is_reverse_;
@@ -625,7 +613,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) {
bool PlanToJsonVisitor::PreVisit(SetProperty &op) {
json self;
self["name"] = "SetProperty";
self["property"] = ToJson(op.property_, *request_router_);
self["property"] = ToJson(op.property_, *dba_);
self["lhs"] = ToJson(op.lhs_);
self["rhs"] = ToJson(op.rhs_);
@@ -662,7 +650,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
json self;
self["name"] = "SetLabels";
self["input_symbol"] = ToJson(op.input_symbol_);
self["labels"] = ToJson(op.labels_, *request_router_);
self["labels"] = ToJson(op.labels_, *dba_);
op.input_->Accept(*this);
self["input"] = PopOutput();
@@ -674,7 +662,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) {
json self;
self["name"] = "RemoveProperty";
self["property"] = ToJson(op.property_, *request_router_);
self["property"] = ToJson(op.property_, *dba_);
self["lhs"] = ToJson(op.lhs_);
op.input_->Accept(*this);
@@ -688,7 +676,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) {
json self;
self["name"] = "RemoveLabels";
self["input_symbol"] = ToJson(op.input_symbol_);
self["labels"] = ToJson(op.labels_, *request_router_);
self["labels"] = ToJson(op.labels_, *dba_);
op.input_->Accept(*this);
self["input"] = PopOutput();

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -18,28 +18,28 @@
#include "query/v2/frontend/ast/ast.hpp"
#include "query/v2/plan/operator.hpp"
#include "query/v2/request_router.hpp"
namespace memgraph::query::v2 {
class DbAccessor;
namespace plan {
class LogicalOperator;
/// Pretty print a `LogicalOperator` plan to a `std::ostream`.
/// RequestRouter is needed for resolving label and property names.
/// DbAccessor is needed for resolving label and property names.
/// Note that `plan_root` isn't modified, but we can't take it as a const
/// because we don't have support for visiting a const LogicalOperator.
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out);
void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out);
/// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`.
inline void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
PrettyPrint(request_router, plan_root, &std::cout);
inline void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root) {
PrettyPrint(dba, plan_root, &std::cout);
}
/// Convert a `LogicalOperator` plan to a JSON representation.
/// DbAccessor is needed for resolving label and property names.
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root);
nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root);
class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
public:
@@ -47,7 +47,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
using HierarchicalLogicalOperatorVisitor::PreVisit;
using HierarchicalLogicalOperatorVisitor::Visit;
PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out);
PlanPrinter(const DbAccessor *dba, std::ostream *out);
bool DefaultPreVisit() override;
@@ -67,7 +67,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(ScanAllByLabelPropertyValue &) override;
bool PreVisit(ScanAllByLabelPropertyRange &) override;
bool PreVisit(ScanAllByLabelProperty &) override;
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
bool PreVisit(ScanAllById &) override;
bool PreVisit(Expand &) override;
bool PreVisit(ExpandVariable &) override;
@@ -114,7 +114,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
void Branch(LogicalOperator &op, const std::string &branch_name = "");
int64_t depth_{0};
const RequestRouterInterface *request_router_{nullptr};
const DbAccessor *dba_{nullptr};
std::ostream *out_{nullptr};
};
@@ -132,20 +132,20 @@ nlohmann::json ToJson(const utils::Bound<Expression *> &bound);
nlohmann::json ToJson(const Symbol &symbol);
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router);
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba);
nlohmann::json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router);
nlohmann::json ToJson(storage::v3::LabelId label, const DbAccessor &dba);
nlohmann::json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router);
nlohmann::json ToJson(storage::v3::PropertyId property, const DbAccessor &dba);
nlohmann::json ToJson(NamedExpression *nexpr);
nlohmann::json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
const RequestRouterInterface &request_router);
const DbAccessor &dba);
nlohmann::json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router);
nlohmann::json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba);
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router);
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba);
nlohmann::json ToJson(const Aggregate::Element &elem);
@@ -160,7 +160,7 @@ nlohmann::json ToJson(const std::vector<T> &items, Args &&...args) {
class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
public:
explicit PlanToJsonVisitor(const RequestRouterInterface *request_router) : request_router_(request_router) {}
explicit PlanToJsonVisitor(const DbAccessor *dba) : dba_(dba) {}
using HierarchicalLogicalOperatorVisitor::PostVisit;
using HierarchicalLogicalOperatorVisitor::PreVisit;
@@ -194,7 +194,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
bool PreVisit(ScanAllByLabelPropertyRange &) override;
bool PreVisit(ScanAllByLabelPropertyValue &) override;
bool PreVisit(ScanAllByLabelProperty &) override;
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
bool PreVisit(ScanAllById &) override;
bool PreVisit(Produce &) override;
bool PreVisit(Accumulate &) override;
@@ -216,7 +216,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
protected:
nlohmann::json output_;
const RequestRouterInterface *request_router_;
const DbAccessor *dba_;
nlohmann::json PopOutput() {
nlohmann::json tmp;

View File

@@ -24,18 +24,18 @@ namespace memgraph::query::v2::plan {
namespace {
uint64_t IndividualCycles(const ProfilingStats &cumulative_stats) {
unsigned long long IndividualCycles(const ProfilingStats &cumulative_stats) {
return cumulative_stats.num_cycles - std::accumulate(cumulative_stats.children.begin(),
cumulative_stats.children.end(), 0ULL,
[](auto acc, auto &stats) { return acc + stats.num_cycles; });
}
double RelativeTime(const uint64_t num_cycles, const uint64_t total_cycles) {
return static_cast<double>(num_cycles) / static_cast<double>(total_cycles);
double RelativeTime(unsigned long long num_cycles, unsigned long long total_cycles) {
return static_cast<double>(num_cycles) / total_cycles;
}
double AbsoluteTime(const uint64_t num_cycles, const uint64_t total_cycles,
const std::chrono::duration<double> total_time) {
double AbsoluteTime(unsigned long long num_cycles, unsigned long long total_cycles,
std::chrono::duration<double> total_time) {
return (RelativeTime(num_cycles, total_cycles) * static_cast<std::chrono::duration<double, std::milli>>(total_time))
.count();
}
@@ -50,29 +50,26 @@ namespace {
class ProfilingStatsToTableHelper {
public:
ProfilingStatsToTableHelper(uint64_t total_cycles, std::chrono::duration<double> total_time)
ProfilingStatsToTableHelper(unsigned long long total_cycles, std::chrono::duration<double> total_time)
: total_cycles_(total_cycles), total_time_(total_time) {}
void Output(const ProfilingStats &cumulative_stats) {
auto cycles = IndividualCycles(cumulative_stats);
auto custom_data_copy = cumulative_stats.custom_data;
ConvertCyclesToTime(custom_data_copy);
rows_.emplace_back(
std::vector<TypedValue>{TypedValue(FormatOperator(cumulative_stats.name)),
TypedValue(cumulative_stats.actual_hits), TypedValue(FormatRelativeTime(cycles)),
TypedValue(FormatAbsoluteTime(cycles)), TypedValue(custom_data_copy.dump())});
rows_.emplace_back(std::vector<TypedValue>{
TypedValue(FormatOperator(cumulative_stats.name)), TypedValue(cumulative_stats.actual_hits),
TypedValue(FormatRelativeTime(cycles)), TypedValue(FormatAbsoluteTime(cycles))});
for (size_t i = 1; i < cumulative_stats.children.size(); ++i) {
Branch(cumulative_stats.children[i]);
}
if (!cumulative_stats.children.empty()) {
if (cumulative_stats.children.size() >= 1) {
Output(cumulative_stats.children[0]);
}
}
std::vector<std::vector<TypedValue>> rows() const { return rows_; }
std::vector<std::vector<TypedValue>> rows() { return rows_; }
private:
void Branch(const ProfilingStats &cumulative_stats) {
@@ -83,28 +80,7 @@ class ProfilingStatsToTableHelper {
--depth_;
}
double AbsoluteTime(const uint64_t cycles) const { return plan::AbsoluteTime(cycles, total_cycles_, total_time_); }
double RelativeTime(const uint64_t cycles) const { return plan::RelativeTime(cycles, total_cycles_); }
void ConvertCyclesToTime(nlohmann::json &custom_data) const {
const auto convert_cycles_in_json = [this](nlohmann::json &json) {
if (!json.is_object()) {
return;
}
if (auto it = json.find(ProfilingStats::kNumCycles); it != json.end()) {
auto num_cycles = it.value().get<uint64_t>();
json[ProfilingStats::kAbsoluteTime] = AbsoluteTime(num_cycles);
json[ProfilingStats::kRelativeTime] = RelativeTime(num_cycles);
}
};
for (auto &json : custom_data) {
convert_cycles_in_json(json);
}
}
std::string Format(const char *str) const {
std::string Format(const char *str) {
std::ostringstream ss;
for (int64_t i = 0; i < depth_; ++i) {
ss << "| ";
@@ -113,21 +89,21 @@ class ProfilingStatsToTableHelper {
return ss.str();
}
std::string Format(const std::string &str) const { return Format(str.c_str()); }
std::string Format(const std::string &str) { return Format(str.c_str()); }
std::string FormatOperator(const char *str) const { return Format(std::string("* ") + str); }
std::string FormatOperator(const char *str) { return Format(std::string("* ") + str); }
std::string FormatRelativeTime(uint64_t num_cycles) const {
return fmt::format("{: 10.6f} %", RelativeTime(num_cycles) * 100);
std::string FormatRelativeTime(unsigned long long num_cycles) {
return fmt::format("{: 10.6f} %", RelativeTime(num_cycles, total_cycles_) * 100);
}
std::string FormatAbsoluteTime(uint64_t num_cycles) const {
return fmt::format("{: 10.6f} ms", AbsoluteTime(num_cycles));
std::string FormatAbsoluteTime(unsigned long long num_cycles) {
return fmt::format("{: 10.6f} ms", AbsoluteTime(num_cycles, total_cycles_, total_time_));
}
int64_t depth_{0};
std::vector<std::vector<TypedValue>> rows_;
uint64_t total_cycles_;
unsigned long long total_cycles_;
std::chrono::duration<double> total_time_;
};
@@ -150,7 +126,7 @@ class ProfilingStatsToJsonHelper {
using json = nlohmann::json;
public:
ProfilingStatsToJsonHelper(uint64_t total_cycles, std::chrono::duration<double> total_time)
ProfilingStatsToJsonHelper(unsigned long long total_cycles, std::chrono::duration<double> total_time)
: total_cycles_(total_cycles), total_time_(total_time) {}
void Output(const ProfilingStats &cumulative_stats) { return Output(cumulative_stats, &json_); }
@@ -175,7 +151,7 @@ class ProfilingStatsToJsonHelper {
}
json json_;
uint64_t total_cycles_;
unsigned long long total_cycles_;
std::chrono::duration<double> total_time_;
};

View File

@@ -26,16 +26,10 @@ namespace plan {
* Stores profiling statistics for a single logical operator.
*/
struct ProfilingStats {
static constexpr std::string_view kNumCycles{"num_cycles"};
static constexpr std::string_view kRelativeTime{"relative_time"};
static constexpr std::string_view kAbsoluteTime{"absolute_time"};
static constexpr std::string_view kActualHits{"actual_hits"};
int64_t actual_hits{0};
uint64_t num_cycles{0};
unsigned long long num_cycles{0};
uint64_t key{0};
const char *name{nullptr};
nlohmann::json custom_data;
// TODO: This should use the allocator for query execution
std::vector<ProfilingStats> children;
};

View File

@@ -1,4 +1,4 @@
// Copyright 2023 Memgraph Ltd.
// Copyright 2022 Memgraph Ltd.
//
// Use of this software is governed by the Business Source License
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
@@ -11,12 +11,10 @@
#include "query/v2/plan/read_write_type_checker.hpp"
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
#define PRE_VISIT(TOp, RWType, continue_visiting) \
/*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \
bool ReadWriteTypeChecker::PreVisit(TOp & /*op*/) { \
UpdateType(RWType); \
return continue_visiting; \
#define PRE_VISIT(TOp, RWType, continue_visiting) \
bool ReadWriteTypeChecker::PreVisit(TOp &op) { \
UpdateType(RWType); \
return continue_visiting; \
}
namespace memgraph::query::v2::plan {
@@ -37,7 +35,7 @@ PRE_VISIT(ScanAllByLabel, RWType::R, true)
PRE_VISIT(ScanAllByLabelPropertyRange, RWType::R, true)
PRE_VISIT(ScanAllByLabelPropertyValue, RWType::R, true)
PRE_VISIT(ScanAllByLabelProperty, RWType::R, true)
PRE_VISIT(ScanByPrimaryKey, RWType::R, true)
PRE_VISIT(ScanAllById, RWType::R, true)
PRE_VISIT(Expand, RWType::R, true)
PRE_VISIT(ExpandVariable, RWType::R, true)

Some files were not shown because too many files have changed in this diff Show More