Compare commits
5 Commits
T1151-MG-d
...
T1061-MG-i
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
18c8473a6b | ||
|
|
bc602bb93c | ||
|
|
a7125894cf | ||
|
|
d46e6e36a2 | ||
|
|
63871d0b03 |
2
.github/workflows/daily_benchmark.yaml
vendored
2
.github/workflows/daily_benchmark.yaml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
256
.github/workflows/diff.yaml
vendored
256
.github/workflows/diff.yaml
vendored
@@ -26,7 +26,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -64,7 +64,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -75,39 +75,14 @@ jobs:
|
||||
- name: Fetch all history for all tags and branches
|
||||
run: git fetch
|
||||
|
||||
- name: Initialize deps
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
- name: Set base branch
|
||||
if: ${{ github.event_name == 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/${{ github.base_ref }}" >> $GITHUB_ENV
|
||||
|
||||
- name: Set base branch # if we manually dispatch or push to master
|
||||
if: ${{ github.event_name != 'pull_request' }}
|
||||
run: |
|
||||
echo "BASE_BRANCH=origin/master" >> $GITHUB_ENV
|
||||
|
||||
- name: Python code analysis
|
||||
run: |
|
||||
CHANGED_FILES=$(git diff -U0 ${{ env.BASE_BRANCH }}... --name-only)
|
||||
for file in ${CHANGED_FILES}; do
|
||||
echo ${file}
|
||||
if [[ ${file} == *.py ]]; then
|
||||
python3 -m black --check --diff ${file}
|
||||
python3 -m isort --check-only --profile "black" --diff ${file}
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Build combined ASAN, UBSAN and coverage binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
cd build
|
||||
cmake -DTEST_COVERAGE=ON -DASAN=ON -DUBSAN=ON ..
|
||||
make -j$THREADS memgraph__unit
|
||||
@@ -135,7 +110,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -170,7 +145,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -189,23 +164,52 @@ jobs:
|
||||
cmake ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run simulation tests
|
||||
- name: Run leftover CTest tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run simulation tests.
|
||||
# Run leftover CTest tests (all except unit and benchmark tests).
|
||||
cd build
|
||||
ctest -R memgraph__simulation --output-on-failure -j$THREADS
|
||||
ctest -E "(memgraph__unit|memgraph__benchmark|memgraph__simulation)" --output-on-failure
|
||||
|
||||
- name: Run single benchmark test
|
||||
- name: Run drivers tests
|
||||
run: |
|
||||
./tests/drivers/run.sh
|
||||
|
||||
- name: Run integration tests
|
||||
run: |
|
||||
cd tests/integration
|
||||
for name in *; do
|
||||
if [ ! -d $name ]; then continue; fi
|
||||
pushd $name >/dev/null
|
||||
echo "Running: $name"
|
||||
if [ -x prepare.sh ]; then
|
||||
./prepare.sh
|
||||
fi
|
||||
if [ -x runner.py ]; then
|
||||
./runner.py
|
||||
elif [ -x runner.sh ]; then
|
||||
./runner.sh
|
||||
fi
|
||||
echo
|
||||
popd >/dev/null
|
||||
done
|
||||
|
||||
- name: Run cppcheck and clang-format
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run simulation tests.
|
||||
cd tests/mgbench
|
||||
./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
|
||||
# Run cppcheck and clang-format.
|
||||
cd tools/github
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
|
||||
release_build:
|
||||
name: "Release build"
|
||||
@@ -217,7 +221,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -236,6 +240,19 @@ jobs:
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run GQL Behave tests
|
||||
run: |
|
||||
cd tests/gql_behave
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
tests/gql_behave/gql_behave_status.csv
|
||||
tests/gql_behave/gql_behave_status.html
|
||||
|
||||
- name: Run unit tests
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
@@ -250,19 +267,10 @@ jobs:
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run simulation tests.
|
||||
# Run unit tests.
|
||||
cd build
|
||||
ctest -R memgraph__simulation --output-on-failure -j$THREADS
|
||||
|
||||
- name: Run single benchmark test
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Run simulation tests.
|
||||
cd tests/mgbench
|
||||
./benchmark.py accesscontrol/small --num-workers-for-import 1 --test-system-arg "split-file splitfiles/accesscontrol_small.shard_configuration bolt-num-workers 1"
|
||||
|
||||
- name: Run e2e tests
|
||||
run: |
|
||||
# TODO(gitbuda): Setup mgclient and pymgclient properly.
|
||||
@@ -270,12 +278,131 @@ jobs:
|
||||
./setup.sh
|
||||
source ve3/bin/activate
|
||||
cd e2e
|
||||
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory ./distributed_queries
|
||||
LD_LIBRARY_PATH=$LD_LIBRARY_PATH:../../libs/mgclient/lib python runner.py --workloads-root-directory .
|
||||
|
||||
- name: Run query performance tests
|
||||
- name: Run stress test (plain)
|
||||
run: |
|
||||
cd tests/manual
|
||||
./query_performance_runner.py
|
||||
cd tests/stress
|
||||
./continuous_integration
|
||||
|
||||
- name: Run stress test (SSL)
|
||||
run: |
|
||||
cd tests/stress
|
||||
./continuous_integration --use-ssl
|
||||
|
||||
- name: Run durability test
|
||||
run: |
|
||||
cd tests/stress
|
||||
source ve3/bin/activate
|
||||
python3 durability --num-steps 5
|
||||
|
||||
- name: Create enterprise DEB package
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
cd build
|
||||
|
||||
# create mgconsole
|
||||
# we use the -B to force the build
|
||||
make -j$THREADS -B mgconsole
|
||||
|
||||
# Create enterprise DEB package.
|
||||
mkdir output && cd output
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
|
||||
- name: Save test data
|
||||
uses: actions/upload-artifact@v2
|
||||
if: always()
|
||||
with:
|
||||
name: "Test data"
|
||||
path: |
|
||||
# multiple paths could be defined
|
||||
build/logs
|
||||
|
||||
release_jepsen_test:
|
||||
name: "Release Jepsen Test"
|
||||
runs-on: [self-hosted, Linux, X64, Debian10, JepsenControl]
|
||||
#continue-on-error: true
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binarie.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS memgraph
|
||||
|
||||
- name: Run Jepsen tests
|
||||
run: |
|
||||
cd tests/jepsen
|
||||
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
path: tests/jepsen/Jepsen.tar.gz
|
||||
|
||||
release_benchmarks:
|
||||
name: "Release benchmarks"
|
||||
runs-on: [self-hosted, Linux, X64, Diff, Gen7]
|
||||
env:
|
||||
THREADS: 24
|
||||
MEMGRAPH_ENTERPRISE_LICENSE: ${{ secrets.MEMGRAPH_ENTERPRISE_LICENSE }}
|
||||
MEMGRAPH_ORGANIZATION_NAME: ${{ secrets.MEMGRAPH_ORGANIZATION_NAME }}
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Build release binaries
|
||||
run: |
|
||||
# Activate toolchain.
|
||||
source /opt/toolchain-v4/activate
|
||||
|
||||
# Initialize dependencies.
|
||||
./init
|
||||
|
||||
# Build only memgraph release binaries.
|
||||
cd build
|
||||
cmake -DCMAKE_BUILD_TYPE=release ..
|
||||
make -j$THREADS
|
||||
|
||||
- name: Run macro benchmarks
|
||||
run: |
|
||||
cd tests/macro_benchmark
|
||||
./harness QuerySuite MemgraphRunner \
|
||||
--groups aggregation 1000_create unwind_create dense_expand match \
|
||||
--no-strict
|
||||
|
||||
- name: Get branch name (merge)
|
||||
if: github.event_name != 'pull_request'
|
||||
@@ -293,8 +420,25 @@ jobs:
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "query_performance" \
|
||||
--benchmark-results-path "../../build/tests/manual/query_performance_benchmark/summary.json" \
|
||||
./main.py --benchmark-name "macro_benchmark" \
|
||||
--benchmark-results-path "../../tests/macro_benchmark/.harness_summary" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
- name: Run mgbench
|
||||
run: |
|
||||
cd tests/mgbench
|
||||
./benchmark.py --num-workers-for-benchmark 12 --export-results benchmark_result.json pokec/medium/*/*
|
||||
|
||||
- name: Upload mgbench results
|
||||
run: |
|
||||
cd tools/bench-graph-client
|
||||
virtualenv -p python3 ve3
|
||||
source ve3/bin/activate
|
||||
pip install -r requirements.txt
|
||||
./main.py --benchmark-name "mgbench" \
|
||||
--benchmark-results-path "../../tests/mgbench/benchmark_result.json" \
|
||||
--github-run-id "${{ github.run_id }}" \
|
||||
--github-run-number "${{ github.run_number }}" \
|
||||
--head-branch-name "${{ env.BRANCH_NAME }}"
|
||||
|
||||
2
.github/workflows/full_clang_tidy.yaml
vendored
2
.github/workflows/full_clang_tidy.yaml
vendored
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
|
||||
20
.github/workflows/package_all.yaml
vendored
20
.github/workflows/package_all.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package centos-7
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-7
|
||||
path: build/output/centos-7/memgraph*.rpm
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package centos-9
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: centos-9
|
||||
path: build/output/centos-9/memgraph*.rpm
|
||||
@@ -51,7 +51,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package debian-10
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-10
|
||||
path: build/output/debian-10/memgraph*.deb
|
||||
@@ -68,7 +68,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package debian-11
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-11
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
@@ -87,7 +87,7 @@ jobs:
|
||||
./run.sh package debian-11 --for-docker
|
||||
./run.sh docker
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: docker
|
||||
path: build/output/docker/memgraph*.tar.gz
|
||||
@@ -104,7 +104,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-18.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-1804
|
||||
path: build/output/ubuntu-18.04/memgraph*.deb
|
||||
@@ -121,7 +121,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-20.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-2004
|
||||
path: build/output/ubuntu-20.04/memgraph*.deb
|
||||
@@ -138,7 +138,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package ubuntu-22.04
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: ubuntu-2204
|
||||
path: build/output/ubuntu-22.04/memgraph*.deb
|
||||
@@ -155,7 +155,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package debian-11 --for-platform
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-11-platform
|
||||
path: build/output/debian-11/memgraph*.deb
|
||||
@@ -172,7 +172,7 @@ jobs:
|
||||
run: |
|
||||
./release/package/run.sh package debian-11-arm
|
||||
- name: "Upload package"
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: debian-11-arm
|
||||
path: build/output/debian-11-arm/memgraph*.deb
|
||||
|
||||
16
.github/workflows/release_centos8.yaml
vendored
16
.github/workflows/release_centos8.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -225,7 +225,7 @@ jobs:
|
||||
rpmlint memgraph*.rpm
|
||||
|
||||
- name: Save enterprise RPM package
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Enterprise RPM package"
|
||||
path: build/output/memgraph*.rpm
|
||||
@@ -262,7 +262,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
|
||||
20
.github/workflows/release_debian10.yaml
vendored
20
.github/workflows/release_debian10.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
@@ -261,7 +261,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
@@ -324,7 +324,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -349,7 +349,7 @@ jobs:
|
||||
./run.sh test --binary ../../build/memgraph --run-args "test-all --node-configs resources/node-config.edn" --ignore-run-stdout-logs --ignore-run-stderr-logs
|
||||
|
||||
- name: Save Jepsen report
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
if: ${{ always() }}
|
||||
with:
|
||||
name: "Jepsen Report"
|
||||
|
||||
2
.github/workflows/release_docker.yaml
vendored
2
.github/workflows/release_docker.yaml
vendored
@@ -19,7 +19,7 @@ jobs:
|
||||
DOCKER_REPOSITORY_NAME: memgraph
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v1
|
||||
|
||||
16
.github/workflows/release_ubuntu2004.yaml
vendored
16
.github/workflows/release_ubuntu2004.yaml
vendored
@@ -17,7 +17,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -55,7 +55,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -97,7 +97,7 @@ jobs:
|
||||
tar -czf code_coverage.tar.gz coverage.json html report.json summary.rmu
|
||||
|
||||
- name: Save code coverage
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/generated/code_coverage.tar.gz
|
||||
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -173,7 +173,7 @@ jobs:
|
||||
./cppcheck_and_clang_format diff
|
||||
|
||||
- name: Save cppcheck and clang-format errors
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Code coverage"
|
||||
path: tools/github/cppcheck_and_clang_format.txt
|
||||
@@ -189,7 +189,7 @@ jobs:
|
||||
|
||||
steps:
|
||||
- name: Set up repository
|
||||
uses: actions/checkout@v3
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
# Number of commits to fetch. `0` indicates all history for all
|
||||
# branches and tags. (default: 1)
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
cpack -G DEB --config ../CPackConfig.cmake
|
||||
|
||||
- name: Save enterprise DEB package
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "Enterprise DEB package"
|
||||
path: build/output/memgraph*.deb
|
||||
@@ -261,7 +261,7 @@ jobs:
|
||||
./continuous_integration
|
||||
|
||||
- name: Save quality assurance status
|
||||
uses: actions/upload-artifact@v3
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: "GQL Behave Status"
|
||||
path: |
|
||||
|
||||
@@ -6,15 +6,18 @@ repos:
|
||||
- id: end-of-file-fixer
|
||||
- id: trailing-whitespace
|
||||
- repo: https://github.com/psf/black
|
||||
rev: 22.10.0
|
||||
rev: 22.3.0
|
||||
hooks:
|
||||
- id: black
|
||||
- repo: https://github.com/pycqa/isort
|
||||
rev: 5.10.1
|
||||
hooks:
|
||||
- id: isort
|
||||
name: isort (python)
|
||||
args: ["--profile", "black"]
|
||||
args: # arguments to configure black
|
||||
- --line-length=120
|
||||
- --include='\.pyi?$'
|
||||
# these folders wont be formatted by black
|
||||
- --exclude="""\.git |
|
||||
\.__pycache__|
|
||||
build|
|
||||
libs|
|
||||
.cache"""
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v13.0.0
|
||||
hooks:
|
||||
|
||||
@@ -182,8 +182,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
# c99-designator is disabled because of required mixture of designated and
|
||||
# non-designated initializers in Python Query Module code (`py_module.cpp`).
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall \
|
||||
-Werror=switch -Werror=switch-bool -Werror=implicit-fallthrough \
|
||||
-Werror=return-type \
|
||||
-Werror=switch -Werror=switch-bool -Werror=return-type \
|
||||
-Werror=return-stack-address \
|
||||
-Wno-c99-designator \
|
||||
-DBOOST_ASIO_USE_TS_EXECUTOR_AS_DEFAULT")
|
||||
|
||||
@@ -83,9 +83,13 @@ modifications:
|
||||
value: "true"
|
||||
override: true
|
||||
|
||||
# - name: "query_modules_directory"
|
||||
# value: "/usr/lib/memgraph/query_modules"
|
||||
# override: true
|
||||
- name: "query_modules_directory"
|
||||
value: "/usr/lib/memgraph/query_modules"
|
||||
override: true
|
||||
|
||||
- name: "auth_module_executable"
|
||||
value: "/usr/lib/memgraph/auth_module/example.py"
|
||||
override: false
|
||||
|
||||
- name: "memory_limit"
|
||||
value: "0"
|
||||
|
||||
7
init
7
init
@@ -140,12 +140,5 @@ done;
|
||||
python3 -m pip install pre-commit
|
||||
python3 -m pre_commit install
|
||||
|
||||
# Install py format tools
|
||||
echo "Install black formatter"
|
||||
python3 -m pip install black==22.10.*
|
||||
echo "Install isort"
|
||||
python3 -m pip install isort==5.10.*
|
||||
|
||||
|
||||
# Link `include/mgp.py` with `release/mgp/mgp.py`
|
||||
ln -v -f include/mgp.py release/mgp/mgp.py
|
||||
|
||||
@@ -116,7 +116,7 @@ declare -A primary_urls=(
|
||||
["pymgclient"]="http://$local_cache_host/git/pymgclient.git"
|
||||
["mgconsole"]="http://$local_cache_host/git/mgconsole.git"
|
||||
["spdlog"]="http://$local_cache_host/git/spdlog"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
|
||||
["nlohmann"]="http://$local_cache_host/file/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="http://$local_cache_host/file/neo4j-community-3.2.3-unix.tar.gz"
|
||||
["librdkafka"]="http://$local_cache_host/git/librdkafka.git"
|
||||
["protobuf"]="http://$local_cache_host/git/protobuf.git"
|
||||
@@ -141,7 +141,7 @@ declare -A secondary_urls=(
|
||||
["pymgclient"]="https://github.com/memgraph/pymgclient.git"
|
||||
["mgconsole"]="http://github.com/memgraph/mgconsole.git"
|
||||
["spdlog"]="https://github.com/gabime/spdlog"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/9d69186291aca4f0137b69c1dee313b391ff564c/single_include/nlohmann/json.hpp"
|
||||
["nlohmann"]="https://raw.githubusercontent.com/nlohmann/json/4f8fba14066156b73f1189a2b8bd568bde5284c5/single_include/nlohmann/json.hpp"
|
||||
["neo4j"]="https://s3-eu-west-1.amazonaws.com/deps.memgraph.io/neo4j-community-3.2.3-unix.tar.gz"
|
||||
["librdkafka"]="https://github.com/edenhill/librdkafka.git"
|
||||
["protobuf"]="https://github.com/protocolbuffers/protobuf.git"
|
||||
@@ -171,7 +171,7 @@ benchmark_tag="v1.6.0"
|
||||
repo_clone_try_double "${primary_urls[gbenchmark]}" "${secondary_urls[gbenchmark]}" "benchmark" "$benchmark_tag" true
|
||||
|
||||
# google test
|
||||
googletest_tag="release-1.12.1"
|
||||
googletest_tag="release-1.8.0"
|
||||
repo_clone_try_double "${primary_urls[gtest]}" "${secondary_urls[gtest]}" "googletest" "$googletest_tag" true
|
||||
|
||||
# libbcrypt
|
||||
|
||||
@@ -1,12 +0,0 @@
|
||||
[tool.black]
|
||||
line-length = 120
|
||||
include = '\.pyi?$'
|
||||
extend-exclude = '''
|
||||
/(
|
||||
| .git
|
||||
| .__pycache__
|
||||
| build
|
||||
| libs
|
||||
| .cache
|
||||
)/
|
||||
'''
|
||||
@@ -21,7 +21,6 @@ add_subdirectory(auth)
|
||||
add_subdirectory(parser)
|
||||
add_subdirectory(expr)
|
||||
add_subdirectory(coordinator)
|
||||
add_subdirectory(functions)
|
||||
|
||||
if (MG_ENTERPRISE)
|
||||
add_subdirectory(audit)
|
||||
@@ -38,13 +37,13 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
|
||||
# Memgraph Single Node v2 Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
set(mg_single_node_v2_sources
|
||||
glue/v2/communication.cpp
|
||||
memgraph.cpp
|
||||
glue/v2/auth.cpp
|
||||
glue/communication.cpp
|
||||
memgraph.cpp
|
||||
glue/auth.cpp
|
||||
)
|
||||
|
||||
set(mg_single_node_v2_libs stdc++fs Threads::Threads
|
||||
telemetry_lib mg-query-v2 mg-communication mg-memory mg-utils mg-auth mg-license mg-settings mg-io mg-coordinator)
|
||||
telemetry_lib mg-query mg-communication mg-memory mg-utils mg-auth mg-license mg-settings)
|
||||
if (MG_ENTERPRISE)
|
||||
# These are enterprise subsystems
|
||||
set(mg_single_node_v2_libs ${mg_single_node_v2_libs} mg-audit)
|
||||
@@ -127,3 +126,19 @@ install(CODE "file(MAKE_DIRECTORY \$ENV{DESTDIR}/var/log/memgraph
|
||||
# ----------------------------------------------------------------------------
|
||||
# Memgraph CSV Import Tool Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
add_executable(mg_import_csv mg_import_csv.cpp)
|
||||
target_link_libraries(mg_import_csv mg-storage-v2)
|
||||
|
||||
# Strip the executable in release build.
|
||||
if (lower_build_type STREQUAL "release")
|
||||
add_custom_command(TARGET mg_import_csv POST_BUILD
|
||||
COMMAND strip -s mg_import_csv
|
||||
COMMENT "Stripping symbols and sections from mg_import_csv")
|
||||
endif()
|
||||
|
||||
install(TARGETS mg_import_csv RUNTIME DESTINATION bin)
|
||||
|
||||
# ----------------------------------------------------------------------------
|
||||
# Memgraph CSV Import Tool Executable
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
@@ -1,68 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstdint>
|
||||
#include <string_view>
|
||||
|
||||
namespace memgraph::common {
|
||||
|
||||
enum class ErrorCode : uint8_t {
|
||||
SERIALIZATION_ERROR,
|
||||
NONEXISTENT_OBJECT,
|
||||
DELETED_OBJECT,
|
||||
VERTEX_HAS_EDGES,
|
||||
PROPERTIES_DISABLED,
|
||||
VERTEX_ALREADY_INSERTED,
|
||||
// Schema Violations
|
||||
SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL,
|
||||
SCHEMA_VERTEX_PROPERTY_WRONG_TYPE,
|
||||
SCHEMA_VERTEX_UPDATE_PRIMARY_KEY,
|
||||
SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL,
|
||||
SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY,
|
||||
SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED,
|
||||
|
||||
OBJECT_NOT_FOUND,
|
||||
};
|
||||
|
||||
constexpr std::string_view ErrorCodeToString(const ErrorCode code) {
|
||||
switch (code) {
|
||||
case ErrorCode::SERIALIZATION_ERROR:
|
||||
return "SERIALIZATION_ERROR";
|
||||
case ErrorCode::NONEXISTENT_OBJECT:
|
||||
return "NONEXISTENT_OBJECT";
|
||||
case ErrorCode::DELETED_OBJECT:
|
||||
return "DELETED_OBJECT";
|
||||
case ErrorCode::VERTEX_HAS_EDGES:
|
||||
return "VERTEX_HAS_EDGES";
|
||||
case ErrorCode::PROPERTIES_DISABLED:
|
||||
return "PROPERTIES_DISABLED";
|
||||
case ErrorCode::VERTEX_ALREADY_INSERTED:
|
||||
return "VERTEX_ALREADY_INSERTED";
|
||||
case ErrorCode::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
|
||||
return "SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL";
|
||||
case ErrorCode::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
|
||||
return "SCHEMA_VERTEX_PROPERTY_WRONG_TYPE";
|
||||
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
|
||||
return "SCHEMA_VERTEX_UPDATE_PRIMARY_KEY";
|
||||
case ErrorCode::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
|
||||
return "SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL";
|
||||
case ErrorCode::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
return "SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY";
|
||||
case ErrorCode::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
return "SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED";
|
||||
case ErrorCode::OBJECT_NOT_FOUND:
|
||||
return "OBJECT_NOT_FOUND";
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace memgraph::common
|
||||
@@ -74,7 +74,7 @@ State RunHandlerV4(Signature signature, TSession &session, State state, Marker m
|
||||
}
|
||||
case Signature::Route: {
|
||||
if constexpr (bolt_minor >= 3) {
|
||||
return HandleRoute<TSession>(session);
|
||||
if (signature == Signature::Route) return HandleRoute<TSession>(session);
|
||||
} else {
|
||||
spdlog::trace("Supported only in bolt v4.3");
|
||||
return State::Close;
|
||||
|
||||
@@ -30,7 +30,6 @@
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/v2/pool.hpp"
|
||||
#include "communication/v2/session.hpp"
|
||||
#include "utils/message.hpp"
|
||||
#include "utils/spin_lock.hpp"
|
||||
#include "utils/synchronized.hpp"
|
||||
|
||||
@@ -59,10 +58,10 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
|
||||
bool IsRunning() const noexcept { return alive_.load(std::memory_order_relaxed); }
|
||||
|
||||
private:
|
||||
Listener(boost::asio::io_context &io_context, TSessionData &data, ServerContext *server_context,
|
||||
Listener(boost::asio::io_context &io_context, TSessionData *data, ServerContext *server_context,
|
||||
tcp::endpoint &endpoint, const std::string_view service_name, const uint64_t inactivity_timeout_sec)
|
||||
: io_context_(io_context),
|
||||
data_(&data),
|
||||
data_(data),
|
||||
server_context_(server_context),
|
||||
acceptor_(io_context_),
|
||||
endpoint_{endpoint},
|
||||
@@ -111,7 +110,7 @@ class Listener final : public std::enable_shared_from_this<Listener<TSession, TS
|
||||
return OnError(ec, "accept");
|
||||
}
|
||||
|
||||
auto session = SessionHandler::Create(std::move(socket), *data_, *server_context_, endpoint_, inactivity_timeout_,
|
||||
auto session = SessionHandler::Create(std::move(socket), data_, *server_context_, endpoint_, inactivity_timeout_,
|
||||
service_name_);
|
||||
session->Start();
|
||||
DoAccept();
|
||||
|
||||
@@ -72,7 +72,7 @@ class Server final {
|
||||
* Constructs and binds server to endpoint, operates on session data and
|
||||
* invokes workers_count workers
|
||||
*/
|
||||
Server(ServerEndpoint &endpoint, TSessionData &session_data, ServerContext *server_context,
|
||||
Server(ServerEndpoint &endpoint, TSessionData *session_data, ServerContext *server_context,
|
||||
const int inactivity_timeout_sec, const std::string_view service_name,
|
||||
size_t workers_count = std::thread::hardware_concurrency())
|
||||
: endpoint_{endpoint},
|
||||
|
||||
@@ -41,7 +41,6 @@
|
||||
#include <boost/beast/websocket/rfc6455.hpp>
|
||||
#include <boost/system/detail/error_code.hpp>
|
||||
|
||||
#include "communication/buffer.hpp"
|
||||
#include "communication/context.hpp"
|
||||
#include "communication/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -140,7 +139,7 @@ class WebsocketSession : public std::enable_shared_from_this<WebsocketSession<TS
|
||||
|
||||
private:
|
||||
// Take ownership of the socket
|
||||
explicit WebsocketSession(tcp::socket &&socket, TSessionData &data, tcp::endpoint endpoint,
|
||||
explicit WebsocketSession(tcp::socket &&socket, TSessionData *data, tcp::endpoint endpoint,
|
||||
std::string_view service_name)
|
||||
: ws_(std::move(socket)),
|
||||
strand_{boost::asio::make_strand(ws_.get_executor())},
|
||||
@@ -312,13 +311,13 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
|
||||
}
|
||||
|
||||
private:
|
||||
explicit Session(tcp::socket &&socket, TSessionData &data, ServerContext &server_context, tcp::endpoint endpoint,
|
||||
explicit Session(tcp::socket &&socket, TSessionData *data, ServerContext &server_context, tcp::endpoint endpoint,
|
||||
const std::chrono::seconds inactivity_timeout_sec, std::string_view service_name)
|
||||
: socket_(CreateSocket(std::move(socket), server_context)),
|
||||
strand_{boost::asio::make_strand(GetExecutor())},
|
||||
output_stream_([this](const uint8_t *data, size_t len, bool have_more) { return Write(data, len, have_more); }),
|
||||
session_(data, endpoint, input_buffer_.read_end(), &output_stream_),
|
||||
data_{&data},
|
||||
data_{data},
|
||||
endpoint_{endpoint},
|
||||
remote_endpoint_{GetRemoteEndpoint()},
|
||||
service_name_{service_name},
|
||||
@@ -374,7 +373,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
|
||||
spdlog::info("Switching {} to websocket connection", remote_endpoint_);
|
||||
if (std::holds_alternative<TCPSocket>(socket_)) {
|
||||
auto sock = std::get<TCPSocket>(std::move(socket_));
|
||||
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), *data_, endpoint_, service_name_)
|
||||
WebsocketSession<TSession, TSessionData>::Create(std::move(sock), data_, endpoint_, service_name_)
|
||||
->DoAccept(parser.release());
|
||||
execution_active_ = false;
|
||||
return;
|
||||
@@ -466,7 +465,7 @@ class Session final : public std::enable_shared_from_this<Session<TSession, TSes
|
||||
if (timeout_timer_.expiry() <= boost::asio::steady_timer::clock_type::now()) {
|
||||
// The deadline has passed. Stop the session. The other actors will
|
||||
// terminate as soon as possible.
|
||||
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_.count());
|
||||
spdlog::info("Shutting down session after {} of inactivity", timeout_seconds_);
|
||||
DoShutdown();
|
||||
} else {
|
||||
// Put the actor back to sleep.
|
||||
|
||||
@@ -18,7 +18,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(HeartbeatRequest &&heartbeat_r
|
||||
|
||||
// add this storage engine to any under-replicated shards that it is not already a part of
|
||||
|
||||
return shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
|
||||
auto initializing_rsms_for_shard_manager =
|
||||
shard_map_.AssignShards(heartbeat_request.from_storage_manager, heartbeat_request.initialized_rsms);
|
||||
|
||||
return HeartbeatResponse{
|
||||
.shards_to_initialize = initializing_rsms_for_shard_manager,
|
||||
};
|
||||
}
|
||||
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(HlcRequest &&hlc_request) {
|
||||
@@ -62,8 +67,12 @@ CoordinatorWriteResponses Coordinator::ApplyWrite(AllocateEdgeIdBatchRequest &&a
|
||||
CoordinatorWriteResponses Coordinator::ApplyWrite(SplitShardRequest &&split_shard_request) {
|
||||
SplitShardResponse res{};
|
||||
|
||||
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
|
||||
split_shard_request.split_key);
|
||||
if (split_shard_request.previous_shard_map_version != shard_map_.shard_map_version) {
|
||||
res.success = false;
|
||||
} else {
|
||||
res.success = shard_map_.SplitShard(split_shard_request.previous_shard_map_version, split_shard_request.label_id,
|
||||
split_shard_request.split_key);
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
@@ -121,6 +121,15 @@ struct InitializeLabelResponse {
|
||||
std::optional<ShardMap> fresher_shard_map;
|
||||
};
|
||||
|
||||
struct HeartbeatRequest {
|
||||
Address from_storage_manager;
|
||||
std::set<boost::uuids::uuid> initialized_rsms;
|
||||
};
|
||||
|
||||
struct HeartbeatResponse {
|
||||
std::vector<ShardToInitialize> shards_to_initialize;
|
||||
};
|
||||
|
||||
using CoordinatorWriteRequests =
|
||||
std::variant<HlcRequest, AllocateEdgeIdBatchRequest, SplitShardRequest, RegisterStorageEngineRequest,
|
||||
DeregisterStorageEngineRequest, InitializeLabelRequest, AllocatePropertyIdsRequest, HeartbeatRequest>;
|
||||
@@ -149,10 +158,10 @@ class Coordinator {
|
||||
|
||||
private:
|
||||
ShardMap shard_map_;
|
||||
uint64_t highest_allocated_timestamp_{0};
|
||||
uint64_t highest_allocated_timestamp_;
|
||||
|
||||
/// Query engines need to periodically request batches of unique edge IDs.
|
||||
uint64_t highest_allocated_edge_id_{0};
|
||||
uint64_t highest_allocated_edge_id_;
|
||||
|
||||
CoordinatorReadResponses HandleRead(GetShardMapRequest && /* get_shard_map_request */) {
|
||||
GetShardMapResponse res;
|
||||
|
||||
@@ -1,168 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <deque>
|
||||
#include <memory>
|
||||
#include <queue>
|
||||
#include <variant>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/coordinator_rsm.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "io/address.hpp"
|
||||
#include "io/future.hpp"
|
||||
#include "io/messages.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
|
||||
namespace memgraph::coordinator::coordinator_worker {
|
||||
|
||||
/// Obligations:
|
||||
/// * ShutDown
|
||||
/// * Cron
|
||||
/// * RouteMessage
|
||||
|
||||
using coordinator::Coordinator;
|
||||
using coordinator::CoordinatorRsm;
|
||||
using io::Address;
|
||||
using io::RequestId;
|
||||
using io::Time;
|
||||
using io::messages::CoordinatorMessages;
|
||||
using msgs::ReadRequests;
|
||||
using msgs::ReadResponses;
|
||||
using msgs::WriteRequests;
|
||||
using msgs::WriteResponses;
|
||||
|
||||
struct ShutDown {};
|
||||
|
||||
struct Cron {};
|
||||
|
||||
struct RouteMessage {
|
||||
CoordinatorMessages message;
|
||||
RequestId request_id;
|
||||
Address to;
|
||||
Address from;
|
||||
};
|
||||
|
||||
using Message = std::variant<RouteMessage, Cron, ShutDown>;
|
||||
|
||||
struct QueueInner {
|
||||
std::mutex mu{};
|
||||
std::condition_variable cv;
|
||||
// TODO(tyler) handle simulator communication std::shared_ptr<std::atomic<int>> blocked;
|
||||
|
||||
// TODO(tyler) investigate using a priority queue that prioritizes messages in a way that
|
||||
// improves overall QoS. For example, maybe we want to schedule raft Append messages
|
||||
// ahead of Read messages or generally writes before reads for lowering the load on the
|
||||
// overall system faster etc... When we do this, we need to make sure to avoid
|
||||
// starvation by sometimes randomizing priorities, rather than following a strict
|
||||
// prioritization.
|
||||
std::deque<Message> queue;
|
||||
|
||||
uint64_t submitted = 0;
|
||||
uint64_t calls_to_pop = 0;
|
||||
};
|
||||
|
||||
/// There are two reasons to implement our own Queue instead of using
|
||||
/// one off-the-shelf:
|
||||
/// 1. we will need to know in the simulator when all threads are waiting
|
||||
/// 2. we will want to implement our own priority queue within this for QoS
|
||||
class Queue {
|
||||
std::shared_ptr<QueueInner> inner_ = std::make_shared<QueueInner>();
|
||||
|
||||
public:
|
||||
void Push(Message &&message) {
|
||||
{
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->submitted++;
|
||||
|
||||
inner_->queue.emplace_back(std::move(message));
|
||||
} // lock dropped before notifying condition variable
|
||||
|
||||
inner_->cv.notify_all();
|
||||
}
|
||||
|
||||
Message Pop() {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
inner_->calls_to_pop++;
|
||||
inner_->cv.notify_all();
|
||||
|
||||
while (inner_->queue.empty()) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
|
||||
Message message = std::move(inner_->queue.front());
|
||||
inner_->queue.pop_front();
|
||||
|
||||
return message;
|
||||
}
|
||||
|
||||
void BlockOnQuiescence() const {
|
||||
MG_ASSERT(inner_.use_count() > 0);
|
||||
std::unique_lock<std::mutex> lock(inner_->mu);
|
||||
|
||||
while (inner_->calls_to_pop <= inner_->submitted) {
|
||||
inner_->cv.wait(lock);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// A CoordinatorWorker owns Raft<CoordinatorRsm> instances. receives messages from the MachineManager.
|
||||
template <typename IoImpl>
|
||||
class CoordinatorWorker {
|
||||
io::Io<IoImpl> io_;
|
||||
Queue queue_;
|
||||
CoordinatorRsm<IoImpl> coordinator_;
|
||||
|
||||
bool Process(ShutDown && /*shut_down*/) { return false; }
|
||||
|
||||
bool Process(Cron && /* cron */) {
|
||||
coordinator_.Cron();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Process(RouteMessage &&route_message) {
|
||||
coordinator_.Handle(std::move(route_message.message), route_message.request_id, route_message.from);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public:
|
||||
CoordinatorWorker(io::Io<IoImpl> io, Queue queue, Coordinator coordinator)
|
||||
: io_(std::move(io)), queue_(std::move(queue)), coordinator_{std::move(io_), {}, std::move(coordinator)} {}
|
||||
|
||||
CoordinatorWorker(CoordinatorWorker &&) noexcept = default;
|
||||
CoordinatorWorker &operator=(CoordinatorWorker &&) noexcept = default;
|
||||
CoordinatorWorker(const CoordinatorWorker &) = delete;
|
||||
CoordinatorWorker &operator=(const CoordinatorWorker &) = delete;
|
||||
~CoordinatorWorker() = default;
|
||||
|
||||
void Run() {
|
||||
bool should_continue = true;
|
||||
while (should_continue) {
|
||||
Message message = queue_.Pop();
|
||||
|
||||
should_continue = std::visit([this](auto &&msg) { return this->Process(std::forward<decltype(msg)>(msg)); },
|
||||
std::move(message));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator::coordinator_worker
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,10 +11,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <compare>
|
||||
#include <ctime>
|
||||
#include <iomanip>
|
||||
|
||||
#include "io/time.hpp"
|
||||
|
||||
@@ -23,26 +20,17 @@ namespace memgraph::coordinator {
|
||||
using Time = memgraph::io::Time;
|
||||
|
||||
/// Hybrid-logical clock
|
||||
struct Hlc final {
|
||||
uint64_t logical_id{0};
|
||||
Time coordinator_wall_clock = Time::min();
|
||||
struct Hlc {
|
||||
uint64_t logical_id;
|
||||
Time coordinator_wall_clock;
|
||||
|
||||
auto operator<=>(const Hlc &other) const noexcept { return logical_id <=> other.logical_id; }
|
||||
auto operator<=>(const Hlc &other) const { return logical_id <=> other.logical_id; }
|
||||
|
||||
bool operator==(const Hlc &other) const noexcept = default;
|
||||
bool operator<(const Hlc &other) const noexcept = default;
|
||||
bool operator==(const uint64_t other) const noexcept { return logical_id == other; }
|
||||
bool operator<(const uint64_t other) const noexcept { return logical_id < other; }
|
||||
bool operator>=(const uint64_t other) const noexcept { return logical_id >= other; }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const Hlc &hlc) {
|
||||
auto wall_clock = std::chrono::system_clock::to_time_t(hlc.coordinator_wall_clock);
|
||||
in << "Hlc { logical_id: " << hlc.logical_id;
|
||||
in << ", coordinator_wall_clock: " << std::put_time(std::localtime(&wall_clock), "%F %T");
|
||||
in << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
bool operator==(const Hlc &other) const = default;
|
||||
bool operator<(const Hlc &other) const = default;
|
||||
bool operator==(const uint64_t other) const { return logical_id == other; }
|
||||
bool operator<(const uint64_t other) const { return logical_id < other; }
|
||||
bool operator>=(const uint64_t other) const { return logical_id >= other; }
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
|
||||
@@ -9,19 +9,8 @@
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "common/types.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "spdlog/spdlog.h"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/temporal.hpp"
|
||||
#include "utils/cast.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
@@ -68,352 +57,6 @@ PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
ShardMap ShardMap::Parse(std::istream &input_stream) {
|
||||
ShardMap shard_map;
|
||||
const auto read_size = [&input_stream] {
|
||||
size_t size{0};
|
||||
input_stream >> size;
|
||||
return size;
|
||||
};
|
||||
|
||||
// Reads a string until the next whitespace
|
||||
const auto read_word = [&input_stream] {
|
||||
std::string word;
|
||||
input_stream >> word;
|
||||
return word;
|
||||
};
|
||||
|
||||
const auto read_names = [&read_size, &read_word] {
|
||||
const auto number_of_names = read_size();
|
||||
spdlog::trace("Reading {} names", number_of_names);
|
||||
std::vector<std::string> names;
|
||||
names.reserve(number_of_names);
|
||||
|
||||
for (auto name_index = 0; name_index < number_of_names; ++name_index) {
|
||||
names.push_back(read_word());
|
||||
spdlog::trace("Read '{}'", names.back());
|
||||
}
|
||||
return names;
|
||||
};
|
||||
|
||||
const auto read_line = [&input_stream] {
|
||||
std::string line;
|
||||
std::getline(input_stream, line);
|
||||
return line;
|
||||
};
|
||||
|
||||
const auto parse_type = [](const std::string &type) {
|
||||
static const auto type_map = std::unordered_map<std::string, common::SchemaType>{
|
||||
{"string", common::SchemaType::STRING}, {"int", common::SchemaType::INT}, {"bool", common::SchemaType::BOOL}};
|
||||
const auto lower_case_type = utils::ToLowerCase(type);
|
||||
auto it = type_map.find(lower_case_type);
|
||||
MG_ASSERT(it != type_map.end(), "Invalid type in split files: {}", type);
|
||||
return it->second;
|
||||
};
|
||||
|
||||
const auto parse_property_value = [](std::string text, const common::SchemaType type) {
|
||||
if (type == common::SchemaType::STRING) {
|
||||
return storage::v3::PropertyValue{std::move(text)};
|
||||
}
|
||||
if (type == common::SchemaType::INT) {
|
||||
size_t processed{0};
|
||||
int64_t value = std::stoll(text, &processed);
|
||||
MG_ASSERT(processed == text.size() || text[processed] == ' ', "Invalid integer format: '{}'", text);
|
||||
return storage::v3::PropertyValue{value};
|
||||
}
|
||||
LOG_FATAL("Not supported type: {}", utils::UnderlyingCast(type));
|
||||
};
|
||||
|
||||
spdlog::debug("Reading properties");
|
||||
const auto properties = read_names();
|
||||
MG_ASSERT(shard_map.AllocatePropertyIds(properties).size() == properties.size(),
|
||||
"Unexpected number of properties created!");
|
||||
|
||||
spdlog::debug("Reading edge types");
|
||||
const auto edge_types = read_names();
|
||||
MG_ASSERT(shard_map.AllocateEdgeTypeIds(edge_types).size() == edge_types.size(),
|
||||
"Unexpected number of properties created!");
|
||||
|
||||
spdlog::debug("Reading primary labels");
|
||||
const auto number_of_primary_labels = read_size();
|
||||
spdlog::debug("Reading {} primary labels", number_of_primary_labels);
|
||||
|
||||
for (auto label_index = 0; label_index < number_of_primary_labels; ++label_index) {
|
||||
const auto primary_label = read_word();
|
||||
spdlog::debug("Reading primary label named '{}'", primary_label);
|
||||
const auto number_of_primary_properties = read_size();
|
||||
spdlog::debug("Reading {} primary properties", number_of_primary_properties);
|
||||
std::vector<std::string> pp_names;
|
||||
std::vector<common::SchemaType> pp_types;
|
||||
pp_names.reserve(number_of_primary_properties);
|
||||
pp_types.reserve(number_of_primary_properties);
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
pp_names.push_back(read_word());
|
||||
spdlog::debug("Reading primary property named '{}'", pp_names.back());
|
||||
pp_types.push_back(parse_type(read_word()));
|
||||
}
|
||||
auto pp_mapping = shard_map.AllocatePropertyIds(pp_names);
|
||||
std::vector<SchemaProperty> schema;
|
||||
schema.reserve(number_of_primary_properties);
|
||||
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
schema.push_back(storage::v3::SchemaProperty{pp_mapping.at(pp_names[property_index]), pp_types[property_index]});
|
||||
}
|
||||
const auto hlc = shard_map.GetHlc();
|
||||
MG_ASSERT(shard_map.InitializeNewLabel(primary_label, schema, 1, hlc).has_value(),
|
||||
"Cannot initialize new label: {}", primary_label);
|
||||
|
||||
const auto number_of_split_points = read_size();
|
||||
spdlog::debug("Reading {} split points", number_of_split_points);
|
||||
|
||||
[[maybe_unused]] const auto remainder_from_last_line = read_line();
|
||||
for (auto split_point_index = 0; split_point_index < number_of_split_points; ++split_point_index) {
|
||||
const auto line = read_line();
|
||||
spdlog::debug("Read split point '{}'", line);
|
||||
MG_ASSERT(line.front() == '[', "Invalid split file format!");
|
||||
MG_ASSERT(line.back() == ']', "Invalid split file format!");
|
||||
std::string_view line_view{line};
|
||||
line_view.remove_prefix(1);
|
||||
line_view.remove_suffix(1);
|
||||
static constexpr std::string_view kDelimiter{","};
|
||||
auto pk_values_as_text = utils::Split(line_view, kDelimiter);
|
||||
std::vector<PropertyValue> pk;
|
||||
pk.reserve(number_of_primary_properties);
|
||||
MG_ASSERT(pk_values_as_text.size() == number_of_primary_properties,
|
||||
"Split point contains invalid number of values '{}'", line);
|
||||
|
||||
for (auto property_index = 0; property_index < number_of_primary_properties; ++property_index) {
|
||||
pk.push_back(parse_property_value(std::move(pk_values_as_text[property_index]), schema[property_index].type));
|
||||
}
|
||||
shard_map.SplitShard(shard_map.GetHlc(), shard_map.labels.at(primary_label), pk);
|
||||
}
|
||||
}
|
||||
|
||||
return shard_map;
|
||||
}
|
||||
|
||||
std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map) {
|
||||
using utils::print_helpers::operator<<;
|
||||
|
||||
in << "ShardMap { shard_map_version: " << shard_map.shard_map_version;
|
||||
in << ", max_property_id: " << shard_map.max_property_id;
|
||||
in << ", max_edge_type_id: " << shard_map.max_edge_type_id;
|
||||
in << ", properties: " << shard_map.properties;
|
||||
in << ", edge_types: " << shard_map.edge_types;
|
||||
in << ", max_label_id: " << shard_map.max_label_id;
|
||||
in << ", labels: " << shard_map.labels;
|
||||
in << ", label_spaces: " << shard_map.label_spaces;
|
||||
in << ", schemas: " << shard_map.schemas;
|
||||
in << "}";
|
||||
return in;
|
||||
}
|
||||
|
||||
Shards ShardMap::GetShardsForLabel(const LabelName &label) const {
|
||||
const auto id = labels.at(label);
|
||||
const auto &shards = label_spaces.at(id).shards;
|
||||
return shards;
|
||||
}
|
||||
|
||||
std::vector<Shards> ShardMap::GetAllShards() const {
|
||||
std::vector<Shards> all_shards;
|
||||
all_shards.reserve(label_spaces.size());
|
||||
std::transform(label_spaces.begin(), label_spaces.end(), std::back_inserter(all_shards),
|
||||
[](const auto &label_space) { return label_space.second.shards; });
|
||||
return all_shards;
|
||||
}
|
||||
|
||||
// TODO(gabor) later we will want to update the wallclock time with
|
||||
// the given Io<impl>'s time as well
|
||||
Hlc ShardMap::IncrementShardMapVersion() noexcept {
|
||||
++shard_map_version.logical_id;
|
||||
return shard_map_version;
|
||||
}
|
||||
|
||||
// TODO(antaljanosbenjamin) use a single map for all name id
|
||||
// mapping and a single counter to maintain the next id
|
||||
std::unordered_map<uint64_t, std::string> ShardMap::IdToNames() {
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
|
||||
const auto map_type_ids = [&id_to_names](const auto &name_to_id_type) {
|
||||
for (const auto &[name, id] : name_to_id_type) {
|
||||
id_to_names.emplace(id.AsUint(), name);
|
||||
}
|
||||
};
|
||||
|
||||
map_type_ids(edge_types);
|
||||
map_type_ids(labels);
|
||||
map_type_ids(properties);
|
||||
|
||||
return id_to_names;
|
||||
}
|
||||
|
||||
Hlc ShardMap::GetHlc() const noexcept { return shard_map_version; }
|
||||
|
||||
boost::uuids::uuid NewShardUuid(uint64_t shard_id) {
|
||||
return boost::uuids::uuid{0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
static_cast<unsigned char>(shard_id >> 56U),
|
||||
static_cast<unsigned char>(shard_id >> 48U),
|
||||
static_cast<unsigned char>(shard_id >> 40U),
|
||||
static_cast<unsigned char>(shard_id >> 32U),
|
||||
static_cast<unsigned char>(shard_id >> 24U),
|
||||
static_cast<unsigned char>(shard_id >> 16U),
|
||||
static_cast<unsigned char>(shard_id >> 8U),
|
||||
static_cast<unsigned char>(shard_id)};
|
||||
}
|
||||
|
||||
HeartbeatResponse ShardMap::AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
|
||||
HeartbeatResponse ret{};
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (auto &[label_id, label_space] : label_spaces) {
|
||||
for (auto it = label_space.shards.begin(); it != label_space.shards.end(); it++) {
|
||||
auto &[low_key, shard] = *it;
|
||||
std::optional<PrimaryKey> high_key;
|
||||
if (const auto next_it = std::next(it); next_it != label_space.shards.end()) {
|
||||
high_key = next_it->first;
|
||||
}
|
||||
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
|
||||
bool shard_assigned_to_machine = false;
|
||||
|
||||
for (auto &peer_metadata : shard.peers) {
|
||||
const bool same_machine = peer_metadata.address.last_known_ip == storage_manager.last_known_ip &&
|
||||
peer_metadata.address.last_known_port == storage_manager.last_known_port;
|
||||
|
||||
if (initialized.contains(peer_metadata.address.unique_id)) {
|
||||
shard_assigned_to_machine = true;
|
||||
|
||||
if (!same_machine) {
|
||||
// set the last known ip and port to the storage manager that has heartbeated it just now
|
||||
mutated = true;
|
||||
peer_metadata.address.last_known_ip = storage_manager.last_known_ip;
|
||||
peer_metadata.address.last_known_port = storage_manager.last_known_port;
|
||||
}
|
||||
|
||||
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
mutated = true;
|
||||
spdlog::info("marking shard as full consensus participant: {}", peer_metadata.address.unique_id);
|
||||
peer_metadata.status = Status::CONSENSUS_PARTICIPANT;
|
||||
}
|
||||
} else if (same_machine && peer_metadata.status == Status::INITIALIZING) {
|
||||
// we are expecting this shard to be initialized (rather than split) on this machine,
|
||||
// so send it an initialization request
|
||||
|
||||
shard_assigned_to_machine = true;
|
||||
spdlog::info("reminding shard manager that they should begin participating in shard");
|
||||
|
||||
ret.shards_to_initialize.push_back(ShardToInitialize{
|
||||
.uuid = peer_metadata.address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
} else if (same_machine && peer_metadata.status == Status::PENDING_SPLIT) {
|
||||
// we are expecting this shard to be split, so send it a split request
|
||||
|
||||
ret.shards_to_split.push_back(ShardToSplit{
|
||||
.shard_to_split_uuid = peer_metadata.split_from,
|
||||
.new_right_side_uuid = peer_metadata.address.unique_id,
|
||||
.split_requested_at = shard.version,
|
||||
.label_id = label_id,
|
||||
.split_key = low_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
} else {
|
||||
MG_ASSERT(
|
||||
!same_machine,
|
||||
"failed to properly handle a new Status type in the heartbeat management and shard assignment code");
|
||||
}
|
||||
}
|
||||
|
||||
if (!shard_assigned_to_machine && shard.peers.size() < label_space.replication_factor) {
|
||||
Address address = storage_manager;
|
||||
|
||||
// NB: increment version for each new uuid for deterministic creation
|
||||
IncrementShardMapVersion();
|
||||
|
||||
address.unique_id = NewShardUuid(shard_map_version.logical_id);
|
||||
|
||||
spdlog::info("assigning shard manager to shard");
|
||||
|
||||
ret.shards_to_initialize.push_back(ShardToInitialize{
|
||||
.uuid = address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = high_key,
|
||||
.schema = schemas[label_id],
|
||||
.config = Config{},
|
||||
.id_to_names = IdToNames(),
|
||||
});
|
||||
|
||||
PeerMetadata peer_metadata = {
|
||||
.address = address,
|
||||
.status = Status::INITIALIZING,
|
||||
};
|
||||
|
||||
shard.peers.emplace_back(peer_metadata);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ShardMap::SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
|
||||
if (previous_shard_map_version != shard_map_version || !label_spaces.contains(label_id)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto &label_space = label_spaces.at(label_id);
|
||||
auto &shards_in_map = label_space.shards;
|
||||
|
||||
MG_ASSERT(!shards_in_map.empty());
|
||||
MG_ASSERT(!shards_in_map.contains(key));
|
||||
|
||||
// Finding the ShardMetadata that the new PrimaryKey should map to.
|
||||
ShardMetadata duplicated_shard = GetShardForKey(label_id, key);
|
||||
|
||||
std::map<boost::uuids::uuid, boost::uuids::uuid> split_mapping = {};
|
||||
|
||||
for (auto &peer_metadata : duplicated_shard.peers) {
|
||||
peer_metadata.status = Status::PENDING_SPLIT;
|
||||
peer_metadata.split_from = peer_metadata.address.unique_id;
|
||||
|
||||
// NB: increment version for each new uuid for deterministic creation
|
||||
IncrementShardMapVersion();
|
||||
|
||||
auto new_uuid = NewShardUuid(shard_map_version.logical_id);
|
||||
|
||||
// store new uuid for the right side of each shard
|
||||
split_mapping.emplace(peer_metadata.address.unique_id, new_uuid);
|
||||
|
||||
peer_metadata.address.unique_id = new_uuid;
|
||||
}
|
||||
|
||||
// Apply the split
|
||||
shards_in_map[key] = duplicated_shard;
|
||||
|
||||
IncrementShardMapVersion();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, Hlc last_shard_map_version) {
|
||||
if (shard_map_version != last_shard_map_version || labels.contains(label_name)) {
|
||||
@@ -425,18 +68,17 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
|
||||
labels.emplace(std::move(label_name), label_id);
|
||||
|
||||
PrimaryKey initial_key = SchemaToMinKey(schema);
|
||||
ShardMetadata empty_shard = {};
|
||||
Shard empty_shard = {};
|
||||
|
||||
Shards shards = {
|
||||
{initial_key, empty_shard},
|
||||
};
|
||||
|
||||
LabelSpace label_space{
|
||||
.schema = schema,
|
||||
.schema = std::move(schema),
|
||||
.shards = shards,
|
||||
.replication_factor = replication_factor,
|
||||
};
|
||||
schemas[label_id] = std::move(schema);
|
||||
|
||||
label_spaces.emplace(label_id, label_space);
|
||||
|
||||
@@ -445,174 +87,4 @@ std::optional<LabelId> ShardMap::InitializeNewLabel(std::string label_name, std:
|
||||
return label_id;
|
||||
}
|
||||
|
||||
void ShardMap::AddServer(Address server_address) {
|
||||
// Find a random place for the server to plug in
|
||||
}
|
||||
|
||||
std::optional<LabelId> ShardMap::GetLabelId(const std::string &label) const {
|
||||
if (const auto it = labels.find(label); it != labels.end()) {
|
||||
return it->second;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string &ShardMap::GetLabelName(const LabelId label) const {
|
||||
if (const auto it =
|
||||
std::ranges::find_if(labels, [label](const auto &name_id_pair) { return name_id_pair.second == label; });
|
||||
it != labels.end()) {
|
||||
return it->first;
|
||||
}
|
||||
throw utils::BasicException("GetLabelName fails on the given label id!");
|
||||
}
|
||||
|
||||
std::optional<PropertyId> ShardMap::GetPropertyId(const std::string &property_name) const {
|
||||
if (const auto it = properties.find(property_name); it != properties.end()) {
|
||||
return it->second;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string &ShardMap::GetPropertyName(const PropertyId property) const {
|
||||
if (const auto it = std::ranges::find_if(
|
||||
properties, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
|
||||
it != properties.end()) {
|
||||
return it->first;
|
||||
}
|
||||
throw utils::BasicException("PropertyId not found!");
|
||||
}
|
||||
|
||||
std::optional<EdgeTypeId> ShardMap::GetEdgeTypeId(const std::string &edge_type) const {
|
||||
if (const auto it = edge_types.find(edge_type); it != edge_types.end()) {
|
||||
return it->second;
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
const std::string &ShardMap::GetEdgeTypeName(const EdgeTypeId property) const {
|
||||
if (const auto it = std::ranges::find_if(
|
||||
edge_types, [property](const auto &name_id_pair) { return name_id_pair.second == property; });
|
||||
it != edge_types.end()) {
|
||||
return it->first;
|
||||
}
|
||||
throw utils::BasicException("EdgeTypeId not found!");
|
||||
}
|
||||
|
||||
Shards ShardMap::GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key,
|
||||
const PrimaryKey &end_key) const {
|
||||
MG_ASSERT(start_key <= end_key);
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
LabelId label_id = labels.at(label_name);
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
const auto &shards_for_label = label_space.shards;
|
||||
|
||||
MG_ASSERT(shards_for_label.begin()->first <= start_key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
auto it = std::prev(shards_for_label.upper_bound(start_key));
|
||||
const auto end_it = shards_for_label.upper_bound(end_key);
|
||||
|
||||
Shards shards{};
|
||||
|
||||
std::copy(it, end_it, std::inserter(shards, shards.end()));
|
||||
|
||||
return shards;
|
||||
}
|
||||
|
||||
ShardMetadata ShardMap::GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
LabelId label_id = labels.at(label_name);
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
MG_ASSERT(label_space.shards.begin()->first <= key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
ShardMetadata ShardMap::GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
MG_ASSERT(label_space.shards.begin()->first <= key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
PropertyMap ShardMap::AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
|
||||
PropertyMap ret{};
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (const auto &property_name : new_properties) {
|
||||
if (properties.contains(property_name)) {
|
||||
auto property_id = properties.at(property_name);
|
||||
ret.emplace(property_name, property_id);
|
||||
} else {
|
||||
mutated = true;
|
||||
|
||||
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
|
||||
ret.emplace(property_name, property_id);
|
||||
properties.emplace(property_name, property_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
EdgeTypeIdMap ShardMap::AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types) {
|
||||
EdgeTypeIdMap ret;
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (const auto &edge_type_name : new_edge_types) {
|
||||
if (edge_types.contains(edge_type_name)) {
|
||||
auto edge_type_id = edge_types.at(edge_type_name);
|
||||
ret.emplace(edge_type_name, edge_type_id);
|
||||
} else {
|
||||
mutated = true;
|
||||
|
||||
const EdgeTypeId edge_type_id = EdgeTypeId::FromUint(++max_edge_type_id);
|
||||
ret.emplace(edge_type_name, edge_type_id);
|
||||
edge_types.emplace(edge_type_name, edge_type_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool ShardMap::ClusterInitialized() const {
|
||||
for (const auto &[label_id, label_space] : label_spaces) {
|
||||
for (const auto &[low_key, shard] : label_space.shards) {
|
||||
if (shard.peers.size() < label_space.replication_factor) {
|
||||
spdlog::info("label_space below desired replication factor");
|
||||
return false;
|
||||
}
|
||||
|
||||
for (const auto &peer_metadata : shard.peers) {
|
||||
if (peer_metadata.status != Status::CONSENSUS_PARTICIPANT) {
|
||||
spdlog::info("shard member not yet a CONSENSUS_PARTICIPANT");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <set>
|
||||
@@ -25,20 +24,14 @@
|
||||
#include "io/address.hpp"
|
||||
#include "storage/v3/config.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
#include "storage/v3/temporal.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/print_helpers.hpp"
|
||||
|
||||
namespace memgraph::coordinator {
|
||||
|
||||
constexpr int64_t kNotExistingId{0};
|
||||
|
||||
using memgraph::io::Address;
|
||||
using memgraph::storage::v3::Config;
|
||||
using memgraph::storage::v3::EdgeTypeId;
|
||||
using memgraph::storage::v3::LabelId;
|
||||
using memgraph::storage::v3::PropertyId;
|
||||
using memgraph::storage::v3::PropertyValue;
|
||||
@@ -47,186 +40,237 @@ using memgraph::storage::v3::SchemaProperty;
|
||||
enum class Status : uint8_t {
|
||||
CONSENSUS_PARTICIPANT,
|
||||
INITIALIZING,
|
||||
PENDING_SPLIT,
|
||||
// TODO(tyler) this will possibly have more states,
|
||||
// depending on the reconfiguration protocol that we
|
||||
// implement.
|
||||
};
|
||||
|
||||
struct PeerMetadata {
|
||||
struct AddressAndStatus {
|
||||
memgraph::io::Address address;
|
||||
Status status;
|
||||
boost::uuids::uuid split_from;
|
||||
|
||||
friend bool operator<(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address < rhs.address; }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const PeerMetadata &peer_metadata) {
|
||||
in << "PeerMetadata { address: ";
|
||||
in << peer_metadata.address;
|
||||
|
||||
if (peer_metadata.status == Status::CONSENSUS_PARTICIPANT) {
|
||||
in << ", status: CONSENSUS_PARTICIPANT";
|
||||
} else if (peer_metadata.status == Status::INITIALIZING) {
|
||||
in << ", status: INITIALIZING";
|
||||
} else if (peer_metadata.status == Status::PENDING_SPLIT) {
|
||||
in << ", status: PENDING_SPLIT";
|
||||
} else {
|
||||
MG_ASSERT(false, "failed to update the operator<< implementation for Status");
|
||||
}
|
||||
|
||||
in << ", split_from: " << peer_metadata.split_from << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
|
||||
friend bool operator==(const PeerMetadata &lhs, const PeerMetadata &rhs) { return lhs.address == rhs.address; }
|
||||
friend bool operator<(const AddressAndStatus &lhs, const AddressAndStatus &rhs) { return lhs.address < rhs.address; }
|
||||
};
|
||||
|
||||
using PrimaryKey = std::vector<PropertyValue>;
|
||||
|
||||
struct ShardMetadata {
|
||||
std::vector<PeerMetadata> peers;
|
||||
uint64_t version;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const ShardMetadata &shard) {
|
||||
using utils::print_helpers::operator<<;
|
||||
|
||||
in << "ShardMetadata { peers: ";
|
||||
in << shard.peers;
|
||||
in << " version: ";
|
||||
in << shard.version;
|
||||
in << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
|
||||
friend bool operator==(const ShardMetadata &lhs, const ShardMetadata &rhs) = default;
|
||||
|
||||
friend bool operator<(const ShardMetadata &lhs, const ShardMetadata &rhs) {
|
||||
if (lhs.peers != rhs.peers) {
|
||||
return lhs.peers < rhs.peers;
|
||||
}
|
||||
|
||||
return lhs.version < rhs.version;
|
||||
}
|
||||
};
|
||||
|
||||
using Shards = std::map<PrimaryKey, ShardMetadata>;
|
||||
using Shard = std::vector<AddressAndStatus>;
|
||||
using Shards = std::map<PrimaryKey, Shard>;
|
||||
using LabelName = std::string;
|
||||
using PropertyName = std::string;
|
||||
using EdgeTypeName = std::string;
|
||||
using PropertyMap = std::map<PropertyName, PropertyId>;
|
||||
using EdgeTypeIdMap = std::map<EdgeTypeName, EdgeTypeId>;
|
||||
|
||||
struct ShardToInitialize {
|
||||
boost::uuids::uuid uuid;
|
||||
LabelId label_id;
|
||||
PrimaryKey min_key;
|
||||
std::optional<PrimaryKey> max_key;
|
||||
std::vector<SchemaProperty> schema;
|
||||
Config config;
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
};
|
||||
|
||||
struct ShardToSplit {
|
||||
boost::uuids::uuid shard_to_split_uuid;
|
||||
boost::uuids::uuid new_right_side_uuid;
|
||||
Hlc split_requested_at;
|
||||
LabelId label_id;
|
||||
PrimaryKey split_key;
|
||||
std::vector<SchemaProperty> schema;
|
||||
Config config;
|
||||
std::unordered_map<uint64_t, std::string> id_to_names;
|
||||
};
|
||||
|
||||
struct HeartbeatRequest {
|
||||
Address from_storage_manager;
|
||||
std::set<boost::uuids::uuid> initialized_rsms;
|
||||
};
|
||||
|
||||
struct HeartbeatResponse {
|
||||
std::vector<ShardToInitialize> shards_to_initialize;
|
||||
std::vector<ShardToSplit> shards_to_split;
|
||||
};
|
||||
|
||||
PrimaryKey SchemaToMinKey(const std::vector<SchemaProperty> &schema);
|
||||
|
||||
struct LabelSpace {
|
||||
std::vector<SchemaProperty> schema;
|
||||
// Maps between the smallest primary key stored in the shard and the shard
|
||||
std::map<PrimaryKey, ShardMetadata> shards;
|
||||
std::map<PrimaryKey, Shard> shards;
|
||||
size_t replication_factor;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LabelSpace &label_space) {
|
||||
using utils::print_helpers::operator<<;
|
||||
|
||||
in << "LabelSpace { schema: ";
|
||||
in << label_space.schema;
|
||||
in << ", shards: ";
|
||||
in << label_space.shards;
|
||||
in << ", replication_factor: " << label_space.replication_factor << "}";
|
||||
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
struct ShardMap {
|
||||
Hlc shard_map_version;
|
||||
uint64_t max_property_id{kNotExistingId};
|
||||
uint64_t max_edge_type_id{kNotExistingId};
|
||||
uint64_t max_property_id;
|
||||
std::map<PropertyName, PropertyId> properties;
|
||||
std::map<EdgeTypeName, EdgeTypeId> edge_types;
|
||||
uint64_t max_label_id{kNotExistingId};
|
||||
uint64_t max_label_id;
|
||||
std::map<LabelName, LabelId> labels;
|
||||
std::map<LabelId, LabelSpace> label_spaces;
|
||||
std::map<LabelId, std::vector<SchemaProperty>> schemas;
|
||||
std::map<LabelName, LabelId> labels;
|
||||
|
||||
[[nodiscard]] static ShardMap Parse(std::istream &input_stream);
|
||||
friend std::ostream &operator<<(std::ostream &in, const ShardMap &shard_map);
|
||||
|
||||
Shards GetShardsForLabel(const LabelName &label) const;
|
||||
|
||||
std::vector<Shards> GetAllShards() const;
|
||||
Shards GetShards(const LabelName &label) {
|
||||
const auto id = labels.at(label);
|
||||
auto &shards = label_spaces.at(id).shards;
|
||||
return shards;
|
||||
}
|
||||
|
||||
// TODO(gabor) later we will want to update the wallclock time with
|
||||
// the given Io<impl>'s time as well
|
||||
Hlc IncrementShardMapVersion() noexcept;
|
||||
Hlc GetHlc() const noexcept;
|
||||
Hlc IncrementShardMapVersion() noexcept {
|
||||
++shard_map_version.logical_id;
|
||||
return shard_map_version;
|
||||
}
|
||||
|
||||
std::unordered_map<uint64_t, std::string> IdToNames();
|
||||
Hlc GetHlc() const noexcept { return shard_map_version; }
|
||||
|
||||
// Returns the shard UUIDs that have been assigned but not yet acknowledged for this storage manager
|
||||
HeartbeatResponse AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized);
|
||||
std::vector<ShardToInitialize> AssignShards(Address storage_manager, std::set<boost::uuids::uuid> initialized) {
|
||||
std::vector<ShardToInitialize> ret{};
|
||||
|
||||
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key);
|
||||
bool mutated = false;
|
||||
|
||||
for (auto &[label_id, label_space] : label_spaces) {
|
||||
for (auto &[low_key, shard] : label_space.shards) {
|
||||
// TODO(tyler) avoid these triple-nested loops by having the heartbeat include better info
|
||||
bool machine_contains_shard = false;
|
||||
|
||||
for (auto &aas : shard) {
|
||||
if (initialized.contains(aas.address.unique_id)) {
|
||||
spdlog::info("marking shard as full consensus participant: {}", aas.address.unique_id);
|
||||
aas.status = Status::CONSENSUS_PARTICIPANT;
|
||||
machine_contains_shard = true;
|
||||
} else {
|
||||
const bool same_machine = aas.address.last_known_ip == storage_manager.last_known_ip &&
|
||||
aas.address.last_known_port == storage_manager.last_known_port;
|
||||
if (same_machine) {
|
||||
machine_contains_shard = true;
|
||||
ret.push_back(ShardToInitialize{
|
||||
.uuid = aas.address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = std::nullopt,
|
||||
.config = Config{},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!machine_contains_shard && shard.size() < label_space.replication_factor) {
|
||||
Address address = storage_manager;
|
||||
|
||||
// TODO(tyler) use deterministic UUID so that coordinators don't diverge here
|
||||
address.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
|
||||
ret.push_back(ShardToInitialize{
|
||||
.uuid = address.unique_id,
|
||||
.label_id = label_id,
|
||||
.min_key = low_key,
|
||||
.max_key = std::nullopt,
|
||||
.config = Config{},
|
||||
});
|
||||
|
||||
AddressAndStatus aas = {
|
||||
.address = address,
|
||||
.status = Status::INITIALIZING,
|
||||
};
|
||||
|
||||
shard.emplace_back(aas);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
bool SplitShard(Hlc previous_shard_map_version, LabelId label_id, const PrimaryKey &key) {
|
||||
if (previous_shard_map_version != shard_map_version) {
|
||||
return false;
|
||||
}
|
||||
|
||||
auto &label_space = label_spaces.at(label_id);
|
||||
auto &shards_in_map = label_space.shards;
|
||||
|
||||
MG_ASSERT(!shards_in_map.empty());
|
||||
MG_ASSERT(!shards_in_map.contains(key));
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
// Finding the Shard that the new PrimaryKey should map to.
|
||||
auto prev = std::prev(shards_in_map.upper_bound(key));
|
||||
Shard duplicated_shard = prev->second;
|
||||
|
||||
// Apply the split
|
||||
shards_in_map[key] = duplicated_shard;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
std::optional<LabelId> InitializeNewLabel(std::string label_name, std::vector<SchemaProperty> schema,
|
||||
size_t replication_factor, Hlc last_shard_map_version);
|
||||
|
||||
void AddServer(Address server_address);
|
||||
void AddServer(Address server_address) {
|
||||
// Find a random place for the server to plug in
|
||||
}
|
||||
|
||||
std::optional<LabelId> GetLabelId(const std::string &label) const;
|
||||
// TODO(antaljanosbenjamin): Remove this and instead use NameIdMapper
|
||||
const std::string &GetLabelName(LabelId label) const;
|
||||
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const;
|
||||
const std::string &GetPropertyName(PropertyId property) const;
|
||||
std::optional<EdgeTypeId> GetEdgeTypeId(const std::string &edge_type) const;
|
||||
const std::string &GetEdgeTypeName(EdgeTypeId property) const;
|
||||
LabelId GetLabelId(const std::string &label) const { return labels.at(label); }
|
||||
|
||||
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const;
|
||||
Shards GetShardsForRange(const LabelName &label_name, const PrimaryKey &start_key, const PrimaryKey &end_key) const {
|
||||
MG_ASSERT(start_key <= end_key);
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
ShardMetadata GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const;
|
||||
LabelId label_id = labels.at(label_name);
|
||||
|
||||
ShardMetadata GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const;
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties);
|
||||
const auto &shards_for_label = label_space.shards;
|
||||
|
||||
EdgeTypeIdMap AllocateEdgeTypeIds(const std::vector<EdgeTypeName> &new_edge_types);
|
||||
MG_ASSERT(shards_for_label.begin()->first <= start_key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
/// Returns true if all shards have the desired number of replicas and they are in
|
||||
/// the CONSENSUS_PARTICIPANT state. Note that this does not necessarily mean that
|
||||
/// there is also an active leader for each shard.
|
||||
bool ClusterInitialized() const;
|
||||
auto it = std::prev(shards_for_label.upper_bound(start_key));
|
||||
const auto end_it = shards_for_label.upper_bound(end_key);
|
||||
|
||||
Shards shards{};
|
||||
|
||||
std::copy(it, end_it, std::inserter(shards, shards.end()));
|
||||
|
||||
return shards;
|
||||
}
|
||||
|
||||
Shard GetShardForKey(const LabelName &label_name, const PrimaryKey &key) const {
|
||||
MG_ASSERT(labels.contains(label_name));
|
||||
|
||||
LabelId label_id = labels.at(label_name);
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
MG_ASSERT(label_space.shards.begin()->first <= key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
Shard GetShardForKey(const LabelId &label_id, const PrimaryKey &key) const {
|
||||
MG_ASSERT(label_spaces.contains(label_id));
|
||||
|
||||
const auto &label_space = label_spaces.at(label_id);
|
||||
|
||||
MG_ASSERT(label_space.shards.begin()->first <= key,
|
||||
"the ShardMap must always contain a minimal key that is less than or equal to any requested key");
|
||||
|
||||
return std::prev(label_space.shards.upper_bound(key))->second;
|
||||
}
|
||||
|
||||
PropertyMap AllocatePropertyIds(const std::vector<PropertyName> &new_properties) {
|
||||
PropertyMap ret{};
|
||||
|
||||
bool mutated = false;
|
||||
|
||||
for (const auto &property_name : new_properties) {
|
||||
if (properties.contains(property_name)) {
|
||||
auto property_id = properties.at(property_name);
|
||||
ret.emplace(property_name, property_id);
|
||||
} else {
|
||||
mutated = true;
|
||||
|
||||
const PropertyId property_id = PropertyId::FromUint(++max_property_id);
|
||||
ret.emplace(property_name, property_id);
|
||||
properties.emplace(property_name, property_id);
|
||||
}
|
||||
}
|
||||
|
||||
if (mutated) {
|
||||
IncrementShardMapVersion();
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
std::optional<PropertyId> GetPropertyId(const std::string &property_name) const {
|
||||
if (properties.contains(property_name)) {
|
||||
return properties.at(property_name);
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::coordinator
|
||||
|
||||
@@ -17,4 +17,4 @@ target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/ast)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/interpret)
|
||||
target_include_directories(mg-expr PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/semantic)
|
||||
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser mg-functions)
|
||||
target_link_libraries(mg-expr cppitertools Boost::headers mg-utils mg-parser)
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
#ifndef MG_AST_INCLUDE_PATH
|
||||
#ifdef MG_CLANG_TIDY_CHECK
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#include "query/v2/bindings/bindings.hpp"
|
||||
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp"
|
||||
#else
|
||||
#error Missing AST include path
|
||||
#endif
|
||||
@@ -21,6 +21,8 @@
|
||||
|
||||
#ifndef MG_INJECTED_NAMESPACE_NAME
|
||||
#ifdef MG_CLANG_TIDY_CHECK
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2
|
||||
#else
|
||||
#error Missing AST namespace
|
||||
#endif
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -51,7 +51,7 @@ constexpr char kId[] = "ID";
|
||||
|
||||
namespace MG_INJECTED_NAMESPACE_NAME {
|
||||
namespace detail {
|
||||
using antlropencypher::v2::MemgraphCypher;
|
||||
using antlropencypher::MemgraphCypher;
|
||||
|
||||
template <typename TVisitor>
|
||||
std::optional<std::pair<Expression *, size_t>> VisitMemoryLimit(MemgraphCypher::MemoryLimitContext *memory_limit_ctx,
|
||||
@@ -211,13 +211,13 @@ inline std::string_view ToString(const PulsarConfigKey key) {
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
using antlropencypher::v2::MemgraphCypher;
|
||||
using antlropencypher::MemgraphCypher;
|
||||
|
||||
struct ParsingContext {
|
||||
bool is_query_cached = false;
|
||||
};
|
||||
|
||||
class CypherMainVisitor : public antlropencypher::v2::MemgraphCypherBaseVisitor {
|
||||
class CypherMainVisitor : public antlropencypher::MemgraphCypherBaseVisitor {
|
||||
public:
|
||||
explicit CypherMainVisitor(ParsingContext context, AstStorage *storage) : context_(context), storage_(storage) {}
|
||||
|
||||
|
||||
@@ -1,279 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iostream>
|
||||
#include <type_traits>
|
||||
|
||||
#include "expr/ast.hpp"
|
||||
#include "expr/typed_value.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
inline constexpr const char *identifier_node_symbol = "MG_SYMBOL_NODE";
|
||||
inline constexpr const char *identifier_edge_symbol = "MG_SYMBOL_EDGE";
|
||||
|
||||
namespace detail {
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream *out, const T &arg) {
|
||||
static_assert(!std::is_convertible<T, Expression *>::value,
|
||||
"This overload shouldn't be called with pointers convertible "
|
||||
"to Expression *. This means your other PrintObject overloads aren't "
|
||||
"being called for certain AST nodes when they should (or perhaps such "
|
||||
"overloads don't exist yet).");
|
||||
*out << arg;
|
||||
}
|
||||
|
||||
inline void PrintObject(std::ostream *out, const std::string &str) { *out << str; }
|
||||
|
||||
inline void PrintObject(std::ostream * /*out*/, Aggregation::Op /*op*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: Aggregation::Op");
|
||||
}
|
||||
|
||||
inline void PrintObject(std::ostream * /*out*/, Expression * /*expr*/);
|
||||
|
||||
inline void PrintObject(std::ostream *out, Identifier *expr) { PrintObject(out, static_cast<Expression *>(expr)); }
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream * /*out*/, const std::vector<T> & /*vec*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: vector<T>");
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream * /*out*/, const std::vector<T, utils::Allocator<T>> & /*vec*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: vector<T, utils::Allocator<T>>");
|
||||
}
|
||||
|
||||
template <typename K, typename V>
|
||||
void PrintObject(std::ostream * /*out*/, const std::map<K, V> & /*map*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: map<K, V>");
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintObject(std::ostream * /*out*/, const utils::pmr::map<utils::pmr::string, T> & /*map*/) {
|
||||
throw utils::NotYetImplemented("PrintObject: map<utils::pmr::string, V>");
|
||||
}
|
||||
|
||||
template <typename T1, typename T2, typename T3>
|
||||
inline void PrintObject(std::ostream *out, const TypedValueT<T1, T2, T3> &value) {
|
||||
using TypedValue = TypedValueT<T1, T2, T3>;
|
||||
switch (value.type()) {
|
||||
case TypedValue::Type::Null:
|
||||
*out << "null";
|
||||
break;
|
||||
case TypedValue::Type::String:
|
||||
PrintObject(out, value.ValueString());
|
||||
break;
|
||||
case TypedValue::Type::Bool:
|
||||
*out << (value.ValueBool() ? "true" : "false");
|
||||
break;
|
||||
case TypedValue::Type::Int:
|
||||
PrintObject(out, value.ValueInt());
|
||||
break;
|
||||
case TypedValue::Type::Double:
|
||||
PrintObject(out, value.ValueDouble());
|
||||
break;
|
||||
case TypedValue::Type::List:
|
||||
PrintObject(out, value.ValueList());
|
||||
break;
|
||||
case TypedValue::Type::Map:
|
||||
PrintObject(out, value.ValueMap());
|
||||
break;
|
||||
case TypedValue::Type::Date:
|
||||
PrintObject(out, value.ValueDate());
|
||||
break;
|
||||
case TypedValue::Type::Duration:
|
||||
PrintObject(out, value.ValueDuration());
|
||||
break;
|
||||
case TypedValue::Type::LocalTime:
|
||||
PrintObject(out, value.ValueLocalTime());
|
||||
break;
|
||||
case TypedValue::Type::LocalDateTime:
|
||||
PrintObject(out, value.ValueLocalDateTime());
|
||||
break;
|
||||
default:
|
||||
MG_ASSERT(false, "PrintObject(std::ostream *out, const TypedValue &value) should not reach here");
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
void PrintOperatorArgs(const std::string & /*name*/, std::ostream *out, bool with_parenthesis, const T &arg) {
|
||||
PrintObject(out, arg);
|
||||
if (with_parenthesis) {
|
||||
*out << ")";
|
||||
}
|
||||
}
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
void PrintOperatorArgs(const std::string &name, std::ostream *out, bool with_parenthesis, const T &arg,
|
||||
const Ts &...args) {
|
||||
PrintObject(out, arg);
|
||||
*out << " " << name << " ";
|
||||
PrintOperatorArgs(name, out, with_parenthesis, args...);
|
||||
}
|
||||
|
||||
template <typename... Ts>
|
||||
void PrintOperator(const std::string &name, std::ostream *out, bool with_parenthesis, const Ts &...args) {
|
||||
if (with_parenthesis) {
|
||||
*out << "(";
|
||||
}
|
||||
PrintOperatorArgs(name, out, with_parenthesis, args...);
|
||||
}
|
||||
|
||||
// new
|
||||
template <typename T>
|
||||
void PrintOperatorArgs(std::ostream *out, const T &arg) {
|
||||
PrintObject(out, arg);
|
||||
}
|
||||
|
||||
template <typename T, typename... Ts>
|
||||
void PrintOperatorArgs(std::ostream *out, const T &arg, const Ts &...args) {
|
||||
PrintObject(out, arg);
|
||||
PrintOperatorArgs(out, args...);
|
||||
}
|
||||
|
||||
template <typename... Ts>
|
||||
void PrintOperator(std::ostream *out, const Ts &...args) {
|
||||
PrintOperatorArgs(out, args...);
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
class ExpressionPrettyPrinter : public ExpressionVisitor<void> {
|
||||
public:
|
||||
explicit ExpressionPrettyPrinter(std::ostream *out) : out_(out) {}
|
||||
|
||||
// Unary operators
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define UNARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE &op) override { detail::PrintOperator(OP_STR, out_, false /*with_parenthesis*/, op.expression_); }
|
||||
|
||||
UNARY_OPERATOR_VISIT(NotOperator, "Not");
|
||||
UNARY_OPERATOR_VISIT(UnaryPlusOperator, "+");
|
||||
UNARY_OPERATOR_VISIT(UnaryMinusOperator, "-");
|
||||
UNARY_OPERATOR_VISIT(IsNullOperator, "IsNull");
|
||||
|
||||
#undef UNARY_OPERATOR_VISIT
|
||||
|
||||
// Binary operators
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define BINARY_OPERATOR_VISIT(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE &op) override { \
|
||||
detail::PrintOperator(OP_STR, out_, true /*with_parenthesis*/, op.expression1_, op.expression2_); \
|
||||
}
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define BINARY_OPERATOR_VISIT_NOT_IMPL(OP_NODE, OP_STR) \
|
||||
/* NOLINTNEXTLINE(bugprone-macro-parentheses) */ \
|
||||
void Visit(OP_NODE & /*op*/) override { throw utils::NotYetImplemented("OP_NODE"); }
|
||||
|
||||
BINARY_OPERATOR_VISIT(OrOperator, "Or");
|
||||
BINARY_OPERATOR_VISIT(XorOperator, "Xor");
|
||||
BINARY_OPERATOR_VISIT(AndOperator, "And");
|
||||
BINARY_OPERATOR_VISIT(AdditionOperator, "+");
|
||||
BINARY_OPERATOR_VISIT(SubtractionOperator, "-");
|
||||
BINARY_OPERATOR_VISIT(MultiplicationOperator, "*");
|
||||
BINARY_OPERATOR_VISIT(DivisionOperator, "/");
|
||||
BINARY_OPERATOR_VISIT(ModOperator, "%");
|
||||
BINARY_OPERATOR_VISIT(NotEqualOperator, "!=");
|
||||
BINARY_OPERATOR_VISIT(EqualOperator, "=");
|
||||
BINARY_OPERATOR_VISIT(LessOperator, "<");
|
||||
BINARY_OPERATOR_VISIT(GreaterOperator, ">");
|
||||
BINARY_OPERATOR_VISIT(LessEqualOperator, "<=");
|
||||
BINARY_OPERATOR_VISIT(GreaterEqualOperator, ">=");
|
||||
BINARY_OPERATOR_VISIT_NOT_IMPL(InListOperator, "In");
|
||||
BINARY_OPERATOR_VISIT_NOT_IMPL(SubscriptOperator, "Subscript");
|
||||
|
||||
#undef BINARY_OPERATOR_VISIT
|
||||
#undef BINARY_OPERATOR_VISIT_NOT_IMPL
|
||||
|
||||
// Other
|
||||
void Visit(ListSlicingOperator & /*op*/) override { throw utils::NotYetImplemented("ListSlicingOperator"); }
|
||||
|
||||
void Visit(IfOperator & /*op*/) override { throw utils::NotYetImplemented("IfOperator"); }
|
||||
|
||||
void Visit(ListLiteral & /*op*/) override { throw utils::NotYetImplemented("ListLiteral"); }
|
||||
|
||||
void Visit(MapLiteral & /*op*/) override { throw utils::NotYetImplemented("MapLiteral"); }
|
||||
|
||||
void Visit(LabelsTest & /*op*/) override { throw utils::NotYetImplemented("LabelsTest"); }
|
||||
|
||||
void Visit(Aggregation & /*op*/) override { throw utils::NotYetImplemented("Aggregation"); }
|
||||
|
||||
void Visit(Function & /*op*/) override { throw utils::NotYetImplemented("Function"); }
|
||||
|
||||
void Visit(Reduce & /*op*/) override { throw utils::NotYetImplemented("Reduce"); }
|
||||
|
||||
void Visit(Coalesce & /*op*/) override { throw utils::NotYetImplemented("Coalesce"); }
|
||||
|
||||
void Visit(Extract & /*op*/) override { throw utils::NotYetImplemented("Extract"); }
|
||||
|
||||
void Visit(All & /*op*/) override { throw utils::NotYetImplemented("All"); }
|
||||
|
||||
void Visit(Single & /*op*/) override { throw utils::NotYetImplemented("Single"); }
|
||||
|
||||
void Visit(Any & /*op*/) override { throw utils::NotYetImplemented("Any"); }
|
||||
|
||||
void Visit(None & /*op*/) override { throw utils::NotYetImplemented("None"); }
|
||||
|
||||
void Visit(Identifier &op) override {
|
||||
auto is_node = true;
|
||||
auto is_edge = false;
|
||||
auto is_other = false;
|
||||
if (is_node) {
|
||||
detail::PrintOperator(out_, identifier_node_symbol);
|
||||
} else if (is_edge) {
|
||||
detail::PrintOperator(out_, identifier_edge_symbol);
|
||||
} else {
|
||||
MG_ASSERT(is_other);
|
||||
detail::PrintOperator(out_, op.name_);
|
||||
}
|
||||
}
|
||||
|
||||
void Visit(PrimitiveLiteral &op) override { detail::PrintObject(out_, op.value_); }
|
||||
|
||||
void Visit(PropertyLookup &op) override { detail::PrintOperator(out_, op.expression_, ".", op.property_.name); }
|
||||
|
||||
void Visit(ParameterLookup & /*op*/) override { throw utils::NotYetImplemented("ParameterLookup"); }
|
||||
|
||||
void Visit(NamedExpression & /*op*/) override { throw utils::NotYetImplemented("NamedExpression"); }
|
||||
|
||||
void Visit(RegexMatch & /*op*/) override { throw utils::NotYetImplemented("RegexMatch"); }
|
||||
|
||||
private:
|
||||
std::ostream *out_;
|
||||
};
|
||||
|
||||
namespace detail {
|
||||
inline void PrintObject(std::ostream *out, Expression *expr) {
|
||||
if (expr) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
} else {
|
||||
*out << "<null>";
|
||||
}
|
||||
}
|
||||
} // namespace detail
|
||||
|
||||
inline void PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(Expression *expr, std::ostream *out) {
|
||||
ExpressionPrettyPrinter printer{out};
|
||||
expr->Accept(printer);
|
||||
}
|
||||
|
||||
inline std::string ExpressiontoStringWhileReplacingNodeAndEdgeSymbols(Expression *expr) {
|
||||
std::ostringstream ss;
|
||||
expr::PrintExpressionToOriginalAndReplaceNodeAndEdgeSymbols(expr, &ss);
|
||||
return ss.str();
|
||||
}
|
||||
} // namespace memgraph::expr
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -24,7 +24,6 @@
|
||||
#include "expr/exceptions.hpp"
|
||||
#include "expr/interpret/frame.hpp"
|
||||
#include "expr/semantic/symbol_table.hpp"
|
||||
#include "functions/awesome_memgraph_functions.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::expr {
|
||||
@@ -36,8 +35,8 @@ template <typename TypedValue, typename EvaluationContext, typename DbAccessor,
|
||||
typename PropertyValue, typename ConvFunctor, typename Error, typename Tag = StorageTag>
|
||||
class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
public:
|
||||
ExpressionEvaluator(Frame *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx, DbAccessor *dba,
|
||||
StorageView view)
|
||||
ExpressionEvaluator(Frame<TypedValue> *frame, const SymbolTable &symbol_table, const EvaluationContext &ctx,
|
||||
DbAccessor *dba, StorageView view)
|
||||
: frame_(frame), symbol_table_(&symbol_table), ctx_(&ctx), dba_(dba), view_(view) {}
|
||||
|
||||
using ExpressionVisitor<TypedValue>::Visit;
|
||||
@@ -47,12 +46,12 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
TypedValue Visit(NamedExpression &named_expression) override {
|
||||
const auto &symbol = symbol_table_->at(named_expression);
|
||||
auto value = named_expression.expression_->Accept(*this);
|
||||
frame_->At(symbol) = value;
|
||||
frame_->at(symbol) = value;
|
||||
return value;
|
||||
}
|
||||
|
||||
TypedValue Visit(Identifier &ident) override {
|
||||
return TypedValue(frame_->At(symbol_table_->at(ident)), ctx_->memory);
|
||||
return TypedValue(frame_->at(symbol_table_->at(ident)), ctx_->memory);
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
@@ -101,28 +100,6 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
#undef BINARY_OPERATOR_VISITOR
|
||||
#undef UNARY_OPERATOR_VISITOR
|
||||
|
||||
void HandleObjectAccessError(Error &shard_error, const std::string_view accessed_object) {
|
||||
switch (shard_error) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access {} on a deleted object.", accessed_object);
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access {} from a node object doesn't exist.", accessed_object);
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
case Error::VERTEX_ALREADY_INSERTED:
|
||||
case Error::OBJECT_NOT_FOUND:
|
||||
throw ExpressionRuntimeException("Unexpected error when accessing {}.", accessed_object);
|
||||
case Error::SCHEMA_NO_SCHEMA_DEFINED_FOR_LABEL:
|
||||
case Error::SCHEMA_VERTEX_PROPERTY_WRONG_TYPE:
|
||||
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_KEY:
|
||||
case Error::SCHEMA_VERTEX_UPDATE_PRIMARY_LABEL:
|
||||
case Error::SCHEMA_VERTEX_SECONDARY_LABEL_IS_PRIMARY:
|
||||
case Error::SCHEMA_VERTEX_PRIMARY_PROPERTIES_UNDEFINED:
|
||||
throw ExpressionRuntimeException("Unexpected schema violation when accessing {}.", accessed_object);
|
||||
}
|
||||
}
|
||||
|
||||
TypedValue Visit(AndOperator &op) override {
|
||||
auto value1 = op.expression1_->Accept(*this);
|
||||
if (value1.IsBool() && !value1.ValueBool()) {
|
||||
@@ -419,7 +396,16 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
has_label = vertex.HasLabel(StorageView::NEW, GetLabel(label));
|
||||
}
|
||||
if (has_label.HasError()) {
|
||||
HandleObjectAccessError(has_label.GetError().code, "labels");
|
||||
switch (has_label.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access labels on a deleted node.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to access labels from a node that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
throw ExpressionRuntimeException("Unexpected error when accessing labels.");
|
||||
}
|
||||
}
|
||||
return *has_label;
|
||||
}
|
||||
@@ -428,7 +414,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
typename TReturnType = std::enable_if_t<std::is_same_v<TTag, QueryEngineTag>, bool>>
|
||||
TReturnType HasLabelImpl(const VertexAccessor &vertex, const LabelIx &label_ix, QueryEngineTag /*tag*/) {
|
||||
auto label = typename VertexAccessor::Label{LabelId::FromUint(label_ix.ix)};
|
||||
return vertex.HasLabel(label);
|
||||
auto has_label = vertex.HasLabel(label);
|
||||
return !has_label;
|
||||
}
|
||||
|
||||
TypedValue Visit(LabelsTest &labels_test) override {
|
||||
@@ -470,7 +457,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
|
||||
TypedValue Visit(Aggregation &aggregation) override {
|
||||
return TypedValue(frame_->At(symbol_table_->at(aggregation)), ctx_->memory);
|
||||
return TypedValue(frame_->at(symbol_table_->at(aggregation)), ctx_->memory);
|
||||
}
|
||||
|
||||
TypedValue Visit(Coalesce &coalesce) override {
|
||||
@@ -491,7 +478,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
}
|
||||
|
||||
TypedValue Visit(Function &function) override {
|
||||
functions::FunctionContext<DbAccessor> function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
|
||||
FunctionContext function_ctx{dba_, ctx_->memory, ctx_->timestamp, &ctx_->counters, view_};
|
||||
// Stack allocate evaluated arguments when there's a small number of them.
|
||||
if (function.arguments_.size() <= 8) {
|
||||
TypedValue arguments[8] = {TypedValue(ctx_->memory), TypedValue(ctx_->memory), TypedValue(ctx_->memory),
|
||||
@@ -528,8 +515,8 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
const auto &accumulator_symbol = symbol_table_->at(*reduce.accumulator_);
|
||||
auto accumulator = reduce.initializer_->Accept(*this);
|
||||
for (const auto &element : list) {
|
||||
frame_->At(accumulator_symbol) = accumulator;
|
||||
frame_->At(element_symbol) = element;
|
||||
frame_->at(accumulator_symbol) = accumulator;
|
||||
frame_->at(element_symbol) = element;
|
||||
accumulator = reduce.expression_->Accept(*this);
|
||||
}
|
||||
return accumulator;
|
||||
@@ -551,7 +538,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
if (element.IsNull()) {
|
||||
result.emplace_back();
|
||||
} else {
|
||||
frame_->At(element_symbol) = element;
|
||||
frame_->at(element_symbol) = element;
|
||||
result.emplace_back(extract.expression_->Accept(*this));
|
||||
}
|
||||
}
|
||||
@@ -571,7 +558,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
bool has_null_elements = false;
|
||||
bool has_value = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->At(symbol) = element;
|
||||
frame_->at(symbol) = element;
|
||||
auto result = all.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of ALL must evaluate to boolean, got {}.", result.type());
|
||||
@@ -608,7 +595,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
bool has_value = false;
|
||||
bool predicate_satisfied = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->At(symbol) = element;
|
||||
frame_->at(symbol) = element;
|
||||
auto result = single.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of SINGLE must evaluate to boolean, got {}.", result.type());
|
||||
@@ -645,7 +632,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
const auto &symbol = symbol_table_->at(*any.identifier_);
|
||||
bool has_value = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->At(symbol) = element;
|
||||
frame_->at(symbol) = element;
|
||||
auto result = any.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of ANY must evaluate to boolean, got {}.", result.type());
|
||||
@@ -677,7 +664,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
const auto &symbol = symbol_table_->at(*none.identifier_);
|
||||
bool has_value = false;
|
||||
for (const auto &element : list) {
|
||||
frame_->At(symbol) = element;
|
||||
frame_->at(symbol) = element;
|
||||
auto result = none.where_->expression_->Accept(*this);
|
||||
if (!result.IsNull() && result.type() != TypedValue::Type::Bool) {
|
||||
throw ExpressionRuntimeException("Predicate of NONE must evaluate to boolean, got {}.", result.type());
|
||||
@@ -731,7 +718,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
TReturnType GetProperty(const TRecordAccessor &record_accessor, PropertyIx prop) {
|
||||
auto maybe_prop = record_accessor.GetProperty(prop.name);
|
||||
// Handler non existent property
|
||||
return conv_(maybe_prop, dba_);
|
||||
return conv_(maybe_prop);
|
||||
}
|
||||
|
||||
template <class TRecordAccessor, class TTag = Tag,
|
||||
@@ -739,7 +726,7 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
TReturnType GetProperty(const TRecordAccessor &record_accessor, const std::string_view name) {
|
||||
auto maybe_prop = record_accessor.GetProperty(std::string(name));
|
||||
// Handler non existent property
|
||||
return conv_(maybe_prop, dba_);
|
||||
return conv_(maybe_prop);
|
||||
}
|
||||
|
||||
template <class TRecordAccessor, class TTag = Tag,
|
||||
@@ -756,7 +743,16 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
maybe_prop = record_accessor.GetProperty(StorageView::NEW, ctx_->properties[prop.ix]);
|
||||
}
|
||||
if (maybe_prop.HasError()) {
|
||||
HandleObjectAccessError(maybe_prop.GetError().code, "property");
|
||||
switch (maybe_prop.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
throw ExpressionRuntimeException("Unexpected error when getting a property.");
|
||||
}
|
||||
}
|
||||
return conv_(*maybe_prop, ctx_->memory);
|
||||
}
|
||||
@@ -775,14 +771,23 @@ class ExpressionEvaluator : public ExpressionVisitor<TypedValue> {
|
||||
maybe_prop = record_accessor.GetProperty(view_, dba_->NameToProperty(name));
|
||||
}
|
||||
if (maybe_prop.HasError()) {
|
||||
HandleObjectAccessError(maybe_prop.GetError().code, "property");
|
||||
switch (maybe_prop.GetError()) {
|
||||
case Error::DELETED_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from a deleted object.");
|
||||
case Error::NONEXISTENT_OBJECT:
|
||||
throw ExpressionRuntimeException("Trying to get a property from an object that doesn't exist.");
|
||||
case Error::SERIALIZATION_ERROR:
|
||||
case Error::VERTEX_HAS_EDGES:
|
||||
case Error::PROPERTIES_DISABLED:
|
||||
throw ExpressionRuntimeException("Unexpected error when getting a property.");
|
||||
}
|
||||
}
|
||||
return conv_(*maybe_prop, ctx_->memory);
|
||||
}
|
||||
|
||||
LabelId GetLabel(LabelIx label) { return ctx_->labels[label.ix]; }
|
||||
|
||||
Frame *frame_;
|
||||
Frame<TypedValue> *frame_;
|
||||
const SymbolTable *symbol_table_;
|
||||
const EvaluationContext *ctx_;
|
||||
DbAccessor *dba_;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -20,43 +20,26 @@
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
template <typename TypedValue>
|
||||
class Frame {
|
||||
public:
|
||||
/// Create a Frame of given size backed by a utils::NewDeleteResource()
|
||||
explicit Frame(size_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
|
||||
explicit Frame(int64_t size) : elems_(size, utils::NewDeleteResource()) { MG_ASSERT(size >= 0); }
|
||||
|
||||
Frame(size_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
|
||||
Frame(int64_t size, utils::MemoryResource *memory) : elems_(size, memory) { MG_ASSERT(size >= 0); }
|
||||
|
||||
TypedValue &operator[](const Symbol &symbol) { return elems_[symbol.position()]; }
|
||||
const TypedValue &operator[](const Symbol &symbol) const { return elems_[symbol.position()]; }
|
||||
|
||||
TypedValue &At(const Symbol &symbol) { return elems_.at(symbol.position()); }
|
||||
const TypedValue &At(const Symbol &symbol) const { return elems_.at(symbol.position()); }
|
||||
TypedValue &at(const Symbol &symbol) { return elems_.at(symbol.position()); }
|
||||
const TypedValue &at(const Symbol &symbol) const { return elems_.at(symbol.position()); }
|
||||
|
||||
uint64_t Id() const { return id_; }
|
||||
void SetId(const uint64_t id) { id_ = id; }
|
||||
|
||||
const utils::pmr::vector<TypedValue> &Elems() const { return elems_; }
|
||||
auto &elems() { return elems_; }
|
||||
|
||||
utils::MemoryResource *GetMemoryResource() const { return elems_.get_allocator().GetMemoryResource(); }
|
||||
|
||||
private:
|
||||
uint64_t id_{0U};
|
||||
utils::pmr::vector<TypedValue> elems_;
|
||||
};
|
||||
|
||||
class FrameWithValidity final : public Frame {
|
||||
public:
|
||||
explicit FrameWithValidity(size_t size) : Frame(size), is_valid_(false) {}
|
||||
|
||||
FrameWithValidity(size_t size, utils::MemoryResource *memory) : Frame(size, memory), is_valid_(false) {}
|
||||
|
||||
bool IsValid() const noexcept { return is_valid_; }
|
||||
void MakeValid() noexcept { is_valid_ = true; }
|
||||
void MakeInvalid() noexcept { is_valid_ = false; }
|
||||
|
||||
private:
|
||||
bool is_valid_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::expr
|
||||
|
||||
@@ -21,7 +21,6 @@
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "expr/typed_value_exception.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
@@ -33,6 +32,16 @@
|
||||
|
||||
namespace memgraph::expr {
|
||||
|
||||
/**
|
||||
* An exception raised by the TypedValue system. Typically when
|
||||
* trying to perform operations (such as addition) on TypedValues
|
||||
* of incompatible Types.
|
||||
*/
|
||||
class TypedValueException : public utils::BasicException {
|
||||
public:
|
||||
using utils::BasicException::BasicException;
|
||||
};
|
||||
|
||||
// TODO: Neo4j does overflow checking. Should we also implement it?
|
||||
/**
|
||||
* Stores a query runtime value and its type.
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
add_library(mg-functions INTERFACE)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,17 +15,9 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
#include "common/errors.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/edge_accessor.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/vertex_accessor.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/temporal.hpp"
|
||||
|
||||
using memgraph::communication::bolt::Value;
|
||||
@@ -71,139 +63,127 @@ query::v2::TypedValue ToTypedValue(const Value &value) {
|
||||
}
|
||||
}
|
||||
|
||||
communication::bolt::Vertex ToBoltVertex(const query::v2::accessors::VertexAccessor &vertex,
|
||||
const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View /*view*/) {
|
||||
auto id = communication::bolt::Id::FromUint(0);
|
||||
|
||||
auto labels = vertex.Labels();
|
||||
std::vector<std::string> new_labels;
|
||||
new_labels.reserve(labels.size());
|
||||
for (const auto &label : labels) {
|
||||
new_labels.push_back(request_router->LabelToName(label.id));
|
||||
}
|
||||
|
||||
auto properties = vertex.Properties();
|
||||
std::map<std::string, Value> new_properties;
|
||||
for (const auto &[prop, property_value] : properties) {
|
||||
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
|
||||
}
|
||||
return communication::bolt::Vertex{id, new_labels, new_properties};
|
||||
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const query::v2::VertexAccessor &vertex,
|
||||
const storage::v3::Shard &db, storage::v3::View view) {
|
||||
return ToBoltVertex(vertex.impl_, db, view);
|
||||
}
|
||||
|
||||
communication::bolt::Edge ToBoltEdge(const query::v2::accessors::EdgeAccessor &edge,
|
||||
const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View /*view*/) {
|
||||
// TODO(jbajic) Fix bolt communication
|
||||
auto id = communication::bolt::Id::FromUint(0);
|
||||
auto from = communication::bolt::Id::FromUint(0);
|
||||
auto to = communication::bolt::Id::FromUint(0);
|
||||
const auto &type = request_router->EdgeTypeToName(edge.EdgeType());
|
||||
|
||||
auto properties = edge.Properties();
|
||||
std::map<std::string, Value> new_properties;
|
||||
for (const auto &[prop, property_value] : properties) {
|
||||
new_properties[request_router->PropertyToName(prop)] = ToBoltValue(property_value);
|
||||
}
|
||||
return communication::bolt::Edge{id, from, to, type, new_properties};
|
||||
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const query::v2::EdgeAccessor &edge,
|
||||
const storage::v3::Shard &db, storage::v3::View view) {
|
||||
return ToBoltEdge(edge.impl_, db, view);
|
||||
}
|
||||
|
||||
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path & /*edge*/,
|
||||
const query::v2::RequestRouterInterface * /*request_router*/,
|
||||
storage::v3::View /*view*/) {
|
||||
// TODO(jbajic) Fix bolt communication
|
||||
MG_ASSERT(false, "Path is unimplemented!");
|
||||
return {};
|
||||
}
|
||||
|
||||
Value ToBoltValue(const query::v2::TypedValue &value, const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View view) {
|
||||
storage::v3::Result<Value> ToBoltValue(const query::v2::TypedValue &value, const storage::v3::Shard &db,
|
||||
storage::v3::View view) {
|
||||
switch (value.type()) {
|
||||
case query::v2::TypedValue::Type::Null:
|
||||
return {};
|
||||
return Value();
|
||||
case query::v2::TypedValue::Type::Bool:
|
||||
return {value.ValueBool()};
|
||||
return Value(value.ValueBool());
|
||||
case query::v2::TypedValue::Type::Int:
|
||||
return {value.ValueInt()};
|
||||
return Value(value.ValueInt());
|
||||
case query::v2::TypedValue::Type::Double:
|
||||
return {value.ValueDouble()};
|
||||
return Value(value.ValueDouble());
|
||||
case query::v2::TypedValue::Type::String:
|
||||
return {std::string(value.ValueString())};
|
||||
return Value(std::string(value.ValueString()));
|
||||
case query::v2::TypedValue::Type::List: {
|
||||
std::vector<Value> values;
|
||||
values.reserve(value.ValueList().size());
|
||||
for (const auto &v : value.ValueList()) {
|
||||
auto value = ToBoltValue(v, request_router, view);
|
||||
values.emplace_back(std::move(value));
|
||||
auto maybe_value = ToBoltValue(v, db, view);
|
||||
if (maybe_value.HasError()) return maybe_value.GetError();
|
||||
values.emplace_back(std::move(*maybe_value));
|
||||
}
|
||||
return {std::move(values)};
|
||||
return Value(std::move(values));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Map: {
|
||||
std::map<std::string, Value> map;
|
||||
for (const auto &kv : value.ValueMap()) {
|
||||
auto value = ToBoltValue(kv.second, request_router, view);
|
||||
map.emplace(kv.first, std::move(value));
|
||||
auto maybe_value = ToBoltValue(kv.second, db, view);
|
||||
if (maybe_value.HasError()) return maybe_value.GetError();
|
||||
map.emplace(kv.first, std::move(*maybe_value));
|
||||
}
|
||||
return {std::move(map)};
|
||||
return Value(std::move(map));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Vertex: {
|
||||
auto vertex = ToBoltVertex(value.ValueVertex(), request_router, view);
|
||||
return {std::move(vertex)};
|
||||
auto maybe_vertex = ToBoltVertex(value.ValueVertex(), db, view);
|
||||
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
|
||||
return Value(std::move(*maybe_vertex));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Edge: {
|
||||
auto edge = ToBoltEdge(value.ValueEdge(), request_router, view);
|
||||
return {std::move(edge)};
|
||||
auto maybe_edge = ToBoltEdge(value.ValueEdge(), db, view);
|
||||
if (maybe_edge.HasError()) return maybe_edge.GetError();
|
||||
return Value(std::move(*maybe_edge));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Path: {
|
||||
auto path = ToBoltPath(value.ValuePath(), request_router, view);
|
||||
return {std::move(path)};
|
||||
auto maybe_path = ToBoltPath(value.ValuePath(), db, view);
|
||||
if (maybe_path.HasError()) return maybe_path.GetError();
|
||||
return Value(std::move(*maybe_path));
|
||||
}
|
||||
case query::v2::TypedValue::Type::Date:
|
||||
return {value.ValueDate()};
|
||||
return Value(value.ValueDate());
|
||||
case query::v2::TypedValue::Type::LocalTime:
|
||||
return {value.ValueLocalTime()};
|
||||
return Value(value.ValueLocalTime());
|
||||
case query::v2::TypedValue::Type::LocalDateTime:
|
||||
return {value.ValueLocalDateTime()};
|
||||
return Value(value.ValueLocalDateTime());
|
||||
case query::v2::TypedValue::Type::Duration:
|
||||
return {value.ValueDuration()};
|
||||
return Value(value.ValueDuration());
|
||||
}
|
||||
}
|
||||
|
||||
Value ToBoltValue(msgs::Value value) {
|
||||
switch (value.type) {
|
||||
case msgs::Value::Type::Null:
|
||||
return {};
|
||||
case msgs::Value::Type::Bool:
|
||||
return {value.bool_v};
|
||||
case msgs::Value::Type::Int64:
|
||||
return {value.int_v};
|
||||
case msgs::Value::Type::Double:
|
||||
return {value.double_v};
|
||||
case msgs::Value::Type::String:
|
||||
return {std::string(value.string_v)};
|
||||
case msgs::Value::Type::List: {
|
||||
std::vector<Value> values;
|
||||
values.reserve(value.list_v.size());
|
||||
for (const auto &v : value.list_v) {
|
||||
auto maybe_value = ToBoltValue(v);
|
||||
values.emplace_back(std::move(maybe_value));
|
||||
}
|
||||
return Value{std::move(values)};
|
||||
}
|
||||
case msgs::Value::Type::Map: {
|
||||
std::map<std::string, Value> map;
|
||||
for (const auto &kv : value.map_v) {
|
||||
auto maybe_value = ToBoltValue(kv.second);
|
||||
map.emplace(kv.first, std::move(maybe_value));
|
||||
}
|
||||
return Value{std::move(map)};
|
||||
}
|
||||
case msgs::Value::Type::Vertex:
|
||||
case msgs::Value::Type::Edge: {
|
||||
throw utils::BasicException("Vertex and Edge not supported!");
|
||||
}
|
||||
// TODO Value to Date types not supported
|
||||
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
|
||||
const storage::v3::Shard &db, storage::v3::View view) {
|
||||
// TODO(jbajic) Fix bolt communication
|
||||
auto id = communication::bolt::Id::FromUint(0);
|
||||
auto maybe_labels = vertex.Labels(view);
|
||||
if (maybe_labels.HasError()) return maybe_labels.GetError();
|
||||
std::vector<std::string> labels;
|
||||
labels.reserve(maybe_labels->size());
|
||||
for (const auto &label : *maybe_labels) {
|
||||
labels.push_back(db.LabelToName(label));
|
||||
}
|
||||
auto maybe_properties = vertex.Properties(view);
|
||||
if (maybe_properties.HasError()) return maybe_properties.GetError();
|
||||
std::map<std::string, Value> properties;
|
||||
for (const auto &prop : *maybe_properties) {
|
||||
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
|
||||
}
|
||||
return communication::bolt::Vertex{id, labels, properties};
|
||||
}
|
||||
|
||||
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
|
||||
const storage::v3::Shard &db, storage::v3::View view) {
|
||||
// TODO(jbajic) Fix bolt communication
|
||||
auto id = communication::bolt::Id::FromUint(0);
|
||||
auto from = communication::bolt::Id::FromUint(0);
|
||||
auto to = communication::bolt::Id::FromUint(0);
|
||||
const auto &type = db.EdgeTypeToName(edge.EdgeType());
|
||||
auto maybe_properties = edge.Properties(view);
|
||||
if (maybe_properties.HasError()) return maybe_properties.GetError();
|
||||
std::map<std::string, Value> properties;
|
||||
for (const auto &prop : *maybe_properties) {
|
||||
properties[db.PropertyToName(prop.first)] = ToBoltValue(prop.second);
|
||||
}
|
||||
return communication::bolt::Edge{id, from, to, type, properties};
|
||||
}
|
||||
|
||||
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
|
||||
storage::v3::View view) {
|
||||
std::vector<communication::bolt::Vertex> vertices;
|
||||
vertices.reserve(path.vertices().size());
|
||||
for (const auto &v : path.vertices()) {
|
||||
auto maybe_vertex = ToBoltVertex(v, db, view);
|
||||
if (maybe_vertex.HasError()) return maybe_vertex.GetError();
|
||||
vertices.emplace_back(std::move(*maybe_vertex));
|
||||
}
|
||||
std::vector<communication::bolt::Edge> edges;
|
||||
edges.reserve(path.edges().size());
|
||||
for (const auto &e : path.edges()) {
|
||||
auto maybe_edge = ToBoltEdge(e, db, view);
|
||||
if (maybe_edge.HasError()) return maybe_edge.GetError();
|
||||
edges.emplace_back(std::move(*maybe_edge));
|
||||
}
|
||||
return communication::bolt::Path(vertices, edges);
|
||||
}
|
||||
|
||||
storage::v3::PropertyValue ToPropertyValue(const Value &value) {
|
||||
|
||||
@@ -13,14 +13,10 @@
|
||||
#pragma once
|
||||
|
||||
#include "communication/bolt/v1/value.hpp"
|
||||
#include "coordinator/shard_map.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::storage::v3 {
|
||||
class EdgeAccessor;
|
||||
@@ -32,37 +28,36 @@ namespace memgraph::glue::v2 {
|
||||
|
||||
/// @param storage::v3::VertexAccessor for converting to
|
||||
/// communication::bolt::Vertex.
|
||||
/// @param query::v2::RequestRouterInterface *request_router getting label and property names.
|
||||
/// @param storage::v3::Shard for getting label and property names.
|
||||
/// @param storage::v3::View for deciding which vertex attributes are visible.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Vertex ToBoltVertex(const storage::v3::VertexAccessor &vertex,
|
||||
const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Vertex> ToBoltVertex(const storage::v3::VertexAccessor &vertex,
|
||||
const storage::v3::Shard &db, storage::v3::View view);
|
||||
|
||||
/// @param storage::v3::EdgeAccessor for converting to communication::bolt::Edge.
|
||||
/// @param query::v2::RequestRouterInterface *request_router getting edge type and property names.
|
||||
/// @param storage::v3::Shard for getting edge type and property names.
|
||||
/// @param storage::v3::View for deciding which edge attributes are visible.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Edge ToBoltEdge(const storage::v3::EdgeAccessor &edge,
|
||||
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Edge> ToBoltEdge(const storage::v3::EdgeAccessor &edge,
|
||||
const storage::v3::Shard &db, storage::v3::View view);
|
||||
|
||||
/// @param query::v2::Path for converting to communication::bolt::Path.
|
||||
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
|
||||
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
|
||||
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Path ToBoltPath(const query::v2::accessors::Path &path,
|
||||
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Path> ToBoltPath(const query::v2::Path &path, const storage::v3::Shard &db,
|
||||
storage::v3::View view);
|
||||
|
||||
/// @param query::v2::TypedValue for converting to communication::bolt::Value.
|
||||
/// @param query::v2::RequestRouterInterface *request_router ToBoltVertex and ToBoltEdge.
|
||||
/// @param storage::v3::Shard for ToBoltVertex and ToBoltEdge.
|
||||
/// @param storage::v3::View for ToBoltVertex and ToBoltEdge.
|
||||
///
|
||||
/// @throw std::bad_alloc
|
||||
communication::bolt::Value ToBoltValue(const query::v2::TypedValue &value,
|
||||
const query::v2::RequestRouterInterface *request_router, storage::v3::View view);
|
||||
storage::v3::Result<communication::bolt::Value> ToBoltValue(const query::v2::TypedValue &value,
|
||||
const storage::v3::Shard &db, storage::v3::View view);
|
||||
|
||||
query::v2::TypedValue ToTypedValue(const communication::bolt::Value &value);
|
||||
|
||||
@@ -70,9 +65,4 @@ communication::bolt::Value ToBoltValue(const storage::v3::PropertyValue &value);
|
||||
|
||||
storage::v3::PropertyValue ToPropertyValue(const communication::bolt::Value &value);
|
||||
|
||||
communication::bolt::Value ToBoltValue(msgs::Value value);
|
||||
|
||||
communication::bolt::Value ToBoltValue(msgs::Value value, const query::v2::RequestRouterInterface *request_router,
|
||||
storage::v3::View view);
|
||||
|
||||
} // namespace memgraph::glue::v2
|
||||
|
||||
@@ -15,38 +15,11 @@
|
||||
|
||||
#include <fmt/format.h>
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
#include <boost/functional/hash.hpp>
|
||||
#include <boost/uuid/uuid.hpp>
|
||||
#include <boost/uuid/uuid_generators.hpp>
|
||||
#include <boost/uuid/uuid_io.hpp>
|
||||
|
||||
#include "utils/logging.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
struct PartialAddress {
|
||||
boost::asio::ip::address ip;
|
||||
uint16_t port;
|
||||
|
||||
friend bool operator==(const PartialAddress &lhs, const PartialAddress &rhs) = default;
|
||||
|
||||
/// unique_id is most dominant for ordering, then ip, then port
|
||||
friend bool operator<(const PartialAddress &lhs, const PartialAddress &rhs) {
|
||||
if (lhs.ip != rhs.ip) {
|
||||
return lhs.ip < rhs.ip;
|
||||
}
|
||||
|
||||
return lhs.port < rhs.port;
|
||||
}
|
||||
|
||||
std::string ToString() const { return fmt::format("PartialAddress {{ ip: {}, port: {} }}", ip.to_string(), port); }
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const PartialAddress &partial_address) {
|
||||
in << partial_address.ToString();
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
struct Address {
|
||||
// It's important for all participants to have a
|
||||
// unique identifier - IP and port alone are not
|
||||
@@ -60,39 +33,18 @@ struct Address {
|
||||
uint16_t last_known_port;
|
||||
|
||||
static Address TestAddress(uint16_t port) {
|
||||
MG_ASSERT(port <= 255);
|
||||
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, static_cast<unsigned char>(port)},
|
||||
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
.last_known_port = port,
|
||||
};
|
||||
}
|
||||
|
||||
// NB: don't use this in test code because it is non-deterministic
|
||||
static Address UniqueLocalAddress() {
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{boost::uuids::random_generator()()},
|
||||
};
|
||||
}
|
||||
|
||||
/// `Coordinator`s have constant UUIDs because there is at most one per ip/port pair.
|
||||
Address ForkLocalCoordinator() {
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
.last_known_ip = last_known_ip,
|
||||
.last_known_port = last_known_port,
|
||||
};
|
||||
}
|
||||
|
||||
/// `ShardManager`s have constant UUIDs because there is at most one per ip/port pair.
|
||||
Address ForkLocalShardManager() {
|
||||
return Address{
|
||||
.unique_id = boost::uuids::uuid{2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
|
||||
.last_known_ip = last_known_ip,
|
||||
.last_known_port = last_known_port,
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns a new ID with the same IP and port but a unique UUID.
|
||||
Address ForkUniqueAddress() {
|
||||
return Address{
|
||||
@@ -102,13 +54,6 @@ struct Address {
|
||||
};
|
||||
}
|
||||
|
||||
PartialAddress ToPartialAddress() const {
|
||||
return PartialAddress{
|
||||
.ip = last_known_ip,
|
||||
.port = last_known_port,
|
||||
};
|
||||
}
|
||||
|
||||
friend bool operator==(const Address &lhs, const Address &rhs) = default;
|
||||
|
||||
/// unique_id is most dominant for ordering, then last_known_ip, then last_known_port
|
||||
@@ -128,36 +73,5 @@ struct Address {
|
||||
return fmt::format("Address {{ unique_id: {}, last_known_ip: {}, last_known_port: {} }}",
|
||||
boost::uuids::to_string(unique_id), last_known_ip.to_string(), last_known_port);
|
||||
}
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const Address &address) {
|
||||
in << address.ToString();
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
}; // namespace memgraph::io
|
||||
|
||||
namespace std {
|
||||
template <>
|
||||
struct hash<memgraph::io::PartialAddress> {
|
||||
size_t operator()(const memgraph::io::PartialAddress &pa) const {
|
||||
using boost::hash_combine;
|
||||
using boost::hash_value;
|
||||
|
||||
// Start with a hash value of 0 .
|
||||
std::size_t seed = 0;
|
||||
|
||||
if (pa.ip.is_v4()) {
|
||||
auto h = std::hash<boost::asio::ip::address_v4>()(pa.ip.to_v4());
|
||||
hash_combine(seed, h);
|
||||
} else {
|
||||
auto h = std::hash<boost::asio::ip::address_v6>()(pa.ip.to_v6());
|
||||
hash_combine(seed, h);
|
||||
}
|
||||
hash_combine(seed, hash_value(pa.port));
|
||||
|
||||
// Return the result.
|
||||
return seed;
|
||||
}
|
||||
};
|
||||
} // namespace std
|
||||
|
||||
@@ -11,16 +11,11 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "utils/exceptions.hpp"
|
||||
namespace memgraph::io {
|
||||
/// Protocol:
|
||||
/// crc32: 4 bytes
|
||||
/// len: 8 bytes
|
||||
/// buffer: <len bytes>
|
||||
|
||||
namespace memgraph::expr {
|
||||
/**
|
||||
* An exception raised by the TypedValue system. Typically when
|
||||
* trying to perform operations (such as addition) on TypedValues
|
||||
* of incompatible Types.
|
||||
*/
|
||||
class TypedValueException : public utils::BasicException {
|
||||
public:
|
||||
using utils::BasicException::BasicException;
|
||||
};
|
||||
} // namespace memgraph::expr
|
||||
std::optional<std::pair<size_t, size_t>> make_frame(char *ptr, size_t len) { return std::nullopt; }
|
||||
} // namespace memgraph::io
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -35,13 +35,10 @@ class Shared {
|
||||
std::optional<T> item_;
|
||||
bool consumed_ = false;
|
||||
bool waiting_ = false;
|
||||
bool filled_ = false;
|
||||
std::function<bool()> wait_notifier_ = nullptr;
|
||||
std::function<void()> fill_notifier_ = nullptr;
|
||||
std::function<bool()> simulator_notifier_ = nullptr;
|
||||
|
||||
public:
|
||||
explicit Shared(std::function<bool()> wait_notifier, std::function<void()> fill_notifier)
|
||||
: wait_notifier_(wait_notifier), fill_notifier_(fill_notifier) {}
|
||||
explicit Shared(std::function<bool()> simulator_notifier) : simulator_notifier_(simulator_notifier) {}
|
||||
Shared() = default;
|
||||
Shared(Shared &&) = delete;
|
||||
Shared &operator=(Shared &&) = delete;
|
||||
@@ -67,7 +64,8 @@ class Shared {
|
||||
waiting_ = true;
|
||||
|
||||
while (!item_) {
|
||||
if (wait_notifier_) [[unlikely]] {
|
||||
bool simulator_progressed = false;
|
||||
if (simulator_notifier_) [[unlikely]] {
|
||||
// We can't hold our own lock while notifying
|
||||
// the simulator because notifying the simulator
|
||||
// involves acquiring the simulator's mutex
|
||||
@@ -79,7 +77,7 @@ class Shared {
|
||||
// so we have to get out of its way to avoid
|
||||
// a cyclical deadlock.
|
||||
lock.unlock();
|
||||
std::invoke(wait_notifier_);
|
||||
simulator_progressed = std::invoke(simulator_notifier_);
|
||||
lock.lock();
|
||||
if (item_) {
|
||||
// item may have been filled while we
|
||||
@@ -87,7 +85,8 @@ class Shared {
|
||||
// the simulator of our waiting_ status.
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
}
|
||||
if (!simulator_progressed) [[likely]] {
|
||||
cv_.wait(lock);
|
||||
}
|
||||
MG_ASSERT(!consumed_, "Future consumed twice!");
|
||||
@@ -118,19 +117,11 @@ class Shared {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
MG_ASSERT(!consumed_, "Promise filled after it was already consumed!");
|
||||
MG_ASSERT(!filled_, "Promise filled twice!");
|
||||
MG_ASSERT(!item_, "Promise filled twice!");
|
||||
|
||||
item_ = std::move(item);
|
||||
filled_ = true;
|
||||
item_ = item;
|
||||
} // lock released before condition variable notification
|
||||
|
||||
if (fill_notifier_) {
|
||||
spdlog::trace("calling fill notifier");
|
||||
std::invoke(fill_notifier_);
|
||||
} else {
|
||||
spdlog::trace("not calling fill notifier");
|
||||
}
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
@@ -157,14 +148,6 @@ class Future {
|
||||
old.consumed_or_moved_ = true;
|
||||
}
|
||||
|
||||
Future &operator=(Future &&old) noexcept {
|
||||
MG_ASSERT(!old.consumed_or_moved_, "Future moved from after already being moved from or consumed.");
|
||||
shared_ = std::move(old.shared_);
|
||||
consumed_or_moved_ = old.consumed_or_moved_;
|
||||
old.consumed_or_moved_ = true;
|
||||
return *this;
|
||||
}
|
||||
|
||||
Future(const Future &) = delete;
|
||||
Future &operator=(const Future &) = delete;
|
||||
~Future() = default;
|
||||
@@ -209,7 +192,7 @@ class Future {
|
||||
template <typename T>
|
||||
class Promise {
|
||||
std::shared_ptr<details::Shared<T>> shared_;
|
||||
bool filled_or_moved_{false};
|
||||
bool filled_or_moved_ = false;
|
||||
|
||||
public:
|
||||
explicit Promise(std::shared_ptr<details::Shared<T>> shared) : shared_(shared) {}
|
||||
@@ -229,13 +212,12 @@ class Promise {
|
||||
Promise(const Promise &) = delete;
|
||||
Promise &operator=(const Promise &) = delete;
|
||||
|
||||
// NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Branch)
|
||||
~Promise() { MG_ASSERT(filled_or_moved_, "Promise destroyed before its associated Future was filled!"); }
|
||||
|
||||
// Fill the expected item into the Future.
|
||||
void Fill(T item) {
|
||||
MG_ASSERT(!filled_or_moved_, "Promise::Fill called on a promise that is already filled or moved!");
|
||||
shared_->Fill(std::move(item));
|
||||
shared_->Fill(item);
|
||||
filled_or_moved_ = true;
|
||||
}
|
||||
|
||||
@@ -262,9 +244,8 @@ std::pair<Future<T>, Promise<T>> FuturePromisePair() {
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifications(std::function<bool()> wait_notifier,
|
||||
std::function<void()> fill_notifier) {
|
||||
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(wait_notifier, fill_notifier);
|
||||
std::pair<Future<T>, Promise<T>> FuturePromisePairWithNotifier(std::function<bool()> simulator_notifier) {
|
||||
std::shared_ptr<details::Shared<T>> shared = std::make_shared<details::Shared<T>>(simulator_notifier);
|
||||
|
||||
Future<T> future = Future<T>(shared);
|
||||
Promise<T> promise = Promise<T>(shared);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -30,11 +30,15 @@ class LocalTransport {
|
||||
explicit LocalTransport(std::shared_ptr<LocalTransportHandle> local_transport_handle)
|
||||
: local_transport_handle_(std::move(local_transport_handle)) {}
|
||||
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
|
||||
std::function<void()> fill_notifier, Duration timeout) {
|
||||
return local_transport_handle_->template SubmitRequest<ResponseT, RequestT>(
|
||||
to_address, from_address, std::move(request), timeout, fill_notifier);
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
|
||||
Duration timeout) {
|
||||
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
|
||||
|
||||
local_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
|
||||
std::move(promise));
|
||||
|
||||
return std::move(future);
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
@@ -43,8 +47,8 @@ class LocalTransport {
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
|
||||
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
|
||||
return local_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
|
||||
}
|
||||
|
||||
Time Now() const { return local_transport_handle_->Now(); }
|
||||
@@ -56,7 +60,5 @@ class LocalTransport {
|
||||
std::random_device rng;
|
||||
return distrib(rng);
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() { return local_transport_handle_->ResponseLatencies(); }
|
||||
};
|
||||
}; // namespace memgraph::io::local_transport
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,7 +19,6 @@
|
||||
|
||||
#include "io/errors.hpp"
|
||||
#include "io/message_conversion.hpp"
|
||||
#include "io/message_histogram_collector.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
|
||||
@@ -29,8 +28,6 @@ class LocalTransportHandle {
|
||||
mutable std::mutex mu_{};
|
||||
mutable std::condition_variable cv_;
|
||||
bool should_shut_down_ = false;
|
||||
MessageHistogramCollector histograms_;
|
||||
RequestId request_id_counter_ = 0;
|
||||
|
||||
// the responses to requests that are being waited on
|
||||
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
|
||||
@@ -57,11 +54,6 @@ class LocalTransportHandle {
|
||||
return should_shut_down_;
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
return histograms_.ResponseLatencies();
|
||||
}
|
||||
|
||||
static Time Now() {
|
||||
auto nano_time = std::chrono::system_clock::now();
|
||||
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
|
||||
@@ -104,17 +96,15 @@ class LocalTransportHandle {
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
|
||||
auto type_info = TypeInfoFor(message);
|
||||
|
||||
std::any message_any(std::move(message));
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
|
||||
std::any message_any(std::forward<M>(message));
|
||||
OpaqueMessage opaque_message{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message_any),
|
||||
.type_info = type_info};
|
||||
.message = std::move(message_any)};
|
||||
|
||||
PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
|
||||
PromiseKey promise_key{
|
||||
.requester_address = to_address, .request_id = opaque_message.request_id, .replier_address = from_address};
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
@@ -125,10 +115,7 @@ class LocalTransportHandle {
|
||||
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
|
||||
promises_.erase(promise_key);
|
||||
|
||||
Duration response_latency = Now() - dop.requested_at;
|
||||
|
||||
dop.promise.Fill(std::move(opaque_message), response_latency);
|
||||
histograms_.Measure(type_info, response_latency);
|
||||
dop.promise.Fill(std::move(opaque_message));
|
||||
} else {
|
||||
spdlog::info("placing message in can_receive_");
|
||||
can_receive_.emplace_back(std::move(opaque_message));
|
||||
@@ -138,39 +125,26 @@ class LocalTransportHandle {
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
|
||||
Duration timeout, std::function<void()> fill_notifier) {
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
|
||||
// set null notifier for when the Future::Wait is called
|
||||
nullptr,
|
||||
// set notifier for when Promise::Fill is called
|
||||
std::move(fill_notifier));
|
||||
|
||||
template <Message RequestT, Message ResponseT>
|
||||
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
|
||||
Duration timeout, ResponsePromise<ResponseT> promise) {
|
||||
const bool port_matches = to_address.last_known_port == from_address.last_known_port;
|
||||
const bool ip_matches = to_address.last_known_ip == from_address.last_known_ip;
|
||||
|
||||
MG_ASSERT(port_matches && ip_matches);
|
||||
const Time deadline = Now() + timeout;
|
||||
|
||||
const auto now = Now();
|
||||
const Time deadline = now + timeout;
|
||||
|
||||
RequestId request_id = 0;
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
request_id = ++request_id_counter_;
|
||||
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
|
||||
PromiseKey promise_key{
|
||||
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
|
||||
OpaquePromise opaque_promise(std::move(promise).ToUnique());
|
||||
DeadlineAndOpaquePromise dop{.requested_at = now, .deadline = deadline, .promise = std::move(opaque_promise)};
|
||||
|
||||
MG_ASSERT(!promises_.contains(promise_key));
|
||||
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
|
||||
promises_.emplace(std::move(promise_key), std::move(dop));
|
||||
} // lock dropped
|
||||
|
||||
Send<RequestT>(to_address, from_address, request_id, std::move(request));
|
||||
|
||||
return std::move(future);
|
||||
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -11,17 +11,20 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/type_info_ref.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
using memgraph::io::Duration;
|
||||
using memgraph::io::Message;
|
||||
using memgraph::io::Time;
|
||||
|
||||
struct PromiseKey {
|
||||
Address requester_address;
|
||||
uint64_t request_id;
|
||||
// TODO(tyler) possibly remove replier_address from promise key
|
||||
// once we want to support DSR.
|
||||
Address replier_address;
|
||||
|
||||
public:
|
||||
friend bool operator<(const PromiseKey &lhs, const PromiseKey &rhs) {
|
||||
@@ -29,7 +32,11 @@ struct PromiseKey {
|
||||
return lhs.requester_address < rhs.requester_address;
|
||||
}
|
||||
|
||||
return lhs.request_id < rhs.request_id;
|
||||
if (lhs.request_id != rhs.request_id) {
|
||||
return lhs.request_id < rhs.request_id;
|
||||
}
|
||||
|
||||
return lhs.replier_address < rhs.replier_address;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -38,8 +45,6 @@ struct OpaqueMessage {
|
||||
Address from_address;
|
||||
uint64_t request_id;
|
||||
std::any message;
|
||||
utils::TypeInfoRef type_info;
|
||||
Time deliverable_at;
|
||||
|
||||
/// Recursively tries to match a specific type from the outer
|
||||
/// variant's parameter pack against the type of the std::any,
|
||||
@@ -87,10 +92,6 @@ struct OpaqueMessage {
|
||||
};
|
||||
}
|
||||
|
||||
std::string demangled_name = "\"" + boost::core::demangle(message.type().name()) + "\"";
|
||||
spdlog::error("failed to cast message of type {} to expected request type (probably in Receive argument types)",
|
||||
demangled_name);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
};
|
||||
@@ -99,7 +100,7 @@ class OpaquePromiseTraitBase {
|
||||
public:
|
||||
virtual const std::type_info *TypeInfo() const = 0;
|
||||
virtual bool IsAwaited(void *ptr) const = 0;
|
||||
virtual void Fill(void *ptr, OpaqueMessage &&, Duration) const = 0;
|
||||
virtual void Fill(void *ptr, OpaqueMessage &&) const = 0;
|
||||
virtual void TimeOut(void *ptr) const = 0;
|
||||
|
||||
virtual ~OpaquePromiseTraitBase() = default;
|
||||
@@ -117,13 +118,12 @@ class OpaquePromiseTrait : public OpaquePromiseTraitBase {
|
||||
|
||||
bool IsAwaited(void *ptr) const override { return static_cast<ResponsePromise<T> *>(ptr)->IsAwaited(); };
|
||||
|
||||
void Fill(void *ptr, OpaqueMessage &&opaque_message, Duration response_latency) const override {
|
||||
void Fill(void *ptr, OpaqueMessage &&opaque_message) const override {
|
||||
T message = std::any_cast<T>(std::move(opaque_message.message));
|
||||
auto response_envelope = ResponseEnvelope<T>{.message = std::move(message),
|
||||
.request_id = opaque_message.request_id,
|
||||
.to_address = opaque_message.to_address,
|
||||
.from_address = opaque_message.from_address,
|
||||
.response_latency = response_latency};
|
||||
.from_address = opaque_message.from_address};
|
||||
auto promise = static_cast<ResponsePromise<T> *>(ptr);
|
||||
auto unique_promise = std::unique_ptr<ResponsePromise<T>>(promise);
|
||||
unique_promise->Fill(std::move(response_envelope));
|
||||
@@ -187,9 +187,9 @@ class OpaquePromise {
|
||||
ptr_ = nullptr;
|
||||
}
|
||||
|
||||
void Fill(OpaqueMessage &&opaque_message, Duration response_latency) {
|
||||
void Fill(OpaqueMessage &&opaque_message) {
|
||||
MG_ASSERT(ptr_ != nullptr);
|
||||
trait_->Fill(ptr_, std::move(opaque_message), response_latency);
|
||||
trait_->Fill(ptr_, std::move(opaque_message));
|
||||
ptr_ = nullptr;
|
||||
}
|
||||
|
||||
@@ -199,24 +199,18 @@ class OpaquePromise {
|
||||
};
|
||||
|
||||
struct DeadlineAndOpaquePromise {
|
||||
Time requested_at;
|
||||
Time deadline;
|
||||
OpaquePromise promise;
|
||||
};
|
||||
|
||||
template <class From>
|
||||
std::type_info const &TypeInfoForVariant(From const &from) {
|
||||
std::type_info const &type_info_for_variant(From const &from) {
|
||||
return std::visit([](auto &&x) -> decltype(auto) { return typeid(x); }, from);
|
||||
}
|
||||
|
||||
template <class T>
|
||||
utils::TypeInfoRef TypeInfoFor(const T & /* t */) {
|
||||
return typeid(T);
|
||||
}
|
||||
|
||||
template <typename From, typename Return, typename Head, typename... Rest>
|
||||
std::optional<Return> ConvertVariantInner(From &&a) {
|
||||
if (typeid(Head) == TypeInfoForVariant(a)) {
|
||||
if (typeid(Head) == type_info_for_variant(a)) {
|
||||
Head concrete = std::get<Head>(std::forward<From>(a));
|
||||
return concrete;
|
||||
}
|
||||
|
||||
@@ -1,132 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <cmath>
|
||||
#include <compare>
|
||||
#include <unordered_map>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
|
||||
#include "io/time.hpp"
|
||||
#include "utils/histogram.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/print_helpers.hpp"
|
||||
#include "utils/type_info_ref.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
struct LatencyHistogramSummary {
|
||||
uint64_t count;
|
||||
Duration p0;
|
||||
Duration p50;
|
||||
Duration p75;
|
||||
Duration p90;
|
||||
Duration p95;
|
||||
Duration p975;
|
||||
Duration p99;
|
||||
Duration p999;
|
||||
Duration p9999;
|
||||
Duration p100;
|
||||
Duration sum;
|
||||
|
||||
friend bool operator==(const LatencyHistogramSummary &lhs, const LatencyHistogramSummary &rhs) = default;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummary &histo) {
|
||||
in << "{ \"count\": " << histo.count;
|
||||
in << ", \"p0\": " << histo.p0.count();
|
||||
in << ", \"p50\": " << histo.p50.count();
|
||||
in << ", \"p75\": " << histo.p75.count();
|
||||
in << ", \"p90\": " << histo.p90.count();
|
||||
in << ", \"p95\": " << histo.p95.count();
|
||||
in << ", \"p975\": " << histo.p975.count();
|
||||
in << ", \"p99\": " << histo.p99.count();
|
||||
in << ", \"p999\": " << histo.p999.count();
|
||||
in << ", \"p9999\": " << histo.p9999.count();
|
||||
in << ", \"p100\": " << histo.p100.count();
|
||||
in << ", \"sum\": " << histo.sum.count();
|
||||
in << " }";
|
||||
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
struct LatencyHistogramSummaries {
|
||||
std::unordered_map<std::string, LatencyHistogramSummary> latencies;
|
||||
|
||||
std::string SummaryTable() {
|
||||
std::string output;
|
||||
|
||||
const auto row = [&output](const auto &c1, const auto &c2, const auto &c3, const auto &c4, const auto &c5,
|
||||
const auto &c6, const auto &c7) {
|
||||
output +=
|
||||
fmt::format("{: >50} | {: >8} | {: >8} | {: >8} | {: >8} | {: >8} | {: >8}\n", c1, c2, c3, c4, c5, c6, c7);
|
||||
};
|
||||
row("name", "count", "min (μs)", "med (μs)", "p99 (μs)", "max (μs)", "sum (ms)");
|
||||
|
||||
for (const auto &[name, histo] : latencies) {
|
||||
row(name, histo.count, histo.p0.count(), histo.p50.count(), histo.p99.count(), histo.p100.count(),
|
||||
histo.sum.count() / 1000);
|
||||
}
|
||||
|
||||
output += "\n";
|
||||
return output;
|
||||
}
|
||||
|
||||
friend bool operator==(const LatencyHistogramSummaries &lhs, const LatencyHistogramSummaries &rhs) = default;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const LatencyHistogramSummaries &histo) {
|
||||
using memgraph::utils::print_helpers::operator<<;
|
||||
in << histo.latencies;
|
||||
return in;
|
||||
}
|
||||
};
|
||||
|
||||
class MessageHistogramCollector {
|
||||
std::unordered_map<utils::TypeInfoRef, utils::Histogram, utils::TypeInfoHasher, utils::TypeInfoEqualTo> histograms_;
|
||||
|
||||
public:
|
||||
void Measure(const std::type_info &type_info, const Duration &duration) {
|
||||
auto &histo = histograms_[type_info];
|
||||
histo.Measure(duration.count());
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() {
|
||||
std::unordered_map<std::string, LatencyHistogramSummary> ret{};
|
||||
|
||||
for (const auto &[type_id, histo] : histograms_) {
|
||||
std::string demangled_name = "\"" + boost::core::demangle(type_id.get().name()) + "\"";
|
||||
|
||||
LatencyHistogramSummary latency_histogram_summary{
|
||||
.count = histo.Count(),
|
||||
.p0 = Duration(static_cast<int64_t>(histo.Percentile(0.0))),
|
||||
.p50 = Duration(static_cast<int64_t>(histo.Percentile(50.0))),
|
||||
.p75 = Duration(static_cast<int64_t>(histo.Percentile(75.0))),
|
||||
.p90 = Duration(static_cast<int64_t>(histo.Percentile(90.0))),
|
||||
.p95 = Duration(static_cast<int64_t>(histo.Percentile(95.0))),
|
||||
.p975 = Duration(static_cast<int64_t>(histo.Percentile(97.5))),
|
||||
.p99 = Duration(static_cast<int64_t>(histo.Percentile(99.0))),
|
||||
.p999 = Duration(static_cast<int64_t>(histo.Percentile(99.9))),
|
||||
.p9999 = Duration(static_cast<int64_t>(histo.Percentile(99.99))),
|
||||
.p100 = Duration(static_cast<int64_t>(histo.Percentile(100.0))),
|
||||
.sum = Duration(histo.Sum()),
|
||||
};
|
||||
|
||||
ret.emplace(demangled_name, latency_histogram_summary);
|
||||
}
|
||||
|
||||
return LatencyHistogramSummaries{.latencies = ret};
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::io
|
||||
@@ -1,93 +0,0 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <condition_variable>
|
||||
#include <functional>
|
||||
#include <mutex>
|
||||
#include <optional>
|
||||
#include <vector>
|
||||
|
||||
namespace memgraph::io {
|
||||
|
||||
class ReadinessToken {
|
||||
size_t id_;
|
||||
|
||||
public:
|
||||
explicit ReadinessToken(size_t id) : id_(id) {}
|
||||
size_t GetId() const { return id_; }
|
||||
};
|
||||
|
||||
class Inner {
|
||||
std::condition_variable cv_;
|
||||
std::mutex mu_;
|
||||
std::vector<ReadinessToken> ready_;
|
||||
std::optional<std::function<bool()>> tick_simulator_;
|
||||
|
||||
public:
|
||||
void Notify(ReadinessToken readiness_token) {
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
ready_.emplace_back(readiness_token);
|
||||
} // mutex dropped
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
ReadinessToken Await() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
while (ready_.empty()) {
|
||||
if (tick_simulator_) [[unlikely]] {
|
||||
// This avoids a deadlock in a similar way that
|
||||
// Future::Wait will release its mutex while
|
||||
// interacting with the simulator, due to
|
||||
// the fact that the simulator may cause
|
||||
// notifications that we are interested in.
|
||||
lock.unlock();
|
||||
std::invoke(tick_simulator_.value());
|
||||
lock.lock();
|
||||
} else {
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
|
||||
ReadinessToken ret = ready_.back();
|
||||
ready_.pop_back();
|
||||
return ret;
|
||||
}
|
||||
|
||||
void InstallSimulatorTicker(std::function<bool()> tick_simulator) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
tick_simulator_ = tick_simulator;
|
||||
}
|
||||
};
|
||||
|
||||
class Notifier {
|
||||
std::shared_ptr<Inner> inner_;
|
||||
|
||||
public:
|
||||
Notifier() : inner_(std::make_shared<Inner>()) {}
|
||||
Notifier(const Notifier &) = default;
|
||||
Notifier &operator=(const Notifier &) = default;
|
||||
Notifier(Notifier &&old) = default;
|
||||
Notifier &operator=(Notifier &&old) = default;
|
||||
~Notifier() = default;
|
||||
|
||||
void Notify(ReadinessToken readiness_token) const { inner_->Notify(readiness_token); }
|
||||
|
||||
ReadinessToken Await() const { return inner_->Await(); }
|
||||
|
||||
void InstallSimulatorTicker(std::function<bool()> tick_simulator) { inner_->InstallSimulatorTicker(tick_simulator); }
|
||||
};
|
||||
|
||||
} // namespace memgraph::io
|
||||
61
src/io/protobuf_transport/protobuf_transport.hpp
Normal file
61
src/io/protobuf_transport/protobuf_transport.hpp
Normal file
@@ -0,0 +1,61 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/message_conversion.hpp"
|
||||
#include "io/protobuf_transport/protobuf_transport_handle.hpp"
|
||||
#include "io/transport.hpp"
|
||||
|
||||
namespace memgraph::io::protobuf_transport {
|
||||
|
||||
class ProtobufTransport {
|
||||
std::shared_ptr<ProtobufTransportHandle> protobuf_transport_handle_;
|
||||
uint16_t listen_port_;
|
||||
|
||||
public:
|
||||
explicit ProtobufTransport(uint16_t listen_port)
|
||||
: protobuf_transport_handle_(std::make_unique<ProtobufTransportHandle>()), listen_port_(listen_port) {}
|
||||
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RequestId request_id, RequestT request,
|
||||
Duration timeout) {
|
||||
auto [future, promise] = memgraph::io::FuturePromisePair<ResponseResult<ResponseT>>();
|
||||
|
||||
protobuf_transport_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
|
||||
std::move(promise));
|
||||
|
||||
return std::move(future);
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address receiver_address, Duration timeout) {
|
||||
return protobuf_transport_handle_->template Receive<Ms...>(receiver_address, timeout);
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
|
||||
return protobuf_transport_handle_->template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
|
||||
}
|
||||
|
||||
Time Now() const { return protobuf_transport_handle_->Now(); }
|
||||
|
||||
bool ShouldShutDown() const { return protobuf_transport_handle_->ShouldShutDown(); }
|
||||
|
||||
template <class D = std::poisson_distribution<>, class Return = uint64_t>
|
||||
Return Rand(D distrib) {
|
||||
std::random_device rng;
|
||||
return distrib(rng);
|
||||
}
|
||||
};
|
||||
|
||||
}; // namespace memgraph::io::protobuf_transport
|
||||
161
src/io/protobuf_transport/protobuf_transport_handle.hpp
Normal file
161
src/io/protobuf_transport/protobuf_transport_handle.hpp
Normal file
@@ -0,0 +1,161 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
|
||||
#include "google/protobuf/message.h"
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/transport.hpp"
|
||||
//#include "protobuf/messages.pb.cc"
|
||||
#include "protobuf/messages.pb.h"
|
||||
|
||||
namespace memgraph::io::protobuf_transport {
|
||||
|
||||
using PbAddress = memgraph::protobuf::Address;
|
||||
using memgraph::protobuf::UberMessage;
|
||||
|
||||
class ProtobufTransportHandle {
|
||||
mutable std::mutex mu_{};
|
||||
mutable std::condition_variable cv_;
|
||||
bool should_shut_down_ = false;
|
||||
|
||||
// the responses to requests that are being waited on
|
||||
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
|
||||
|
||||
// messages that are sent to servers that may later receive them
|
||||
std::vector<std::string> can_receive_;
|
||||
|
||||
// serialized outbound messages
|
||||
std::vector<std::string> outbox_;
|
||||
|
||||
public:
|
||||
~ProtobufTransportHandle() {
|
||||
for (auto &&[pk, promise] : promises_) {
|
||||
std::move(promise.promise).TimeOut();
|
||||
}
|
||||
promises_.clear();
|
||||
}
|
||||
|
||||
void ShutDown() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
should_shut_down_ = true;
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
bool ShouldShutDown() const {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
return should_shut_down_;
|
||||
}
|
||||
|
||||
static Time Now() {
|
||||
auto nano_time = std::chrono::system_clock::now();
|
||||
return std::chrono::time_point_cast<std::chrono::microseconds>(nano_time);
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(Address /* receiver_address */, Duration timeout) {
|
||||
std::unique_lock lock(mu_);
|
||||
|
||||
Time before = Now();
|
||||
|
||||
spdlog::info("can_receive_ size: {}", can_receive_.size());
|
||||
|
||||
while (can_receive_.empty()) {
|
||||
Time now = Now();
|
||||
|
||||
// protection against non-monotonic timesources
|
||||
auto maxed_now = std::max(now, before);
|
||||
auto elapsed = maxed_now - before;
|
||||
|
||||
if (timeout < elapsed) {
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
Duration relative_timeout = timeout - elapsed;
|
||||
|
||||
std::cv_status cv_status_value = cv_.wait_for(lock, relative_timeout);
|
||||
|
||||
if (cv_status_value == std::cv_status::timeout) {
|
||||
return TimedOut{};
|
||||
}
|
||||
}
|
||||
|
||||
auto current_message = std::move(can_receive_.back());
|
||||
can_receive_.pop_back();
|
||||
|
||||
auto m_opt = std::move(current_message).Take<Ms...>();
|
||||
|
||||
return std::move(m_opt).value();
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M &&message) {
|
||||
PromiseKey promise_key{.requester_address = to_address, .request_id = request_id, .replier_address = from_address};
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
if (promises_.contains(promise_key)) {
|
||||
// hair-pin local message optimization
|
||||
spdlog::info("using message to fill local promise");
|
||||
DeadlineAndOpaquePromise dop = std::move(promises_.at(promise_key));
|
||||
promises_.erase(promise_key);
|
||||
|
||||
std::any message_any(std::forward<M>(message));
|
||||
OpaqueMessage opaque_message{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message_any)};
|
||||
|
||||
dop.promise.Fill(std::move(opaque_message));
|
||||
} else {
|
||||
spdlog::info("placing message in outbox");
|
||||
|
||||
// serialize protobuf message and place it in the outbox
|
||||
|
||||
std::string bytes;
|
||||
bool success = message.SerializeToString(&bytes);
|
||||
MG_ASSERT(success);
|
||||
|
||||
outbox_.emplace_back(std::move(bytes));
|
||||
}
|
||||
} // lock dropped
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
template <Message RequestT, Message ResponseT>
|
||||
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, RequestT &&request,
|
||||
Duration timeout, ResponsePromise<ResponseT> promise) {
|
||||
const Time deadline = Now() + timeout;
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
PromiseKey promise_key{
|
||||
.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
|
||||
OpaquePromise opaque_promise(std::move(promise).ToUnique());
|
||||
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
|
||||
promises_.emplace(std::move(promise_key), std::move(dop));
|
||||
} // lock dropped
|
||||
|
||||
Send(to_address, from_address, request_id, std::forward<RequestT>(request));
|
||||
}
|
||||
};
|
||||
|
||||
} // namespace memgraph::io::protobuf_transport
|
||||
@@ -11,19 +11,15 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <functional>
|
||||
#include <typeinfo>
|
||||
#include <chrono>
|
||||
#include <condition_variable>
|
||||
#include <iostream>
|
||||
#include <map>
|
||||
#include <mutex>
|
||||
|
||||
namespace memgraph::utils {
|
||||
#include "google/protobuf/message.h"
|
||||
|
||||
using TypeInfoRef = std::reference_wrapper<const std::type_info>;
|
||||
#include "io/address.hpp"
|
||||
#include "io/transport.hpp"
|
||||
|
||||
struct TypeInfoHasher {
|
||||
std::size_t operator()(TypeInfoRef code) const { return code.get().hash_code(); }
|
||||
};
|
||||
|
||||
struct TypeInfoEqualTo {
|
||||
bool operator()(TypeInfoRef lhs, TypeInfoRef rhs) const { return lhs.get() == rhs.get(); }
|
||||
};
|
||||
|
||||
} // namespace memgraph::utils
|
||||
namespace memgraph::io::protobuf_transport {}
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,12 +19,9 @@
|
||||
#include <map>
|
||||
#include <set>
|
||||
#include <thread>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
|
||||
#include "io/message_conversion.hpp"
|
||||
#include "io/simulator/simulator.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
@@ -91,46 +88,6 @@ struct ReadResponse {
|
||||
std::optional<Address> retry_leader;
|
||||
};
|
||||
|
||||
template <class... ReadReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadResponse<std::variant<ReadReturn...>> &response) {
|
||||
return TypeInfoForVariant(response.read_return);
|
||||
}
|
||||
|
||||
template <class ReadReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadResponse<ReadReturn> & /* response */) {
|
||||
return typeid(ReadReturn);
|
||||
}
|
||||
|
||||
template <class ReadOperation>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadRequest<ReadOperation> & /* request */) {
|
||||
return typeid(ReadOperation);
|
||||
}
|
||||
|
||||
template <class... ReadOperations>
|
||||
utils::TypeInfoRef TypeInfoFor(const ReadRequest<std::variant<ReadOperations...>> &request) {
|
||||
return TypeInfoForVariant(request.operation);
|
||||
}
|
||||
|
||||
template <class... WriteReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteResponse<std::variant<WriteReturn...>> &response) {
|
||||
return TypeInfoForVariant(response.write_return);
|
||||
}
|
||||
|
||||
template <class WriteReturn>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteResponse<WriteReturn> & /* response */) {
|
||||
return typeid(WriteReturn);
|
||||
}
|
||||
|
||||
template <class WriteOperation>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteRequest<WriteOperation> & /* request */) {
|
||||
return typeid(WriteOperation);
|
||||
}
|
||||
|
||||
template <class... WriteOperations>
|
||||
utils::TypeInfoRef TypeInfoFor(const WriteRequest<std::variant<WriteOperations...>> &request) {
|
||||
return TypeInfoForVariant(request.operation);
|
||||
}
|
||||
|
||||
/// AppendRequest is a raft-level message that the Leader
|
||||
/// periodically broadcasts to all Follower peers. This
|
||||
/// serves three main roles:
|
||||
@@ -192,7 +149,7 @@ struct PendingClientRequest {
|
||||
|
||||
struct Leader {
|
||||
std::map<Address, FollowerTracker> followers;
|
||||
std::map<LogIndex, PendingClientRequest> pending_client_requests;
|
||||
std::unordered_map<LogIndex, PendingClientRequest> pending_client_requests;
|
||||
Time last_broadcast = Time::min();
|
||||
|
||||
std::string static ToString() { return "\tLeader \t"; }
|
||||
@@ -247,7 +204,7 @@ to a CAS operation.
|
||||
template <typename WriteOperation, typename ReadOperation, typename ReplicatedState, typename WriteResponseValue,
|
||||
typename ReadResponseValue>
|
||||
concept Rsm = requires(ReplicatedState state, WriteOperation w, ReadOperation r) {
|
||||
{ state.Read(std::move(r)) } -> std::same_as<ReadResponseValue>;
|
||||
{ state.Read(r) } -> std::same_as<ReadResponseValue>;
|
||||
{ state.Apply(w) } -> std::same_as<WriteResponseValue>;
|
||||
};
|
||||
|
||||
@@ -403,7 +360,7 @@ class Raft {
|
||||
const PendingClientRequest client_request = std::move(leader.pending_client_requests.at(apply_index));
|
||||
leader.pending_client_requests.erase(apply_index);
|
||||
|
||||
WriteResponse<WriteResponseValue> resp{
|
||||
const WriteResponse<WriteResponseValue> resp{
|
||||
.success = true,
|
||||
.write_return = std::move(write_return),
|
||||
.raft_index = apply_index,
|
||||
@@ -555,7 +512,7 @@ class Raft {
|
||||
for (const auto &peer : peers_) {
|
||||
// request_id not necessary to set because it's not a Future-backed Request.
|
||||
static constexpr auto request_id = 0;
|
||||
io_.template Send(peer, request_id, VoteRequest{request});
|
||||
io_.template Send<VoteRequest>(peer, request_id, request);
|
||||
outstanding_votes.insert(peer);
|
||||
}
|
||||
|
||||
@@ -591,7 +548,7 @@ class Raft {
|
||||
const Time now = io_.Now();
|
||||
const Duration broadcast_timeout = RandomTimeout(kMinimumBroadcastTimeout, kMaximumBroadcastTimeout);
|
||||
|
||||
if (now > leader.last_broadcast + broadcast_timeout) {
|
||||
if (now - leader.last_broadcast > broadcast_timeout) {
|
||||
BroadcastAppendEntries(leader.followers);
|
||||
leader.last_broadcast = now;
|
||||
}
|
||||
@@ -625,12 +582,13 @@ class Raft {
|
||||
MG_ASSERT(std::max(req.term, state_.term) == req.term);
|
||||
}
|
||||
|
||||
io_.Send(from_address, request_id,
|
||||
VoteResponse{
|
||||
.term = std::max(req.term, state_.term),
|
||||
.committed_log_size = state_.committed_log_size,
|
||||
.vote_granted = new_leader,
|
||||
});
|
||||
const VoteResponse res{
|
||||
.term = std::max(req.term, state_.term),
|
||||
.committed_log_size = state_.committed_log_size,
|
||||
.vote_granted = new_leader,
|
||||
};
|
||||
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
if (new_leader) {
|
||||
// become a follower
|
||||
@@ -692,7 +650,7 @@ class Raft {
|
||||
|
||||
return Leader{
|
||||
.followers = std::move(followers),
|
||||
.pending_client_requests = std::map<LogIndex, PendingClientRequest>(),
|
||||
.pending_client_requests = std::unordered_map<LogIndex, PendingClientRequest>(),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -718,10 +676,6 @@ class Raft {
|
||||
.log_size = state_.log.size(),
|
||||
};
|
||||
|
||||
static_assert(std::is_trivially_copyable_v<AppendResponse>,
|
||||
"This function copies this message, therefore it is important to be trivially copyable. Otherwise it "
|
||||
"should be moved");
|
||||
|
||||
if constexpr (std::is_same<ALL, Leader>()) {
|
||||
MG_ASSERT(req.term != state_.term, "Multiple leaders are acting under the term ", req.term);
|
||||
}
|
||||
@@ -740,7 +694,7 @@ class Raft {
|
||||
// become follower of this leader, reply with our log status
|
||||
state_.term = req.term;
|
||||
|
||||
io_.Send(from_address, request_id, AppendResponse{res});
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
Log("becoming Follower of Leader ", from_address.last_known_port, " at term ", req.term);
|
||||
return Follower{
|
||||
@@ -751,7 +705,7 @@ class Raft {
|
||||
|
||||
if (req.term < state_.term) {
|
||||
// nack this request from an old leader
|
||||
io_.Send(from_address, request_id, AppendResponse{res});
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -812,7 +766,7 @@ class Raft {
|
||||
|
||||
Log("returning log_size of ", res.log_size);
|
||||
|
||||
io_.Send(from_address, request_id, AppendResponse{res});
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -860,20 +814,18 @@ class Raft {
|
||||
// Leaders are able to immediately respond to the requester (with a ReadResponseValue) applied to the ReplicatedState
|
||||
std::optional<Role> Handle(Leader & /* variable */, ReadRequest<ReadOperation> &&req, RequestId request_id,
|
||||
Address from_address) {
|
||||
auto type_info = TypeInfoFor(req);
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
Log("handling ReadOperation<" + demangled_name + ">");
|
||||
ReadOperation &read_operation = req.operation;
|
||||
Log("handling ReadOperation");
|
||||
ReadOperation read_operation = req.operation;
|
||||
|
||||
ReadResponseValue read_return = replicated_state_.Read(std::move(read_operation));
|
||||
ReadResponseValue read_return = replicated_state_.Read(read_operation);
|
||||
|
||||
ReadResponse<ReadResponseValue> resp{
|
||||
const ReadResponse<ReadResponseValue> resp{
|
||||
.success = true,
|
||||
.read_return = std::move(read_return),
|
||||
.retry_leader = std::nullopt,
|
||||
};
|
||||
|
||||
io_.Send(from_address, request_id, std::move(resp));
|
||||
io_.Send(from_address, request_id, resp);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -882,11 +834,11 @@ class Raft {
|
||||
std::optional<Role> Handle(Candidate & /* variable */, ReadRequest<ReadOperation> && /* variable */,
|
||||
RequestId request_id, Address from_address) {
|
||||
Log("received ReadOperation - not redirecting because no Leader is known");
|
||||
ReadResponse<ReadResponseValue> res{
|
||||
const ReadResponse<ReadResponseValue> res{
|
||||
.success = false,
|
||||
};
|
||||
|
||||
io_.Send(from_address, request_id, std::move(res));
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
Cron();
|
||||
|
||||
@@ -898,12 +850,12 @@ class Raft {
|
||||
Address from_address) {
|
||||
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
|
||||
|
||||
ReadResponse<ReadResponseValue> res{
|
||||
const ReadResponse<ReadResponseValue> res{
|
||||
.success = false,
|
||||
.retry_leader = follower.leader_address,
|
||||
};
|
||||
|
||||
io_.Send(from_address, request_id, std::move(res));
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -917,12 +869,12 @@ class Raft {
|
||||
Address from_address) {
|
||||
Log("redirecting client to known Leader with port ", follower.leader_address.last_known_port);
|
||||
|
||||
WriteResponse<WriteResponseValue> res{
|
||||
const WriteResponse<WriteResponseValue> res{
|
||||
.success = false,
|
||||
.retry_leader = follower.leader_address,
|
||||
};
|
||||
|
||||
io_.Send(from_address, request_id, std::move(res));
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
@@ -931,11 +883,11 @@ class Raft {
|
||||
RequestId request_id, Address from_address) {
|
||||
Log("received WriteRequest - not redirecting because no Leader is known");
|
||||
|
||||
WriteResponse<WriteResponseValue> res{
|
||||
const WriteResponse<WriteResponseValue> res{
|
||||
.success = false,
|
||||
};
|
||||
|
||||
io_.Send(from_address, request_id, std::move(res));
|
||||
io_.Send(from_address, request_id, res);
|
||||
|
||||
Cron();
|
||||
|
||||
@@ -945,9 +897,7 @@ class Raft {
|
||||
// only leaders actually handle replication requests from clients
|
||||
std::optional<Role> Handle(Leader &leader, WriteRequest<WriteOperation> &&req, RequestId request_id,
|
||||
Address from_address) {
|
||||
auto type_info = TypeInfoFor(req);
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
Log("handling WriteRequest<" + demangled_name + ">");
|
||||
Log("handling WriteRequest");
|
||||
|
||||
// we are the leader. add item to log and send Append to peers
|
||||
MG_ASSERT(state_.term >= LastLogTerm());
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -13,13 +13,9 @@
|
||||
|
||||
#include <iostream>
|
||||
#include <optional>
|
||||
#include <type_traits>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/rsm/raft.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
@@ -38,14 +34,6 @@ using memgraph::io::rsm::WriteRequest;
|
||||
using memgraph::io::rsm::WriteResponse;
|
||||
using memgraph::utils::BasicResult;
|
||||
|
||||
template <typename RequestT, typename ResponseT>
|
||||
struct AsyncRequest {
|
||||
Time start_time;
|
||||
RequestT request;
|
||||
Notifier notifier;
|
||||
ResponseFuture<ResponseT> future;
|
||||
};
|
||||
|
||||
template <typename IoImpl, typename WriteRequestT, typename WriteResponseT, typename ReadRequestT,
|
||||
typename ReadResponseT>
|
||||
class RsmClient {
|
||||
@@ -55,28 +43,21 @@ class RsmClient {
|
||||
Address leader_;
|
||||
ServerPool server_addrs_;
|
||||
|
||||
/// State for single async read/write operations. In the future this could become a map
|
||||
/// of async operations that can be accessed via an ID etc...
|
||||
std::unordered_map<size_t, AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>>> async_reads_;
|
||||
std::unordered_map<size_t, AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>>> async_writes_;
|
||||
|
||||
void SelectRandomLeader() {
|
||||
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
|
||||
size_t addr_index = io_.Rand(addr_distrib);
|
||||
leader_ = server_addrs_[addr_index];
|
||||
|
||||
spdlog::debug("selecting a random leader at index {} with address {}", addr_index, leader_.ToString());
|
||||
}
|
||||
|
||||
template <typename ResponseT>
|
||||
void PossiblyRedirectLeader(const ResponseT &response) {
|
||||
if (response.retry_leader) {
|
||||
MG_ASSERT(!response.success, "retry_leader should never be set for successful responses");
|
||||
leader_ = response.retry_leader.value();
|
||||
spdlog::debug("client redirected to leader server {}", leader_.ToString());
|
||||
}
|
||||
if (!response.success) {
|
||||
SelectRandomLeader();
|
||||
} else if (!response.success) {
|
||||
std::uniform_int_distribution<size_t> addr_distrib(0, (server_addrs_.size() - 1));
|
||||
size_t addr_index = io_.Rand(addr_distrib);
|
||||
leader_ = server_addrs_[addr_index];
|
||||
|
||||
spdlog::debug(
|
||||
"client NOT redirected to leader server despite our success failing to be processed (it probably was sent to "
|
||||
"a RSM Candidate) trying a random one at index {} with address {}",
|
||||
addr_index, leader_.ToString());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,173 +65,71 @@ class RsmClient {
|
||||
RsmClient(Io<IoImpl> io, Address leader, ServerPool server_addrs)
|
||||
: io_{io}, leader_{leader}, server_addrs_{server_addrs} {}
|
||||
|
||||
RsmClient(const RsmClient &) = delete;
|
||||
RsmClient &operator=(const RsmClient &) = delete;
|
||||
RsmClient(RsmClient &&) noexcept = default;
|
||||
RsmClient &operator=(RsmClient &&) noexcept = default;
|
||||
|
||||
RsmClient() = delete;
|
||||
~RsmClient() = default;
|
||||
|
||||
BasicResult<TimedOut, WriteResponseT> SendWriteRequest(WriteRequestT req) {
|
||||
Notifier notifier;
|
||||
const ReadinessToken readiness_token{0};
|
||||
SendAsyncWriteRequest(std::move(req), notifier, readiness_token);
|
||||
auto poll_result = AwaitAsyncWriteRequest(readiness_token);
|
||||
while (!poll_result) {
|
||||
poll_result = AwaitAsyncWriteRequest(readiness_token);
|
||||
}
|
||||
return poll_result.value();
|
||||
WriteRequest<WriteRequestT> client_req;
|
||||
client_req.operation = req;
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const Time before = io_.Now();
|
||||
|
||||
do {
|
||||
spdlog::debug("client sending WriteRequest to Leader {}", leader_.ToString());
|
||||
ResponseFuture<WriteResponse<WriteResponseT>> response_future =
|
||||
io_.template Request<WriteRequest<WriteRequestT>, WriteResponse<WriteResponseT>>(leader_, client_req);
|
||||
ResponseResult<WriteResponse<WriteResponseT>> response_result = std::move(response_future).Wait();
|
||||
|
||||
if (response_result.HasError()) {
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
return response_result.GetError();
|
||||
}
|
||||
|
||||
ResponseEnvelope<WriteResponse<WriteResponseT>> &&response_envelope = std::move(response_result.GetValue());
|
||||
WriteResponse<WriteResponseT> &&write_response = std::move(response_envelope.message);
|
||||
|
||||
if (write_response.success) {
|
||||
return std::move(write_response.write_return);
|
||||
}
|
||||
|
||||
PossiblyRedirectLeader(write_response);
|
||||
} while (io_.Now() < before + overall_timeout);
|
||||
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
BasicResult<TimedOut, ReadResponseT> SendReadRequest(ReadRequestT req) {
|
||||
Notifier notifier;
|
||||
const ReadinessToken readiness_token{0};
|
||||
SendAsyncReadRequest(std::move(req), notifier, readiness_token);
|
||||
auto poll_result = AwaitAsyncReadRequest(readiness_token);
|
||||
while (!poll_result) {
|
||||
poll_result = AwaitAsyncReadRequest(readiness_token);
|
||||
}
|
||||
return poll_result.value();
|
||||
}
|
||||
|
||||
/// AsyncRead methods
|
||||
void SendAsyncReadRequest(ReadRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
|
||||
ReadRequest<ReadRequestT> read_req = {.operation = req};
|
||||
|
||||
AsyncRequest<ReadRequestT, ReadResponse<ReadResponseT>> async_request{
|
||||
.start_time = io_.Now(),
|
||||
.request = std::move(req),
|
||||
.notifier = notifier,
|
||||
.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
|
||||
leader_, std::move(read_req), notifier, readiness_token),
|
||||
};
|
||||
|
||||
async_reads_.emplace(readiness_token.GetId(), std::move(async_request));
|
||||
}
|
||||
|
||||
void ResendAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
|
||||
ReadRequest<ReadRequestT> read_req = {.operation = async_request.request};
|
||||
|
||||
async_request.future = io_.template RequestWithNotification<ReadResponse<ReadResponseT>, ReadRequest<ReadRequestT>>(
|
||||
leader_, std::move(read_req), async_request.notifier, readiness_token);
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> PollAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
|
||||
if (!async_request.future.IsReady()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return AwaitAsyncReadRequest(readiness_token);
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, ReadResponseT>> AwaitAsyncReadRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_reads_.at(readiness_token.GetId());
|
||||
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(async_request.future).Wait();
|
||||
ReadRequest<ReadRequestT> read_req;
|
||||
read_req.operation = req;
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
|
||||
const bool result_has_error = get_response_result.HasError();
|
||||
const Time before = io_.Now();
|
||||
|
||||
if (result_has_error && past_time_out) {
|
||||
// TODO static assert the exact type of error.
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
async_reads_.erase(readiness_token.GetId());
|
||||
return TimedOut{};
|
||||
}
|
||||
do {
|
||||
spdlog::debug("client sending ReadRequest to Leader {}", leader_.ToString());
|
||||
|
||||
ResponseFuture<ReadResponse<ReadResponseT>> get_response_future =
|
||||
io_.template Request<ReadRequest<ReadRequestT>, ReadResponse<ReadResponseT>>(leader_, read_req);
|
||||
|
||||
// receive response
|
||||
ResponseResult<ReadResponse<ReadResponseT>> get_response_result = std::move(get_response_future).Wait();
|
||||
|
||||
if (get_response_result.HasError()) {
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
return get_response_result.GetError();
|
||||
}
|
||||
|
||||
if (!result_has_error) {
|
||||
ResponseEnvelope<ReadResponse<ReadResponseT>> &&get_response_envelope = std::move(get_response_result.GetValue());
|
||||
ReadResponse<ReadResponseT> &&read_get_response = std::move(get_response_envelope.message);
|
||||
|
||||
PossiblyRedirectLeader(read_get_response);
|
||||
|
||||
if (read_get_response.success) {
|
||||
async_reads_.erase(readiness_token.GetId());
|
||||
spdlog::debug("returning read_return for RSM request");
|
||||
return std::move(read_get_response.read_return);
|
||||
}
|
||||
} else {
|
||||
SelectRandomLeader();
|
||||
}
|
||||
|
||||
ResendAsyncReadRequest(readiness_token);
|
||||
PossiblyRedirectLeader(read_get_response);
|
||||
} while (io_.Now() < before + overall_timeout);
|
||||
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
/// AsyncWrite methods
|
||||
void SendAsyncWriteRequest(WriteRequestT &&req, Notifier notifier, ReadinessToken readiness_token) {
|
||||
WriteRequest<WriteRequestT> write_req = {.operation = req};
|
||||
|
||||
AsyncRequest<WriteRequestT, WriteResponse<WriteResponseT>> async_request{
|
||||
.start_time = io_.Now(),
|
||||
.request = std::move(req),
|
||||
.notifier = notifier,
|
||||
.future = io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
|
||||
leader_, std::move(write_req), notifier, readiness_token),
|
||||
};
|
||||
|
||||
async_writes_.emplace(readiness_token.GetId(), std::move(async_request));
|
||||
}
|
||||
|
||||
void ResendAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
|
||||
WriteRequest<WriteRequestT> write_req = {.operation = async_request.request};
|
||||
|
||||
async_request.future =
|
||||
io_.template RequestWithNotification<WriteResponse<WriteResponseT>, WriteRequest<WriteRequestT>>(
|
||||
leader_, std::move(write_req), async_request.notifier, readiness_token);
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> PollAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
|
||||
if (!async_request.future.IsReady()) {
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
return AwaitAsyncWriteRequest(readiness_token);
|
||||
}
|
||||
|
||||
std::optional<BasicResult<TimedOut, WriteResponseT>> AwaitAsyncWriteRequest(const ReadinessToken &readiness_token) {
|
||||
auto &async_request = async_writes_.at(readiness_token.GetId());
|
||||
ResponseResult<WriteResponse<WriteResponseT>> get_response_result = std::move(async_request.future).Wait();
|
||||
|
||||
const Duration overall_timeout = io_.GetDefaultTimeout();
|
||||
const bool past_time_out = io_.Now() > async_request.start_time + overall_timeout;
|
||||
const bool result_has_error = get_response_result.HasError();
|
||||
|
||||
if (result_has_error && past_time_out) {
|
||||
// TODO static assert the exact type of error.
|
||||
spdlog::debug("client timed out while trying to communicate with leader server {}", leader_.ToString());
|
||||
async_writes_.erase(readiness_token.GetId());
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
if (!result_has_error) {
|
||||
ResponseEnvelope<WriteResponse<WriteResponseT>> &&get_response_envelope =
|
||||
std::move(get_response_result.GetValue());
|
||||
WriteResponse<WriteResponseT> &&write_get_response = std::move(get_response_envelope.message);
|
||||
|
||||
PossiblyRedirectLeader(write_get_response);
|
||||
|
||||
if (write_get_response.success) {
|
||||
async_writes_.erase(readiness_token.GetId());
|
||||
return std::move(write_get_response.write_return);
|
||||
}
|
||||
} else {
|
||||
SelectRandomLeader();
|
||||
}
|
||||
|
||||
ResendAsyncWriteRequest(readiness_token);
|
||||
|
||||
return std::nullopt;
|
||||
return TimedOut{};
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -29,8 +29,6 @@ class Simulator {
|
||||
explicit Simulator(SimulatorConfig config)
|
||||
: rng_(std::mt19937{config.rng_seed}), simulator_handle_{std::make_shared<SimulatorHandle>(config)} {}
|
||||
|
||||
~Simulator() { ShutDown(); }
|
||||
|
||||
void ShutDown() { simulator_handle_->ShutDown(); }
|
||||
|
||||
Io<SimulatorTransport> RegisterNew() {
|
||||
@@ -41,7 +39,7 @@ class Simulator {
|
||||
Io<SimulatorTransport> Register(Address address) {
|
||||
std::uniform_int_distribution<uint64_t> seed_distrib;
|
||||
uint64_t seed = seed_distrib(rng_);
|
||||
return Io{SimulatorTransport(simulator_handle_, address, seed), address};
|
||||
return Io{SimulatorTransport{simulator_handle_, address, seed}, address};
|
||||
}
|
||||
|
||||
void IncrementServerCountAndWaitForQuiescentState(Address address) {
|
||||
@@ -49,14 +47,5 @@ class Simulator {
|
||||
}
|
||||
|
||||
SimulatorStats Stats() { return simulator_handle_->Stats(); }
|
||||
|
||||
std::shared_ptr<SimulatorHandle> GetSimulatorHandle() const { return simulator_handle_; }
|
||||
|
||||
std::function<bool()> GetSimulatorTickClosure() {
|
||||
std::function<bool()> tick_closure = [handle_copy = simulator_handle_] {
|
||||
return handle_copy->MaybeTickSimulator();
|
||||
};
|
||||
return tick_closure;
|
||||
}
|
||||
};
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -26,6 +26,5 @@ struct SimulatorConfig {
|
||||
uint64_t rng_seed = 0;
|
||||
Time start_time = Time::min();
|
||||
Time abort_time = Time::max();
|
||||
Duration message_delay = std::chrono::microseconds(100);
|
||||
};
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -16,19 +16,15 @@
|
||||
#include "io/simulator/simulator_stats.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
|
||||
using memgraph::io::Duration;
|
||||
using memgraph::io::Time;
|
||||
|
||||
void SimulatorHandle::ShutDown() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
should_shut_down_ = true;
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
std::move(dop).promise.TimeOut();
|
||||
it = promises_.erase(it);
|
||||
}
|
||||
can_receive_.clear();
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
@@ -37,11 +33,6 @@ bool SimulatorHandle::ShouldShutDown() const {
|
||||
return should_shut_down_;
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries SimulatorHandle::ResponseLatencies() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
return histograms_.ResponseLatencies();
|
||||
}
|
||||
|
||||
void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address address) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
server_addresses_.insert(address);
|
||||
@@ -52,84 +43,47 @@ void SimulatorHandle::IncrementServerCountAndWaitForQuiescentState(Address addre
|
||||
const bool all_servers_blocked = blocked_servers == server_addresses_.size();
|
||||
|
||||
if (all_servers_blocked) {
|
||||
spdlog::trace("quiescent state detected - {} out of {} servers now blocked on receive", blocked_servers,
|
||||
server_addresses_.size());
|
||||
return;
|
||||
}
|
||||
|
||||
spdlog::trace("not returning from quiescent because we see {} blocked out of {}", blocked_servers,
|
||||
server_addresses_.size());
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
|
||||
bool SortInFlight(const std::pair<Address, OpaqueMessage> &lhs, const std::pair<Address, OpaqueMessage> &rhs) {
|
||||
// NB: never sort based on the request ID etc...
|
||||
// This should only be used from std::stable_sort
|
||||
// because by comparing on the from_address alone,
|
||||
// we expect the sender ordering to remain
|
||||
// deterministic.
|
||||
const auto &[addr_1, om_1] = lhs;
|
||||
const auto &[addr_2, om_2] = rhs;
|
||||
return om_1.from_address < om_2.from_address;
|
||||
}
|
||||
|
||||
bool SimulatorHandle::MaybeTickSimulator() {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
const size_t blocked_servers = blocked_on_receive_.size();
|
||||
|
||||
if (should_shut_down_ || blocked_servers < server_addresses_.size()) {
|
||||
if (blocked_servers < server_addresses_.size()) {
|
||||
// we only need to advance the simulator when all
|
||||
// servers have reached a quiescent state, blocked
|
||||
// on their own futures or receive methods.
|
||||
return false;
|
||||
}
|
||||
|
||||
// We allow the simulator to progress the state of the system only
|
||||
// after all servers are blocked on receive.
|
||||
spdlog::trace("~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ simulator tick ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~");
|
||||
stats_.simulator_ticks++;
|
||||
blocked_on_receive_.clear();
|
||||
|
||||
cv_.notify_all();
|
||||
|
||||
bool timed_anything_out = TimeoutPromisesPastDeadline();
|
||||
|
||||
if (timed_anything_out) {
|
||||
spdlog::trace("simulator progressing: timed out a request");
|
||||
}
|
||||
|
||||
const Duration clock_advance = std::chrono::microseconds{time_distrib_(rng_)};
|
||||
|
||||
// We don't always want to advance the clock with every message that we deliver because
|
||||
// when we advance it for every message, it causes timeouts to occur for any "request path"
|
||||
// over a certain length. Alternatively, we don't want to simply deliver multiple messages
|
||||
// in a single simulator tick because that would reduce the amount of concurrent message
|
||||
// mixing that may naturally occur in production. This approach is to mod the random clock
|
||||
// advance by a prime number (hopefully avoiding most harmonic effects that would be introduced
|
||||
// by only advancing the clock by an even amount etc...) and only advancing the clock close to
|
||||
// half of the time.
|
||||
if (clock_advance.count() % 97 > 49) {
|
||||
spdlog::trace("simulator progressing: clock advanced by {}", clock_advance.count());
|
||||
cluster_wide_time_microseconds_ += clock_advance;
|
||||
stats_.elapsed_time = cluster_wide_time_microseconds_ - config_.start_time;
|
||||
}
|
||||
|
||||
if (cluster_wide_time_microseconds_ >= config_.abort_time) {
|
||||
spdlog::error(
|
||||
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
|
||||
"in an expected amount of time. The SimulatorConfig.rng_seed for this run is {}",
|
||||
config_.rng_seed);
|
||||
throw utils::BasicException{"Cluster has executed beyond its configured abort_time"};
|
||||
}
|
||||
TimeoutPromisesPastDeadline();
|
||||
|
||||
if (in_flight_.empty()) {
|
||||
// return early here because there are no messages to schedule
|
||||
|
||||
// We tick the clock forward when all servers are blocked but
|
||||
// there are no in-flight messages to schedule delivery of.
|
||||
std::poisson_distribution<> time_distrib(50);
|
||||
Duration clock_advance = std::chrono::microseconds{time_distrib(rng_)};
|
||||
cluster_wide_time_microseconds_ += clock_advance;
|
||||
|
||||
MG_ASSERT(cluster_wide_time_microseconds_ < config_.abort_time,
|
||||
"Cluster has executed beyond its configured abort_time, and something may be failing to make progress "
|
||||
"in an expected amount of time.");
|
||||
return true;
|
||||
}
|
||||
|
||||
std::stable_sort(in_flight_.begin(), in_flight_.end(), SortInFlight);
|
||||
|
||||
if (config_.scramble_messages && in_flight_.size() > 1) {
|
||||
if (config_.scramble_messages) {
|
||||
// scramble messages
|
||||
std::uniform_int_distribution<size_t> swap_distrib(0, in_flight_.size() - 1);
|
||||
const size_t swap_index = swap_distrib(rng_);
|
||||
@@ -139,14 +93,17 @@ bool SimulatorHandle::MaybeTickSimulator() {
|
||||
auto [to_address, opaque_message] = std::move(in_flight_.back());
|
||||
in_flight_.pop_back();
|
||||
|
||||
const int drop_threshold = drop_distrib_(rng_);
|
||||
std::uniform_int_distribution<int> drop_distrib(0, 99);
|
||||
const int drop_threshold = drop_distrib(rng_);
|
||||
const bool should_drop = drop_threshold < config_.drop_percent;
|
||||
|
||||
if (should_drop) {
|
||||
stats_.dropped_messages++;
|
||||
}
|
||||
|
||||
PromiseKey promise_key{.requester_address = to_address, .request_id = opaque_message.request_id};
|
||||
PromiseKey promise_key{.requester_address = to_address,
|
||||
.request_id = opaque_message.request_id,
|
||||
.replier_address = opaque_message.from_address};
|
||||
|
||||
if (promises_.contains(promise_key)) {
|
||||
// complete waiting promise if it's there
|
||||
@@ -158,25 +115,16 @@ bool SimulatorHandle::MaybeTickSimulator() {
|
||||
if (should_drop || normal_timeout) {
|
||||
stats_.timed_out_requests++;
|
||||
dop.promise.TimeOut();
|
||||
spdlog::trace("simulator timing out request ");
|
||||
} else {
|
||||
stats_.total_responses++;
|
||||
Duration response_latency = cluster_wide_time_microseconds_ - dop.requested_at;
|
||||
auto type_info = opaque_message.type_info;
|
||||
dop.promise.Fill(std::move(opaque_message), response_latency);
|
||||
histograms_.Measure(type_info, response_latency);
|
||||
spdlog::trace("simulator replying to request");
|
||||
dop.promise.Fill(std::move(opaque_message));
|
||||
}
|
||||
} else if (should_drop) {
|
||||
// don't add it anywhere, let it drop
|
||||
spdlog::trace("simulator silently dropping request");
|
||||
} else {
|
||||
// add to can_receive_ if not
|
||||
spdlog::trace("simulator adding message to can_receive_ from {} to {}", opaque_message.from_address.last_known_port,
|
||||
opaque_message.to_address.last_known_port);
|
||||
const auto &[om_vec, inserted] =
|
||||
can_receive_.try_emplace(to_address.ToPartialAddress(), std::deque<OpaqueMessage>());
|
||||
om_vec->second.emplace_front(std::move(opaque_message));
|
||||
const auto &[om_vec, inserted] = can_receive_.try_emplace(to_address, std::vector<OpaqueMessage>());
|
||||
om_vec->second.emplace_back(std::move(opaque_message));
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -22,12 +22,9 @@
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/core/demangle.hpp>
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/message_conversion.hpp"
|
||||
#include "io/message_histogram_collector.hpp"
|
||||
#include "io/simulator/simulator_config.hpp"
|
||||
#include "io/simulator/simulator_stats.hpp"
|
||||
#include "io/time.hpp"
|
||||
@@ -46,7 +43,7 @@ class SimulatorHandle {
|
||||
std::map<PromiseKey, DeadlineAndOpaquePromise> promises_;
|
||||
|
||||
// messages that are sent to servers that may later receive them
|
||||
std::map<PartialAddress, std::deque<OpaqueMessage>> can_receive_;
|
||||
std::map<Address, std::vector<OpaqueMessage>> can_receive_;
|
||||
|
||||
Time cluster_wide_time_microseconds_;
|
||||
bool should_shut_down_ = false;
|
||||
@@ -54,45 +51,29 @@ class SimulatorHandle {
|
||||
std::set<Address> blocked_on_receive_;
|
||||
std::set<Address> server_addresses_;
|
||||
std::mt19937 rng_;
|
||||
std::uniform_int_distribution<int> time_distrib_{0, 30000};
|
||||
std::uniform_int_distribution<int> drop_distrib_{0, 99};
|
||||
SimulatorConfig config_;
|
||||
MessageHistogramCollector histograms_;
|
||||
RequestId request_id_counter_{0};
|
||||
|
||||
bool TimeoutPromisesPastDeadline() {
|
||||
bool timed_anything_out = false;
|
||||
void TimeoutPromisesPastDeadline() {
|
||||
const Time now = cluster_wide_time_microseconds_;
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
if (dop.deadline < now && config_.perform_timeouts) {
|
||||
spdlog::trace("timing out request from requester {}.", promise_key.requester_address.ToString());
|
||||
if (dop.deadline < now) {
|
||||
spdlog::info("timing out request from requester {} to replier {}.", promise_key.requester_address.ToString(),
|
||||
promise_key.replier_address.ToString());
|
||||
std::move(dop).promise.TimeOut();
|
||||
it = promises_.erase(it);
|
||||
|
||||
stats_.timed_out_requests++;
|
||||
timed_anything_out = true;
|
||||
} else {
|
||||
++it;
|
||||
}
|
||||
}
|
||||
return timed_anything_out;
|
||||
}
|
||||
|
||||
public:
|
||||
explicit SimulatorHandle(SimulatorConfig config)
|
||||
: cluster_wide_time_microseconds_(config.start_time), rng_(config.rng_seed), config_(config) {}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies();
|
||||
|
||||
~SimulatorHandle() {
|
||||
for (auto it = promises_.begin(); it != promises_.end();) {
|
||||
auto &[promise_key, dop] = *it;
|
||||
std::move(dop).promise.TimeOut();
|
||||
it = promises_.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
void IncrementServerCountAndWaitForQuiescentState(Address address);
|
||||
|
||||
/// This method causes most of the interesting simulation logic to happen, wrt network behavior.
|
||||
@@ -105,122 +86,80 @@ class SimulatorHandle {
|
||||
|
||||
bool ShouldShutDown() const;
|
||||
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> SubmitRequest(Address to_address, Address from_address, RValueRef<RequestT> request,
|
||||
Duration timeout, std::function<bool()> &&maybe_tick_simulator,
|
||||
std::function<void()> &&fill_notifier) {
|
||||
auto type_info = TypeInfoFor(request);
|
||||
std::string demangled_name = boost::core::demangle(type_info.get().name());
|
||||
spdlog::trace("simulator sending request {} to {}", demangled_name, to_address);
|
||||
template <Message Request, Message Response>
|
||||
void SubmitRequest(Address to_address, Address from_address, RequestId request_id, Request &&request,
|
||||
Duration timeout, ResponsePromise<Response> &&promise) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
auto [future, promise] = memgraph::io::FuturePromisePairWithNotifications<ResponseResult<ResponseT>>(
|
||||
// set notifier for when the Future::Wait is called
|
||||
std::move(maybe_tick_simulator),
|
||||
// set notifier for when Promise::Fill is called
|
||||
std::move(fill_notifier));
|
||||
const Time deadline = cluster_wide_time_microseconds_ + timeout;
|
||||
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
std::any message(request);
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message)};
|
||||
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
|
||||
|
||||
RequestId request_id = ++request_id_counter_;
|
||||
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id, .replier_address = to_address};
|
||||
OpaquePromise opaque_promise(std::move(promise).ToUnique());
|
||||
DeadlineAndOpaquePromise dop{.deadline = deadline, .promise = std::move(opaque_promise)};
|
||||
promises_.emplace(std::move(promise_key), std::move(dop));
|
||||
|
||||
const Time deadline = cluster_wide_time_microseconds_ + timeout;
|
||||
|
||||
std::any message(std::move(request));
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message),
|
||||
.type_info = type_info,
|
||||
.deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
|
||||
in_flight_.emplace_back(std::make_pair(to_address, std::move(om)));
|
||||
|
||||
PromiseKey promise_key{.requester_address = from_address, .request_id = request_id};
|
||||
OpaquePromise opaque_promise(std::move(promise).ToUnique());
|
||||
DeadlineAndOpaquePromise dop{
|
||||
.requested_at = cluster_wide_time_microseconds_,
|
||||
.deadline = deadline,
|
||||
.promise = std::move(opaque_promise),
|
||||
};
|
||||
|
||||
MG_ASSERT(!promises_.contains(promise_key));
|
||||
|
||||
promises_.emplace(std::move(promise_key), std::move(dop));
|
||||
|
||||
stats_.total_messages++;
|
||||
stats_.total_requests++;
|
||||
} // lock dropped here
|
||||
stats_.total_messages++;
|
||||
stats_.total_requests++;
|
||||
|
||||
cv_.notify_all();
|
||||
|
||||
return std::move(future);
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
requires(sizeof...(Ms) > 0) RequestResult<Ms...> Receive(const Address &receiver, Duration timeout) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
|
||||
blocked_on_receive_.emplace(receiver);
|
||||
|
||||
const Time deadline = cluster_wide_time_microseconds_ + timeout;
|
||||
|
||||
auto partial_address = receiver.ToPartialAddress();
|
||||
|
||||
while (!should_shut_down_ && (cluster_wide_time_microseconds_ < deadline)) {
|
||||
if (can_receive_.contains(partial_address)) {
|
||||
std::deque<OpaqueMessage> &can_rx = can_receive_.at(partial_address);
|
||||
|
||||
bool contains_items = !can_rx.empty();
|
||||
bool can_receive = contains_items && can_rx.back().deliverable_at <= cluster_wide_time_microseconds_;
|
||||
|
||||
if (can_receive) {
|
||||
if (can_receive_.contains(receiver)) {
|
||||
std::vector<OpaqueMessage> &can_rx = can_receive_.at(receiver);
|
||||
if (!can_rx.empty()) {
|
||||
OpaqueMessage message = std::move(can_rx.back());
|
||||
can_rx.pop_back();
|
||||
|
||||
// TODO(tyler) search for item in can_receive_ that matches the desired types, rather
|
||||
// than asserting that the last item in can_rx matches.
|
||||
auto m_opt = std::move(message).Take<Ms...>();
|
||||
MG_ASSERT(m_opt.has_value(), "Wrong message type received compared to the expected type");
|
||||
|
||||
blocked_on_receive_.erase(receiver);
|
||||
|
||||
return std::move(m_opt).value();
|
||||
}
|
||||
if (contains_items) {
|
||||
auto count = can_rx.back().deliverable_at.time_since_epoch().count();
|
||||
auto now_count = cluster_wide_time_microseconds_.time_since_epoch().count();
|
||||
spdlog::trace("can't receive message yet due to artificial latency. deliverable_at: {}, now: {}", count,
|
||||
now_count);
|
||||
}
|
||||
}
|
||||
|
||||
if (!should_shut_down_) {
|
||||
if (!blocked_on_receive_.contains(receiver)) {
|
||||
blocked_on_receive_.emplace(receiver);
|
||||
spdlog::trace("blocking receiver {}", receiver.ToPartialAddress().port);
|
||||
cv_.notify_all();
|
||||
}
|
||||
lock.unlock();
|
||||
bool made_progress = MaybeTickSimulator();
|
||||
lock.lock();
|
||||
if (!should_shut_down_ && !made_progress) {
|
||||
cv_.wait(lock);
|
||||
}
|
||||
}
|
||||
spdlog::trace("timing out receiver {}", receiver.ToPartialAddress().port);
|
||||
|
||||
blocked_on_receive_.erase(receiver);
|
||||
|
||||
return TimedOut{};
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, RValueRef<M> message) {
|
||||
spdlog::trace("sending message from {} to {}", from_address.last_known_port, to_address.last_known_port);
|
||||
auto type_info = TypeInfoFor(message);
|
||||
{
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
std::any message_any(std::move(message));
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message_any),
|
||||
.type_info = type_info,
|
||||
.deliverable_at = cluster_wide_time_microseconds_ + config_.message_delay};
|
||||
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
|
||||
void Send(Address to_address, Address from_address, RequestId request_id, M message) {
|
||||
std::unique_lock<std::mutex> lock(mu_);
|
||||
std::any message_any(std::move(message));
|
||||
OpaqueMessage om{.to_address = to_address,
|
||||
.from_address = from_address,
|
||||
.request_id = request_id,
|
||||
.message = std::move(message_any)};
|
||||
in_flight_.emplace_back(std::make_pair(std::move(to_address), std::move(om)));
|
||||
|
||||
stats_.total_messages++;
|
||||
} // lock dropped before cv notification
|
||||
stats_.total_messages++;
|
||||
|
||||
cv_.notify_all();
|
||||
}
|
||||
|
||||
@@ -13,10 +13,6 @@
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "io/time.hpp"
|
||||
|
||||
namespace memgraph::io::simulator {
|
||||
struct SimulatorStats {
|
||||
uint64_t total_messages = 0;
|
||||
@@ -25,22 +21,5 @@ struct SimulatorStats {
|
||||
uint64_t total_requests = 0;
|
||||
uint64_t total_responses = 0;
|
||||
uint64_t simulator_ticks = 0;
|
||||
Duration elapsed_time;
|
||||
|
||||
friend bool operator==(const SimulatorStats & /* lhs */, const SimulatorStats & /* rhs */) = default;
|
||||
|
||||
friend std::ostream &operator<<(std::ostream &in, const SimulatorStats &stats) {
|
||||
auto elapsed_ms = std::chrono::duration_cast<std::chrono::milliseconds>(stats.elapsed_time).count();
|
||||
|
||||
std::string formated = fmt::format(
|
||||
"SimulatorStats {{ total_messages: {}, dropped_messages: {}, timed_out_requests: {}, total_requests: {}, "
|
||||
"total_responses: {}, simulator_ticks: {}, elapsed_time: {}ms }}",
|
||||
stats.total_messages, stats.dropped_messages, stats.timed_out_requests, stats.total_requests,
|
||||
stats.total_responses, stats.simulator_ticks, elapsed_ms);
|
||||
|
||||
in << formated;
|
||||
|
||||
return in;
|
||||
}
|
||||
};
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -15,7 +15,6 @@
|
||||
#include <utility>
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/simulator/simulator_handle.hpp"
|
||||
#include "io/time.hpp"
|
||||
|
||||
@@ -26,22 +25,24 @@ using memgraph::io::Time;
|
||||
|
||||
class SimulatorTransport {
|
||||
std::shared_ptr<SimulatorHandle> simulator_handle_;
|
||||
Address address_;
|
||||
const Address address_;
|
||||
std::mt19937 rng_;
|
||||
|
||||
public:
|
||||
SimulatorTransport(std::shared_ptr<SimulatorHandle> simulator_handle, Address address, uint64_t seed)
|
||||
: simulator_handle_(simulator_handle), address_(address), rng_(std::mt19937{seed}) {}
|
||||
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, RValueRef<RequestT> request,
|
||||
std::function<void()> notification, Duration timeout) {
|
||||
std::function<bool()> tick_simulator = [handle_copy = simulator_handle_] {
|
||||
return handle_copy->MaybeTickSimulator();
|
||||
};
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, Address from_address, uint64_t request_id, RequestT request,
|
||||
Duration timeout) {
|
||||
std::function<bool()> maybe_tick_simulator = [this] { return simulator_handle_->MaybeTickSimulator(); };
|
||||
auto [future, promise] =
|
||||
memgraph::io::FuturePromisePairWithNotifier<ResponseResult<ResponseT>>(maybe_tick_simulator);
|
||||
|
||||
return simulator_handle_->template SubmitRequest<ResponseT, RequestT>(
|
||||
to_address, from_address, std::move(request), timeout, std::move(tick_simulator), std::move(notification));
|
||||
simulator_handle_->SubmitRequest(to_address, from_address, request_id, std::move(request), timeout,
|
||||
std::move(promise));
|
||||
|
||||
return std::move(future);
|
||||
}
|
||||
|
||||
template <Message... Ms>
|
||||
@@ -50,8 +51,8 @@ class SimulatorTransport {
|
||||
}
|
||||
|
||||
template <Message M>
|
||||
void Send(Address to_address, Address from_address, uint64_t request_id, RValueRef<M> message) {
|
||||
return simulator_handle_->template Send<M>(to_address, from_address, request_id, std::move(message));
|
||||
void Send(Address to_address, Address from_address, uint64_t request_id, M message) {
|
||||
return simulator_handle_->template Send<M>(to_address, from_address, request_id, message);
|
||||
}
|
||||
|
||||
Time Now() const { return simulator_handle_->Now(); }
|
||||
@@ -62,7 +63,5 @@ class SimulatorTransport {
|
||||
Return Rand(D distrib) {
|
||||
return distrib(rng_);
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() { return simulator_handle_->ResponseLatencies(); }
|
||||
};
|
||||
}; // namespace memgraph::io::simulator
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,10 +19,7 @@
|
||||
#include "io/address.hpp"
|
||||
#include "io/errors.hpp"
|
||||
#include "io/future.hpp"
|
||||
#include "io/message_histogram_collector.hpp"
|
||||
#include "io/notifier.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "utils/concepts.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
namespace memgraph::io {
|
||||
@@ -33,15 +30,7 @@ using memgraph::utils::BasicResult;
|
||||
// reasonable constraints around message types over time,
|
||||
// as we adapt things to use Thrift-generated message types.
|
||||
template <typename T>
|
||||
concept Message = std::movable<T> && std::copyable<T>;
|
||||
|
||||
template <utils::Object T>
|
||||
struct RValueRefEnforcer {
|
||||
using Type = T &&;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
using RValueRef = typename RValueRefEnforcer<T>::Type;
|
||||
concept Message = std::same_as<T, std::decay_t<T>>;
|
||||
|
||||
using RequestId = uint64_t;
|
||||
|
||||
@@ -51,7 +40,6 @@ struct ResponseEnvelope {
|
||||
RequestId request_id;
|
||||
Address to_address;
|
||||
Address from_address;
|
||||
Duration response_latency;
|
||||
};
|
||||
|
||||
template <Message M>
|
||||
@@ -78,6 +66,7 @@ template <typename I>
|
||||
class Io {
|
||||
I implementation_;
|
||||
Address address_;
|
||||
RequestId request_id_counter_ = 0;
|
||||
Duration default_timeout_ = std::chrono::microseconds{100000};
|
||||
|
||||
public:
|
||||
@@ -91,44 +80,22 @@ class Io {
|
||||
Duration GetDefaultTimeout() { return default_timeout_; }
|
||||
|
||||
/// Issue a request with an explicit timeout in microseconds provided. This tends to be used by clients.
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RValueRef<RequestT> request, Duration timeout) {
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> RequestWithTimeout(Address address, RequestT request, Duration timeout) {
|
||||
const RequestId request_id = ++request_id_counter_;
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = nullptr;
|
||||
return implementation_.template Request<ResponseT, RequestT>(address, from_address, std::move(request),
|
||||
fill_notifier, timeout);
|
||||
return implementation_.template Request<RequestT, ResponseT>(address, from_address, request_id, request, timeout);
|
||||
}
|
||||
|
||||
/// Issue a request that times out after the default timeout. This tends
|
||||
/// to be used by clients.
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, RValueRef<RequestT> request) {
|
||||
template <Message RequestT, Message ResponseT>
|
||||
ResponseFuture<ResponseT> Request(Address to_address, RequestT request) {
|
||||
const RequestId request_id = ++request_id_counter_;
|
||||
const Duration timeout = default_timeout_;
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = nullptr;
|
||||
return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
|
||||
fill_notifier, timeout);
|
||||
}
|
||||
|
||||
/// Issue a request that will notify a Notifier when it is filled or times out.
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> RequestWithNotification(Address to_address, RValueRef<RequestT> request, Notifier notifier,
|
||||
ReadinessToken readiness_token) {
|
||||
const Duration timeout = default_timeout_;
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
|
||||
return implementation_.template Request<ResponseT, RequestT>(to_address, from_address, std::move(request),
|
||||
fill_notifier, timeout);
|
||||
}
|
||||
|
||||
/// Issue a request that will notify a Notifier when it is filled or times out.
|
||||
template <Message ResponseT, Message RequestT>
|
||||
ResponseFuture<ResponseT> RequestWithNotificationAndTimeout(Address to_address, RequestT &&request, Notifier notifier,
|
||||
ReadinessToken readiness_token, Duration timeout) {
|
||||
const Address from_address = address_;
|
||||
std::function<void()> fill_notifier = [notifier, readiness_token]() { notifier.Notify(readiness_token); };
|
||||
return implementation_.template Request<ResponseT>(to_address, from_address, std::forward<RequestT>(request),
|
||||
fill_notifier, timeout);
|
||||
return implementation_.template Request<RequestT, ResponseT>(to_address, from_address, request_id,
|
||||
std::move(request), timeout);
|
||||
}
|
||||
|
||||
/// Wait for an explicit number of microseconds for a request of one of the
|
||||
@@ -150,9 +117,9 @@ class Io {
|
||||
/// responses are not necessarily expected, and for servers to respond to requests.
|
||||
/// If you need reliable delivery, this must be built on-top. TCP is not enough for most use cases.
|
||||
template <Message M>
|
||||
void Send(Address to_address, RequestId request_id, M &&message) {
|
||||
void Send(Address to_address, RequestId request_id, M message) {
|
||||
Address from_address = address_;
|
||||
return implementation_.template Send<M>(to_address, from_address, request_id, std::forward<M>(message));
|
||||
return implementation_.template Send<M>(to_address, from_address, request_id, std::move(message));
|
||||
}
|
||||
|
||||
/// The current system time. This time source should be preferred over any other,
|
||||
@@ -172,15 +139,6 @@ class Io {
|
||||
Address GetAddress() { return address_; }
|
||||
void SetAddress(Address address) { address_ = address; }
|
||||
|
||||
Io<I> ForkLocal(boost::uuids::uuid uuid) {
|
||||
Address new_address{
|
||||
.unique_id = uuid,
|
||||
.last_known_ip = address_.last_known_ip,
|
||||
.last_known_port = address_.last_known_port,
|
||||
};
|
||||
return Io(implementation_, new_address);
|
||||
}
|
||||
|
||||
LatencyHistogramSummaries ResponseLatencies() { return implementation_.ResponseLatencies(); }
|
||||
Io<I> ForkLocal() { return Io(implementation_, address_.ForkUniqueAddress()); }
|
||||
};
|
||||
}; // namespace memgraph::io
|
||||
|
||||
@@ -11,11 +11,7 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
|
||||
#include <boost/asio/ip/tcp.hpp>
|
||||
|
||||
#include "io/address.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/schemas.hpp"
|
||||
@@ -41,8 +37,6 @@ struct MachineConfig {
|
||||
bool is_query_engine;
|
||||
boost::asio::ip::address listen_ip;
|
||||
uint16_t listen_port;
|
||||
size_t shard_worker_threads = std::max(static_cast<unsigned int>(1), std::thread::hardware_concurrency());
|
||||
bool sync_message_handling = false;
|
||||
};
|
||||
|
||||
} // namespace memgraph::machine_manager
|
||||
|
||||
@@ -11,43 +11,39 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "coordinator/coordinator_rsm.hpp"
|
||||
#include "coordinator/coordinator_worker.hpp"
|
||||
#include "io/message_conversion.hpp"
|
||||
#include "io/messages.hpp"
|
||||
#include "io/rsm/rsm_client.hpp"
|
||||
#include "io/time.hpp"
|
||||
#include "machine_manager/machine_config.hpp"
|
||||
#include "storage/v3/shard_manager.hpp"
|
||||
#include <coordinator/coordinator_rsm.hpp>
|
||||
#include <io/message_conversion.hpp>
|
||||
#include <io/messages.hpp>
|
||||
#include <io/rsm/rsm_client.hpp>
|
||||
#include <io/time.hpp>
|
||||
#include <machine_manager/machine_config.hpp>
|
||||
#include <storage/v3/shard_manager.hpp>
|
||||
|
||||
namespace memgraph::machine_manager {
|
||||
|
||||
using coordinator::Coordinator;
|
||||
using coordinator::CoordinatorReadRequests;
|
||||
using coordinator::CoordinatorReadResponses;
|
||||
using coordinator::CoordinatorRsm;
|
||||
using coordinator::CoordinatorWriteRequests;
|
||||
using coordinator::CoordinatorWriteResponses;
|
||||
using coordinator::coordinator_worker::CoordinatorWorker;
|
||||
using CoordinatorRouteMessage = coordinator::coordinator_worker::RouteMessage;
|
||||
using CoordinatorQueue = coordinator::coordinator_worker::Queue;
|
||||
using io::ConvertVariant;
|
||||
using io::Duration;
|
||||
using io::RequestId;
|
||||
using io::Time;
|
||||
using io::messages::CoordinatorMessages;
|
||||
using io::messages::ShardManagerMessages;
|
||||
using io::messages::ShardMessages;
|
||||
using io::messages::StorageReadRequest;
|
||||
using io::messages::StorageWriteRequest;
|
||||
using io::rsm::AppendRequest;
|
||||
using io::rsm::AppendResponse;
|
||||
using io::rsm::ReadRequest;
|
||||
using io::rsm::VoteRequest;
|
||||
using io::rsm::VoteResponse;
|
||||
using io::rsm::WriteRequest;
|
||||
using io::rsm::WriteResponse;
|
||||
using storage::v3::ShardManager;
|
||||
using memgraph::coordinator::Coordinator;
|
||||
using memgraph::coordinator::CoordinatorReadRequests;
|
||||
using memgraph::coordinator::CoordinatorReadResponses;
|
||||
using memgraph::coordinator::CoordinatorRsm;
|
||||
using memgraph::coordinator::CoordinatorWriteRequests;
|
||||
using memgraph::coordinator::CoordinatorWriteResponses;
|
||||
using memgraph::io::ConvertVariant;
|
||||
using memgraph::io::Duration;
|
||||
using memgraph::io::RequestId;
|
||||
using memgraph::io::Time;
|
||||
using memgraph::io::messages::CoordinatorMessages;
|
||||
using memgraph::io::messages::ShardManagerMessages;
|
||||
using memgraph::io::messages::ShardMessages;
|
||||
using memgraph::io::messages::StorageReadRequest;
|
||||
using memgraph::io::messages::StorageWriteRequest;
|
||||
using memgraph::io::rsm::AppendRequest;
|
||||
using memgraph::io::rsm::AppendResponse;
|
||||
using memgraph::io::rsm::ReadRequest;
|
||||
using memgraph::io::rsm::VoteRequest;
|
||||
using memgraph::io::rsm::VoteResponse;
|
||||
using memgraph::io::rsm::WriteRequest;
|
||||
using memgraph::io::rsm::WriteResponse;
|
||||
using memgraph::storage::v3::ShardManager;
|
||||
|
||||
/// The MachineManager is responsible for:
|
||||
/// * starting the entire system and ensuring that high-level
|
||||
@@ -66,11 +62,9 @@ template <typename IoImpl>
|
||||
class MachineManager {
|
||||
io::Io<IoImpl> io_;
|
||||
MachineConfig config_;
|
||||
Address coordinator_address_;
|
||||
CoordinatorQueue coordinator_queue_;
|
||||
std::jthread coordinator_handle_;
|
||||
CoordinatorRsm<IoImpl> coordinator_;
|
||||
ShardManager<IoImpl> shard_manager_;
|
||||
Time next_cron_ = Time::min();
|
||||
Time next_cron_;
|
||||
|
||||
public:
|
||||
// TODO initialize ShardManager with "real" coordinator addresses instead of io.GetAddress
|
||||
@@ -78,49 +72,20 @@ class MachineManager {
|
||||
MachineManager(io::Io<IoImpl> io, MachineConfig config, Coordinator coordinator)
|
||||
: io_(io),
|
||||
config_(config),
|
||||
coordinator_address_(io.GetAddress().ForkLocalCoordinator()),
|
||||
shard_manager_{io.ForkLocal(io.GetAddress().ForkLocalShardManager().unique_id), config.shard_worker_threads,
|
||||
coordinator_address_} {
|
||||
auto coordinator_io = io.ForkLocal(coordinator_address_.unique_id);
|
||||
CoordinatorWorker coordinator_worker{coordinator_io, coordinator_queue_, coordinator};
|
||||
coordinator_handle_ = std::jthread([coordinator = std::move(coordinator_worker)]() mutable { coordinator.Run(); });
|
||||
}
|
||||
coordinator_{std::move(io.ForkLocal()), {}, std::move(coordinator)},
|
||||
shard_manager_(ShardManager{io.ForkLocal(), coordinator_.GetAddress()}) {}
|
||||
|
||||
MachineManager(MachineManager &&) noexcept = default;
|
||||
MachineManager &operator=(MachineManager &&) noexcept = default;
|
||||
MachineManager(const MachineManager &) = delete;
|
||||
MachineManager &operator=(const MachineManager &) = delete;
|
||||
|
||||
~MachineManager() {
|
||||
if (coordinator_handle_.joinable()) {
|
||||
coordinator_queue_.Push(coordinator::coordinator_worker::ShutDown{});
|
||||
coordinator_handle_.join();
|
||||
}
|
||||
}
|
||||
|
||||
Address CoordinatorAddress() { return coordinator_address_; }
|
||||
Address CoordinatorAddress() { return coordinator_.GetAddress(); }
|
||||
|
||||
void Run() {
|
||||
while (true) {
|
||||
MaybeBlockOnSyncHandling();
|
||||
|
||||
if (io_.ShouldShutDown()) {
|
||||
break;
|
||||
}
|
||||
|
||||
while (!io_.ShouldShutDown()) {
|
||||
const auto now = io_.Now();
|
||||
|
||||
uint64_t now_us = now.time_since_epoch().count();
|
||||
uint64_t next_us = next_cron_.time_since_epoch().count();
|
||||
|
||||
if (now >= next_cron_) {
|
||||
spdlog::info("now {} >= next_cron_ {}", now_us, next_us);
|
||||
next_cron_ = Cron();
|
||||
} else {
|
||||
spdlog::info("now {} < next_cron_ {}", now_us, next_us);
|
||||
}
|
||||
|
||||
Duration receive_timeout = std::max(next_cron_, now) - now;
|
||||
Duration receive_timeout = next_cron_ - now;
|
||||
|
||||
// Note: this parameter pack must be kept in-sync with the ReceiveWithTimeout parameter pack below
|
||||
using AllMessages =
|
||||
@@ -129,7 +94,7 @@ class MachineManager {
|
||||
WriteResponse<CoordinatorWriteResponses>, ReadRequest<StorageReadRequest>,
|
||||
AppendRequest<StorageWriteRequest>, WriteRequest<StorageWriteRequest>>;
|
||||
|
||||
spdlog::info("MM waiting on Receive on address {}", io_.GetAddress().ToString());
|
||||
spdlog::info("MM waiting on Receive");
|
||||
|
||||
// Note: this parameter pack must be kept in-sync with the AllMessages parameter pack above
|
||||
auto request_result = io_.template ReceiveWithTimeout<
|
||||
@@ -140,6 +105,7 @@ class MachineManager {
|
||||
|
||||
if (request_result.HasError()) {
|
||||
// time to do Cron
|
||||
spdlog::info("MM got timeout");
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -148,7 +114,8 @@ class MachineManager {
|
||||
spdlog::info("MM got message to {}", request_envelope.to_address.ToString());
|
||||
|
||||
// If message is for the coordinator, cast it to subset and pass it to the coordinator
|
||||
bool to_coordinator = coordinator_address_ == request_envelope.to_address;
|
||||
bool to_coordinator = coordinator_.GetAddress() == request_envelope.to_address;
|
||||
spdlog::info("coordinator: {}", coordinator_.GetAddress().ToString());
|
||||
if (to_coordinator) {
|
||||
std::optional<CoordinatorMessages> conversion_attempt =
|
||||
ConvertVariant<AllMessages, ReadRequest<CoordinatorReadRequests>, AppendRequest<CoordinatorWriteRequests>,
|
||||
@@ -161,13 +128,8 @@ class MachineManager {
|
||||
|
||||
CoordinatorMessages &&cm = std::move(conversion_attempt.value());
|
||||
|
||||
CoordinatorRouteMessage route_message{
|
||||
.message = std::move(cm),
|
||||
.request_id = request_envelope.request_id,
|
||||
.to = request_envelope.to_address,
|
||||
.from = request_envelope.from_address,
|
||||
};
|
||||
coordinator_queue_.Push(std::move(route_message));
|
||||
coordinator_.Handle(std::forward<CoordinatorMessages>(cm), request_envelope.request_id,
|
||||
request_envelope.from_address);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -194,7 +156,7 @@ class MachineManager {
|
||||
AppendResponse, WriteRequest<StorageWriteRequest>, VoteRequest, VoteResponse>(
|
||||
std::move(request_envelope.message));
|
||||
|
||||
MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {} - incorrect message type",
|
||||
MG_ASSERT(conversion_attempt.has_value(), "shard rsm message conversion failed for {}",
|
||||
request_envelope.to_address.ToString());
|
||||
|
||||
spdlog::info("got shard rsm message");
|
||||
@@ -206,27 +168,9 @@ class MachineManager {
|
||||
}
|
||||
|
||||
private:
|
||||
// This method exists for controlling concurrency
|
||||
// during deterministic simulation testing.
|
||||
void MaybeBlockOnSyncHandling() {
|
||||
if (!config_.sync_message_handling) {
|
||||
return;
|
||||
}
|
||||
|
||||
// block on coordinator
|
||||
coordinator_queue_.BlockOnQuiescence();
|
||||
|
||||
// block on shards
|
||||
shard_manager_.BlockOnQuiescence();
|
||||
}
|
||||
|
||||
Time Cron() {
|
||||
spdlog::info("running MachineManager::Cron, address {}", io_.GetAddress().ToString());
|
||||
coordinator_queue_.Push(coordinator::coordinator_worker::Cron{});
|
||||
MaybeBlockOnSyncHandling();
|
||||
Time ret = shard_manager_.Cron();
|
||||
MaybeBlockOnSyncHandling();
|
||||
return ret;
|
||||
return shard_manager_.Cron();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
826
src/memgraph.cpp
826
src/memgraph.cpp
File diff suppressed because it is too large
Load Diff
@@ -23,7 +23,7 @@ add_custom_command(
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
|
||||
COMMAND
|
||||
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
|
||||
-Dlanguage=Cpp -visitor -package antlropencypher::v2
|
||||
-Dlanguage=Cpp -visitor -package antlropencypher
|
||||
-o ${opencypher_generated}
|
||||
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
|
||||
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,10 +14,10 @@
|
||||
#include <string>
|
||||
|
||||
#include "antlr4-runtime.h"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "parser/opencypher/generated/MemgraphCypher.h"
|
||||
#include "parser/opencypher/generated/MemgraphCypherLexer.h"
|
||||
#include "utils/concepts.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
|
||||
namespace memgraph::frontend::opencypher {
|
||||
|
||||
@@ -32,9 +32,11 @@ class SyntaxException : public utils::BasicException {
|
||||
* This thing must me a class since parser.cypher() returns pointer and there is
|
||||
* no way for us to get ownership over the object.
|
||||
*/
|
||||
enum class ParserOpTag : uint8_t { CYPHER, EXPRESSION };
|
||||
enum class ParserOpTag : uint8_t {
|
||||
CYPHER, EXPRESSION
|
||||
};
|
||||
|
||||
template <ParserOpTag Tag = ParserOpTag::CYPHER>
|
||||
template<ParserOpTag Tag = ParserOpTag::CYPHER>
|
||||
class Parser {
|
||||
public:
|
||||
/**
|
||||
@@ -44,9 +46,10 @@ class Parser {
|
||||
Parser(const std::string query) : query_(std::move(query)) {
|
||||
parser_.removeErrorListeners();
|
||||
parser_.addErrorListener(&error_listener_);
|
||||
if constexpr (Tag == ParserOpTag::CYPHER) {
|
||||
if constexpr(Tag == ParserOpTag::CYPHER) {
|
||||
tree_ = parser_.cypher();
|
||||
} else {
|
||||
}
|
||||
else {
|
||||
tree_ = parser_.expression();
|
||||
}
|
||||
if (parser_.getNumberOfSyntaxErrors()) {
|
||||
@@ -72,11 +75,11 @@ class Parser {
|
||||
FirstMessageErrorListener error_listener_;
|
||||
std::string query_;
|
||||
antlr4::ANTLRInputStream input_{query_};
|
||||
antlropencypher::v2::MemgraphCypherLexer lexer_{&input_};
|
||||
antlropencypher::MemgraphCypherLexer lexer_{&input_};
|
||||
antlr4::CommonTokenStream tokens_{&lexer_};
|
||||
|
||||
// generate ast
|
||||
antlropencypher::v2::MemgraphCypher parser_{&tokens_};
|
||||
antlropencypher::MemgraphCypher parser_{&tokens_};
|
||||
antlr4::tree::ParseTree *tree_ = nullptr;
|
||||
};
|
||||
} // namespace memgraph::frontend::opencypher
|
||||
|
||||
1053
src/protobuf/messages.pb.cc
Normal file
1053
src/protobuf/messages.pb.cc
Normal file
File diff suppressed because it is too large
Load Diff
1231
src/protobuf/messages.pb.h
Normal file
1231
src/protobuf/messages.pb.h
Normal file
File diff suppressed because it is too large
Load Diff
24
src/protobuf/messages.proto
Normal file
24
src/protobuf/messages.proto
Normal file
@@ -0,0 +1,24 @@
|
||||
syntax = "proto3";
|
||||
|
||||
package memgraph.protobuf;
|
||||
|
||||
message Address {
|
||||
bytes unique_id = 1;
|
||||
// this is either 4 bytes for ipv4 or 16 bytes for ipv6
|
||||
bytes last_known_ip = 2;
|
||||
uint32 last_known_port = 3;
|
||||
}
|
||||
|
||||
message TestRequest {
|
||||
uint64 request_id = 1;
|
||||
string content = 2;
|
||||
}
|
||||
|
||||
message UberMessage {
|
||||
uint64 request_id = 1;
|
||||
Address to_address = 2;
|
||||
Address from_address = 3;
|
||||
oneof specific_message {
|
||||
TestRequest test_request = 4;
|
||||
}
|
||||
}
|
||||
@@ -48,20 +48,18 @@ add_dependencies(mg-query generate_lcp_query)
|
||||
target_include_directories(mg-query PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_link_libraries(mg-query dl cppitertools Boost::headers)
|
||||
target_link_libraries(mg-query mg-integrations-pulsar mg-integrations-kafka mg-storage-v2 mg-license mg-utils mg-kvstore mg-memory)
|
||||
|
||||
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
|
||||
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
|
||||
endif()
|
||||
|
||||
if("${MG_PYTHON_VERSION}" STREQUAL "")
|
||||
find_package(Python3 3.5 REQUIRED COMPONENTS Development)
|
||||
else()
|
||||
find_package(Python3 "${MG_PYTHON_VERSION}" EXACT REQUIRED COMPONENTS Development)
|
||||
endif()
|
||||
|
||||
target_link_libraries(mg-query Python3::Python)
|
||||
|
||||
# Generate Antlr openCypher parser
|
||||
|
||||
set(opencypher_frontend ${CMAKE_CURRENT_SOURCE_DIR}/frontend/opencypher)
|
||||
set(opencypher_generated ${opencypher_frontend}/generated)
|
||||
set(opencypher_lexer_grammar ${opencypher_frontend}/grammar/MemgraphCypherLexer.g4)
|
||||
@@ -84,15 +82,15 @@ add_custom_command(
|
||||
OUTPUT ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include}
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory ${opencypher_generated}
|
||||
COMMAND
|
||||
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
|
||||
-Dlanguage=Cpp -visitor -package antlropencypher
|
||||
-o ${opencypher_generated}
|
||||
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
|
||||
java -jar ${CMAKE_SOURCE_DIR}/libs/antlr-4.10.1-complete.jar
|
||||
-Dlanguage=Cpp -visitor -package antlropencypher
|
||||
-o ${opencypher_generated}
|
||||
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
|
||||
WORKING_DIRECTORY "${CMAKE_BINARY_DIR}"
|
||||
DEPENDS
|
||||
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
|
||||
${opencypher_frontend}/grammar/CypherLexer.g4
|
||||
${opencypher_frontend}/grammar/Cypher.g4)
|
||||
${opencypher_lexer_grammar} ${opencypher_parser_grammar}
|
||||
${opencypher_frontend}/grammar/CypherLexer.g4
|
||||
${opencypher_frontend}/grammar/Cypher.g4)
|
||||
|
||||
add_custom_target(generate_opencypher_parser
|
||||
DEPENDS ${antlr_opencypher_generated_src} ${antlr_opencypher_generated_include})
|
||||
|
||||
@@ -9,8 +9,10 @@ set(mg_query_v2_sources
|
||||
${lcp_query_v2_cpp_files}
|
||||
common.cpp
|
||||
cypher_query_interpreter.cpp
|
||||
dump.cpp
|
||||
frontend/semantic/required_privileges.cpp
|
||||
frontend/stripped.cpp
|
||||
interpret/awesome_memgraph_functions.cpp
|
||||
interpreter.cpp
|
||||
metadata.cpp
|
||||
plan/operator.cpp
|
||||
@@ -21,10 +23,18 @@ set(mg_query_v2_sources
|
||||
plan/rewrite/index_lookup.cpp
|
||||
plan/rule_based_planner.cpp
|
||||
plan/variable_start_planner.cpp
|
||||
# procedure/mg_procedure_impl.cpp
|
||||
# procedure/mg_procedure_helpers.cpp
|
||||
# procedure/module.cpp
|
||||
# procedure/py_module.cpp
|
||||
serialization/property_value.cpp
|
||||
# stream/streams.cpp
|
||||
# stream/sources.cpp
|
||||
# stream/common.cpp
|
||||
# trigger.cpp
|
||||
# trigger_context.cpp
|
||||
bindings/typed_value.cpp
|
||||
accessors.cpp
|
||||
multiframe.cpp)
|
||||
accessors.cpp)
|
||||
|
||||
find_package(Boost REQUIRED)
|
||||
|
||||
@@ -33,8 +43,8 @@ add_dependencies(mg-query-v2 generate_lcp_query_v2)
|
||||
target_include_directories(mg-query-v2 PUBLIC ${CMAKE_SOURCE_DIR}/include)
|
||||
target_include_directories(mg-query-v2 PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/bindings)
|
||||
target_link_libraries(mg-query-v2 dl cppitertools Boost::headers)
|
||||
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory mg-coordinator)
|
||||
target_link_libraries(mg-query-v2 mg-expr mg-functions)
|
||||
target_link_libraries(mg-query-v2 mg-integrations-pulsar mg-integrations-kafka mg-storage-v3 mg-license mg-utils mg-kvstore mg-memory)
|
||||
target_link_libraries(mg-query-v2 mg-expr)
|
||||
|
||||
if(NOT "${MG_PYTHON_PATH}" STREQUAL "")
|
||||
set(Python3_ROOT_DIR "${MG_PYTHON_PATH}")
|
||||
|
||||
@@ -10,67 +10,37 @@
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
|
||||
namespace memgraph::query::v2::accessors {
|
||||
EdgeAccessor::EdgeAccessor(Edge edge, const RequestRouterInterface *request_router)
|
||||
: edge(std::move(edge)), request_router_(request_router) {}
|
||||
EdgeAccessor::EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props)
|
||||
: edge(std::move(edge)), properties(std::move(props)) {}
|
||||
|
||||
EdgeTypeId EdgeAccessor::EdgeType() const { return edge.type.id; }
|
||||
uint64_t EdgeAccessor::EdgeType() const { return edge.type.id; }
|
||||
|
||||
const std::vector<std::pair<PropertyId, Value>> &EdgeAccessor::Properties() const { return edge.properties; }
|
||||
|
||||
Value EdgeAccessor::GetProperty(const std::string &prop_name) const {
|
||||
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
|
||||
if (!maybe_prop) {
|
||||
return {};
|
||||
}
|
||||
const auto prop_id = *maybe_prop;
|
||||
auto it = std::find_if(edge.properties.begin(), edge.properties.end(), [&](auto &pr) { return prop_id == pr.first; });
|
||||
return it->second;
|
||||
std::vector<std::pair<PropertyId, Value>> EdgeAccessor::Properties() const {
|
||||
return properties;
|
||||
// std::map<std::string, TypedValue> res;
|
||||
// for (const auto &[name, value] : *properties) {
|
||||
// res[name] = ValueToTypedValue(value);
|
||||
// }
|
||||
// return res;
|
||||
}
|
||||
|
||||
const Edge &EdgeAccessor::GetEdge() const { return edge; }
|
||||
|
||||
bool EdgeAccessor::IsCycle() const { return edge.src == edge.dst; };
|
||||
|
||||
size_t EdgeAccessor::CypherId() const { return edge.id.gid; }
|
||||
|
||||
VertexAccessor EdgeAccessor::To() const {
|
||||
return VertexAccessor(Vertex{edge.dst}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
Value EdgeAccessor::GetProperty(const std::string & /*prop_name*/) const {
|
||||
// TODO(kostasrim) fix this
|
||||
return {};
|
||||
}
|
||||
|
||||
VertexAccessor EdgeAccessor::From() const {
|
||||
return VertexAccessor(Vertex{edge.src}, std::vector<std::pair<PropertyId, msgs::Value>>{}, request_router_);
|
||||
}
|
||||
Edge EdgeAccessor::GetEdge() const { return edge; }
|
||||
|
||||
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
|
||||
const RequestRouterInterface *request_router)
|
||||
: vertex(std::move(v)), properties(std::move(props)), request_router_(request_router) {}
|
||||
VertexAccessor EdgeAccessor::To() const { return VertexAccessor(Vertex{edge.dst}, {}); }
|
||||
|
||||
VertexAccessor::VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props,
|
||||
const RequestRouterInterface *request_router)
|
||||
: vertex(std::move(v)), request_router_(request_router) {
|
||||
properties.reserve(props.size());
|
||||
for (auto &[id, value] : props) {
|
||||
properties.emplace_back(std::make_pair(id, std::move(value)));
|
||||
}
|
||||
}
|
||||
VertexAccessor EdgeAccessor::From() const { return VertexAccessor(Vertex{edge.src}, {}); }
|
||||
|
||||
VertexAccessor::VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props,
|
||||
const RequestRouterInterface *request_router)
|
||||
: vertex(std::move(v)), request_router_(request_router) {
|
||||
properties.reserve(props.size());
|
||||
for (const auto &[id, value] : props) {
|
||||
properties.emplace_back(std::make_pair(id, value));
|
||||
}
|
||||
}
|
||||
|
||||
Label VertexAccessor::PrimaryLabel() const { return vertex.id.first; }
|
||||
|
||||
const msgs::VertexId &VertexAccessor::Id() const { return vertex.id; }
|
||||
VertexAccessor::VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props)
|
||||
: vertex(std::move(v)), properties(std::move(props)) {}
|
||||
|
||||
std::vector<Label> VertexAccessor::Labels() const { return vertex.labels; }
|
||||
|
||||
@@ -79,23 +49,25 @@ bool VertexAccessor::HasLabel(Label &label) const {
|
||||
[label](const auto &l) { return l.id == label.id; }) != vertex.labels.end();
|
||||
}
|
||||
|
||||
const std::vector<std::pair<PropertyId, Value>> &VertexAccessor::Properties() const { return properties; }
|
||||
std::vector<std::pair<PropertyId, Value>> VertexAccessor::Properties() const {
|
||||
// std::map<std::string, TypedValue> res;
|
||||
// for (const auto &[name, value] : *properties) {
|
||||
// res[name] = ValueToTypedValue(value);
|
||||
// }
|
||||
// return res;
|
||||
return properties;
|
||||
}
|
||||
|
||||
Value VertexAccessor::GetProperty(PropertyId prop_id) const {
|
||||
auto it = std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; });
|
||||
if (it == properties.end()) {
|
||||
return {};
|
||||
}
|
||||
return it->second;
|
||||
return std::find_if(properties.begin(), properties.end(), [&](auto &pr) { return prop_id == pr.first; })->second;
|
||||
// return ValueToTypedValue(properties[prop_name]);
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
Value VertexAccessor::GetProperty(const std::string &prop_name) const {
|
||||
auto maybe_prop = request_router_->MaybeNameToProperty(prop_name);
|
||||
if (!maybe_prop) {
|
||||
return {};
|
||||
}
|
||||
return GetProperty(*maybe_prop);
|
||||
Value VertexAccessor::GetProperty(const std::string & /*prop_name*/) const {
|
||||
// TODO(kostasrim) Add string mapping
|
||||
return {};
|
||||
// return ValueToTypedValue(properties[prop_name]);
|
||||
}
|
||||
|
||||
msgs::Vertex VertexAccessor::GetVertex() const { return vertex; }
|
||||
|
||||
@@ -11,7 +11,6 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
@@ -24,80 +23,108 @@
|
||||
#include "utils/memory.hpp"
|
||||
#include "utils/memory_tracker.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
class RequestRouterInterface;
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
namespace memgraph::query::v2::accessors {
|
||||
|
||||
using Value = msgs::Value;
|
||||
using Edge = msgs::Edge;
|
||||
using Vertex = msgs::Vertex;
|
||||
using Label = msgs::Label;
|
||||
using PropertyId = msgs::PropertyId;
|
||||
using EdgeTypeId = msgs::EdgeTypeId;
|
||||
using Value = memgraph::msgs::Value;
|
||||
using Edge = memgraph::msgs::Edge;
|
||||
using Vertex = memgraph::msgs::Vertex;
|
||||
using Label = memgraph::msgs::Label;
|
||||
using PropertyId = memgraph::msgs::PropertyId;
|
||||
|
||||
class VertexAccessor;
|
||||
|
||||
class EdgeAccessor final {
|
||||
public:
|
||||
explicit EdgeAccessor(Edge edge, const RequestRouterInterface *request_router);
|
||||
EdgeAccessor(Edge edge, std::vector<std::pair<PropertyId, Value>> props);
|
||||
|
||||
[[nodiscard]] EdgeTypeId EdgeType() const;
|
||||
uint64_t EdgeType() const;
|
||||
|
||||
[[nodiscard]] const std::vector<std::pair<PropertyId, Value>> &Properties() const;
|
||||
std::vector<std::pair<PropertyId, Value>> Properties() const;
|
||||
|
||||
[[nodiscard]] Value GetProperty(const std::string &prop_name) const;
|
||||
Value GetProperty(const std::string &prop_name) const;
|
||||
|
||||
[[nodiscard]] const Edge &GetEdge() const;
|
||||
Edge GetEdge() const;
|
||||
|
||||
[[nodiscard]] bool IsCycle() const;
|
||||
// Dummy function
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
inline size_t CypherId() const { return 10; }
|
||||
|
||||
[[nodiscard]] size_t CypherId() const;
|
||||
// bool HasSrcAccessor const { return src == nullptr; }
|
||||
// bool HasDstAccessor const { return dst == nullptr; }
|
||||
|
||||
[[nodiscard]] VertexAccessor To() const;
|
||||
[[nodiscard]] VertexAccessor From() const;
|
||||
VertexAccessor To() const;
|
||||
VertexAccessor From() const;
|
||||
|
||||
friend bool operator==(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return lhs.edge == rhs.edge; }
|
||||
friend bool operator==(const EdgeAccessor &lhs, const EdgeAccessor &rhs) {
|
||||
return lhs.edge == rhs.edge && lhs.properties == rhs.properties;
|
||||
}
|
||||
|
||||
friend bool operator!=(const EdgeAccessor &lhs, const EdgeAccessor &rhs) { return !(lhs == rhs); }
|
||||
|
||||
private:
|
||||
Edge edge;
|
||||
const RequestRouterInterface *request_router_;
|
||||
std::vector<std::pair<PropertyId, Value>> properties;
|
||||
};
|
||||
|
||||
class VertexAccessor final {
|
||||
public:
|
||||
using PropertyId = msgs::PropertyId;
|
||||
using Label = msgs::Label;
|
||||
using VertexId = msgs::VertexId;
|
||||
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props,
|
||||
const RequestRouterInterface *request_router);
|
||||
VertexAccessor(Vertex v, std::vector<std::pair<PropertyId, Value>> props);
|
||||
|
||||
VertexAccessor(Vertex v, std::map<PropertyId, Value> &&props, const RequestRouterInterface *request_router);
|
||||
VertexAccessor(Vertex v, const std::map<PropertyId, Value> &props, const RequestRouterInterface *request_router);
|
||||
std::vector<Label> Labels() const;
|
||||
|
||||
[[nodiscard]] Label PrimaryLabel() const;
|
||||
bool HasLabel(Label &label) const;
|
||||
|
||||
[[nodiscard]] const msgs::VertexId &Id() const;
|
||||
std::vector<std::pair<PropertyId, Value>> Properties() const;
|
||||
|
||||
[[nodiscard]] std::vector<Label> Labels() const;
|
||||
Value GetProperty(PropertyId prop_id) const;
|
||||
Value GetProperty(const std::string &prop_name) const;
|
||||
|
||||
[[nodiscard]] bool HasLabel(Label &label) const;
|
||||
|
||||
[[nodiscard]] const std::vector<std::pair<PropertyId, Value>> &Properties() const;
|
||||
|
||||
[[nodiscard]] Value GetProperty(PropertyId prop_id) const;
|
||||
[[nodiscard]] Value GetProperty(const std::string &prop_name) const;
|
||||
|
||||
[[nodiscard]] msgs::Vertex GetVertex() const;
|
||||
msgs::Vertex GetVertex() const;
|
||||
|
||||
// Dummy function
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] size_t InDegree() const { throw utils::NotYetImplemented("InDegree() not yet implemented"); }
|
||||
inline size_t CypherId() const { return 10; }
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
[[nodiscard]] size_t OutDegree() const { throw utils::NotYetImplemented("OutDegree() not yet implemented"); }
|
||||
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
// auto maybe_edges = impl_.InEdges(view, edge_types);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
//
|
||||
// auto InEdges(storage::View view) const { return InEdges(view, {}); }
|
||||
//
|
||||
// auto InEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types, const VertexAccessor &dest)
|
||||
// const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
// auto maybe_edges = impl_.InEdges(view, edge_types, &dest.impl_);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
//
|
||||
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types) const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
// auto maybe_edges = impl_.OutEdges(view, edge_types);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
//
|
||||
// auto OutEdges(storage::View view) const { return OutEdges(view, {}); }
|
||||
//
|
||||
// auto OutEdges(storage::View view, const std::vector<storage::EdgeTypeId> &edge_types,
|
||||
// const VertexAccessor &dest) const
|
||||
// -> storage::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
// auto maybe_edges = impl_.OutEdges(view, edge_types, &dest.impl_);
|
||||
// if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
// return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
// }
|
||||
|
||||
// storage::Result<size_t> InDegree(storage::View view) const { return impl_.InDegree(view); }
|
||||
//
|
||||
// storage::Result<size_t> OutDegree(storage::View view) const { return impl_.OutDegree(view); }
|
||||
//
|
||||
|
||||
friend bool operator==(const VertexAccessor &lhs, const VertexAccessor &rhs) {
|
||||
return lhs.vertex == rhs.vertex && lhs.properties == rhs.properties;
|
||||
@@ -108,9 +135,12 @@ class VertexAccessor final {
|
||||
private:
|
||||
Vertex vertex;
|
||||
std::vector<std::pair<PropertyId, Value>> properties;
|
||||
const RequestRouterInterface *request_router_;
|
||||
};
|
||||
|
||||
// inline VertexAccessor EdgeAccessor::To() const { return VertexAccessor(impl_.ToVertex()); }
|
||||
|
||||
// inline VertexAccessor EdgeAccessor::From() const { return VertexAccessor(impl_.FromVertex()); }
|
||||
|
||||
// Highly mocked interface. Won't work if used.
|
||||
class Path {
|
||||
public:
|
||||
@@ -151,14 +181,7 @@ class Path {
|
||||
friend bool operator==(const Path & /*lhs*/, const Path & /*rhs*/) { return true; };
|
||||
utils::MemoryResource *GetMemoryResource() { return mem; }
|
||||
|
||||
auto &vertices() { return vertices_; }
|
||||
auto &edges() { return edges_; }
|
||||
const auto &vertices() const { return vertices_; }
|
||||
const auto &edges() const { return edges_; }
|
||||
|
||||
private:
|
||||
std::vector<VertexAccessor> vertices_;
|
||||
std::vector<EdgeAccessor> edges_;
|
||||
utils::MemoryResource *mem = utils::NewDeleteResource();
|
||||
};
|
||||
} // namespace memgraph::query::v2::accessors
|
||||
|
||||
@@ -11,9 +11,5 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#ifdef MG_AST_INCLUDE_PATH
|
||||
#error You are probably trying to include files from expr from both the storage and query engines! You will have a rough time kid!
|
||||
#endif
|
||||
|
||||
#define MG_AST_INCLUDE_PATH "query/v2/frontend/ast/ast.hpp" // NOLINT(cppcoreguidelines-macro-usage)
|
||||
#define MG_INJECTED_NAMESPACE_NAME memgraph::query::v2 // NOLINT(cppcoreguidelines-macro-usage)
|
||||
|
||||
@@ -21,27 +21,24 @@
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "storage/v3/conversions.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
class RequestRouterInterface;
|
||||
|
||||
inline const auto lam = [](const auto &val) { return ValueToTypedValue(val); };
|
||||
namespace detail {
|
||||
class Callable {
|
||||
public:
|
||||
auto operator()(const storage::v3::PropertyValue &val) const {
|
||||
return storage::v3::PropertyToTypedValue<TypedValue>(val);
|
||||
};
|
||||
auto operator()(const msgs::Value &val, RequestRouterInterface *request_router) const {
|
||||
return ValueToTypedValue(val, request_router);
|
||||
auto operator()(const memgraph::storage::v3::PropertyValue &val) const {
|
||||
return memgraph::storage::v3::PropertyToTypedValue<TypedValue>(val);
|
||||
};
|
||||
auto operator()(const msgs::Value &val) const { return ValueToTypedValue(val); };
|
||||
};
|
||||
|
||||
} // namespace detail
|
||||
using ExpressionEvaluator = expr::ExpressionEvaluator<TypedValue, query::v2::EvaluationContext, RequestRouterInterface,
|
||||
storage::v3::View, storage::v3::LabelId, msgs::Value,
|
||||
detail::Callable, common::ErrorCode, expr::QueryEngineTag>;
|
||||
using ExpressionEvaluator =
|
||||
memgraph::expr::ExpressionEvaluator<TypedValue, memgraph::query::v2::EvaluationContext, DbAccessor,
|
||||
storage::v3::View, storage::v3::LabelId, msgs::Value, detail::Callable,
|
||||
memgraph::storage::v3::Error, memgraph::expr::QueryEngineTag>;
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -13,10 +13,9 @@
|
||||
|
||||
#include "query/v2/bindings/bindings.hpp"
|
||||
|
||||
#include "expr/interpret/frame.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "expr/interpret/frame.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
using Frame = memgraph::expr::Frame;
|
||||
using FrameWithValidity = memgraph::expr::FrameWithValidity;
|
||||
} // namespace memgraph::query::v2
|
||||
using Frame = memgraph::expr::Frame<TypedValue>;
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
|
||||
#include "query/v2/bindings/symbol.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/exceptions.hpp"
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/path.hpp"
|
||||
@@ -27,6 +28,7 @@
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/result.hpp"
|
||||
#include "storage/v3/schema_validator.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/exceptions.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -80,5 +82,102 @@ inline void ExpectType(const Symbol &symbol, const TypedValue &value, TypedValue
|
||||
throw QueryRuntimeException("Expected a {} for '{}', but got {}.", expected, symbol.name(), value.type());
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
concept AccessorWithSetProperty = requires(T accessor, const storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue new_value) {
|
||||
{ accessor.SetProperty(key, new_value) } -> std::same_as<storage::v3::Result<storage::v3::PropertyValue>>;
|
||||
};
|
||||
|
||||
template <typename T>
|
||||
concept AccessorWithSetPropertyAndValidate = requires(T accessor, const storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue new_value) {
|
||||
{
|
||||
accessor.SetPropertyAndValidate(key, new_value)
|
||||
} -> std::same_as<storage::v3::ResultSchema<storage::v3::PropertyValue>>;
|
||||
};
|
||||
|
||||
template <typename TRecordAccessor>
|
||||
concept RecordAccessor =
|
||||
AccessorWithSetProperty<TRecordAccessor> || AccessorWithSetPropertyAndValidate<TRecordAccessor>;
|
||||
|
||||
inline void HandleSchemaViolation(const storage::v3::SchemaViolation &schema_violation, const DbAccessor &dba) {
|
||||
switch (schema_violation.status) {
|
||||
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_HAS_NO_PRIMARY_PROPERTY: {
|
||||
throw SchemaViolationException(
|
||||
fmt::format("Primary key {} not defined on label :{}",
|
||||
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type),
|
||||
dba.LabelToName(schema_violation.label)));
|
||||
}
|
||||
case storage::v3::SchemaViolation::ValidationStatus::NO_SCHEMA_DEFINED_FOR_LABEL: {
|
||||
throw SchemaViolationException(
|
||||
fmt::format("Label :{} is not a primary label", dba.LabelToName(schema_violation.label)));
|
||||
}
|
||||
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_PROPERTY_WRONG_TYPE: {
|
||||
throw SchemaViolationException(
|
||||
fmt::format("Wrong type of property {} in schema :{}, should be of type {}",
|
||||
*schema_violation.violated_property_value, dba.LabelToName(schema_violation.label),
|
||||
storage::v3::SchemaTypeToString(schema_violation.violated_schema_property->type)));
|
||||
}
|
||||
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_KEY: {
|
||||
throw SchemaViolationException(fmt::format("Updating of primary key {} on schema :{} not supported",
|
||||
*schema_violation.violated_property_value,
|
||||
dba.LabelToName(schema_violation.label)));
|
||||
}
|
||||
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_UPDATE_PRIMARY_LABEL: {
|
||||
throw SchemaViolationException(fmt::format(
|
||||
"Adding primary label as secondary or removing primary label:", *schema_violation.violated_property_value,
|
||||
dba.LabelToName(schema_violation.label)));
|
||||
}
|
||||
case storage::v3::SchemaViolation::ValidationStatus::VERTEX_SECONDARY_LABEL_IS_PRIMARY: {
|
||||
throw SchemaViolationException(fmt::format("Cannot create vertex where primary label is secondary:{}",
|
||||
dba.LabelToName(schema_violation.label)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
inline void HandleErrorOnPropertyUpdate(const storage::v3::Error error) {
|
||||
switch (error) {
|
||||
case storage::v3::Error::SERIALIZATION_ERROR:
|
||||
throw TransactionSerializationException();
|
||||
case storage::v3::Error::DELETED_OBJECT:
|
||||
throw QueryRuntimeException("Trying to set properties on a deleted object.");
|
||||
case storage::v3::Error::PROPERTIES_DISABLED:
|
||||
throw QueryRuntimeException("Can't set property because properties on edges are disabled.");
|
||||
case storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
throw QueryRuntimeException("Unexpected error when setting a property.");
|
||||
}
|
||||
}
|
||||
|
||||
/// Set a property `value` mapped with given `key` on a `record`.
|
||||
///
|
||||
/// @throw QueryRuntimeException if value cannot be set as a property value
|
||||
template <RecordAccessor T>
|
||||
storage::v3::PropertyValue PropsSetChecked(T *record, const DbAccessor &dba, const storage::v3::PropertyId &key,
|
||||
const TypedValue &value) {
|
||||
try {
|
||||
if constexpr (std::is_same_v<T, VertexAccessor>) {
|
||||
const auto maybe_old_value = record->SetPropertyAndValidate(key, storage::v3::TypedToPropertyValue(value));
|
||||
if (maybe_old_value.HasError()) {
|
||||
std::visit(utils::Overloaded{[](const storage::v3::Error error) { HandleErrorOnPropertyUpdate(error); },
|
||||
[&dba](const storage::v3::SchemaViolation &schema_violation) {
|
||||
HandleSchemaViolation(schema_violation, dba);
|
||||
}},
|
||||
maybe_old_value.GetError());
|
||||
}
|
||||
return std::move(*maybe_old_value);
|
||||
} else {
|
||||
// No validation on edge properties
|
||||
const auto maybe_old_value = record->SetProperty(key, storage::v3::TypedToPropertyValue(value));
|
||||
if (maybe_old_value.HasError()) {
|
||||
HandleErrorOnPropertyUpdate(maybe_old_value.GetError());
|
||||
}
|
||||
return std::move(*maybe_old_value);
|
||||
}
|
||||
} catch (const expr::TypedValueException &) {
|
||||
throw QueryRuntimeException("'{}' cannot be used as a property value.", value.type());
|
||||
}
|
||||
}
|
||||
|
||||
int64_t QueryTimestamp();
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -13,35 +13,17 @@
|
||||
|
||||
#include <type_traits>
|
||||
|
||||
#include "io/local_transport/local_transport.hpp"
|
||||
#include "query/v2/bindings/symbol_table.hpp"
|
||||
#include "query/v2/common.hpp"
|
||||
#include "query/v2/metadata.hpp"
|
||||
#include "query/v2/parameters.hpp"
|
||||
#include "query/v2/plan/profile.hpp"
|
||||
//#include "query/v2/trigger.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/shard_request_manager.hpp"
|
||||
#include "utils/async_timer.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
// Used to store range of ids that are available
|
||||
// Used for edge id assignment
|
||||
class IdAllocator {
|
||||
public:
|
||||
IdAllocator() = default;
|
||||
IdAllocator(uint64_t low, uint64_t high) : current_edge_id_{low}, max_id_{high} {};
|
||||
|
||||
uint64_t AllocateId() {
|
||||
MG_ASSERT(current_edge_id_ < max_id_, "Current Edge Id went above max id");
|
||||
return current_edge_id_++;
|
||||
}
|
||||
|
||||
private:
|
||||
uint64_t current_edge_id_;
|
||||
uint64_t max_id_;
|
||||
};
|
||||
|
||||
struct EvaluationContext {
|
||||
/// Memory for allocations during evaluation of a *single* Pull call.
|
||||
///
|
||||
@@ -61,27 +43,20 @@ struct EvaluationContext {
|
||||
};
|
||||
|
||||
inline std::vector<storage::v3::PropertyId> NamesToProperties(const std::vector<std::string> &property_names,
|
||||
RequestRouterInterface *request_router) {
|
||||
DbAccessor *dba) {
|
||||
std::vector<storage::v3::PropertyId> properties;
|
||||
// TODO Fix by using reference
|
||||
properties.reserve(property_names.size());
|
||||
if (request_router != nullptr) {
|
||||
for (const auto &name : property_names) {
|
||||
properties.push_back(request_router->NameToProperty(name));
|
||||
}
|
||||
for (const auto &name : property_names) {
|
||||
properties.push_back(dba->NameToProperty(name));
|
||||
}
|
||||
return properties;
|
||||
}
|
||||
|
||||
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names,
|
||||
RequestRouterInterface *request_router) {
|
||||
inline std::vector<storage::v3::LabelId> NamesToLabels(const std::vector<std::string> &label_names, DbAccessor *dba) {
|
||||
std::vector<storage::v3::LabelId> labels;
|
||||
labels.reserve(label_names.size());
|
||||
// TODO Fix by using reference
|
||||
if (request_router != nullptr) {
|
||||
for (const auto &name : label_names) {
|
||||
labels.push_back(request_router->NameToLabel(name));
|
||||
}
|
||||
for (const auto &name : label_names) {
|
||||
labels.push_back(dba->NameToLabel(name));
|
||||
}
|
||||
return labels;
|
||||
}
|
||||
@@ -96,9 +71,9 @@ struct ExecutionContext {
|
||||
plan::ProfilingStats stats;
|
||||
plan::ProfilingStats *stats_root{nullptr};
|
||||
ExecutionStats execution_stats;
|
||||
// TriggerContextCollector *trigger_context_collector{nullptr};
|
||||
utils::AsyncTimer timer;
|
||||
RequestRouterInterface *request_router{nullptr};
|
||||
IdAllocator *edge_ids_alloc;
|
||||
std::unique_ptr<msgs::ShardRequestManagerInterface> shard_request_manager{nullptr};
|
||||
};
|
||||
|
||||
static_assert(std::is_move_assignable_v<ExecutionContext>, "ExecutionContext must be move assignable!");
|
||||
|
||||
@@ -12,12 +12,11 @@
|
||||
#pragma once
|
||||
#include "bindings/typed_value.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInterface *request_router) {
|
||||
inline TypedValue ValueToTypedValue(const msgs::Value &value) {
|
||||
using Value = msgs::Value;
|
||||
switch (value.type) {
|
||||
case Value::Type::Null:
|
||||
@@ -35,7 +34,7 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
|
||||
std::vector<TypedValue> dst;
|
||||
dst.reserve(lst.size());
|
||||
for (const auto &elem : lst) {
|
||||
dst.push_back(ValueToTypedValue(elem, request_router));
|
||||
dst.push_back(ValueToTypedValue(elem));
|
||||
}
|
||||
return TypedValue(std::move(dst));
|
||||
}
|
||||
@@ -43,23 +42,20 @@ inline TypedValue ValueToTypedValue(const msgs::Value &value, RequestRouterInter
|
||||
const auto &value_map = value.map_v;
|
||||
std::map<std::string, TypedValue> dst;
|
||||
for (const auto &[key, val] : value_map) {
|
||||
dst[key] = ValueToTypedValue(val, request_router);
|
||||
dst[key] = ValueToTypedValue(val);
|
||||
}
|
||||
return TypedValue(std::move(dst));
|
||||
}
|
||||
case Value::Type::Vertex:
|
||||
return TypedValue(accessors::VertexAccessor(
|
||||
value.vertex_v, std::vector<std::pair<storage::v3::PropertyId, msgs::Value>>{}, request_router));
|
||||
return TypedValue(accessors::VertexAccessor(value.vertex_v, {}));
|
||||
case Value::Type::Edge:
|
||||
return TypedValue(accessors::EdgeAccessor(value.edge_v, request_router));
|
||||
return TypedValue(accessors::EdgeAccessor(value.edge_v, {}));
|
||||
case Value::Type::Path:
|
||||
break;
|
||||
}
|
||||
throw std::runtime_error("Incorrect type in conversion");
|
||||
}
|
||||
|
||||
inline const auto ValueToTypedValueFunctor = [](const msgs::Value &value, RequestRouterInterface *request_router) {
|
||||
return ValueToTypedValue(value, request_router);
|
||||
};
|
||||
|
||||
inline msgs::Value TypedValueToValue(const TypedValue &value) {
|
||||
using Value = msgs::Value;
|
||||
switch (value.type()) {
|
||||
@@ -95,10 +91,7 @@ inline msgs::Value TypedValueToValue(const TypedValue &value) {
|
||||
case TypedValue::Type::Edge:
|
||||
return Value(value.ValueEdge().GetEdge());
|
||||
case TypedValue::Type::Path:
|
||||
case TypedValue::Type::LocalTime:
|
||||
case TypedValue::Type::LocalDateTime:
|
||||
case TypedValue::Type::Date:
|
||||
case TypedValue::Type::Duration:
|
||||
default:
|
||||
break;
|
||||
}
|
||||
throw std::runtime_error("Incorrect type in conversion");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,12 +11,11 @@
|
||||
|
||||
#include "query/v2/cypher_query_interpreter.hpp"
|
||||
#include "query/v2/bindings/symbol_generator.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_HIDDEN_bool(query_v2_cost_planner, true, "Use the cost-estimating query planner.");
|
||||
DEFINE_HIDDEN_bool(query_cost_planner, true, "Use the cost-estimating query planner.");
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_VALIDATED_int32(query_v2_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
|
||||
DEFINE_VALIDATED_int32(query_plan_cache_ttl, 60, "Time to live for cached query plans, in seconds.",
|
||||
FLAG_IN_RANGE(0, std::numeric_limits<int32_t>::max()));
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
@@ -118,19 +117,19 @@ ParsedQuery ParseQuery(const std::string &query_string, const std::map<std::stri
|
||||
}
|
||||
|
||||
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
auto vertex_counts = plan::MakeVertexCountCache(request_router);
|
||||
auto vertex_counts = plan::MakeVertexCountCache(db_accessor);
|
||||
auto symbol_table = expr::MakeSymbolTable(query, predefined_identifiers);
|
||||
auto planning_context = plan::MakePlanningContext(&ast_storage, &symbol_table, query, &vertex_counts);
|
||||
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_v2_cost_planner);
|
||||
auto [root, cost] = plan::MakeLogicalPlan(&planning_context, parameters, FLAGS_query_cost_planner);
|
||||
return std::make_unique<SingleNodeLogicalPlan>(std::move(root), cost, std::move(ast_storage),
|
||||
std::move(symbol_table));
|
||||
}
|
||||
|
||||
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, utils::SkipList<PlanCacheEntry> *plan_cache,
|
||||
RequestRouterInterface *request_router,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers) {
|
||||
std::optional<utils::SkipList<PlanCacheEntry>::Accessor> plan_cache_access;
|
||||
if (plan_cache) {
|
||||
@@ -146,7 +145,7 @@ std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_stor
|
||||
}
|
||||
|
||||
auto plan = std::make_shared<CachedPlan>(
|
||||
MakeLogicalPlan(std::move(ast_storage), query, parameters, request_router, predefined_identifiers));
|
||||
MakeLogicalPlan(std::move(ast_storage), query, parameters, db_accessor, predefined_identifiers));
|
||||
if (plan_cache_access) {
|
||||
plan_cache_access->insert({hash, plan});
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -22,9 +22,9 @@
|
||||
#include "utils/timer.hpp"
|
||||
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_bool(query_v2_cost_planner);
|
||||
DECLARE_bool(query_cost_planner);
|
||||
// NOLINTNEXTLINE (cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_int32(query_v2_plan_cache_ttl);
|
||||
DECLARE_int32(query_plan_cache_ttl);
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
@@ -58,7 +58,7 @@ class CachedPlan {
|
||||
|
||||
bool IsExpired() const {
|
||||
// NOLINTNEXTLINE (modernize-use-nullptr)
|
||||
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_v2_plan_cache_ttl);
|
||||
return cache_timer_.Elapsed() > std::chrono::seconds(FLAGS_query_plan_cache_ttl);
|
||||
};
|
||||
|
||||
private:
|
||||
@@ -132,7 +132,7 @@ class SingleNodeLogicalPlan final : public LogicalPlan {
|
||||
};
|
||||
|
||||
std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery *query, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers);
|
||||
|
||||
/**
|
||||
@@ -145,7 +145,7 @@ std::unique_ptr<LogicalPlan> MakeLogicalPlan(AstStorage ast_storage, CypherQuery
|
||||
*/
|
||||
std::shared_ptr<CachedPlan> CypherQueryToPlan(uint64_t hash, AstStorage ast_storage, CypherQuery *query,
|
||||
const Parameters ¶meters, utils::SkipList<PlanCacheEntry> *plan_cache,
|
||||
RequestRouterInterface *request_router,
|
||||
DbAccessor *db_accessor,
|
||||
const std::vector<Identifier *> &predefined_identifiers = {});
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -36,7 +36,7 @@
|
||||
// This cannot be avoided by simple include orderings so we
|
||||
// simply undefine those macros as we're sure that libkrb5
|
||||
// won't and can't be used anywhere in the query engine.
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/storage.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/result.hpp"
|
||||
|
||||
@@ -64,11 +64,24 @@ class EdgeAccessor final {
|
||||
|
||||
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
|
||||
|
||||
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
return impl_.GetProperty(key, view);
|
||||
}
|
||||
|
||||
storage::v3::Result<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue &value) {
|
||||
return impl_.SetProperty(key, value);
|
||||
}
|
||||
|
||||
storage::v3::Result<storage::v3::PropertyValue> RemoveProperty(storage::v3::PropertyId key) {
|
||||
return SetProperty(key, storage::v3::PropertyValue());
|
||||
}
|
||||
|
||||
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
|
||||
return impl_.ClearProperties();
|
||||
}
|
||||
|
||||
VertexAccessor To() const;
|
||||
|
||||
VertexAccessor From() const;
|
||||
@@ -100,19 +113,51 @@ class VertexAccessor final {
|
||||
|
||||
auto PrimaryKey(storage::v3::View view) const { return impl_.PrimaryKey(view); }
|
||||
|
||||
storage::v3::ShardResult<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
|
||||
storage::v3::ResultSchema<bool> AddLabel(storage::v3::LabelId label) { return impl_.AddLabelAndValidate(label); }
|
||||
|
||||
storage::v3::ResultSchema<bool> AddLabelAndValidate(storage::v3::LabelId label) {
|
||||
return impl_.AddLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::ResultSchema<bool> RemoveLabel(storage::v3::LabelId label) {
|
||||
return impl_.RemoveLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::ResultSchema<bool> RemoveLabelAndValidate(storage::v3::LabelId label) {
|
||||
return impl_.RemoveLabelAndValidate(label);
|
||||
}
|
||||
|
||||
storage::v3::Result<bool> HasLabel(storage::v3::View view, storage::v3::LabelId label) const {
|
||||
return impl_.HasLabel(label, view);
|
||||
}
|
||||
|
||||
auto Properties(storage::v3::View view) const { return impl_.Properties(view); }
|
||||
|
||||
storage::v3::ShardResult<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
storage::v3::Result<storage::v3::PropertyValue> GetProperty(storage::v3::View view,
|
||||
storage::v3::PropertyId key) const {
|
||||
return impl_.GetProperty(key, view);
|
||||
}
|
||||
|
||||
storage::v3::ResultSchema<storage::v3::PropertyValue> SetProperty(storage::v3::PropertyId key,
|
||||
const storage::v3::PropertyValue &value) {
|
||||
return impl_.SetPropertyAndValidate(key, value);
|
||||
}
|
||||
|
||||
storage::v3::ResultSchema<storage::v3::PropertyValue> SetPropertyAndValidate(
|
||||
storage::v3::PropertyId key, const storage::v3::PropertyValue &value) {
|
||||
return impl_.SetPropertyAndValidate(key, value);
|
||||
}
|
||||
|
||||
storage::v3::ResultSchema<storage::v3::PropertyValue> RemovePropertyAndValidate(storage::v3::PropertyId key) {
|
||||
return SetPropertyAndValidate(key, storage::v3::PropertyValue{});
|
||||
}
|
||||
|
||||
storage::v3::Result<std::map<storage::v3::PropertyId, storage::v3::PropertyValue>> ClearProperties() {
|
||||
return impl_.ClearProperties();
|
||||
}
|
||||
|
||||
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
auto maybe_edges = impl_.InEdges(view, edge_types);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
@@ -122,7 +167,7 @@ class VertexAccessor final {
|
||||
|
||||
auto InEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
|
||||
const VertexAccessor &dest) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.InEdges(view)))> {
|
||||
const auto dest_id = dest.impl_.Id(view).GetValue();
|
||||
auto maybe_edges = impl_.InEdges(view, edge_types, &dest_id);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
@@ -130,7 +175,7 @@ class VertexAccessor final {
|
||||
}
|
||||
|
||||
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
auto maybe_edges = impl_.OutEdges(view, edge_types);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
@@ -140,16 +185,16 @@ class VertexAccessor final {
|
||||
|
||||
auto OutEdges(storage::v3::View view, const std::vector<storage::v3::EdgeTypeId> &edge_types,
|
||||
const VertexAccessor &dest) const
|
||||
-> storage::v3::ShardResult<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
-> storage::v3::Result<decltype(iter::imap(MakeEdgeAccessor, *impl_.OutEdges(view)))> {
|
||||
const auto dest_id = dest.impl_.Id(view).GetValue();
|
||||
auto maybe_edges = impl_.OutEdges(view, edge_types, &dest_id);
|
||||
if (maybe_edges.HasError()) return maybe_edges.GetError();
|
||||
return iter::imap(MakeEdgeAccessor, std::move(*maybe_edges));
|
||||
}
|
||||
|
||||
storage::v3::ShardResult<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
|
||||
storage::v3::Result<size_t> InDegree(storage::v3::View view) const { return impl_.InDegree(view); }
|
||||
|
||||
storage::v3::ShardResult<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
|
||||
storage::v3::Result<size_t> OutDegree(storage::v3::View view) const { return impl_.OutDegree(view); }
|
||||
|
||||
// TODO(jbajic) Fix Remove Gid
|
||||
static int64_t CypherId() { return 1; }
|
||||
@@ -174,7 +219,195 @@ inline VertexAccessor EdgeAccessor::From() const { return *static_cast<VertexAcc
|
||||
inline bool EdgeAccessor::IsCycle() const { return To() == From(); }
|
||||
|
||||
class DbAccessor final {
|
||||
storage::v3::Shard::Accessor *accessor_;
|
||||
|
||||
class VerticesIterable final {
|
||||
storage::v3::VerticesIterable iterable_;
|
||||
|
||||
public:
|
||||
class Iterator final {
|
||||
storage::v3::VerticesIterable::Iterator it_;
|
||||
|
||||
public:
|
||||
explicit Iterator(storage::v3::VerticesIterable::Iterator it) : it_(it) {}
|
||||
|
||||
VertexAccessor operator*() const { return VertexAccessor(*it_); }
|
||||
|
||||
Iterator &operator++() {
|
||||
++it_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
bool operator==(const Iterator &other) const { return it_ == other.it_; }
|
||||
|
||||
bool operator!=(const Iterator &other) const { return !(other == *this); }
|
||||
};
|
||||
|
||||
explicit VerticesIterable(storage::v3::VerticesIterable iterable) : iterable_(std::move(iterable)) {}
|
||||
|
||||
Iterator begin() { return Iterator(iterable_.begin()); }
|
||||
|
||||
Iterator end() { return Iterator(iterable_.end()); }
|
||||
};
|
||||
|
||||
public:
|
||||
explicit DbAccessor(storage::v3::Shard::Accessor *accessor) : accessor_(accessor) {}
|
||||
|
||||
// TODO(jbajic) Fix Remove Gid
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
std::optional<VertexAccessor> FindVertex(uint64_t /*unused*/) { return std::nullopt; }
|
||||
|
||||
std::optional<VertexAccessor> FindVertex(storage::v3::PrimaryKey &primary_key, storage::v3::View view) {
|
||||
auto maybe_vertex = accessor_->FindVertex(primary_key, view);
|
||||
if (maybe_vertex) return VertexAccessor(*maybe_vertex);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
VerticesIterable Vertices(storage::v3::View view) { return VerticesIterable(accessor_->Vertices(view)); }
|
||||
|
||||
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label) {
|
||||
return VerticesIterable(accessor_->Vertices(label, view));
|
||||
}
|
||||
|
||||
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property) {
|
||||
return VerticesIterable(accessor_->Vertices(label, property, view));
|
||||
}
|
||||
|
||||
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
|
||||
const storage::v3::PropertyValue &value) {
|
||||
return VerticesIterable(accessor_->Vertices(label, property, value, view));
|
||||
}
|
||||
|
||||
VerticesIterable Vertices(storage::v3::View view, storage::v3::LabelId label, storage::v3::PropertyId property,
|
||||
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) {
|
||||
return VerticesIterable(accessor_->Vertices(label, property, lower, upper, view));
|
||||
}
|
||||
|
||||
storage::v3::ResultSchema<VertexAccessor> InsertVertexAndValidate(
|
||||
const storage::v3::LabelId primary_label, const std::vector<storage::v3::LabelId> &labels,
|
||||
const std::vector<std::pair<storage::v3::PropertyId, storage::v3::PropertyValue>> &properties) {
|
||||
auto maybe_vertex_acc = accessor_->CreateVertexAndValidate(primary_label, labels, properties);
|
||||
if (maybe_vertex_acc.HasError()) {
|
||||
return {std::move(maybe_vertex_acc.GetError())};
|
||||
}
|
||||
return VertexAccessor{maybe_vertex_acc.GetValue()};
|
||||
}
|
||||
|
||||
storage::v3::Result<EdgeAccessor> InsertEdge(VertexAccessor *from, VertexAccessor *to,
|
||||
const storage::v3::EdgeTypeId &edge_type) {
|
||||
static constexpr auto kDummyGid = storage::v3::Gid::FromUint(0);
|
||||
auto maybe_edge = accessor_->CreateEdge(from->impl_.Id(storage::v3::View::NEW).GetValue(),
|
||||
to->impl_.Id(storage::v3::View::NEW).GetValue(), edge_type, kDummyGid);
|
||||
if (maybe_edge.HasError()) return storage::v3::Result<EdgeAccessor>(maybe_edge.GetError());
|
||||
return EdgeAccessor(*maybe_edge);
|
||||
}
|
||||
|
||||
storage::v3::Result<std::optional<EdgeAccessor>> RemoveEdge(EdgeAccessor *edge) {
|
||||
auto res = accessor_->DeleteEdge(edge->impl_.FromVertex(), edge->impl_.ToVertex(), edge->impl_.Gid());
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
|
||||
const auto &value = res.GetValue();
|
||||
if (!value) {
|
||||
return std::optional<EdgeAccessor>{};
|
||||
}
|
||||
|
||||
return std::make_optional<EdgeAccessor>(*value);
|
||||
}
|
||||
|
||||
storage::v3::Result<std::optional<std::pair<VertexAccessor, std::vector<EdgeAccessor>>>> DetachRemoveVertex(
|
||||
VertexAccessor *vertex_accessor) {
|
||||
using ReturnType = std::pair<VertexAccessor, std::vector<EdgeAccessor>>;
|
||||
|
||||
auto res = accessor_->DetachDeleteVertex(&vertex_accessor->impl_);
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
|
||||
const auto &value = res.GetValue();
|
||||
if (!value) {
|
||||
return std::optional<ReturnType>{};
|
||||
}
|
||||
|
||||
const auto &[vertex, edges] = *value;
|
||||
|
||||
std::vector<EdgeAccessor> deleted_edges;
|
||||
deleted_edges.reserve(edges.size());
|
||||
std::transform(edges.begin(), edges.end(), std::back_inserter(deleted_edges),
|
||||
[](const auto &deleted_edge) { return EdgeAccessor{deleted_edge}; });
|
||||
|
||||
return std::make_optional<ReturnType>(vertex, std::move(deleted_edges));
|
||||
}
|
||||
|
||||
storage::v3::Result<std::optional<VertexAccessor>> RemoveVertex(VertexAccessor *vertex_accessor) {
|
||||
auto res = accessor_->DeleteVertex(&vertex_accessor->impl_);
|
||||
if (res.HasError()) {
|
||||
return res.GetError();
|
||||
}
|
||||
|
||||
const auto &value = res.GetValue();
|
||||
if (!value) {
|
||||
return std::optional<VertexAccessor>{};
|
||||
}
|
||||
|
||||
return {std::make_optional<VertexAccessor>(*value)};
|
||||
}
|
||||
|
||||
// TODO(jbajic) Query engine should have a map of labels, properties and edge
|
||||
// types
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
storage::v3::PropertyId NameToProperty(const std::string_view name) { return accessor_->NameToProperty(name); }
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
storage::v3::LabelId NameToLabel(const std::string_view name) { return accessor_->NameToLabel(name); }
|
||||
|
||||
// NOLINTNEXTLINE(readability-convert-member-functions-to-static)
|
||||
storage::v3::EdgeTypeId NameToEdgeType(const std::string_view name) { return accessor_->NameToEdgeType(name); }
|
||||
|
||||
const std::string &PropertyToName(storage::v3::PropertyId prop) const { return accessor_->PropertyToName(prop); }
|
||||
|
||||
const std::string &LabelToName(storage::v3::LabelId label) const { return accessor_->LabelToName(label); }
|
||||
|
||||
const std::string &EdgeTypeToName(storage::v3::EdgeTypeId type) const { return accessor_->EdgeTypeToName(type); }
|
||||
|
||||
void AdvanceCommand() { accessor_->AdvanceCommand(); }
|
||||
|
||||
void Commit() { return accessor_->Commit(coordinator::Hlc{}); }
|
||||
|
||||
void Abort() { accessor_->Abort(); }
|
||||
|
||||
bool LabelIndexExists(storage::v3::LabelId label) const { return accessor_->LabelIndexExists(label); }
|
||||
|
||||
bool LabelPropertyIndexExists(storage::v3::LabelId label, storage::v3::PropertyId prop) const {
|
||||
return accessor_->LabelPropertyIndexExists(label, prop);
|
||||
}
|
||||
|
||||
int64_t VerticesCount() const { return accessor_->ApproximateVertexCount(); }
|
||||
|
||||
int64_t VerticesCount(storage::v3::LabelId label) const { return accessor_->ApproximateVertexCount(label); }
|
||||
|
||||
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property) const {
|
||||
return accessor_->ApproximateVertexCount(label, property);
|
||||
}
|
||||
|
||||
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
|
||||
const storage::v3::PropertyValue &value) const {
|
||||
return accessor_->ApproximateVertexCount(label, property, value);
|
||||
}
|
||||
|
||||
int64_t VerticesCount(storage::v3::LabelId label, storage::v3::PropertyId property,
|
||||
const std::optional<utils::Bound<storage::v3::PropertyValue>> &lower,
|
||||
const std::optional<utils::Bound<storage::v3::PropertyValue>> &upper) const {
|
||||
return accessor_->ApproximateVertexCount(label, property, lower, upper);
|
||||
}
|
||||
|
||||
storage::v3::IndicesInfo ListAllIndices() const { return accessor_->ListAllIndices(); }
|
||||
|
||||
const storage::v3::SchemaValidator &GetSchemaValidator() const { return accessor_->GetSchemaValidator(); }
|
||||
|
||||
storage::v3::SchemasInfo ListAllSchemas() const { return accessor_->ListAllSchemas(); }
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
483
src/query/v2/dump.cpp
Normal file
483
src/query/v2/dump.cpp
Normal file
@@ -0,0 +1,483 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "query/v2/dump.hpp"
|
||||
|
||||
#include <iomanip>
|
||||
#include <limits>
|
||||
#include <map>
|
||||
#include <optional>
|
||||
#include <ostream>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <fmt/format.h>
|
||||
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/exceptions.hpp"
|
||||
#include "query/v2/stream.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/storage.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/string.hpp"
|
||||
#include "utils/temporal.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
namespace {
|
||||
|
||||
// Property that is used to make a difference among vertices. It is added to
|
||||
// property set of vertices to match edges and removed after the entire graph
|
||||
// is built.
|
||||
const char *kInternalPropertyId = "__mg_id__";
|
||||
|
||||
// Label that is attached to each vertex and is used for easier creation of
|
||||
// index on internal property id.
|
||||
const char *kInternalVertexLabel = "__mg_vertex__";
|
||||
|
||||
/// A helper function that escapes label, edge type and property names.
|
||||
std::string EscapeName(const std::string_view value) {
|
||||
std::string out;
|
||||
out.reserve(value.size() + 2);
|
||||
out.append(1, '`');
|
||||
for (auto c : value) {
|
||||
if (c == '`') {
|
||||
out.append("``");
|
||||
} else {
|
||||
out.append(1, c);
|
||||
}
|
||||
}
|
||||
out.append(1, '`');
|
||||
return out;
|
||||
}
|
||||
|
||||
void DumpPreciseDouble(std::ostream *os, double value) {
|
||||
// A temporary stream is used to keep precision of the original output
|
||||
// stream unchanged.
|
||||
std::ostringstream temp_oss;
|
||||
temp_oss << std::setprecision(std::numeric_limits<double>::max_digits10) << value;
|
||||
*os << temp_oss.str();
|
||||
}
|
||||
|
||||
namespace {
|
||||
void DumpDate(std::ostream &os, const storage::v3::TemporalData &value) {
|
||||
utils::Date date(value.microseconds);
|
||||
os << "DATE(\"" << date << "\")";
|
||||
}
|
||||
|
||||
void DumpLocalTime(std::ostream &os, const storage::v3::TemporalData &value) {
|
||||
utils::LocalTime lt(value.microseconds);
|
||||
os << "LOCALTIME(\"" << lt << "\")";
|
||||
}
|
||||
|
||||
void DumpLocalDateTime(std::ostream &os, const storage::v3::TemporalData &value) {
|
||||
utils::LocalDateTime ldt(value.microseconds);
|
||||
os << "LOCALDATETIME(\"" << ldt << "\")";
|
||||
}
|
||||
|
||||
void DumpDuration(std::ostream &os, const storage::v3::TemporalData &value) {
|
||||
utils::Duration dur(value.microseconds);
|
||||
os << "DURATION(\"" << dur << "\")";
|
||||
}
|
||||
|
||||
void DumpTemporalData(std::ostream &os, const storage::v3::TemporalData &value) {
|
||||
switch (value.type) {
|
||||
case storage::v3::TemporalType::Date: {
|
||||
DumpDate(os, value);
|
||||
return;
|
||||
}
|
||||
case storage::v3::TemporalType::LocalTime: {
|
||||
DumpLocalTime(os, value);
|
||||
return;
|
||||
}
|
||||
case storage::v3::TemporalType::LocalDateTime: {
|
||||
DumpLocalDateTime(os, value);
|
||||
return;
|
||||
}
|
||||
case storage::v3::TemporalType::Duration: {
|
||||
DumpDuration(os, value);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
} // namespace
|
||||
|
||||
void DumpPropertyValue(std::ostream *os, const storage::v3::PropertyValue &value) {
|
||||
switch (value.type()) {
|
||||
case storage::v3::PropertyValue::Type::Null:
|
||||
*os << "Null";
|
||||
return;
|
||||
case storage::v3::PropertyValue::Type::Bool:
|
||||
*os << (value.ValueBool() ? "true" : "false");
|
||||
return;
|
||||
case storage::v3::PropertyValue::Type::String:
|
||||
*os << utils::Escape(value.ValueString());
|
||||
return;
|
||||
case storage::v3::PropertyValue::Type::Int:
|
||||
*os << value.ValueInt();
|
||||
return;
|
||||
case storage::v3::PropertyValue::Type::Double:
|
||||
DumpPreciseDouble(os, value.ValueDouble());
|
||||
return;
|
||||
case storage::v3::PropertyValue::Type::List: {
|
||||
*os << "[";
|
||||
const auto &list = value.ValueList();
|
||||
utils::PrintIterable(*os, list, ", ", [](auto &os, const auto &item) { DumpPropertyValue(&os, item); });
|
||||
*os << "]";
|
||||
return;
|
||||
}
|
||||
case storage::v3::PropertyValue::Type::Map: {
|
||||
*os << "{";
|
||||
const auto &map = value.ValueMap();
|
||||
utils::PrintIterable(*os, map, ", ", [](auto &os, const auto &kv) {
|
||||
os << EscapeName(kv.first) << ": ";
|
||||
DumpPropertyValue(&os, kv.second);
|
||||
});
|
||||
*os << "}";
|
||||
return;
|
||||
}
|
||||
case storage::v3::PropertyValue::Type::TemporalData: {
|
||||
DumpTemporalData(*os, value.ValueTemporalData());
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void DumpProperties(std::ostream *os, query::v2::DbAccessor *dba,
|
||||
const std::map<storage::v3::PropertyId, storage::v3::PropertyValue> &store,
|
||||
std::optional<int64_t> property_id = std::nullopt) {
|
||||
*os << "{";
|
||||
if (property_id) {
|
||||
*os << kInternalPropertyId << ": " << *property_id;
|
||||
if (store.size() > 0) *os << ", ";
|
||||
}
|
||||
utils::PrintIterable(*os, store, ", ", [&dba](auto &os, const auto &kv) {
|
||||
os << EscapeName(dba->PropertyToName(kv.first)) << ": ";
|
||||
DumpPropertyValue(&os, kv.second);
|
||||
});
|
||||
*os << "}";
|
||||
}
|
||||
|
||||
void DumpVertex(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::VertexAccessor &vertex) {
|
||||
*os << "CREATE (";
|
||||
*os << ":" << kInternalVertexLabel;
|
||||
auto maybe_labels = vertex.Labels(storage::v3::View::OLD);
|
||||
if (maybe_labels.HasError()) {
|
||||
switch (maybe_labels.GetError()) {
|
||||
case storage::v3::Error::DELETED_OBJECT:
|
||||
throw query::v2::QueryRuntimeException("Trying to get labels from a deleted node.");
|
||||
case storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
throw query::v2::QueryRuntimeException("Trying to get labels from a node that doesn't exist.");
|
||||
case storage::v3::Error::SERIALIZATION_ERROR:
|
||||
case storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case storage::v3::Error::PROPERTIES_DISABLED:
|
||||
throw query::v2::QueryRuntimeException("Unexpected error when getting labels.");
|
||||
}
|
||||
}
|
||||
for (const auto &label : *maybe_labels) {
|
||||
*os << ":" << EscapeName(dba->LabelToName(label));
|
||||
}
|
||||
*os << " ";
|
||||
auto maybe_props = vertex.Properties(storage::v3::View::OLD);
|
||||
if (maybe_props.HasError()) {
|
||||
switch (maybe_props.GetError()) {
|
||||
case storage::v3::Error::DELETED_OBJECT:
|
||||
throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
|
||||
case storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
throw query::v2::QueryRuntimeException("Trying to get properties from a node that doesn't exist.");
|
||||
case storage::v3::Error::SERIALIZATION_ERROR:
|
||||
case storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case storage::v3::Error::PROPERTIES_DISABLED:
|
||||
throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
|
||||
}
|
||||
}
|
||||
DumpProperties(os, dba, *maybe_props, vertex.CypherId());
|
||||
*os << ");";
|
||||
}
|
||||
|
||||
void DumpEdge(std::ostream *os, query::v2::DbAccessor *dba, const query::v2::EdgeAccessor &edge) {
|
||||
*os << "MATCH ";
|
||||
*os << "(u:" << kInternalVertexLabel << "), ";
|
||||
*os << "(v:" << kInternalVertexLabel << ")";
|
||||
*os << " WHERE ";
|
||||
*os << "u." << kInternalPropertyId << " = " << edge.From().CypherId();
|
||||
*os << " AND ";
|
||||
*os << "v." << kInternalPropertyId << " = " << edge.To().CypherId() << " ";
|
||||
*os << "CREATE (u)-[";
|
||||
*os << ":" << EscapeName(dba->EdgeTypeToName(edge.EdgeType()));
|
||||
auto maybe_props = edge.Properties(storage::v3::View::OLD);
|
||||
if (maybe_props.HasError()) {
|
||||
switch (maybe_props.GetError()) {
|
||||
case storage::v3::Error::DELETED_OBJECT:
|
||||
throw query::v2::QueryRuntimeException("Trying to get properties from a deleted object.");
|
||||
case storage::v3::Error::NONEXISTENT_OBJECT:
|
||||
throw query::v2::QueryRuntimeException("Trying to get properties from an edge that doesn't exist.");
|
||||
case storage::v3::Error::SERIALIZATION_ERROR:
|
||||
case storage::v3::Error::VERTEX_HAS_EDGES:
|
||||
case storage::v3::Error::PROPERTIES_DISABLED:
|
||||
throw query::v2::QueryRuntimeException("Unexpected error when getting properties.");
|
||||
}
|
||||
}
|
||||
if (maybe_props->size() > 0) {
|
||||
*os << " ";
|
||||
DumpProperties(os, dba, *maybe_props);
|
||||
}
|
||||
*os << "]->(v);";
|
||||
}
|
||||
|
||||
void DumpLabelIndex(std::ostream *os, query::v2::DbAccessor *dba, const storage::v3::LabelId label) {
|
||||
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << ";";
|
||||
}
|
||||
|
||||
void DumpLabelPropertyIndex(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
|
||||
storage::v3::PropertyId property) {
|
||||
*os << "CREATE INDEX ON :" << EscapeName(dba->LabelToName(label)) << "(" << EscapeName(dba->PropertyToName(property))
|
||||
<< ");";
|
||||
}
|
||||
|
||||
void DumpExistenceConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
|
||||
storage::v3::PropertyId property) {
|
||||
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT EXISTS (u."
|
||||
<< EscapeName(dba->PropertyToName(property)) << ");";
|
||||
}
|
||||
|
||||
void DumpUniqueConstraint(std::ostream *os, query::v2::DbAccessor *dba, storage::v3::LabelId label,
|
||||
const std::set<storage::v3::PropertyId> &properties) {
|
||||
*os << "CREATE CONSTRAINT ON (u:" << EscapeName(dba->LabelToName(label)) << ") ASSERT ";
|
||||
utils::PrintIterable(*os, properties, ", ", [&dba](auto &stream, const auto &property) {
|
||||
stream << "u." << EscapeName(dba->PropertyToName(property));
|
||||
});
|
||||
*os << " IS UNIQUE;";
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
PullPlanDump::PullPlanDump(DbAccessor *dba)
|
||||
: dba_(dba),
|
||||
vertices_iterable_(dba->Vertices(storage::v3::View::OLD)),
|
||||
pull_chunks_{// Dump all label indices
|
||||
CreateLabelIndicesPullChunk(),
|
||||
// Dump all label property indices
|
||||
CreateLabelPropertyIndicesPullChunk(),
|
||||
// Create internal index for faster edge creation
|
||||
CreateInternalIndexPullChunk(),
|
||||
// Dump all vertices
|
||||
CreateVertexPullChunk(),
|
||||
// Dump all edges
|
||||
CreateEdgePullChunk(),
|
||||
// Drop the internal index
|
||||
CreateDropInternalIndexPullChunk(),
|
||||
// Internal index cleanup
|
||||
CreateInternalIndexCleanupPullChunk()} {}
|
||||
|
||||
bool PullPlanDump::Pull(AnyStream *stream, std::optional<int> n) {
|
||||
// Iterate all functions that stream some results.
|
||||
// Each function should return number of results it streamed after it
|
||||
// finishes. If the function did not finish streaming all the results,
|
||||
// std::nullopt should be returned because n results have already been sent.
|
||||
while (current_chunk_index_ < pull_chunks_.size() && (!n || *n > 0)) {
|
||||
const auto maybe_streamed_count = pull_chunks_[current_chunk_index_](stream, n);
|
||||
|
||||
if (!maybe_streamed_count) {
|
||||
// n wasn't large enough to stream all the results from the current chunk
|
||||
break;
|
||||
}
|
||||
|
||||
if (n) {
|
||||
// chunk finished streaming its results
|
||||
// subtract number of results streamed in current pull
|
||||
// so we know how many results we need to stream from future
|
||||
// chunks.
|
||||
*n -= *maybe_streamed_count;
|
||||
}
|
||||
|
||||
++current_chunk_index_;
|
||||
}
|
||||
return current_chunk_index_ == pull_chunks_.size();
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateLabelIndicesPullChunk() {
|
||||
// Dump all label indices
|
||||
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
|
||||
// Delay the construction of indices vectors
|
||||
if (!indices_info_) {
|
||||
indices_info_.emplace(dba_->ListAllIndices());
|
||||
}
|
||||
const auto &label = indices_info_->label;
|
||||
|
||||
size_t local_counter = 0;
|
||||
while (global_index < label.size() && (!n || local_counter < *n)) {
|
||||
std::ostringstream os;
|
||||
DumpLabelIndex(&os, dba_, label[global_index]);
|
||||
stream->Result({TypedValue(os.str())});
|
||||
|
||||
++global_index;
|
||||
++local_counter;
|
||||
}
|
||||
|
||||
if (global_index == label.size()) {
|
||||
return local_counter;
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
};
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateLabelPropertyIndicesPullChunk() {
|
||||
return [this, global_index = 0U](AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
|
||||
// Delay the construction of indices vectors
|
||||
if (!indices_info_) {
|
||||
indices_info_.emplace(dba_->ListAllIndices());
|
||||
}
|
||||
const auto &label_property = indices_info_->label_property;
|
||||
|
||||
size_t local_counter = 0;
|
||||
while (global_index < label_property.size() && (!n || local_counter < *n)) {
|
||||
std::ostringstream os;
|
||||
const auto &label_property_index = label_property[global_index];
|
||||
DumpLabelPropertyIndex(&os, dba_, label_property_index.first, label_property_index.second);
|
||||
stream->Result({TypedValue(os.str())});
|
||||
|
||||
++global_index;
|
||||
++local_counter;
|
||||
}
|
||||
|
||||
if (global_index == label_property.size()) {
|
||||
return local_counter;
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
};
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexPullChunk() {
|
||||
return [this](AnyStream *stream, std::optional<int>) mutable -> std::optional<size_t> {
|
||||
if (vertices_iterable_.begin() != vertices_iterable_.end()) {
|
||||
std::ostringstream os;
|
||||
os << "CREATE INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
|
||||
stream->Result({TypedValue(os.str())});
|
||||
internal_index_created_ = true;
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateVertexPullChunk() {
|
||||
return [this, maybe_current_iter = std::optional<VertexAccessorIterableIterator>{}](
|
||||
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
|
||||
// Delay the call of begin() function
|
||||
// If multiple begins are called before an iteration,
|
||||
// one iteration will make the rest of iterators be in undefined
|
||||
// states.
|
||||
if (!maybe_current_iter) {
|
||||
maybe_current_iter.emplace(vertices_iterable_.begin());
|
||||
}
|
||||
|
||||
auto ¤t_iter{*maybe_current_iter};
|
||||
|
||||
size_t local_counter = 0;
|
||||
while (current_iter != vertices_iterable_.end() && (!n || local_counter < *n)) {
|
||||
std::ostringstream os;
|
||||
DumpVertex(&os, dba_, *current_iter);
|
||||
stream->Result({TypedValue(os.str())});
|
||||
++local_counter;
|
||||
++current_iter;
|
||||
}
|
||||
if (current_iter == vertices_iterable_.end()) {
|
||||
return local_counter;
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
};
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateEdgePullChunk() {
|
||||
return [this, maybe_current_vertex_iter = std::optional<VertexAccessorIterableIterator>{},
|
||||
// we need to save the iterable which contains list of accessor so
|
||||
// our saved iterator is valid in the next run
|
||||
maybe_edge_iterable = std::shared_ptr<EdgeAccessorIterable>{nullptr},
|
||||
maybe_current_edge_iter = std::optional<EdgeAccessorIterableIterator>{}](
|
||||
AnyStream *stream, std::optional<int> n) mutable -> std::optional<size_t> {
|
||||
// Delay the call of begin() function
|
||||
// If multiple begins are called before an iteration,
|
||||
// one iteration will make the rest of iterators be in undefined
|
||||
// states.
|
||||
if (!maybe_current_vertex_iter) {
|
||||
maybe_current_vertex_iter.emplace(vertices_iterable_.begin());
|
||||
}
|
||||
|
||||
auto ¤t_vertex_iter{*maybe_current_vertex_iter};
|
||||
size_t local_counter = 0U;
|
||||
for (; current_vertex_iter != vertices_iterable_.end() && (!n || local_counter < *n); ++current_vertex_iter) {
|
||||
const auto &vertex = *current_vertex_iter;
|
||||
// If we have a saved iterable from a previous pull
|
||||
// we need to use the same iterable
|
||||
if (!maybe_edge_iterable) {
|
||||
maybe_edge_iterable = std::make_shared<EdgeAccessorIterable>(vertex.OutEdges(storage::v3::View::OLD));
|
||||
}
|
||||
auto &maybe_edges = *maybe_edge_iterable;
|
||||
MG_ASSERT(maybe_edges.HasValue(), "Invalid database state!");
|
||||
auto current_edge_iter = maybe_current_edge_iter ? *maybe_current_edge_iter : maybe_edges->begin();
|
||||
for (; current_edge_iter != maybe_edges->end() && (!n || local_counter < *n); ++current_edge_iter) {
|
||||
std::ostringstream os;
|
||||
DumpEdge(&os, dba_, *current_edge_iter);
|
||||
stream->Result({TypedValue(os.str())});
|
||||
|
||||
++local_counter;
|
||||
}
|
||||
|
||||
if (current_edge_iter != maybe_edges->end()) {
|
||||
maybe_current_edge_iter.emplace(current_edge_iter);
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
maybe_current_edge_iter = std::nullopt;
|
||||
maybe_edge_iterable = nullptr;
|
||||
}
|
||||
|
||||
if (current_vertex_iter == vertices_iterable_.end()) {
|
||||
return local_counter;
|
||||
}
|
||||
|
||||
return std::nullopt;
|
||||
};
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateDropInternalIndexPullChunk() {
|
||||
return [this](AnyStream *stream, std::optional<int>) {
|
||||
if (internal_index_created_) {
|
||||
std::ostringstream os;
|
||||
os << "DROP INDEX ON :" << kInternalVertexLabel << "(" << kInternalPropertyId << ");";
|
||||
stream->Result({TypedValue(os.str())});
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
}
|
||||
|
||||
PullPlanDump::PullChunk PullPlanDump::CreateInternalIndexCleanupPullChunk() {
|
||||
return [this](AnyStream *stream, std::optional<int>) {
|
||||
if (internal_index_created_) {
|
||||
std::ostringstream os;
|
||||
os << "MATCH (u) REMOVE u:" << kInternalVertexLabel << ", u." << kInternalPropertyId << ";";
|
||||
stream->Result({TypedValue(os.str())});
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
};
|
||||
}
|
||||
|
||||
void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream) { PullPlanDump(dba).Pull(stream, {}); }
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
63
src/query/v2/dump.hpp
Normal file
63
src/query/v2/dump.hpp
Normal file
@@ -0,0 +1,63 @@
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ostream>
|
||||
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/stream.hpp"
|
||||
#include "storage/v3/storage.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
void DumpDatabaseToCypherQueries(query::v2::DbAccessor *dba, AnyStream *stream);
|
||||
|
||||
struct PullPlanDump {
|
||||
explicit PullPlanDump(query::v2::DbAccessor *dba);
|
||||
|
||||
/// Pull the dump results lazily
|
||||
/// @return true if all results were returned, false otherwise
|
||||
bool Pull(AnyStream *stream, std::optional<int> n);
|
||||
|
||||
private:
|
||||
query::v2::DbAccessor *dba_ = nullptr;
|
||||
|
||||
std::optional<storage::v3::IndicesInfo> indices_info_ = std::nullopt;
|
||||
|
||||
using VertexAccessorIterable = decltype(std::declval<query::v2::DbAccessor>().Vertices(storage::v3::View::OLD));
|
||||
using VertexAccessorIterableIterator = decltype(std::declval<VertexAccessorIterable>().begin());
|
||||
|
||||
using EdgeAccessorIterable = decltype(std::declval<VertexAccessor>().OutEdges(storage::v3::View::OLD));
|
||||
using EdgeAccessorIterableIterator = decltype(std::declval<EdgeAccessorIterable>().GetValue().begin());
|
||||
|
||||
VertexAccessorIterable vertices_iterable_;
|
||||
bool internal_index_created_ = false;
|
||||
|
||||
size_t current_chunk_index_ = 0;
|
||||
|
||||
using PullChunk = std::function<std::optional<size_t>(AnyStream *stream, std::optional<int> n)>;
|
||||
// We define every part of the dump query in a self contained function.
|
||||
// Each functions is responsible of keeping track of its execution status.
|
||||
// If a function did finish its execution, it should return number of results
|
||||
// it streamed so we know how many rows should be pulled from the next
|
||||
// function, otherwise std::nullopt is returned.
|
||||
std::vector<PullChunk> pull_chunks_;
|
||||
|
||||
PullChunk CreateLabelIndicesPullChunk();
|
||||
PullChunk CreateLabelPropertyIndicesPullChunk();
|
||||
PullChunk CreateInternalIndexPullChunk();
|
||||
PullChunk CreateVertexPullChunk();
|
||||
PullChunk CreateEdgePullChunk();
|
||||
PullChunk CreateDropInternalIndexPullChunk();
|
||||
PullChunk CreateInternalIndexCleanupPullChunk();
|
||||
};
|
||||
} // namespace memgraph::query::v2
|
||||
@@ -224,4 +224,12 @@ class VersionInfoInMulticommandTxException : public QueryException {
|
||||
: QueryException("Version info query not allowed in multicommand transactions.") {}
|
||||
};
|
||||
|
||||
/**
|
||||
* An exception for an illegal operation that violates schema
|
||||
*/
|
||||
class SchemaViolationException : public QueryRuntimeException {
|
||||
public:
|
||||
using QueryRuntimeException::QueryRuntimeException;
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -18,15 +18,12 @@
|
||||
#include <vector>
|
||||
|
||||
#include "query/v2/bindings/ast_visitor.hpp"
|
||||
#include "common/types.hpp"
|
||||
#include "query/v2/bindings/symbol.hpp"
|
||||
#include "functions/awesome_memgraph_functions.hpp"
|
||||
#include "query/v2/interpret/awesome_memgraph_functions.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/path.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "utils/typeinfo.hpp"
|
||||
#include "query/v2/conversions.hpp"
|
||||
|
||||
cpp<#
|
||||
|
||||
@@ -838,15 +835,13 @@ cpp<#
|
||||
:slk-load (slk-load-ast-vector "Expression"))
|
||||
(function-name "std::string" :scope :public)
|
||||
(function "std::function<TypedValue(const TypedValue *, int64_t,
|
||||
const functions::FunctionContext<RequestRouterInterface> &)>"
|
||||
const FunctionContext &)>"
|
||||
:scope :public
|
||||
:dont-save t
|
||||
:clone :copy
|
||||
:slk-load (lambda (member)
|
||||
#>cpp
|
||||
self->${member} = functions::NameToFunction<TypedValue,
|
||||
functions::FunctionContext<RequestRouterInterface>,
|
||||
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(self->function_name_);
|
||||
self->${member} = query::v2::NameToFunction(self->function_name_);
|
||||
cpp<#)))
|
||||
(:public
|
||||
#>cpp
|
||||
@@ -869,8 +864,7 @@ cpp<#
|
||||
const std::vector<Expression *> &arguments)
|
||||
: arguments_(arguments),
|
||||
function_name_(function_name),
|
||||
function_(functions::NameToFunction<TypedValue, functions::FunctionContext<RequestRouterInterface>,
|
||||
functions::QueryEngineTag, decltype(ValueToTypedValueFunctor)>(function_name_)) {
|
||||
function_(NameToFunction(function_name_)) {
|
||||
if (!function_) {
|
||||
throw SemanticException("Function '{}' doesn't exist.", function_name);
|
||||
}
|
||||
|
||||
1185
src/query/v2/interpret/awesome_memgraph_functions.cpp
Normal file
1185
src/query/v2/interpret/awesome_memgraph_functions.cpp
Normal file
File diff suppressed because it is too large
Load Diff
@@ -16,13 +16,12 @@
|
||||
#include <unordered_map>
|
||||
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "storage/v3/view.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
class RequestRouterInterface;
|
||||
class DbAccessor;
|
||||
|
||||
namespace {
|
||||
const char kStartsWith[] = "STARTSWITH";
|
||||
@@ -32,9 +31,7 @@ const char kId[] = "ID";
|
||||
} // namespace
|
||||
|
||||
struct FunctionContext {
|
||||
// TODO(kostasrim) consider optional here. RequestRouter does not exist on the storage.
|
||||
// DbAccessor *db_accessor;
|
||||
RequestRouterInterface *request_router;
|
||||
DbAccessor *db_accessor;
|
||||
utils::MemoryResource *memory;
|
||||
int64_t timestamp;
|
||||
std::unordered_map<std::string, int64_t> *counters;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -19,13 +19,9 @@
|
||||
#include <cstdint>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <memory>
|
||||
#include <optional>
|
||||
|
||||
#include "coordinator/coordinator_client.hpp"
|
||||
#include "expr/ast/ast_visitor.hpp"
|
||||
#include "io/local_transport/local_system.hpp"
|
||||
#include "io/local_transport/local_transport.hpp"
|
||||
#include "memory/memory_control.hpp"
|
||||
#include "parser/opencypher/parser.hpp"
|
||||
#include "query/v2/bindings/eval.hpp"
|
||||
@@ -37,17 +33,17 @@
|
||||
#include "query/v2/context.hpp"
|
||||
#include "query/v2/cypher_query_interpreter.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/dump.hpp"
|
||||
#include "query/v2/exceptions.hpp"
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/frontend/semantic/required_privileges.hpp"
|
||||
#include "query/v2/metadata.hpp"
|
||||
#include "query/v2/multiframe.hpp"
|
||||
#include "query/v2/plan/planner.hpp"
|
||||
#include "query/v2/plan/profile.hpp"
|
||||
#include "query/v2/plan/vertex_count_cache.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "storage/v3/shard.hpp"
|
||||
#include "storage/v3/storage.hpp"
|
||||
#include "utils/algorithm.hpp"
|
||||
#include "utils/csv_parsing.hpp"
|
||||
#include "utils/event_counter.hpp"
|
||||
@@ -64,11 +60,7 @@
|
||||
#include "utils/tsc.hpp"
|
||||
#include "utils/variant_helpers.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_bool(use_multi_frame, false, "Whether to use MultiFrame or not");
|
||||
|
||||
namespace EventCounter {
|
||||
|
||||
extern Event ReadQuery;
|
||||
extern Event WriteQuery;
|
||||
extern Event ReadWriteQuery;
|
||||
@@ -78,7 +70,6 @@ extern const Event LabelPropertyIndexCreated;
|
||||
|
||||
extern const Event StreamsCreated;
|
||||
extern const Event TriggersCreated;
|
||||
|
||||
} // namespace EventCounter
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
@@ -123,6 +114,17 @@ std::optional<TResult> GetOptionalValue(query::v2::Expression *expression, Expre
|
||||
return {};
|
||||
};
|
||||
|
||||
std::optional<std::string> GetOptionalStringValue(query::v2::Expression *expression, ExpressionEvaluator &evaluator) {
|
||||
if (expression != nullptr) {
|
||||
auto value = expression->Accept(evaluator);
|
||||
MG_ASSERT(value.IsNull() || value.IsString());
|
||||
if (value.IsString()) {
|
||||
return {std::string(value.ValueString().begin(), value.ValueString().end())};
|
||||
}
|
||||
}
|
||||
return {};
|
||||
};
|
||||
|
||||
class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
|
||||
public:
|
||||
explicit ReplQueryHandler(storage::v3::Shard * /*db*/) {}
|
||||
@@ -149,18 +151,18 @@ class ReplQueryHandler final : public query::v2::ReplicationQueryHandler {
|
||||
/// @throw QueryRuntimeException if an error ocurred.
|
||||
|
||||
Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router) {
|
||||
DbAccessor *db_accessor) {
|
||||
// Empty frame for evaluation of password expression. This is OK since
|
||||
// password should be either null or string literal and it's evaluation
|
||||
// should not depend on frame.
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
// the argument to Callback.
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD);
|
||||
|
||||
std::string username = auth_query->user_;
|
||||
std::string rolename = auth_query->role_;
|
||||
@@ -318,16 +320,16 @@ Callback HandleAuthQuery(AuthQuery *auth_query, AuthQueryHandler *auth, const Pa
|
||||
}
|
||||
|
||||
Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters ¶meters,
|
||||
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
|
||||
InterpreterContext *interpreter_context, DbAccessor *db_accessor,
|
||||
std::vector<Notification> *notifications) {
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
// the argument to Callback.
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD);
|
||||
|
||||
Callback callback;
|
||||
switch (repl_query->action_) {
|
||||
@@ -453,9 +455,23 @@ Callback HandleReplicationQuery(ReplicationQuery *repl_query, const Parameters &
|
||||
}
|
||||
}
|
||||
|
||||
Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶meters,
|
||||
RequestRouterInterface *request_router) {
|
||||
expr::Frame frame(0);
|
||||
std::optional<std::string> StringPointerToOptional(const std::string *str) {
|
||||
return str == nullptr ? std::nullopt : std::make_optional(*str);
|
||||
}
|
||||
|
||||
std::vector<std::string> EvaluateTopicNames(ExpressionEvaluator &evaluator,
|
||||
std::variant<Expression *, std::vector<std::string>> topic_variant) {
|
||||
return std::visit(utils::Overloaded{[&](Expression *expression) {
|
||||
auto topic_names = expression->Accept(evaluator);
|
||||
MG_ASSERT(topic_names.IsString());
|
||||
return utils::Split(topic_names.ValueString(), ",");
|
||||
},
|
||||
[&](std::vector<std::string> topic_names) { return topic_names; }},
|
||||
std::move(topic_variant));
|
||||
}
|
||||
|
||||
Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶meters, DbAccessor *db_accessor) {
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
// TODO: MemoryResource for EvaluationContext, it should probably be passed as
|
||||
@@ -464,7 +480,7 @@ Callback HandleSettingQuery(SettingQuery *setting_query, const Parameters ¶m
|
||||
std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::system_clock::now().time_since_epoch())
|
||||
.count();
|
||||
evaluation_context.parameters = parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, db_accessor, storage::v3::View::OLD);
|
||||
|
||||
Callback callback;
|
||||
switch (setting_query->action_) {
|
||||
@@ -655,21 +671,16 @@ struct PullPlanVector {
|
||||
struct PullPlan {
|
||||
explicit PullPlan(std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router = nullptr,
|
||||
// TriggerContextCollector *trigger_context_collector = nullptr,
|
||||
std::optional<size_t> memory_limit = {});
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> PullMultiple(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary);
|
||||
|
||||
private:
|
||||
std::shared_ptr<CachedPlan> plan_ = nullptr;
|
||||
plan::UniqueCursorPtr cursor_ = nullptr;
|
||||
expr::FrameWithValidity frame_;
|
||||
MultiFrame multi_frame_;
|
||||
expr::Frame<TypedValue> frame_;
|
||||
ExecutionContext ctx_;
|
||||
std::optional<size_t> memory_limit_;
|
||||
|
||||
@@ -689,137 +700,29 @@ struct PullPlan {
|
||||
|
||||
PullPlan::PullPlan(const std::shared_ptr<CachedPlan> plan, const Parameters ¶meters, const bool is_profile_query,
|
||||
DbAccessor *dba, InterpreterContext *interpreter_context, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router, const std::optional<size_t> memory_limit)
|
||||
const std::optional<size_t> memory_limit)
|
||||
// TriggerContextCollector *trigger_context_collector, const std::optional<size_t> memory_limit)
|
||||
: plan_(plan),
|
||||
cursor_(plan->plan().MakeCursor(execution_memory)),
|
||||
frame_(plan->symbol_table().max_position(), execution_memory),
|
||||
multi_frame_(plan->symbol_table().max_position(), FLAGS_default_multi_frame_size, execution_memory),
|
||||
memory_limit_(memory_limit) {
|
||||
ctx_.db_accessor = dba;
|
||||
ctx_.symbol_table = plan->symbol_table();
|
||||
ctx_.evaluation_context.timestamp = QueryTimestamp();
|
||||
ctx_.evaluation_context.parameters = parameters;
|
||||
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, request_router);
|
||||
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, request_router);
|
||||
ctx_.evaluation_context.properties = NamesToProperties(plan->ast_storage().properties_, dba);
|
||||
ctx_.evaluation_context.labels = NamesToLabels(plan->ast_storage().labels_, dba);
|
||||
if (interpreter_context->config.execution_timeout_sec > 0) {
|
||||
ctx_.timer = utils::AsyncTimer{interpreter_context->config.execution_timeout_sec};
|
||||
}
|
||||
ctx_.is_shutting_down = &interpreter_context->is_shutting_down;
|
||||
ctx_.is_profile_query = is_profile_query;
|
||||
ctx_.request_router = request_router;
|
||||
ctx_.edge_ids_alloc = &interpreter_context->edge_ids_alloc;
|
||||
}
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::PullMultiple(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary) {
|
||||
// Set up temporary memory for a single Pull. Initial memory comes from the
|
||||
// stack. 256 KiB should fit on the stack and should be more than enough for a
|
||||
// single `Pull`.
|
||||
MG_ASSERT(!n.has_value(), "should pull all!");
|
||||
static constexpr size_t stack_size = 256UL * 1024UL;
|
||||
char stack_data[stack_size];
|
||||
utils::ResourceWithOutOfMemoryException resource_with_exception;
|
||||
utils::MonotonicBufferResource monotonic_memory(&stack_data[0], stack_size, &resource_with_exception);
|
||||
// We can throw on every query because a simple queries for deleting will use only
|
||||
// the stack allocated buffer.
|
||||
// Also, we want to throw only when the query engine requests more memory and not the storage
|
||||
// so we add the exception to the allocator.
|
||||
// TODO (mferencevic): Tune the parameters accordingly.
|
||||
utils::PoolResource pool_memory(128, 1024, &monotonic_memory);
|
||||
std::optional<utils::LimitedMemoryResource> maybe_limited_resource;
|
||||
|
||||
if (memory_limit_) {
|
||||
maybe_limited_resource.emplace(&pool_memory, *memory_limit_);
|
||||
ctx_.evaluation_context.memory = &*maybe_limited_resource;
|
||||
} else {
|
||||
ctx_.evaluation_context.memory = &pool_memory;
|
||||
}
|
||||
|
||||
// Returns true if a result was pulled.
|
||||
const auto pull_result = [&]() -> bool { return cursor_->PullMultiple(multi_frame_, ctx_); };
|
||||
|
||||
const auto stream_values = [&output_symbols, &stream](const Frame &frame) {
|
||||
// TODO: The streamed values should also probably use the above memory.
|
||||
std::vector<TypedValue> values;
|
||||
values.reserve(output_symbols.size());
|
||||
|
||||
for (const auto &symbol : output_symbols) {
|
||||
values.emplace_back(frame[symbol]);
|
||||
}
|
||||
|
||||
stream->Result(values);
|
||||
};
|
||||
|
||||
// Get the execution time of all possible result pulls and streams.
|
||||
utils::Timer timer;
|
||||
|
||||
int i = 0;
|
||||
if (has_unsent_results_ && !output_symbols.empty()) {
|
||||
// stream unsent results from previous pull
|
||||
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
|
||||
stream_values(frame);
|
||||
frame.MakeInvalid();
|
||||
++i;
|
||||
if (i == n) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (; !n || i < n;) {
|
||||
if (!pull_result()) {
|
||||
break;
|
||||
}
|
||||
|
||||
if (!output_symbols.empty()) {
|
||||
for (auto &frame : multi_frame_.GetValidFramesConsumer()) {
|
||||
stream_values(frame);
|
||||
frame.MakeInvalid();
|
||||
++i;
|
||||
if (i == n) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
multi_frame_.MakeAllFramesInvalid();
|
||||
}
|
||||
}
|
||||
|
||||
// If we finished because we streamed the requested n results,
|
||||
// we try to pull the next result to see if there is more.
|
||||
// If there is additional result, we leave the pulled result in the frame
|
||||
// and set the flag to true.
|
||||
has_unsent_results_ = i == n && pull_result();
|
||||
|
||||
execution_time_ += timer.Elapsed();
|
||||
|
||||
if (has_unsent_results_) {
|
||||
return std::nullopt;
|
||||
}
|
||||
summary->insert_or_assign("plan_execution_time", execution_time_.count());
|
||||
// We are finished with pulling all the data, therefore we can send any
|
||||
// metadata about the results i.e. notifications and statistics
|
||||
const bool is_any_counter_set =
|
||||
std::any_of(ctx_.execution_stats.counters.begin(), ctx_.execution_stats.counters.end(),
|
||||
[](const auto &counter) { return counter > 0; });
|
||||
if (is_any_counter_set) {
|
||||
std::map<std::string, TypedValue> stats;
|
||||
for (size_t i = 0; i < ctx_.execution_stats.counters.size(); ++i) {
|
||||
stats.emplace(ExecutionStatsKeyToString(ExecutionStats::Key(i)), ctx_.execution_stats.counters[i]);
|
||||
}
|
||||
summary->insert_or_assign("stats", std::move(stats));
|
||||
}
|
||||
cursor_->Shutdown();
|
||||
ctx_.profile_execution_time = execution_time_;
|
||||
return GetStatsWithTotalTime(ctx_);
|
||||
// ctx_.trigger_context_collector = trigger_context_collector;
|
||||
}
|
||||
|
||||
std::optional<plan::ProfilingStatsWithTotalTime> PullPlan::Pull(AnyStream *stream, std::optional<int> n,
|
||||
const std::vector<Symbol> &output_symbols,
|
||||
std::map<std::string, TypedValue> *summary) {
|
||||
if (FLAGS_use_multi_frame) {
|
||||
return PullMultiple(stream, n, output_symbols, summary);
|
||||
}
|
||||
// Set up temporary memory for a single Pull. Initial memory comes from the
|
||||
// stack. 256 KiB should fit on the stack and should be more than enough for a
|
||||
// single `Pull`.
|
||||
@@ -910,26 +813,13 @@ using RWType = plan::ReadWriteTypeChecker::RWType;
|
||||
} // namespace
|
||||
|
||||
InterpreterContext::InterpreterContext(storage::v3::Shard *db, const InterpreterConfig config,
|
||||
const std::filesystem::path & /*data_directory*/,
|
||||
std::unique_ptr<RequestRouterFactory> request_router_factory,
|
||||
coordinator::Address coordinator_addr)
|
||||
: db(db),
|
||||
config(config),
|
||||
coordinator_address{coordinator_addr},
|
||||
request_router_factory_{std::move(request_router_factory)} {}
|
||||
const std::filesystem::path &data_directory)
|
||||
// : db(db), trigger_store(data_directory / "triggers"), config(config), streams{this, data_directory /
|
||||
// "streams"} {}
|
||||
: db(db), config(config) {}
|
||||
|
||||
Interpreter::Interpreter(InterpreterContext *interpreter_context) : interpreter_context_(interpreter_context) {
|
||||
MG_ASSERT(interpreter_context_, "Interpreter context must not be NULL");
|
||||
|
||||
request_router_ =
|
||||
interpreter_context_->request_router_factory_->CreateRequestRouter(interpreter_context_->coordinator_address);
|
||||
|
||||
// Get edge ids
|
||||
const auto edge_ids_alloc_min_max_pair =
|
||||
request_router_->AllocateInitialEdgeIds(interpreter_context_->coordinator_address);
|
||||
if (edge_ids_alloc_min_max_pair) {
|
||||
interpreter_context_->edge_ids_alloc = {edge_ids_alloc_min_max_pair->first, edge_ids_alloc_min_max_pair->second};
|
||||
}
|
||||
}
|
||||
|
||||
PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper) {
|
||||
@@ -942,6 +832,14 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
}
|
||||
in_explicit_transaction_ = true;
|
||||
expect_rollback_ = false;
|
||||
|
||||
db_accessor_ = std::make_unique<storage::v3::Shard::Accessor>(
|
||||
interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride()));
|
||||
execution_db_accessor_.emplace(db_accessor_.get());
|
||||
|
||||
// if (interpreter_context_->trigger_store.HasTriggers()) {
|
||||
// trigger_context_collector_.emplace(interpreter_context_->trigger_store.GetEventTypes());
|
||||
// }
|
||||
};
|
||||
} else if (query_upper == "COMMIT") {
|
||||
handler = [this] {
|
||||
@@ -988,17 +886,16 @@ PreparedQuery Interpreter::PrepareTransactionQuery(std::string_view query_upper)
|
||||
|
||||
PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications,
|
||||
RequestRouterInterface *request_router) {
|
||||
utils::MemoryResource *execution_memory, std::vector<Notification> *notifications) {
|
||||
// TriggerContextCollector *trigger_context_collector = nullptr) {
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_query.query);
|
||||
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parsed_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::v3::View::OLD);
|
||||
const auto memory_limit =
|
||||
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
if (memory_limit) {
|
||||
@@ -1013,9 +910,10 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
"convert the parsed row values to the appropriate type. This can be done using the built-in "
|
||||
"conversion functions such as ToInteger, ToFloat, ToBoolean etc.");
|
||||
}
|
||||
|
||||
auto plan = CypherQueryToPlan(parsed_query.stripped_query.hash(), std::move(parsed_query.ast_storage), cypher_query,
|
||||
parsed_query.parameters,
|
||||
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
|
||||
parsed_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
|
||||
|
||||
summary->insert_or_assign("cost_estimate", plan->cost());
|
||||
auto rw_type_checker = plan::ReadWriteTypeChecker();
|
||||
@@ -1034,7 +932,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
utils::FindOr(parsed_query.stripped_query.named_expressions(), symbol.token_position(), symbol.name()).first);
|
||||
}
|
||||
auto pull_plan = std::make_shared<PullPlan>(plan, parsed_query.parameters, false, dba, interpreter_context,
|
||||
execution_memory, request_router, memory_limit);
|
||||
execution_memory, memory_limit);
|
||||
// execution_memory, trigger_context_collector, memory_limit);
|
||||
return PreparedQuery{std::move(header), std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::move(pull_plan), output_symbols = std::move(output_symbols), summary](
|
||||
@@ -1048,7 +946,7 @@ PreparedQuery PrepareCypherQuery(ParsedQuery parsed_query, std::map<std::string,
|
||||
}
|
||||
|
||||
PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary,
|
||||
InterpreterContext *interpreter_context, RequestRouterInterface *request_router,
|
||||
InterpreterContext *interpreter_context, DbAccessor *dba,
|
||||
utils::MemoryResource *execution_memory) {
|
||||
const std::string kExplainQueryStart = "explain ";
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kExplainQueryStart),
|
||||
@@ -1067,20 +965,19 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in EXPLAIN");
|
||||
|
||||
auto cypher_query_plan =
|
||||
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
|
||||
cypher_query, parsed_inner_query.parameters,
|
||||
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
|
||||
auto cypher_query_plan = CypherQueryToPlan(
|
||||
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
|
||||
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
|
||||
|
||||
std::stringstream printed_plan;
|
||||
plan::PrettyPrint(*request_router, &cypher_query_plan->plan(), &printed_plan);
|
||||
plan::PrettyPrint(*dba, &cypher_query_plan->plan(), &printed_plan);
|
||||
|
||||
std::vector<std::vector<TypedValue>> printed_plan_rows;
|
||||
for (const auto &row : utils::Split(utils::RTrim(printed_plan.str()), "\n")) {
|
||||
printed_plan_rows.push_back(std::vector<TypedValue>{TypedValue(row)});
|
||||
}
|
||||
|
||||
summary->insert_or_assign("explain", plan::PlanToJson(*request_router, &cypher_query_plan->plan()).dump());
|
||||
summary->insert_or_assign("explain", plan::PlanToJson(*dba, &cypher_query_plan->plan()).dump());
|
||||
|
||||
return PreparedQuery{{"QUERY PLAN"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
@@ -1096,8 +993,7 @@ PreparedQuery PrepareExplainQuery(ParsedQuery parsed_query, std::map<std::string
|
||||
|
||||
PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router = nullptr) {
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory) {
|
||||
const std::string kProfileQueryStart = "profile ";
|
||||
|
||||
MG_ASSERT(utils::StartsWith(utils::ToLowerCase(parsed_query.stripped_query.query()), kProfileQueryStart),
|
||||
@@ -1135,26 +1031,25 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
|
||||
auto *cypher_query = utils::Downcast<CypherQuery>(parsed_inner_query.query);
|
||||
MG_ASSERT(cypher_query, "Cypher grammar should not allow other queries in PROFILE");
|
||||
expr::Frame frame(0);
|
||||
expr::Frame<TypedValue> frame(0);
|
||||
SymbolTable symbol_table;
|
||||
EvaluationContext evaluation_context;
|
||||
evaluation_context.timestamp = QueryTimestamp();
|
||||
evaluation_context.parameters = parsed_inner_query.parameters;
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, request_router, storage::v3::View::OLD);
|
||||
ExpressionEvaluator evaluator(&frame, symbol_table, evaluation_context, dba, storage::v3::View::OLD);
|
||||
const auto memory_limit =
|
||||
expr::EvaluateMemoryLimit(&evaluator, cypher_query->memory_limit_, cypher_query->memory_scale_);
|
||||
|
||||
auto cypher_query_plan =
|
||||
CypherQueryToPlan(parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage),
|
||||
cypher_query, parsed_inner_query.parameters,
|
||||
parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, request_router);
|
||||
auto cypher_query_plan = CypherQueryToPlan(
|
||||
parsed_inner_query.stripped_query.hash(), std::move(parsed_inner_query.ast_storage), cypher_query,
|
||||
parsed_inner_query.parameters, parsed_inner_query.is_cacheable ? &interpreter_context->plan_cache : nullptr, dba);
|
||||
auto rw_type_checker = plan::ReadWriteTypeChecker();
|
||||
rw_type_checker.InferRWType(const_cast<plan::LogicalOperator &>(cypher_query_plan->plan()));
|
||||
|
||||
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME", "CUSTOM DATA"},
|
||||
return PreparedQuery{{"OPERATOR", "ACTUAL HITS", "RELATIVE TIME", "ABSOLUTE TIME"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[plan = std::move(cypher_query_plan), parameters = std::move(parsed_inner_query.parameters),
|
||||
summary, dba, interpreter_context, execution_memory, memory_limit, request_router,
|
||||
summary, dba, interpreter_context, execution_memory, memory_limit,
|
||||
// We want to execute the query we are profiling lazily, so we delay
|
||||
// the construction of the corresponding context.
|
||||
stats_and_total_time = std::optional<plan::ProfilingStatsWithTotalTime>{},
|
||||
@@ -1163,7 +1058,7 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
// No output symbols are given so that nothing is streamed.
|
||||
if (!stats_and_total_time) {
|
||||
stats_and_total_time = PullPlan(plan, parameters, true, dba, interpreter_context,
|
||||
execution_memory, request_router, memory_limit)
|
||||
execution_memory, memory_limit)
|
||||
.Pull(stream, {}, {}, summary);
|
||||
pull_plan = std::make_shared<PullPlanVector>(ProfilingStatsToTable(*stats_and_total_time));
|
||||
}
|
||||
@@ -1182,7 +1077,16 @@ PreparedQuery PrepareProfileQuery(ParsedQuery parsed_query, bool in_explicit_tra
|
||||
|
||||
PreparedQuery PrepareDumpQuery(ParsedQuery parsed_query, std::map<std::string, TypedValue> *summary, DbAccessor *dba,
|
||||
utils::MemoryResource *execution_memory) {
|
||||
throw QueryRuntimeException("Dump query is not supported!");
|
||||
return PreparedQuery{{"QUERY"},
|
||||
std::move(parsed_query.required_privileges),
|
||||
[pull_plan = std::make_shared<PullPlanDump>(dba)](
|
||||
AnyStream *stream, std::optional<int> n) -> std::optional<QueryHandlerResult> {
|
||||
if (pull_plan->Pull(stream, n)) {
|
||||
return QueryHandlerResult::COMMIT;
|
||||
}
|
||||
return std::nullopt;
|
||||
},
|
||||
RWType::R};
|
||||
}
|
||||
|
||||
PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
@@ -1289,15 +1193,14 @@ PreparedQuery PrepareIndexQuery(ParsedQuery parsed_query, bool in_explicit_trans
|
||||
|
||||
PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transaction,
|
||||
std::map<std::string, TypedValue> *summary, InterpreterContext *interpreter_context,
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory,
|
||||
RequestRouterInterface *request_router) {
|
||||
DbAccessor *dba, utils::MemoryResource *execution_memory) {
|
||||
if (in_explicit_transaction) {
|
||||
throw UserModificationInMulticommandTxException();
|
||||
}
|
||||
|
||||
auto *auth_query = utils::Downcast<AuthQuery>(parsed_query.query);
|
||||
|
||||
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, request_router);
|
||||
auto callback = HandleAuthQuery(auth_query, interpreter_context->auth, parsed_query.parameters, dba);
|
||||
|
||||
SymbolTable symbol_table;
|
||||
std::vector<Symbol> output_symbols;
|
||||
@@ -1326,14 +1229,14 @@ PreparedQuery PrepareAuthQuery(ParsedQuery parsed_query, bool in_explicit_transa
|
||||
|
||||
PreparedQuery PrepareReplicationQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
std::vector<Notification> *notifications, InterpreterContext *interpreter_context,
|
||||
RequestRouterInterface *request_router) {
|
||||
DbAccessor *dba) {
|
||||
if (in_explicit_transaction) {
|
||||
throw ReplicationModificationInMulticommandTxException();
|
||||
}
|
||||
|
||||
auto *replication_query = utils::Downcast<ReplicationQuery>(parsed_query.query);
|
||||
auto callback = HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context,
|
||||
request_router, notifications);
|
||||
auto callback =
|
||||
HandleReplicationQuery(replication_query, parsed_query.parameters, interpreter_context, dba, notifications);
|
||||
|
||||
return PreparedQuery{callback.header, std::move(parsed_query.required_privileges),
|
||||
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
|
||||
@@ -1421,15 +1324,14 @@ PreparedQuery PrepareCreateSnapshotQuery(ParsedQuery parsed_query, bool in_expli
|
||||
throw SemanticException("CreateSnapshot query is not supported!");
|
||||
}
|
||||
|
||||
PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction,
|
||||
RequestRouterInterface *request_router) {
|
||||
PreparedQuery PrepareSettingQuery(ParsedQuery parsed_query, const bool in_explicit_transaction, DbAccessor *dba) {
|
||||
if (in_explicit_transaction) {
|
||||
throw SettingConfigInMulticommandTxException{};
|
||||
}
|
||||
|
||||
auto *setting_query = utils::Downcast<SettingQuery>(parsed_query.query);
|
||||
MG_ASSERT(setting_query);
|
||||
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, request_router);
|
||||
auto callback = HandleSettingQuery(setting_query, parsed_query.parameters, dba);
|
||||
|
||||
return PreparedQuery{std::move(callback.header), std::move(parsed_query.required_privileges),
|
||||
[callback_fn = std::move(callback.fn), pull_plan = std::shared_ptr<PullPlanVector>{nullptr}](
|
||||
@@ -1623,10 +1525,15 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
ParsedQuery parsed_query =
|
||||
ParseQuery(query_string, params, &interpreter_context_->ast_cache, interpreter_context_->config.query);
|
||||
query_execution->summary["parsing_time"] = parsing_timer.Elapsed().count();
|
||||
|
||||
// Some queries require an active transaction in order to be prepared.
|
||||
if (!in_explicit_transaction_ &&
|
||||
(utils::Downcast<CypherQuery>(parsed_query.query) || utils::Downcast<ExplainQuery>(parsed_query.query) ||
|
||||
utils::Downcast<ProfileQuery>(parsed_query.query))) {
|
||||
request_router_->StartTransaction();
|
||||
utils::Downcast<ProfileQuery>(parsed_query.query) || utils::Downcast<DumpQuery>(parsed_query.query) ||
|
||||
utils::Downcast<TriggerQuery>(parsed_query.query))) {
|
||||
db_accessor_ = std::make_unique<storage::v3::Shard::Accessor>(
|
||||
interpreter_context_->db->Access(coordinator::Hlc{}, GetIsolationLevelOverride()));
|
||||
execution_db_accessor_.emplace(db_accessor_.get());
|
||||
}
|
||||
|
||||
utils::Timer planning_timer;
|
||||
@@ -1635,14 +1542,14 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
if (utils::Downcast<CypherQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareCypherQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*execution_db_accessor_, &query_execution->execution_memory,
|
||||
&query_execution->notifications, request_router_.get());
|
||||
&query_execution->notifications);
|
||||
} else if (utils::Downcast<ExplainQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareExplainQuery(std::move(parsed_query), &query_execution->summary, interpreter_context_,
|
||||
&*request_router_, &query_execution->execution_memory_with_exception);
|
||||
&*execution_db_accessor_, &query_execution->execution_memory_with_exception);
|
||||
} else if (utils::Downcast<ProfileQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareProfileQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
|
||||
interpreter_context_, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory_with_exception, request_router_.get());
|
||||
&query_execution->execution_memory_with_exception);
|
||||
} else if (utils::Downcast<DumpQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareDumpQuery(std::move(parsed_query), &query_execution->summary, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory);
|
||||
@@ -1652,7 +1559,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
} else if (utils::Downcast<AuthQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareAuthQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
|
||||
interpreter_context_, &*execution_db_accessor_,
|
||||
&query_execution->execution_memory_with_exception, request_router_.get());
|
||||
&query_execution->execution_memory_with_exception);
|
||||
} else if (utils::Downcast<InfoQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareInfoQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->summary,
|
||||
interpreter_context_, interpreter_context_->db,
|
||||
@@ -1663,7 +1570,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
} else if (utils::Downcast<ReplicationQuery>(parsed_query.query)) {
|
||||
prepared_query =
|
||||
PrepareReplicationQuery(std::move(parsed_query), in_explicit_transaction_, &query_execution->notifications,
|
||||
interpreter_context_, request_router_.get());
|
||||
interpreter_context_, &*execution_db_accessor_);
|
||||
} else if (utils::Downcast<LockPathQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareLockPathQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_,
|
||||
&*execution_db_accessor_);
|
||||
@@ -1680,7 +1587,7 @@ Interpreter::PrepareResult Interpreter::Prepare(const std::string &query_string,
|
||||
prepared_query =
|
||||
PrepareCreateSnapshotQuery(std::move(parsed_query), in_explicit_transaction_, interpreter_context_);
|
||||
} else if (utils::Downcast<SettingQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, request_router_.get());
|
||||
prepared_query = PrepareSettingQuery(std::move(parsed_query), in_explicit_transaction_, &*execution_db_accessor_);
|
||||
} else if (utils::Downcast<VersionQuery>(parsed_query.query)) {
|
||||
prepared_query = PrepareVersionQuery(std::move(parsed_query), in_explicit_transaction_);
|
||||
} else if (utils::Downcast<SchemaQuery>(parsed_query.query)) {
|
||||
@@ -1721,7 +1628,6 @@ void Interpreter::Commit() {
|
||||
// For now, we will not check if there are some unfinished queries.
|
||||
// We should document clearly that all results should be pulled to complete
|
||||
// a query.
|
||||
request_router_->Commit();
|
||||
if (!db_accessor_) return;
|
||||
|
||||
const auto reset_necessary_members = [this]() {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -12,11 +12,7 @@
|
||||
#pragma once
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
#include <cstdint>
|
||||
|
||||
#include "coordinator/coordinator.hpp"
|
||||
#include "coordinator/coordinator_client.hpp"
|
||||
#include "io/transport.hpp"
|
||||
#include "query/v2/auth_checker.hpp"
|
||||
#include "query/v2/bindings/cypher_main_visitor.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
@@ -30,7 +26,6 @@
|
||||
#include "query/v2/metadata.hpp"
|
||||
#include "query/v2/plan/operator.hpp"
|
||||
#include "query/v2/plan/read_write_type_checker.hpp"
|
||||
#include "query/v2/requests.hpp"
|
||||
#include "query/v2/stream.hpp"
|
||||
#include "storage/v3/isolation_level.hpp"
|
||||
#include "storage/v3/name_id_mapper.hpp"
|
||||
@@ -170,9 +165,7 @@ struct PreparedQuery {
|
||||
*/
|
||||
struct InterpreterContext {
|
||||
explicit InterpreterContext(storage::v3::Shard *db, InterpreterConfig config,
|
||||
const std::filesystem::path &data_directory,
|
||||
std::unique_ptr<RequestRouterFactory> request_router_factory,
|
||||
coordinator::Address coordinator_addr);
|
||||
const std::filesystem::path &data_directory);
|
||||
|
||||
storage::v3::Shard *db;
|
||||
|
||||
@@ -186,26 +179,22 @@ struct InterpreterContext {
|
||||
utils::SkipList<PlanCacheEntry> plan_cache;
|
||||
|
||||
const InterpreterConfig config;
|
||||
IdAllocator edge_ids_alloc;
|
||||
|
||||
coordinator::Address coordinator_address;
|
||||
std::unique_ptr<RequestRouterFactory> request_router_factory_;
|
||||
|
||||
storage::v3::LabelId NameToLabelId(std::string_view label_name) {
|
||||
return storage::v3::LabelId::FromUint(query_id_mapper_.NameToId(label_name));
|
||||
return storage::v3::LabelId::FromUint(query_id_mapper.NameToId(label_name));
|
||||
}
|
||||
|
||||
storage::v3::PropertyId NameToPropertyId(std::string_view property_name) {
|
||||
return storage::v3::PropertyId::FromUint(query_id_mapper_.NameToId(property_name));
|
||||
return storage::v3::PropertyId::FromUint(query_id_mapper.NameToId(property_name));
|
||||
}
|
||||
|
||||
storage::v3::EdgeTypeId NameToEdgeTypeId(std::string_view edge_type_name) {
|
||||
return storage::v3::EdgeTypeId::FromUint(query_id_mapper_.NameToId(edge_type_name));
|
||||
return storage::v3::EdgeTypeId::FromUint(query_id_mapper.NameToId(edge_type_name));
|
||||
}
|
||||
|
||||
private:
|
||||
// TODO Replace with local map of labels, properties and edge type ids
|
||||
storage::v3::NameIdMapper query_id_mapper_;
|
||||
storage::v3::NameIdMapper query_id_mapper;
|
||||
};
|
||||
|
||||
/// Function that is used to tell all active interpreters that they should stop
|
||||
@@ -294,16 +283,11 @@ class Interpreter final {
|
||||
*/
|
||||
void Abort();
|
||||
|
||||
const RequestRouterInterface *GetRequestRouter() const { return request_router_.get(); }
|
||||
void InstallSimulatorTicker(std::function<bool()> &&tick_simulator) {
|
||||
request_router_->InstallSimulatorTicker(tick_simulator);
|
||||
}
|
||||
|
||||
private:
|
||||
struct QueryExecution {
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
utils::MonotonicBufferResource execution_memory{kExecutionMemoryBlockSize};
|
||||
utils::ResourceWithOutOfMemoryException execution_memory_with_exception{&execution_memory};
|
||||
std::optional<PreparedQuery> prepared_query;
|
||||
|
||||
std::map<std::string, TypedValue> summary;
|
||||
std::vector<Notification> notifications;
|
||||
@@ -340,10 +324,9 @@ class Interpreter final {
|
||||
|
||||
// This cannot be std::optional because we need to move this accessor later on into a lambda capture
|
||||
// which is assigned to std::function. std::function requires every object to be copyable, so we
|
||||
// move this unique_ptr into a shared_ptr.
|
||||
// move this unique_ptr into a shrared_ptr.
|
||||
std::unique_ptr<storage::v3::Shard::Accessor> db_accessor_;
|
||||
std::optional<DbAccessor> execution_db_accessor_;
|
||||
std::unique_ptr<RequestRouterInterface> request_router_;
|
||||
bool in_explicit_transaction_{false};
|
||||
bool expect_rollback_{false};
|
||||
|
||||
|
||||
@@ -1,153 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#include "query/v2/multiframe.hpp"
|
||||
|
||||
#include <algorithm>
|
||||
#include <iterator>
|
||||
|
||||
#include "query/v2/bindings/frame.hpp"
|
||||
#include "utils/pmr/vector.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DEFINE_uint64(default_multi_frame_size, 100, "Default size of MultiFrame");
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
static_assert(std::forward_iterator<ValidFramesReader::Iterator>);
|
||||
static_assert(std::forward_iterator<ValidFramesModifier::Iterator>);
|
||||
static_assert(std::forward_iterator<ValidFramesConsumer::Iterator>);
|
||||
static_assert(std::forward_iterator<InvalidFramesPopulator::Iterator>);
|
||||
|
||||
MultiFrame::MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory)
|
||||
: frames_(utils::pmr::vector<FrameWithValidity>(
|
||||
number_of_frames, FrameWithValidity(size_of_frame, execution_memory), execution_memory)) {
|
||||
MG_ASSERT(number_of_frames > 0);
|
||||
}
|
||||
|
||||
MultiFrame::MultiFrame(const MultiFrame &other) : frames_{other.frames_} {}
|
||||
|
||||
// NOLINTNEXTLINE (bugprone-exception-escape)
|
||||
MultiFrame::MultiFrame(MultiFrame &&other) noexcept : frames_(std::move(other.frames_)) {}
|
||||
|
||||
FrameWithValidity &MultiFrame::GetFirstFrame() {
|
||||
MG_ASSERT(!frames_.empty());
|
||||
return frames_.front();
|
||||
}
|
||||
|
||||
void MultiFrame::MakeAllFramesInvalid() noexcept {
|
||||
std::for_each(frames_.begin(), frames_.end(), [](auto &frame) { frame.MakeInvalid(); });
|
||||
}
|
||||
|
||||
bool MultiFrame::HasValidFrame() const noexcept {
|
||||
return std::any_of(frames_.begin(), frames_.end(), [](const auto &frame) { return frame.IsValid(); });
|
||||
}
|
||||
|
||||
bool MultiFrame::HasInvalidFrame() const noexcept {
|
||||
return std::any_of(frames_.rbegin(), frames_.rend(), [](const auto &frame) { return !frame.IsValid(); });
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE (bugprone-exception-escape)
|
||||
void MultiFrame::DefragmentValidFrames() noexcept {
|
||||
static constexpr auto kIsValid = [](const FrameWithValidity &frame) { return frame.IsValid(); };
|
||||
static constexpr auto kIsInvalid = [](const FrameWithValidity &frame) { return !frame.IsValid(); };
|
||||
auto first_invalid_frame = std::find_if(frames_.begin(), frames_.end(), kIsInvalid);
|
||||
auto following_first_valid = std::find_if(first_invalid_frame, frames_.end(), kIsValid);
|
||||
while (first_invalid_frame != frames_.end() && following_first_valid != frames_.end()) {
|
||||
std::swap(*first_invalid_frame, *following_first_valid);
|
||||
first_invalid_frame++;
|
||||
first_invalid_frame = std::find_if(first_invalid_frame, frames_.end(), kIsInvalid);
|
||||
following_first_valid++;
|
||||
following_first_valid = std::find_if(following_first_valid, frames_.end(), kIsValid);
|
||||
}
|
||||
}
|
||||
|
||||
ValidFramesReader MultiFrame::GetValidFramesReader() { return ValidFramesReader{*this}; }
|
||||
|
||||
ValidFramesModifier MultiFrame::GetValidFramesModifier() { return ValidFramesModifier{*this}; }
|
||||
|
||||
ValidFramesConsumer MultiFrame::GetValidFramesConsumer() { return ValidFramesConsumer{*this}; }
|
||||
|
||||
InvalidFramesPopulator MultiFrame::GetInvalidFramesPopulator() { return InvalidFramesPopulator{*this}; }
|
||||
|
||||
ValidFramesReader::ValidFramesReader(MultiFrame &multiframe) : multiframe_(&multiframe) {
|
||||
/*
|
||||
From: https://en.cppreference.com/w/cpp/algorithm/find
|
||||
Returns an iterator to the first element in the range [first, last) that satisfies specific criteria:
|
||||
find_if searches for an element for which predicate p returns true
|
||||
Return value
|
||||
Iterator to the first element satisfying the condition or last if no such element is found.
|
||||
|
||||
-> this is what we want. We want the "after" last valid frame (weather this is vector::end or and invalid frame).
|
||||
*/
|
||||
auto it = std::find_if(multiframe.frames_.begin(), multiframe.frames_.end(),
|
||||
[](const auto &frame) { return !frame.IsValid(); });
|
||||
after_last_valid_frame_ = multiframe_->frames_.data() + std::distance(multiframe.frames_.begin(), it);
|
||||
}
|
||||
|
||||
ValidFramesReader::Iterator ValidFramesReader::begin() {
|
||||
if (multiframe_->frames_[0].IsValid()) {
|
||||
return Iterator{&multiframe_->frames_[0]};
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
ValidFramesReader::Iterator ValidFramesReader::end() { return Iterator{after_last_valid_frame_}; }
|
||||
|
||||
ValidFramesModifier::ValidFramesModifier(MultiFrame &multiframe) : multiframe_(&multiframe) {}
|
||||
|
||||
ValidFramesModifier::Iterator ValidFramesModifier::begin() {
|
||||
if (multiframe_->frames_[0].IsValid()) {
|
||||
return Iterator{&multiframe_->frames_[0], *this};
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
ValidFramesModifier::Iterator ValidFramesModifier::end() {
|
||||
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
|
||||
}
|
||||
|
||||
ValidFramesConsumer::ValidFramesConsumer(MultiFrame &multiframe) : multiframe_(&multiframe) {}
|
||||
|
||||
// NOLINTNEXTLINE (bugprone-exception-escape)
|
||||
ValidFramesConsumer::~ValidFramesConsumer() noexcept {
|
||||
// TODO Possible optimisation: only DefragmentValidFrames if one frame has been invalidated? Only if does not
|
||||
// cost too much to store it
|
||||
multiframe_->DefragmentValidFrames();
|
||||
}
|
||||
|
||||
ValidFramesConsumer::Iterator ValidFramesConsumer::begin() {
|
||||
if (multiframe_->frames_[0].IsValid()) {
|
||||
return Iterator{&multiframe_->frames_[0], *this};
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
ValidFramesConsumer::Iterator ValidFramesConsumer::end() {
|
||||
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size(), *this};
|
||||
}
|
||||
|
||||
InvalidFramesPopulator::InvalidFramesPopulator(MultiFrame &multiframe) : multiframe_(&multiframe) {}
|
||||
|
||||
InvalidFramesPopulator::Iterator InvalidFramesPopulator::begin() {
|
||||
for (auto &frame : multiframe_->frames_) {
|
||||
if (!frame.IsValid()) {
|
||||
return Iterator{&frame};
|
||||
}
|
||||
}
|
||||
return end();
|
||||
}
|
||||
|
||||
InvalidFramesPopulator::Iterator InvalidFramesPopulator::end() {
|
||||
return Iterator{multiframe_->frames_.data() + multiframe_->frames_.size()};
|
||||
}
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
@@ -1,308 +0,0 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
// License, and you may not use this file except in compliance with the Business Source License.
|
||||
//
|
||||
// As of the Change Date specified in that file, in accordance with
|
||||
// the Business Source License, use of this software will be governed
|
||||
// by the Apache License, Version 2.0, included in the file
|
||||
// licenses/APL.txt.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <iterator>
|
||||
|
||||
#include <gflags/gflags.h>
|
||||
|
||||
#include "query/v2/bindings/frame.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-avoid-non-const-global-variables)
|
||||
DECLARE_uint64(default_multi_frame_size);
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
|
||||
class ValidFramesConsumer;
|
||||
class ValidFramesModifier;
|
||||
class ValidFramesReader;
|
||||
class InvalidFramesPopulator;
|
||||
|
||||
class MultiFrame {
|
||||
public:
|
||||
friend class ValidFramesConsumer;
|
||||
friend class ValidFramesModifier;
|
||||
friend class ValidFramesReader;
|
||||
friend class InvalidFramesPopulator;
|
||||
|
||||
MultiFrame(size_t size_of_frame, size_t number_of_frames, utils::MemoryResource *execution_memory);
|
||||
~MultiFrame() = default;
|
||||
|
||||
// Assigning and moving the MultiFrame is not allowed if any accessor from the above ones are alive.
|
||||
MultiFrame(const MultiFrame &other);
|
||||
MultiFrame(MultiFrame &&other) noexcept;
|
||||
MultiFrame &operator=(const MultiFrame &other) = delete;
|
||||
MultiFrame &operator=(MultiFrame &&other) noexcept = delete;
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
|
||||
* state in the MultiFrame.
|
||||
* Iteration goes in a deterministic order.
|
||||
* One can't modify the validity of the Frame nor its content with this implementation.
|
||||
*/
|
||||
ValidFramesReader GetValidFramesReader();
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
|
||||
* state in the MultiFrame.
|
||||
* Iteration goes in a deterministic order.
|
||||
* One can't modify the validity of the Frame with this implementation. One can modify its content.
|
||||
*/
|
||||
ValidFramesModifier GetValidFramesModifier();
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in a valid
|
||||
* state in the MultiFrame.
|
||||
* Iteration goes in a deterministic order.
|
||||
* One can modify the validity of the Frame with this implementation.
|
||||
* If you do not plan to modify the validity of the Frames, use GetValidFramesReader/GetValidFramesModifer instead as
|
||||
* this is faster.
|
||||
*/
|
||||
ValidFramesConsumer GetValidFramesConsumer();
|
||||
|
||||
/*
|
||||
* Returns a object on which one can iterate in a for-loop. By doing so, you will only get Frames that are in an
|
||||
* invalid state in the MultiFrame. Iteration goes in a deterministic order. One can modify the validity of
|
||||
* the Frame with this implementation.
|
||||
*/
|
||||
InvalidFramesPopulator GetInvalidFramesPopulator();
|
||||
|
||||
/**
|
||||
* Return the first Frame of the MultiFrame. This is only meant to be used in very specific cases. Please consider
|
||||
* using the iterators instead.
|
||||
* The Frame can be valid or invalid.
|
||||
*/
|
||||
FrameWithValidity &GetFirstFrame();
|
||||
|
||||
void MakeAllFramesInvalid() noexcept;
|
||||
|
||||
bool HasValidFrame() const noexcept;
|
||||
bool HasInvalidFrame() const noexcept;
|
||||
|
||||
inline utils::MemoryResource *GetMemoryResource() { return frames_[0].GetMemoryResource(); }
|
||||
|
||||
private:
|
||||
void DefragmentValidFrames() noexcept;
|
||||
|
||||
utils::pmr::vector<FrameWithValidity> frames_;
|
||||
};
|
||||
|
||||
class ValidFramesReader {
|
||||
public:
|
||||
explicit ValidFramesReader(MultiFrame &multiframe);
|
||||
|
||||
~ValidFramesReader() = default;
|
||||
ValidFramesReader(const ValidFramesReader &other) = delete;
|
||||
ValidFramesReader(ValidFramesReader &&other) noexcept = default;
|
||||
ValidFramesReader &operator=(const ValidFramesReader &other) = delete;
|
||||
ValidFramesReader &operator=(ValidFramesReader &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = const Frame;
|
||||
using pointer = value_type *;
|
||||
using reference = const Frame &;
|
||||
|
||||
Iterator() = default;
|
||||
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
Iterator &operator++() {
|
||||
ptr_++;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
ptr_++;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
FrameWithValidity *after_last_valid_frame_;
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
class ValidFramesModifier {
|
||||
public:
|
||||
explicit ValidFramesModifier(MultiFrame &multiframe);
|
||||
|
||||
~ValidFramesModifier() = default;
|
||||
ValidFramesModifier(const ValidFramesModifier &other) = delete;
|
||||
ValidFramesModifier(ValidFramesModifier &&other) noexcept = default;
|
||||
ValidFramesModifier &operator=(const ValidFramesModifier &other) = delete;
|
||||
ValidFramesModifier &operator=(ValidFramesModifier &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = Frame;
|
||||
using pointer = value_type *;
|
||||
using reference = Frame &;
|
||||
|
||||
Iterator() = default;
|
||||
Iterator(FrameWithValidity *ptr, ValidFramesModifier &iterator_wrapper)
|
||||
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
// Prefix increment
|
||||
Iterator &operator++() {
|
||||
do {
|
||||
ptr_++;
|
||||
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
++*this;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
ValidFramesModifier *iterator_wrapper_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
class ValidFramesConsumer {
|
||||
public:
|
||||
explicit ValidFramesConsumer(MultiFrame &multiframe);
|
||||
|
||||
~ValidFramesConsumer() noexcept;
|
||||
ValidFramesConsumer(const ValidFramesConsumer &other) = delete;
|
||||
ValidFramesConsumer(ValidFramesConsumer &&other) noexcept = default;
|
||||
ValidFramesConsumer &operator=(const ValidFramesConsumer &other) = delete;
|
||||
ValidFramesConsumer &operator=(ValidFramesConsumer &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = FrameWithValidity;
|
||||
using pointer = value_type *;
|
||||
using reference = FrameWithValidity &;
|
||||
|
||||
Iterator() = default;
|
||||
Iterator(FrameWithValidity *ptr, ValidFramesConsumer &iterator_wrapper)
|
||||
: ptr_(ptr), iterator_wrapper_(&iterator_wrapper) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
Iterator &operator++() {
|
||||
do {
|
||||
ptr_++;
|
||||
} while (*this != iterator_wrapper_->end() && !ptr_->IsValid());
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
++*this;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
ValidFramesConsumer *iterator_wrapper_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
class InvalidFramesPopulator {
|
||||
public:
|
||||
explicit InvalidFramesPopulator(MultiFrame &multiframe);
|
||||
~InvalidFramesPopulator() = default;
|
||||
|
||||
InvalidFramesPopulator(const InvalidFramesPopulator &other) = delete;
|
||||
InvalidFramesPopulator(InvalidFramesPopulator &&other) noexcept = default;
|
||||
InvalidFramesPopulator &operator=(const InvalidFramesPopulator &other) = delete;
|
||||
InvalidFramesPopulator &operator=(InvalidFramesPopulator &&other) noexcept = default;
|
||||
|
||||
struct Iterator {
|
||||
using iterator_category = std::forward_iterator_tag;
|
||||
using difference_type = std::ptrdiff_t;
|
||||
using value_type = FrameWithValidity;
|
||||
using pointer = value_type *;
|
||||
using reference = FrameWithValidity &;
|
||||
|
||||
Iterator() = default;
|
||||
explicit Iterator(FrameWithValidity *ptr) : ptr_(ptr) {}
|
||||
|
||||
reference operator*() const { return *ptr_; }
|
||||
pointer operator->() { return ptr_; }
|
||||
|
||||
Iterator &operator++() {
|
||||
ptr_->MakeValid();
|
||||
ptr_++;
|
||||
return *this;
|
||||
}
|
||||
|
||||
// NOLINTNEXTLINE(cert-dcl21-cpp)
|
||||
Iterator operator++(int) {
|
||||
auto old = *this;
|
||||
++ptr_;
|
||||
return old;
|
||||
}
|
||||
|
||||
friend bool operator==(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ == rhs.ptr_; };
|
||||
friend bool operator!=(const Iterator &lhs, const Iterator &rhs) { return lhs.ptr_ != rhs.ptr_; };
|
||||
|
||||
private:
|
||||
FrameWithValidity *ptr_{nullptr};
|
||||
};
|
||||
|
||||
Iterator begin();
|
||||
Iterator end();
|
||||
|
||||
private:
|
||||
MultiFrame *multiframe_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
@@ -12,8 +12,8 @@
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include "storage/v3/property_value.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
@@ -32,7 +32,7 @@ struct Parameters {
|
||||
* @param position Token position in query of value.
|
||||
* @param value
|
||||
*/
|
||||
void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace(position, value); }
|
||||
void Add(int position, const storage::v3::PropertyValue &value) { storage_.emplace_back(position, value); }
|
||||
|
||||
/**
|
||||
* Returns the value found for the given token position.
|
||||
@@ -41,11 +41,23 @@ struct Parameters {
|
||||
* @return Value for the given token position.
|
||||
*/
|
||||
const storage::v3::PropertyValue &AtTokenPosition(int position) const {
|
||||
auto found = storage_.find(position);
|
||||
auto found = std::find_if(storage_.begin(), storage_.end(), [&](const auto &a) { return a.first == position; });
|
||||
MG_ASSERT(found != storage_.end(), "Token position must be present in container");
|
||||
return found->second;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the position-th stripped value. Asserts that this
|
||||
* container has at least (position + 1) elements.
|
||||
*
|
||||
* @param position Which stripped param is sought.
|
||||
* @return Token position and value for sought param.
|
||||
*/
|
||||
const std::pair<int, storage::v3::PropertyValue> &At(int position) const {
|
||||
MG_ASSERT(position < static_cast<int>(storage_.size()), "Invalid position");
|
||||
return storage_[position];
|
||||
}
|
||||
|
||||
/** Returns the number of arguments in this container */
|
||||
auto size() const { return storage_.size(); }
|
||||
|
||||
@@ -53,7 +65,7 @@ struct Parameters {
|
||||
auto end() const { return storage_.end(); }
|
||||
|
||||
private:
|
||||
std::unordered_map<int, storage::v3::PropertyValue> storage_;
|
||||
std::vector<std::pair<int, storage::v3::PropertyValue>> storage_;
|
||||
};
|
||||
|
||||
} // namespace memgraph::query::v2
|
||||
|
||||
@@ -14,8 +14,6 @@
|
||||
#include <functional>
|
||||
#include <utility>
|
||||
|
||||
#include "query/db_accessor.hpp"
|
||||
#include "query/v2/accessors.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "utils/logging.hpp"
|
||||
#include "utils/memory.hpp"
|
||||
@@ -30,8 +28,6 @@ namespace memgraph::query::v2 {
|
||||
*/
|
||||
class Path {
|
||||
public:
|
||||
using VertexAccessor = accessors::VertexAccessor;
|
||||
using EdgeAccessor = accessors::EdgeAccessor;
|
||||
/** Allocator type so that STL containers are aware that we need one */
|
||||
using allocator_type = utils::Allocator<char>;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -149,6 +149,8 @@ class CostEstimator : public HierarchicalLogicalOperatorVisitor {
|
||||
return true;
|
||||
}
|
||||
|
||||
// TODO: Cost estimate ScanAllById?
|
||||
|
||||
// For the given op first increments the cardinality and then cost.
|
||||
#define POST_VISIT_CARD_FIRST(NAME) \
|
||||
bool PostVisit(NAME &) override { \
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -22,13 +22,12 @@
|
||||
#include <variant>
|
||||
#include <vector>
|
||||
|
||||
#include "expr/semantic/symbol.hpp"
|
||||
#include "query/v2/common.hpp"
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "expr/semantic/symbol.hpp"
|
||||
#include "query/v2/bindings/typed_value.hpp"
|
||||
#include "query/v2/bindings/frame.hpp"
|
||||
#include "query/v2/bindings/symbol_table.hpp"
|
||||
#include "query/v2/multiframe.hpp"
|
||||
#include "storage/v3/id_types.hpp"
|
||||
#include "utils/bound.hpp"
|
||||
#include "utils/fnv.hpp"
|
||||
@@ -72,24 +71,6 @@ class Cursor {
|
||||
/// @throws QueryRuntimeException if something went wrong with execution
|
||||
virtual bool Pull(Frame &, ExecutionContext &) = 0;
|
||||
|
||||
/// Run an iteration of a @c LogicalOperator with MultiFrame.
|
||||
///
|
||||
/// Since operators may be chained, the iteration may pull results from
|
||||
/// multiple operators.
|
||||
///
|
||||
/// @param MultiFrame May be read from or written to while performing the
|
||||
/// iteration.
|
||||
/// @param ExecutionContext Used to get the position of symbols in frame and
|
||||
/// other information.
|
||||
/// @return True if the operator was able to populate at least one Frame on the MultiFrame,
|
||||
/// thus if an operator returns true, that means there is at least one valid Frame in the
|
||||
/// MultiFrame.
|
||||
///
|
||||
/// @throws QueryRuntimeException if something went wrong with execution
|
||||
virtual bool PullMultiple(MultiFrame &, ExecutionContext &) {MG_ASSERT(false, "PullMultipleIsNotImplemented"); return false; }
|
||||
|
||||
virtual void PushDown(const MultiFrame&) { MG_ASSERT(false, "PushDownIsNotImplemented"); }
|
||||
|
||||
/// Resets the Cursor to its initial state.
|
||||
virtual void Reset() = 0;
|
||||
|
||||
@@ -129,7 +110,7 @@ class ScanAllByLabel;
|
||||
class ScanAllByLabelPropertyRange;
|
||||
class ScanAllByLabelPropertyValue;
|
||||
class ScanAllByLabelProperty;
|
||||
class ScanByPrimaryKey;
|
||||
class ScanAllById;
|
||||
class Expand;
|
||||
class ExpandVariable;
|
||||
class ConstructNamedPath;
|
||||
@@ -160,7 +141,7 @@ class Foreach;
|
||||
using LogicalOperatorCompositeVisitor = utils::CompositeVisitor<
|
||||
Once, CreateNode, CreateExpand, ScanAll, ScanAllByLabel,
|
||||
ScanAllByLabelPropertyRange, ScanAllByLabelPropertyValue,
|
||||
ScanAllByLabelProperty, ScanByPrimaryKey,
|
||||
ScanAllByLabelProperty, ScanAllById,
|
||||
Expand, ExpandVariable, ConstructNamedPath, Filter, Produce, Delete,
|
||||
SetProperty, SetProperties, SetLabels, RemoveProperty, RemoveLabels,
|
||||
EdgeUniquenessFilter, Accumulate, Aggregate, Skip, Limit, OrderBy, Merge,
|
||||
@@ -351,14 +332,11 @@ and false on every following Pull.")
|
||||
class OnceCursor : public Cursor {
|
||||
public:
|
||||
OnceCursor() {}
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void PushDown(const MultiFrame&) override;
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
private:
|
||||
std::optional<MultiFrame> pushed_down_multi_frame_;
|
||||
bool did_pull_{false};
|
||||
};
|
||||
cpp<#)
|
||||
@@ -863,21 +841,19 @@ given label and property.
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
(lcp:define-class scan-by-primary-key (scan-all)
|
||||
((label "::storage::v3::LabelId" :scope :public)
|
||||
(primary-key "std::vector<Expression*>" :scope :public)
|
||||
(expression "Expression *" :scope :public
|
||||
|
||||
|
||||
(lcp:define-class scan-all-by-id (scan-all)
|
||||
((expression "Expression *" :scope :public
|
||||
:slk-save #'slk-save-ast-pointer
|
||||
:slk-load (slk-load-ast-pointer "Expression")))
|
||||
(:documentation
|
||||
"ScanAll producing a single node with specified by the label and primary key")
|
||||
"ScanAll producing a single node with ID equal to evaluated expression")
|
||||
(:public
|
||||
#>cpp
|
||||
ScanByPrimaryKey() {}
|
||||
ScanByPrimaryKey(const std::shared_ptr<LogicalOperator> &input,
|
||||
Symbol output_symbol,
|
||||
storage::v3::LabelId label,
|
||||
std::vector<query::v2::Expression*> primary_key,
|
||||
ScanAllById() {}
|
||||
ScanAllById(const std::shared_ptr<LogicalOperator> &input,
|
||||
Symbol output_symbol, Expression *expression,
|
||||
storage::v3::View view = storage::v3::View::OLD);
|
||||
|
||||
bool Accept(HierarchicalLogicalOperatorVisitor &visitor) override;
|
||||
@@ -1180,7 +1156,6 @@ a boolean value.")
|
||||
public:
|
||||
FilterCursor(const Filter &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1232,7 +1207,6 @@ RETURN clause) the Produce's pull succeeds exactly once.")
|
||||
public:
|
||||
ProduceCursor(const Produce &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1280,7 +1254,6 @@ Has a flag for using DETACH DELETE when deleting vertices.")
|
||||
public:
|
||||
DeleteCursor(const Delete &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1574,7 +1547,6 @@ edge lists).")
|
||||
EdgeUniquenessFilterCursor(const EdgeUniquenessFilter &,
|
||||
utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
bool PullMultiple(MultiFrame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
@@ -1971,6 +1943,27 @@ and returns true, once.")
|
||||
input_ = input;
|
||||
}
|
||||
cpp<#)
|
||||
(:private
|
||||
#>cpp
|
||||
class OptionalCursor : public Cursor {
|
||||
public:
|
||||
OptionalCursor(const Optional &, utils::MemoryResource *);
|
||||
bool Pull(Frame &, ExecutionContext &) override;
|
||||
void Shutdown() override;
|
||||
void Reset() override;
|
||||
|
||||
private:
|
||||
const Optional &self_;
|
||||
const UniqueCursorPtr input_cursor_;
|
||||
const UniqueCursorPtr optional_cursor_;
|
||||
// indicates if the next Pull from this cursor should
|
||||
// perform a Pull from the input_cursor_
|
||||
// this is true when:
|
||||
// - first pulling from this Cursor
|
||||
// - previous Pull from this cursor exhausted the optional_cursor_
|
||||
bool pull_input_{true};
|
||||
};
|
||||
cpp<#)
|
||||
(:serialize (:slk))
|
||||
(:clone))
|
||||
|
||||
|
||||
@@ -398,7 +398,7 @@ void Filters::AnalyzeAndStoreFilter(Expression *expr, const SymbolTable &symbol_
|
||||
auto add_id_equal = [&](auto *maybe_id_fun, auto *val_expr) -> bool {
|
||||
auto *id_fun = utils::Downcast<Function>(maybe_id_fun);
|
||||
if (!id_fun) return false;
|
||||
if (id_fun->function_name_ != functions::kId) return false;
|
||||
if (id_fun->function_name_ != kId) return false;
|
||||
if (id_fun->arguments_.size() != 1U) return false;
|
||||
auto *ident = utils::Downcast<Identifier>(id_fun->arguments_.front());
|
||||
if (!ident) return false;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -14,13 +14,11 @@
|
||||
|
||||
#include "query/v2/bindings/pretty_print.hpp"
|
||||
#include "query/v2/db_accessor.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
#include "utils/string.hpp"
|
||||
|
||||
namespace memgraph::query::v2::plan {
|
||||
|
||||
PlanPrinter::PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out)
|
||||
: request_router_(request_router), out_(out) {}
|
||||
PlanPrinter::PlanPrinter(const DbAccessor *dba, std::ostream *out) : dba_(dba), out_(out) {}
|
||||
|
||||
#define PRE_VISIT(TOp) \
|
||||
bool PlanPrinter::PreVisit(TOp &) { \
|
||||
@@ -34,7 +32,7 @@ bool PlanPrinter::PreVisit(CreateExpand &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* CreateExpand (" << op.input_symbol_.name() << ")"
|
||||
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
|
||||
<< op.edge_info_.symbol.name() << ":" << request_router_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
|
||||
<< op.edge_info_.symbol.name() << ":" << dba_->EdgeTypeToName(op.edge_info_.edge_type) << "]"
|
||||
<< (op.edge_info_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
|
||||
<< op.node_info_.symbol.name() << ")";
|
||||
});
|
||||
@@ -54,7 +52,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAll &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabel"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
|
||||
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << ")";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -62,8 +60,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabel &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabelPropertyValue"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
|
||||
<< request_router_->PropertyToName(op.property_) << "})";
|
||||
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
|
||||
<< dba_->PropertyToName(op.property_) << "})";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -71,8 +69,8 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyValue &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabelPropertyRange"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
|
||||
<< request_router_->PropertyToName(op.property_) << "})";
|
||||
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
|
||||
<< dba_->PropertyToName(op.property_) << "})";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -80,16 +78,16 @@ bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelPropertyRange &op) {
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanAllByLabelProperty &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanAllByLabelProperty"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << " {"
|
||||
<< request_router_->PropertyToName(op.property_) << "})";
|
||||
<< " (" << op.output_symbol_.name() << " :" << dba_->LabelToName(op.label_) << " {"
|
||||
<< dba_->PropertyToName(op.property_) << "})";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
bool PlanPrinter::PreVisit(query::v2::plan::ScanByPrimaryKey &op) {
|
||||
bool PlanPrinter::PreVisit(ScanAllById &op) {
|
||||
WithPrintLn([&](auto &out) {
|
||||
out << "* ScanByPrimaryKey"
|
||||
<< " (" << op.output_symbol_.name() << " :" << request_router_->LabelToName(op.label_) << ")";
|
||||
out << "* ScanAllById"
|
||||
<< " (" << op.output_symbol_.name() << ")";
|
||||
});
|
||||
return true;
|
||||
}
|
||||
@@ -100,7 +98,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::Expand &op) {
|
||||
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
|
||||
<< op.common_.edge_symbol.name();
|
||||
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
|
||||
stream << ":" << request_router_->EdgeTypeToName(edge_type);
|
||||
stream << ":" << dba_->EdgeTypeToName(edge_type);
|
||||
});
|
||||
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
|
||||
<< op.common_.node_symbol.name() << ")";
|
||||
@@ -129,7 +127,7 @@ bool PlanPrinter::PreVisit(query::v2::plan::ExpandVariable &op) {
|
||||
<< (op.common_.direction == query::v2::EdgeAtom::Direction::IN ? "<-" : "-") << "["
|
||||
<< op.common_.edge_symbol.name();
|
||||
utils::PrintIterable(*out_, op.common_.edge_types, "|", [this](auto &stream, const auto &edge_type) {
|
||||
stream << ":" << request_router_->EdgeTypeToName(edge_type);
|
||||
stream << ":" << dba_->EdgeTypeToName(edge_type);
|
||||
});
|
||||
*out_ << "]" << (op.common_.direction == query::v2::EdgeAtom::Direction::OUT ? "->" : "-") << "("
|
||||
<< op.common_.node_symbol.name() << ")";
|
||||
@@ -263,14 +261,14 @@ void PlanPrinter::Branch(query::v2::plan::LogicalOperator &op, const std::string
|
||||
--depth_;
|
||||
}
|
||||
|
||||
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out) {
|
||||
PlanPrinter printer(&request_router, out);
|
||||
void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out) {
|
||||
PlanPrinter printer(&dba, out);
|
||||
// FIXME(mtomic): We should make visitors that take const arguments.
|
||||
const_cast<LogicalOperator *>(plan_root)->Accept(printer);
|
||||
}
|
||||
|
||||
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
|
||||
impl::PlanToJsonVisitor visitor(&request_router);
|
||||
nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root) {
|
||||
impl::PlanToJsonVisitor visitor(&dba);
|
||||
// FIXME(mtomic): We should make visitors that take const arguments.
|
||||
const_cast<LogicalOperator *>(plan_root)->Accept(visitor);
|
||||
return visitor.output();
|
||||
@@ -348,17 +346,11 @@ json ToJson(const utils::Bound<Expression *> &bound) {
|
||||
|
||||
json ToJson(const Symbol &symbol) { return symbol.name(); }
|
||||
|
||||
json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router) {
|
||||
return request_router.EdgeTypeToName(edge_type);
|
||||
}
|
||||
json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba) { return dba.EdgeTypeToName(edge_type); }
|
||||
|
||||
json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router) {
|
||||
return request_router.LabelToName(label);
|
||||
}
|
||||
json ToJson(storage::v3::LabelId label, const DbAccessor &dba) { return dba.LabelToName(label); }
|
||||
|
||||
json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router) {
|
||||
return request_router.PropertyToName(property);
|
||||
}
|
||||
json ToJson(storage::v3::PropertyId property, const DbAccessor &dba) { return dba.PropertyToName(property); }
|
||||
|
||||
json ToJson(NamedExpression *nexpr) {
|
||||
json json;
|
||||
@@ -367,30 +359,29 @@ json ToJson(NamedExpression *nexpr) {
|
||||
return json;
|
||||
}
|
||||
|
||||
json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
|
||||
const RequestRouterInterface &request_router) {
|
||||
json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties, const DbAccessor &dba) {
|
||||
json json;
|
||||
for (const auto &prop_pair : properties) {
|
||||
json.emplace(ToJson(prop_pair.first, request_router), ToJson(prop_pair.second));
|
||||
json.emplace(ToJson(prop_pair.first, dba), ToJson(prop_pair.second));
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router) {
|
||||
json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba) {
|
||||
json self;
|
||||
self["symbol"] = ToJson(node_info.symbol);
|
||||
self["labels"] = ToJson(node_info.labels, request_router);
|
||||
self["labels"] = ToJson(node_info.labels, dba);
|
||||
const auto *props = std::get_if<PropertiesMapList>(&node_info.properties);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba);
|
||||
return self;
|
||||
}
|
||||
|
||||
json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router) {
|
||||
json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba) {
|
||||
json self;
|
||||
self["symbol"] = ToJson(edge_info.symbol);
|
||||
const auto *props = std::get_if<PropertiesMapList>(&edge_info.properties);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, request_router);
|
||||
self["edge_type"] = ToJson(edge_info.edge_type, request_router);
|
||||
self["properties"] = ToJson(props ? *props : PropertiesMapList{}, dba);
|
||||
self["edge_type"] = ToJson(edge_info.edge_type, dba);
|
||||
self["direction"] = ToString(edge_info.direction);
|
||||
return self;
|
||||
}
|
||||
@@ -432,7 +423,7 @@ bool PlanToJsonVisitor::PreVisit(ScanAll &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabel";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *dba_);
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -445,8 +436,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabel &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabelPropertyRange";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *dba_);
|
||||
self["property"] = ToJson(op.property_, *dba_);
|
||||
self["lower_bound"] = op.lower_bound_ ? ToJson(*op.lower_bound_) : json();
|
||||
self["upper_bound"] = op.upper_bound_ ? ToJson(*op.upper_bound_) : json();
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
@@ -461,8 +452,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyRange &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabelPropertyValue";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *dba_);
|
||||
self["property"] = ToJson(op.property_, *dba_);
|
||||
self["expression"] = ToJson(op.expression_);
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
@@ -476,8 +467,8 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelPropertyValue &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
|
||||
json self;
|
||||
self["name"] = "ScanAllByLabelProperty";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["label"] = ToJson(op.label_, *dba_);
|
||||
self["property"] = ToJson(op.property_, *dba_);
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -487,15 +478,12 @@ bool PlanToJsonVisitor::PreVisit(ScanAllByLabelProperty &op) {
|
||||
return false;
|
||||
}
|
||||
|
||||
bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(ScanAllById &op) {
|
||||
json self;
|
||||
self["name"] = "ScanByPrimaryKey";
|
||||
self["label"] = ToJson(op.label_, *request_router_);
|
||||
self["name"] = "ScanAllById";
|
||||
self["output_symbol"] = ToJson(op.output_symbol_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
|
||||
output_ = std::move(self);
|
||||
return false;
|
||||
}
|
||||
@@ -503,7 +491,7 @@ bool PlanToJsonVisitor::PreVisit(ScanByPrimaryKey &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(CreateNode &op) {
|
||||
json self;
|
||||
self["name"] = "CreateNode";
|
||||
self["node_info"] = ToJson(op.node_info_, *request_router_);
|
||||
self["node_info"] = ToJson(op.node_info_, *dba_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
@@ -516,8 +504,8 @@ bool PlanToJsonVisitor::PreVisit(CreateExpand &op) {
|
||||
json self;
|
||||
self["name"] = "CreateExpand";
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["node_info"] = ToJson(op.node_info_, *request_router_);
|
||||
self["edge_info"] = ToJson(op.edge_info_, *request_router_);
|
||||
self["node_info"] = ToJson(op.node_info_, *dba_);
|
||||
self["edge_info"] = ToJson(op.edge_info_, *dba_);
|
||||
self["existing_node"] = op.existing_node_;
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -533,7 +521,7 @@ bool PlanToJsonVisitor::PreVisit(Expand &op) {
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["node_symbol"] = ToJson(op.common_.node_symbol);
|
||||
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *dba_);
|
||||
self["direction"] = ToString(op.common_.direction);
|
||||
self["existing_node"] = op.common_.existing_node;
|
||||
|
||||
@@ -550,7 +538,7 @@ bool PlanToJsonVisitor::PreVisit(ExpandVariable &op) {
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["node_symbol"] = ToJson(op.common_.node_symbol);
|
||||
self["edge_symbol"] = ToJson(op.common_.edge_symbol);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *request_router_);
|
||||
self["edge_types"] = ToJson(op.common_.edge_types, *dba_);
|
||||
self["direction"] = ToString(op.common_.direction);
|
||||
self["type"] = ToString(op.type_);
|
||||
self["is_reverse"] = op.is_reverse_;
|
||||
@@ -625,7 +613,7 @@ bool PlanToJsonVisitor::PreVisit(Delete &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(SetProperty &op) {
|
||||
json self;
|
||||
self["name"] = "SetProperty";
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *dba_);
|
||||
self["lhs"] = ToJson(op.lhs_);
|
||||
self["rhs"] = ToJson(op.rhs_);
|
||||
|
||||
@@ -662,7 +650,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
|
||||
json self;
|
||||
self["name"] = "SetLabels";
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["labels"] = ToJson(op.labels_, *request_router_);
|
||||
self["labels"] = ToJson(op.labels_, *dba_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
@@ -674,7 +662,7 @@ bool PlanToJsonVisitor::PreVisit(SetLabels &op) {
|
||||
bool PlanToJsonVisitor::PreVisit(RemoveProperty &op) {
|
||||
json self;
|
||||
self["name"] = "RemoveProperty";
|
||||
self["property"] = ToJson(op.property_, *request_router_);
|
||||
self["property"] = ToJson(op.property_, *dba_);
|
||||
self["lhs"] = ToJson(op.lhs_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
@@ -688,7 +676,7 @@ bool PlanToJsonVisitor::PreVisit(RemoveLabels &op) {
|
||||
json self;
|
||||
self["name"] = "RemoveLabels";
|
||||
self["input_symbol"] = ToJson(op.input_symbol_);
|
||||
self["labels"] = ToJson(op.labels_, *request_router_);
|
||||
self["labels"] = ToJson(op.labels_, *dba_);
|
||||
|
||||
op.input_->Accept(*this);
|
||||
self["input"] = PopOutput();
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -18,28 +18,28 @@
|
||||
|
||||
#include "query/v2/frontend/ast/ast.hpp"
|
||||
#include "query/v2/plan/operator.hpp"
|
||||
#include "query/v2/request_router.hpp"
|
||||
|
||||
namespace memgraph::query::v2 {
|
||||
class DbAccessor;
|
||||
|
||||
namespace plan {
|
||||
|
||||
class LogicalOperator;
|
||||
|
||||
/// Pretty print a `LogicalOperator` plan to a `std::ostream`.
|
||||
/// RequestRouter is needed for resolving label and property names.
|
||||
/// DbAccessor is needed for resolving label and property names.
|
||||
/// Note that `plan_root` isn't modified, but we can't take it as a const
|
||||
/// because we don't have support for visiting a const LogicalOperator.
|
||||
void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root, std::ostream *out);
|
||||
void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root, std::ostream *out);
|
||||
|
||||
/// Overload of `PrettyPrint` which defaults the `std::ostream` to `std::cout`.
|
||||
inline void PrettyPrint(const RequestRouterInterface &request_router, const LogicalOperator *plan_root) {
|
||||
PrettyPrint(request_router, plan_root, &std::cout);
|
||||
inline void PrettyPrint(const DbAccessor &dba, const LogicalOperator *plan_root) {
|
||||
PrettyPrint(dba, plan_root, &std::cout);
|
||||
}
|
||||
|
||||
/// Convert a `LogicalOperator` plan to a JSON representation.
|
||||
/// DbAccessor is needed for resolving label and property names.
|
||||
nlohmann::json PlanToJson(const RequestRouterInterface &request_router, const LogicalOperator *plan_root);
|
||||
nlohmann::json PlanToJson(const DbAccessor &dba, const LogicalOperator *plan_root);
|
||||
|
||||
class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
public:
|
||||
@@ -47,7 +47,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
using HierarchicalLogicalOperatorVisitor::PreVisit;
|
||||
using HierarchicalLogicalOperatorVisitor::Visit;
|
||||
|
||||
PlanPrinter(const RequestRouterInterface *request_router, std::ostream *out);
|
||||
PlanPrinter(const DbAccessor *dba, std::ostream *out);
|
||||
|
||||
bool DefaultPreVisit() override;
|
||||
|
||||
@@ -67,7 +67,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(ScanAllByLabelPropertyValue &) override;
|
||||
bool PreVisit(ScanAllByLabelPropertyRange &) override;
|
||||
bool PreVisit(ScanAllByLabelProperty &) override;
|
||||
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
|
||||
bool PreVisit(ScanAllById &) override;
|
||||
|
||||
bool PreVisit(Expand &) override;
|
||||
bool PreVisit(ExpandVariable &) override;
|
||||
@@ -114,7 +114,7 @@ class PlanPrinter : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
void Branch(LogicalOperator &op, const std::string &branch_name = "");
|
||||
|
||||
int64_t depth_{0};
|
||||
const RequestRouterInterface *request_router_{nullptr};
|
||||
const DbAccessor *dba_{nullptr};
|
||||
std::ostream *out_{nullptr};
|
||||
};
|
||||
|
||||
@@ -132,20 +132,20 @@ nlohmann::json ToJson(const utils::Bound<Expression *> &bound);
|
||||
|
||||
nlohmann::json ToJson(const Symbol &symbol);
|
||||
|
||||
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(storage::v3::EdgeTypeId edge_type, const DbAccessor &dba);
|
||||
|
||||
nlohmann::json ToJson(storage::v3::LabelId label, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(storage::v3::LabelId label, const DbAccessor &dba);
|
||||
|
||||
nlohmann::json ToJson(storage::v3::PropertyId property, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(storage::v3::PropertyId property, const DbAccessor &dba);
|
||||
|
||||
nlohmann::json ToJson(NamedExpression *nexpr);
|
||||
|
||||
nlohmann::json ToJson(const std::vector<std::pair<storage::v3::PropertyId, Expression *>> &properties,
|
||||
const RequestRouterInterface &request_router);
|
||||
const DbAccessor &dba);
|
||||
|
||||
nlohmann::json ToJson(const NodeCreationInfo &node_info, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(const NodeCreationInfo &node_info, const DbAccessor &dba);
|
||||
|
||||
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const RequestRouterInterface &request_router);
|
||||
nlohmann::json ToJson(const EdgeCreationInfo &edge_info, const DbAccessor &dba);
|
||||
|
||||
nlohmann::json ToJson(const Aggregate::Element &elem);
|
||||
|
||||
@@ -160,7 +160,7 @@ nlohmann::json ToJson(const std::vector<T> &items, Args &&...args) {
|
||||
|
||||
class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
public:
|
||||
explicit PlanToJsonVisitor(const RequestRouterInterface *request_router) : request_router_(request_router) {}
|
||||
explicit PlanToJsonVisitor(const DbAccessor *dba) : dba_(dba) {}
|
||||
|
||||
using HierarchicalLogicalOperatorVisitor::PostVisit;
|
||||
using HierarchicalLogicalOperatorVisitor::PreVisit;
|
||||
@@ -194,7 +194,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
bool PreVisit(ScanAllByLabelPropertyRange &) override;
|
||||
bool PreVisit(ScanAllByLabelPropertyValue &) override;
|
||||
bool PreVisit(ScanAllByLabelProperty &) override;
|
||||
bool PreVisit(ScanByPrimaryKey & /*unused*/) override;
|
||||
bool PreVisit(ScanAllById &) override;
|
||||
|
||||
bool PreVisit(Produce &) override;
|
||||
bool PreVisit(Accumulate &) override;
|
||||
@@ -216,7 +216,7 @@ class PlanToJsonVisitor : public virtual HierarchicalLogicalOperatorVisitor {
|
||||
|
||||
protected:
|
||||
nlohmann::json output_;
|
||||
const RequestRouterInterface *request_router_;
|
||||
const DbAccessor *dba_;
|
||||
|
||||
nlohmann::json PopOutput() {
|
||||
nlohmann::json tmp;
|
||||
|
||||
@@ -24,18 +24,18 @@ namespace memgraph::query::v2::plan {
|
||||
|
||||
namespace {
|
||||
|
||||
uint64_t IndividualCycles(const ProfilingStats &cumulative_stats) {
|
||||
unsigned long long IndividualCycles(const ProfilingStats &cumulative_stats) {
|
||||
return cumulative_stats.num_cycles - std::accumulate(cumulative_stats.children.begin(),
|
||||
cumulative_stats.children.end(), 0ULL,
|
||||
[](auto acc, auto &stats) { return acc + stats.num_cycles; });
|
||||
}
|
||||
|
||||
double RelativeTime(const uint64_t num_cycles, const uint64_t total_cycles) {
|
||||
return static_cast<double>(num_cycles) / static_cast<double>(total_cycles);
|
||||
double RelativeTime(unsigned long long num_cycles, unsigned long long total_cycles) {
|
||||
return static_cast<double>(num_cycles) / total_cycles;
|
||||
}
|
||||
|
||||
double AbsoluteTime(const uint64_t num_cycles, const uint64_t total_cycles,
|
||||
const std::chrono::duration<double> total_time) {
|
||||
double AbsoluteTime(unsigned long long num_cycles, unsigned long long total_cycles,
|
||||
std::chrono::duration<double> total_time) {
|
||||
return (RelativeTime(num_cycles, total_cycles) * static_cast<std::chrono::duration<double, std::milli>>(total_time))
|
||||
.count();
|
||||
}
|
||||
@@ -50,29 +50,26 @@ namespace {
|
||||
|
||||
class ProfilingStatsToTableHelper {
|
||||
public:
|
||||
ProfilingStatsToTableHelper(uint64_t total_cycles, std::chrono::duration<double> total_time)
|
||||
ProfilingStatsToTableHelper(unsigned long long total_cycles, std::chrono::duration<double> total_time)
|
||||
: total_cycles_(total_cycles), total_time_(total_time) {}
|
||||
|
||||
void Output(const ProfilingStats &cumulative_stats) {
|
||||
auto cycles = IndividualCycles(cumulative_stats);
|
||||
auto custom_data_copy = cumulative_stats.custom_data;
|
||||
ConvertCyclesToTime(custom_data_copy);
|
||||
|
||||
rows_.emplace_back(
|
||||
std::vector<TypedValue>{TypedValue(FormatOperator(cumulative_stats.name)),
|
||||
TypedValue(cumulative_stats.actual_hits), TypedValue(FormatRelativeTime(cycles)),
|
||||
TypedValue(FormatAbsoluteTime(cycles)), TypedValue(custom_data_copy.dump())});
|
||||
rows_.emplace_back(std::vector<TypedValue>{
|
||||
TypedValue(FormatOperator(cumulative_stats.name)), TypedValue(cumulative_stats.actual_hits),
|
||||
TypedValue(FormatRelativeTime(cycles)), TypedValue(FormatAbsoluteTime(cycles))});
|
||||
|
||||
for (size_t i = 1; i < cumulative_stats.children.size(); ++i) {
|
||||
Branch(cumulative_stats.children[i]);
|
||||
}
|
||||
|
||||
if (!cumulative_stats.children.empty()) {
|
||||
if (cumulative_stats.children.size() >= 1) {
|
||||
Output(cumulative_stats.children[0]);
|
||||
}
|
||||
}
|
||||
|
||||
std::vector<std::vector<TypedValue>> rows() const { return rows_; }
|
||||
std::vector<std::vector<TypedValue>> rows() { return rows_; }
|
||||
|
||||
private:
|
||||
void Branch(const ProfilingStats &cumulative_stats) {
|
||||
@@ -83,28 +80,7 @@ class ProfilingStatsToTableHelper {
|
||||
--depth_;
|
||||
}
|
||||
|
||||
double AbsoluteTime(const uint64_t cycles) const { return plan::AbsoluteTime(cycles, total_cycles_, total_time_); }
|
||||
|
||||
double RelativeTime(const uint64_t cycles) const { return plan::RelativeTime(cycles, total_cycles_); }
|
||||
|
||||
void ConvertCyclesToTime(nlohmann::json &custom_data) const {
|
||||
const auto convert_cycles_in_json = [this](nlohmann::json &json) {
|
||||
if (!json.is_object()) {
|
||||
return;
|
||||
}
|
||||
if (auto it = json.find(ProfilingStats::kNumCycles); it != json.end()) {
|
||||
auto num_cycles = it.value().get<uint64_t>();
|
||||
json[ProfilingStats::kAbsoluteTime] = AbsoluteTime(num_cycles);
|
||||
json[ProfilingStats::kRelativeTime] = RelativeTime(num_cycles);
|
||||
}
|
||||
};
|
||||
|
||||
for (auto &json : custom_data) {
|
||||
convert_cycles_in_json(json);
|
||||
}
|
||||
}
|
||||
|
||||
std::string Format(const char *str) const {
|
||||
std::string Format(const char *str) {
|
||||
std::ostringstream ss;
|
||||
for (int64_t i = 0; i < depth_; ++i) {
|
||||
ss << "| ";
|
||||
@@ -113,21 +89,21 @@ class ProfilingStatsToTableHelper {
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
std::string Format(const std::string &str) const { return Format(str.c_str()); }
|
||||
std::string Format(const std::string &str) { return Format(str.c_str()); }
|
||||
|
||||
std::string FormatOperator(const char *str) const { return Format(std::string("* ") + str); }
|
||||
std::string FormatOperator(const char *str) { return Format(std::string("* ") + str); }
|
||||
|
||||
std::string FormatRelativeTime(uint64_t num_cycles) const {
|
||||
return fmt::format("{: 10.6f} %", RelativeTime(num_cycles) * 100);
|
||||
std::string FormatRelativeTime(unsigned long long num_cycles) {
|
||||
return fmt::format("{: 10.6f} %", RelativeTime(num_cycles, total_cycles_) * 100);
|
||||
}
|
||||
|
||||
std::string FormatAbsoluteTime(uint64_t num_cycles) const {
|
||||
return fmt::format("{: 10.6f} ms", AbsoluteTime(num_cycles));
|
||||
std::string FormatAbsoluteTime(unsigned long long num_cycles) {
|
||||
return fmt::format("{: 10.6f} ms", AbsoluteTime(num_cycles, total_cycles_, total_time_));
|
||||
}
|
||||
|
||||
int64_t depth_{0};
|
||||
std::vector<std::vector<TypedValue>> rows_;
|
||||
uint64_t total_cycles_;
|
||||
unsigned long long total_cycles_;
|
||||
std::chrono::duration<double> total_time_;
|
||||
};
|
||||
|
||||
@@ -150,7 +126,7 @@ class ProfilingStatsToJsonHelper {
|
||||
using json = nlohmann::json;
|
||||
|
||||
public:
|
||||
ProfilingStatsToJsonHelper(uint64_t total_cycles, std::chrono::duration<double> total_time)
|
||||
ProfilingStatsToJsonHelper(unsigned long long total_cycles, std::chrono::duration<double> total_time)
|
||||
: total_cycles_(total_cycles), total_time_(total_time) {}
|
||||
|
||||
void Output(const ProfilingStats &cumulative_stats) { return Output(cumulative_stats, &json_); }
|
||||
@@ -175,7 +151,7 @@ class ProfilingStatsToJsonHelper {
|
||||
}
|
||||
|
||||
json json_;
|
||||
uint64_t total_cycles_;
|
||||
unsigned long long total_cycles_;
|
||||
std::chrono::duration<double> total_time_;
|
||||
};
|
||||
|
||||
|
||||
@@ -26,16 +26,10 @@ namespace plan {
|
||||
* Stores profiling statistics for a single logical operator.
|
||||
*/
|
||||
struct ProfilingStats {
|
||||
static constexpr std::string_view kNumCycles{"num_cycles"};
|
||||
static constexpr std::string_view kRelativeTime{"relative_time"};
|
||||
static constexpr std::string_view kAbsoluteTime{"absolute_time"};
|
||||
static constexpr std::string_view kActualHits{"actual_hits"};
|
||||
|
||||
int64_t actual_hits{0};
|
||||
uint64_t num_cycles{0};
|
||||
unsigned long long num_cycles{0};
|
||||
uint64_t key{0};
|
||||
const char *name{nullptr};
|
||||
nlohmann::json custom_data;
|
||||
// TODO: This should use the allocator for query execution
|
||||
std::vector<ProfilingStats> children;
|
||||
};
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Copyright 2023 Memgraph Ltd.
|
||||
// Copyright 2022 Memgraph Ltd.
|
||||
//
|
||||
// Use of this software is governed by the Business Source License
|
||||
// included in the file licenses/BSL.txt; by using this file, you agree to be bound by the terms of the Business Source
|
||||
@@ -11,12 +11,10 @@
|
||||
|
||||
#include "query/v2/plan/read_write_type_checker.hpp"
|
||||
|
||||
// NOLINTNEXTLINE(cppcoreguidelines-macro-usage)
|
||||
#define PRE_VISIT(TOp, RWType, continue_visiting) \
|
||||
/*NOLINTNEXTLINE(bugprone-macro-parentheses)*/ \
|
||||
bool ReadWriteTypeChecker::PreVisit(TOp & /*op*/) { \
|
||||
UpdateType(RWType); \
|
||||
return continue_visiting; \
|
||||
#define PRE_VISIT(TOp, RWType, continue_visiting) \
|
||||
bool ReadWriteTypeChecker::PreVisit(TOp &op) { \
|
||||
UpdateType(RWType); \
|
||||
return continue_visiting; \
|
||||
}
|
||||
|
||||
namespace memgraph::query::v2::plan {
|
||||
@@ -37,7 +35,7 @@ PRE_VISIT(ScanAllByLabel, RWType::R, true)
|
||||
PRE_VISIT(ScanAllByLabelPropertyRange, RWType::R, true)
|
||||
PRE_VISIT(ScanAllByLabelPropertyValue, RWType::R, true)
|
||||
PRE_VISIT(ScanAllByLabelProperty, RWType::R, true)
|
||||
PRE_VISIT(ScanByPrimaryKey, RWType::R, true)
|
||||
PRE_VISIT(ScanAllById, RWType::R, true)
|
||||
|
||||
PRE_VISIT(Expand, RWType::R, true)
|
||||
PRE_VISIT(ExpandVariable, RWType::R, true)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user