Add support for Amazon Linux 2 and stop generating C++ using Lisp/LCP (#814)

2023-03-14 19:24:55 +01:00
parent 6f51141148
commit c4167bafdd
29 changed files with 7155 additions and 212 deletions
--- a/environment/os/amzn-2.sh
+++ b/environment/os/amzn-2.sh
@@ -0,0 +1,156 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+source "$DIR/../util.sh"
+
+check_operating_system "amzn-2"
+check_architecture "x86_64"
+
+TOOLCHAIN_BUILD_DEPS=(
+    gcc gcc-c++ make # generic build tools
+    wget # used for archive download
+    gnupg2 # used for archive signature verification
+    tar gzip bzip2 xz unzip # used for archive unpacking
+    zlib-devel # zlib library used for all builds
+    expat-devel xz-devel python3-devel texinfo
+    curl libcurl-devel # for cmake
+    readline-devel # for cmake and llvm
+    libffi-devel libxml2-devel # for llvm
+    libedit-devel pcre-devel automake bison # for swig
+    file
+    openssl-devel
+    gmp-devel
+    gperf
+    diffutils
+    patch
+    libipt libipt-devel # intel
+    perl # for openssl
+)
+
+TOOLCHAIN_RUN_DEPS=(
+    make # generic build tools
+    tar gzip bzip2 xz # used for archive unpacking
+    zlib # zlib library used for all builds
+    expat xz-libs python3 # for gdb
+    readline # for cmake and llvm
+    libffi libxml2 # for llvm
+    openssl-devel
+)
+
+MEMGRAPH_BUILD_DEPS=(
+    git # source code control
+    make # build system
+    wget # for downloading libs
+    libuuid-devel java-11-openjdk # required by antlr
+    readline-devel # for memgraph console
+    python3-devel # for query modules
+    openssl-devel
+    libseccomp-devel
+    python3 python3-pip nmap-ncat # for tests
+    #
+    # IMPORTANT: python3-yaml does NOT exist on CentOS
+    # Install it using `pip3 install PyYAML`
+    #
+    PyYAML # Package name here does not correspond to the yum package!
+    libcurl-devel # mg-requests
+    rpm-build rpmlint # for RPM package building
+    doxygen graphviz # source documentation generators
+    which nodejs golang zip unzip java-11-openjdk-devel # for driver tests
+    autoconf # for jemalloc code generation
+    libtool  # for protobuf code generation
+)
+
+list() {
+    echo "$1"
+}
+
+check() {
+    local missing=""
+    # On Fedora yum/dnf and python10 use newer glibc which is not compatible
+    # with ours, so we need to momentarely disable env
+    local OLD_LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-""}
+    LD_LIBRARY_PATH=""
+    for pkg in $1; do
+        if [ "$pkg" == "PyYAML" ]; then
+            if ! python3 -c "import yaml" >/dev/null 2>/dev/null; then
+                missing="$pkg $missing"
+            fi
+            continue
+        fi
+        if ! yum list installed "$pkg" >/dev/null 2>/dev/null; then
+            missing="$pkg $missing"
+        fi
+    done
+    if [ "$missing" != "" ]; then
+        echo "MISSING PACKAGES: $missing"
+        exit 1
+    fi
+    LD_LIBRARY_PATH=${OLD_LD_LIBRARY_PATH}
+}
+
+install() {
+    cd "$DIR"
+    if [ "$EUID" -ne 0 ]; then
+        echo "Please run as root."
+        exit 1
+    fi
+    # If GitHub Actions runner is installed, append LANG to the environment.
+    # Python related tests don't work without the LANG export.
+    if [ -d "/home/gh/actions-runner" ]; then
+        echo "LANG=en_US.utf8" >> /home/gh/actions-runner/.env
+    else
+        echo "NOTE: export LANG=en_US.utf8"
+    fi
+    yum update -y
+    for pkg in $1; do
+        if [ "$pkg" == libipt ]; then
+            if ! yum list installed libipt >/dev/null 2>/dev/null; then
+                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == libipt-devel ]; then
+            if ! yum list installed libipt-devel >/dev/null 2>/dev/null; then
+                yum install -y http://repo.okay.com.mx/centos/8/x86_64/release/libipt-devel-1.6.1-8.el8.x86_64.rpm
+            fi
+            continue
+        fi
+        if [ "$pkg" == nodejs ]; then
+            curl -sL https://rpm.nodesource.com/setup_16.x | bash -
+            if ! yum list installed nodejs >/dev/null 2>/dev/null; then
+                yum install -y nodejs
+            fi
+            continue
+        fi
+        if [ "$pkg" == PyYAML ]; then
+            if [ -z ${SUDO_USER+x} ]; then # Running as root (e.g. Docker).
+                pip3 install --user PyYAML
+            else # Running using sudo.
+                sudo -H -u "$SUDO_USER" bash -c "pip3 install --user PyYAML"
+            fi
+            continue
+        fi
+        if [ "$pkg" == nodejs ]; then
+            curl -sL https://rpm.nodesource.com/setup_16.x | bash -
+            if ! yum list installed nodejs >/dev/null 2>/dev/null; then
+                yum install -y nodejs
+            fi
+            continue
+        fi
+        if [ "$pkg" == java-11-openjdk ]; then
+            amazon-linux-extras install -y java-openjdk11
+            continue
+        fi
+        if [ "$pkg" == java-11-openjdk-devel ]; then
+            amazon-linux-extras install -y java-openjdk11
+            yum install -y java-11-openjdk-devel
+            continue
+        fi
+        yum install -y "$pkg"
+    done
+}
+
+deps=$2"[*]"
+"$1" "${!deps}"
--- a/environment/toolchain/v4.sh
+++ b/environment/toolchain/v4.sh
@@ -415,6 +415,34 @@ if [ ! -f $PREFIX/bin/gdb ]; then
                --with-intel-pt \
                --enable-tui \
                --with-python=python3
+    elif [[ "${DISTRO}" == "amzn-2" ]]; then
+        # Remove readline, gdb does not compile
+        env \
+            CC=gcc \
+            CXX=g++ \
+            CFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+            CXXFLAGS="-g -O2 -fstack-protector-strong -Wformat -Werror=format-security" \
+            CPPFLAGS="-Wdate-time -D_FORTIFY_SOURCE=2 -fPIC" \
+            LDFLAGS="-Wl,-z,relro" \
+            PYTHON="" \
+            ../configure \
+                --build=x86_64-linux-gnu \
+                --host=x86_64-linux-gnu \
+                --prefix=$PREFIX \
+                --disable-maintainer-mode \
+                --disable-dependency-tracking \
+                --disable-silent-rules \
+                --disable-gdbtk \
+                --disable-shared \
+                --without-guile \
+                --with-system-gdbinit=$PREFIX/etc/gdb/gdbinit \
+                --with-expat \
+                --with-system-zlib \
+                --with-lzma \
+                --with-babeltrace \
+                --with-intel-pt \
+                --enable-tui \
+                --with-python=python3
    else
        # https://buildd.debian.org/status/fetch.php?pkg=gdb&arch=amd64&ver=8.2.1-2&stamp=1550831554&raw=0
        env \
@@ -1143,119 +1171,121 @@ if [ ! -f $PREFIX/include/libaio.h ]; then
    popd
 fi

-log_tool_name "folly $FBLIBS_VERSION"
-if [ ! -d $PREFIX/include/folly ]; then
-    if [ -d folly-$FBLIBS_VERSION ]; then
-        rm -rf folly-$FBLIBS_VERSION
+if [[ "${DISTRO}" != "amzn-2" ]]; then
+    log_tool_name "folly $FBLIBS_VERSION"
+    if [ ! -d $PREFIX/include/folly ]; then
+        if [ -d folly-$FBLIBS_VERSION ]; then
+            rm -rf folly-$FBLIBS_VERSION
+        fi
+        mkdir folly-$FBLIBS_VERSION
+        tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
+        pushd folly-$FBLIBS_VERSION
+        patch -p1 < ../../folly.patch
+        # build is used by facebook builder
+        mkdir _build
+        pushd _build
+        cmake .. $COMMON_CMAKE_FLAGS \
+            -DBOOST_LINK_STATIC=ON \
+            -DBUILD_TESTS=OFF \
+            -DGFLAGS_NOTHREADS=OFF \
+            -DCXX_STD="c++20"
+        make -j$CPUS install
+        popd && popd
    fi
-    mkdir folly-$FBLIBS_VERSION
-    tar -xzf ../archives/folly-$FBLIBS_VERSION.tar.gz -C folly-$FBLIBS_VERSION
-    pushd folly-$FBLIBS_VERSION
-    patch -p1 < ../../folly.patch
-    # build is used by facebook builder
-    mkdir _build
-    pushd _build
-    cmake .. $COMMON_CMAKE_FLAGS \
-        -DBOOST_LINK_STATIC=ON \
-        -DBUILD_TESTS=OFF \
-        -DGFLAGS_NOTHREADS=OFF \
-        -DCXX_STD="c++20"
-    make -j$CPUS install
-    popd && popd
-fi

-log_tool_name "fizz $FBLIBS_VERSION"
-if [ ! -d $PREFIX/include/fizz ]; then
-    if [ -d fizz-$FBLIBS_VERSION ]; then
-        rm -rf fizz-$FBLIBS_VERSION
+    log_tool_name "fizz $FBLIBS_VERSION"
+    if [ ! -d $PREFIX/include/fizz ]; then
+        if [ -d fizz-$FBLIBS_VERSION ]; then
+            rm -rf fizz-$FBLIBS_VERSION
+        fi
+        mkdir fizz-$FBLIBS_VERSION
+        tar -xzf ../archives/fizz-$FBLIBS_VERSION.tar.gz -C fizz-$FBLIBS_VERSION
+        pushd fizz-$FBLIBS_VERSION
+        # build is used by facebook builder
+        mkdir _build
+        pushd _build
+        cmake ../fizz $COMMON_CMAKE_FLAGS \
+            -DBUILD_TESTS=OFF \
+            -DBUILD_EXAMPLES=OFF \
+            -DGFLAGS_NOTHREADS=OFF
+        make -j$CPUS install
+        popd && popd
    fi
-    mkdir fizz-$FBLIBS_VERSION
-    tar -xzf ../archives/fizz-$FBLIBS_VERSION.tar.gz -C fizz-$FBLIBS_VERSION
-    pushd fizz-$FBLIBS_VERSION
-    # build is used by facebook builder
-    mkdir _build
-    pushd _build
-    cmake ../fizz $COMMON_CMAKE_FLAGS \
-        -DBUILD_TESTS=OFF \
-        -DBUILD_EXAMPLES=OFF \
-        -DGFLAGS_NOTHREADS=OFF
-    make -j$CPUS install
-    popd && popd
-fi

-log_tool_name "wangle FBLIBS_VERSION"
-if [ ! -d $PREFIX/include/wangle ]; then
-    if [ -d wangle-$FBLIBS_VERSION ]; then
-        rm -rf wangle-$FBLIBS_VERSION
+    log_tool_name "wangle FBLIBS_VERSION"
+    if [ ! -d $PREFIX/include/wangle ]; then
+        if [ -d wangle-$FBLIBS_VERSION ]; then
+            rm -rf wangle-$FBLIBS_VERSION
+        fi
+        mkdir wangle-$FBLIBS_VERSION
+        tar -xzf ../archives/wangle-$FBLIBS_VERSION.tar.gz -C wangle-$FBLIBS_VERSION
+        pushd wangle-$FBLIBS_VERSION
+        # build is used by facebook builder
+        mkdir _build
+        pushd _build
+        cmake ../wangle $COMMON_CMAKE_FLAGS \
+            -DBUILD_TESTS=OFF \
+            -DBUILD_EXAMPLES=OFF \
+            -DGFLAGS_NOTHREADS=OFF
+        make -j$CPUS install
+        popd && popd
    fi
-    mkdir wangle-$FBLIBS_VERSION
-    tar -xzf ../archives/wangle-$FBLIBS_VERSION.tar.gz -C wangle-$FBLIBS_VERSION
-    pushd wangle-$FBLIBS_VERSION
-    # build is used by facebook builder
-    mkdir _build
-    pushd _build
-    cmake ../wangle $COMMON_CMAKE_FLAGS \
-        -DBUILD_TESTS=OFF \
-        -DBUILD_EXAMPLES=OFF \
-        -DGFLAGS_NOTHREADS=OFF
-    make -j$CPUS install
-    popd && popd
-fi

-log_tool_name "proxygen $FBLIBS_VERSION"
-if [ ! -d $PREFIX/include/proxygen ]; then
-    if [ -d proxygen-$FBLIBS_VERSION ]; then
-        rm -rf proxygen-$FBLIBS_VERSION
+    log_tool_name "proxygen $FBLIBS_VERSION"
+    if [ ! -d $PREFIX/include/proxygen ]; then
+        if [ -d proxygen-$FBLIBS_VERSION ]; then
+            rm -rf proxygen-$FBLIBS_VERSION
+        fi
+        mkdir proxygen-$FBLIBS_VERSION
+        tar -xzf ../archives/proxygen-$FBLIBS_VERSION.tar.gz -C proxygen-$FBLIBS_VERSION
+        pushd proxygen-$FBLIBS_VERSION
+        patch -p1 < ../../proxygen.patch
+        # build is used by facebook builder
+        mkdir _build
+        pushd _build
+        cmake .. $COMMON_CMAKE_FLAGS \
+            -DBUILD_TESTS=OFF \
+            -DBUILD_SAMPLES=OFF \
+            -DGFLAGS_NOTHREADS=OFF \
+            -DBUILD_QUIC=OFF
+        make -j$CPUS install
+        popd && popd
    fi
-    mkdir proxygen-$FBLIBS_VERSION
-    tar -xzf ../archives/proxygen-$FBLIBS_VERSION.tar.gz -C proxygen-$FBLIBS_VERSION
-    pushd proxygen-$FBLIBS_VERSION
-    patch -p1 < ../../proxygen.patch
-    # build is used by facebook builder
-    mkdir _build
-    pushd _build
-    cmake .. $COMMON_CMAKE_FLAGS \
-        -DBUILD_TESTS=OFF \
-        -DBUILD_SAMPLES=OFF \
-        -DGFLAGS_NOTHREADS=OFF \
-        -DBUILD_QUIC=OFF
-    make -j$CPUS install
-    popd && popd
-fi

-log_tool_name "flex $FBLIBS_VERSION"
-if [ ! -f $PREFIX/include/FlexLexer.h ]; then
-    if [ -d flex-$FLEX_VERSION ]; then
-        rm -rf flex-$FLEX_VERSION
+    log_tool_name "flex $FBLIBS_VERSION"
+    if [ ! -f $PREFIX/include/FlexLexer.h ]; then
+        if [ -d flex-$FLEX_VERSION ]; then
+            rm -rf flex-$FLEX_VERSION
+        fi
+        tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
+        pushd flex-$FLEX_VERSION
+        ./configure $COMMON_CONFIGURE_FLAGS
+        make -j$CPUS install
+        popd
    fi
-    tar -xzf ../archives/flex-$FLEX_VERSION.tar.gz
-    pushd flex-$FLEX_VERSION
-    ./configure $COMMON_CONFIGURE_FLAGS
-    make -j$CPUS install
-    popd
-fi

-log_tool_name "fbthrift $FBLIBS_VERSION"
-if [ ! -d $PREFIX/include/thrift ]; then
-    if [ -d fbthrift-$FBLIBS_VERSION ]; then
-        rm -rf fbthrift-$FBLIBS_VERSION
+    log_tool_name "fbthrift $FBLIBS_VERSION"
+    if [ ! -d $PREFIX/include/thrift ]; then
+        if [ -d fbthrift-$FBLIBS_VERSION ]; then
+            rm -rf fbthrift-$FBLIBS_VERSION
+        fi
+        git clone --depth 1 --branch v$FBLIBS_VERSION https://github.com/facebook/fbthrift.git fbthrift-$FBLIBS_VERSION
+        pushd fbthrift-$FBLIBS_VERSION
+        # build is used by facebook builder
+        mkdir _build
+        pushd _build
+        if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
+            CMAKE_CXX_FLAGS="-fsized-deallocation"
+        else
+            CMAKE_CXX_FLAGS="-fsized-deallocation -stdlib=libc++"
+        fi
+        cmake .. $COMMON_CMAKE_FLAGS \
+            -Denable_tests=OFF \
+            -DGFLAGS_NOTHREADS=OFF \
+            -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"
+        make -j$CPUS install
+        popd
    fi
-    git clone --depth 1 --branch v$FBLIBS_VERSION https://github.com/facebook/fbthrift.git fbthrift-$FBLIBS_VERSION
-    pushd fbthrift-$FBLIBS_VERSION
-    # build is used by facebook builder
-    mkdir _build
-    pushd _build
-    if [ "$TOOLCHAIN_STDCXX" = "libstdc++" ]; then
-        CMAKE_CXX_FLAGS="-fsized-deallocation"
-    else
-        CMAKE_CXX_FLAGS="-fsized-deallocation -stdlib=libc++"
-    fi
-    cmake .. $COMMON_CMAKE_FLAGS \
-        -Denable_tests=OFF \
-        -DGFLAGS_NOTHREADS=OFF \
-        -DCMAKE_CXX_FLAGS="$CMAKE_CXX_FLAGS"
-    make -j$CPUS install
-    popd
 fi

 popd