From aa85a6f77ba5b800507d0595abd6e4c83297e44c Mon Sep 17 00:00:00 2001 From: Guofang Date: Thu, 24 Sep 2026 15:17:12 +0800 Subject: [PATCH] Build Apache Arrow and Parquet from source in debian13 image Add pinned xsimd 14.2.0, Apache Thrift v0.22.0 (C++ library only), Snappy 1.2.2, LZ4 v1.10.0, and Apache Arrow 25.0.1 with Parquet as static PIC builds. Arrow uses ARROW_DEPENDENCY_SOURCE=SYSTEM so no dependencies are fetched implicitly, disables the bundled jemalloc and mimalloc allocators, and enables Parquet encryption via OpenSSL. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01M5XtQocmT1zExJBt4oh3YK --- INCLUDED_LIBRARIES.md | 9 +++- debian13/Dockerfile | 101 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/INCLUDED_LIBRARIES.md b/INCLUDED_LIBRARIES.md index 23faf06..29ea4cd 100644 --- a/INCLUDED_LIBRARIES.md +++ b/INCLUDED_LIBRARIES.md @@ -45,7 +45,14 @@ | libyaml | `0.2.5` | | libxml2 | `2.15.3` | | libproxyprotocol | `v1.3.3` | +| xsimd | `14.2.0` | +| Apache Thrift | `v0.22.0` | +| Snappy | `1.2.2` | +| LZ4 | `v1.10.0` | +| Apache Arrow / Parquet | `25.0.1` | -The image includes position-independent static builds of these libraries. Protobuf and libxml2 enable zlib support, and LibDwarf enables compressed debug section support using zlib and Zstd. ZeroMQ enables CURVE security using static libsodium. CZMQ enables UUID support using static util-linux libuuid while disabling undeclared optional integrations. librdkafka includes SSL, zlib, Zstd, and bundled LZ4 support while disabling undeclared optional integrations. spdlog uses external fmt, glog uses gflags, and all Meta libraries share the same release tag. +The image includes position-independent static builds of these libraries. Protobuf and libxml2 enable zlib support, and LibDwarf enables compressed debug section support using zlib and Zstd. ZeroMQ enables CURVE security using static libsodium. CZMQ enables UUID support using static util-linux libuuid while disabling undeclared optional integrations. librdkafka includes SSL, zlib, Zstd, and bundled LZ4 support while disabling undeclared optional integrations. spdlog uses external fmt, glog uses gflags, and all Meta libraries share the same release tag. Apache Arrow is built with Parquet (including OpenSSL-based Parquet encryption) and Snappy, LZ4, zlib, and Zstd compression using only the pinned system dependencies above (`ARROW_DEPENDENCY_SOURCE=SYSTEM`), with the bundled jemalloc and mimalloc allocators disabled. Apache Thrift only builds its C++ library, and its headers sit next to fbthrift's under `include/thrift` without overlapping. + +**Pending on rocky9 and alpine3.24:** xsimd, Apache Thrift, Snappy, LZ4, and Apache Arrow / Parquet are currently built only in the `debian13` image. **Alpine-only exception:** the `alpine3.24` image does not build Folly, Fizz, Wangle, mvfst, fbthrift, or CacheLib. These aren't on Meta's supported-OS list, and Folly's portability layer reaches for glibc-only macros/symbols with no musl branch (e.g. `off64_t`/`lseek64`/`pread64` in `folly/portability/Unistd.h`, `__ELF_NATIVE_CLASS` in `folly/debugging/symbolizer/Elf.cpp`), so patching each one as it surfaces risks introducing subtle bugs in code this project doesn't own. `debian13` and `rocky9` are glibc-based and build the full list above, Meta libraries included. diff --git a/debian13/Dockerfile b/debian13/Dockerfile index 147b17c..8930571 100644 --- a/debian13/Dockerfile +++ b/debian13/Dockerfile @@ -768,4 +768,105 @@ RUN curl --fail --location --output libproxyprotocol.tar.gz \ && cp src/src/proxy_protocol.h /usr/local/include/ \ && rm -rf src +RUN curl --fail --location --output xsimd.tar.gz \ + https://wangguofang.net/github.com/xtensor-stack/xsimd/archive/14.2.0.tar.gz \ + && mkdir src \ + && tar --extract --gzip --file=xsimd.tar.gz --directory=src --strip-components=1 \ + && rm xsimd.tar.gz \ + && cmake -G Ninja -S src -B build -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TESTS=OFF \ + -DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu \ + && cmake --build build --parallel "$(nproc)" --target install \ + && rm -rf build src + +RUN curl --fail --location --output thrift.tar.gz \ + https://wangguofang.net/github.com/apache/thrift/archive/v0.22.0.tar.gz \ + && mkdir src \ + && tar --extract --gzip --file=thrift.tar.gz --directory=src --strip-components=1 \ + && rm thrift.tar.gz \ + && cmake -G Ninja -S src -B build -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_COMPILER=OFF \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_TUTORIALS=OFF \ + -DCMAKE_INSTALL_DIR=lib/x86_64-linux-gnu/cmake \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DLIB_INSTALL_DIR=lib/x86_64-linux-gnu \ + -DPKGCONFIG_INSTALL_DIR=lib/x86_64-linux-gnu/pkgconfig \ + -DWITH_AS3=OFF \ + -DWITH_C_GLIB=OFF \ + -DWITH_CPP=ON \ + -DWITH_JAVA=OFF \ + -DWITH_JAVASCRIPT=OFF \ + -DWITH_LIBEVENT=OFF \ + -DWITH_NODEJS=OFF \ + -DWITH_OPENSSL=OFF \ + -DWITH_PYTHON=OFF \ + -DWITH_QT5=OFF \ + -DWITH_ZLIB=OFF \ + && cmake --build build --parallel "$(nproc)" --target install \ + && rm -rf build src + +RUN curl --fail --location --output snappy.tar.gz \ + https://wangguofang.net/github.com/google/snappy/archive/1.2.2.tar.gz \ + && mkdir src \ + && tar --extract --gzip --file=snappy.tar.gz --directory=src --strip-components=1 \ + && rm snappy.tar.gz \ + && cmake -G Ninja -S src -B build -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DSNAPPY_BUILD_BENCHMARKS=OFF \ + -DSNAPPY_BUILD_TESTS=OFF \ + && cmake --build build --parallel "$(nproc)" --target install \ + && rm -rf build src + +RUN curl --fail --location --output lz4.tar.gz \ + https://wangguofang.net/github.com/lz4/lz4/archive/v1.10.0.tar.gz \ + && mkdir src \ + && tar --extract --gzip --file=lz4.tar.gz --directory=src --strip-components=1 \ + && rm lz4.tar.gz \ + && cmake -G Ninja -S src/build/cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DBUILD_STATIC_LIBS=ON \ + -DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DLZ4_BUILD_CLI=OFF \ + -DLZ4_POSITION_INDEPENDENT_LIB=ON \ + && cmake --build build --parallel "$(nproc)" --target install \ + && rm -rf build src + +RUN curl --fail --location --output arrow.tar.gz \ + https://wangguofang.net/github.com/apache/arrow/archive/apache-arrow-25.0.1.tar.gz \ + && mkdir src \ + && tar --extract --gzip --file=arrow.tar.gz --directory=src --strip-components=1 \ + && rm arrow.tar.gz \ + && cmake -G Ninja -S src/cpp -B build -DCMAKE_BUILD_TYPE=Release \ + -DARROW_BUILD_BENCHMARKS=OFF \ + -DARROW_BUILD_EXAMPLES=OFF \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_BUILD_STATIC=ON \ + -DARROW_BUILD_TESTS=OFF \ + -DARROW_BUILD_UTILITIES=OFF \ + -DARROW_DEPENDENCY_SOURCE=SYSTEM \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_IPC=ON \ + -DARROW_JEMALLOC=OFF \ + -DARROW_MIMALLOC=OFF \ + -DARROW_PARQUET=ON \ + -DARROW_POSITION_INDEPENDENT_CODE=ON \ + -DARROW_WITH_BROTLI=OFF \ + -DARROW_WITH_BZ2=OFF \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DPARQUET_BUILD_EXAMPLES=OFF \ + -DPARQUET_BUILD_EXECUTABLES=OFF \ + -DPARQUET_REQUIRE_ENCRYPTION=ON \ + && cmake --build build --parallel "$(nproc)" --target install \ + && rm -rf build src + CMD ["/bin/bash"]