From e31a97d39f0700d3243cb90e1d0a99966eb5fbe5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 15:54:16 +0800 Subject: [PATCH 01/31] feat(cpp): add native installation packages --- .github/workflows/cpp-packaging.yml | 130 +++++++++++++ cpp/CMakeLists.txt | 171 +++++++++++++++++- cpp/cmake/TsFileConfig.cmake.in | 29 +++ cpp/cmake/TsFilePublicHeaders.cmake | 47 +++++ cpp/cmake/libtsfile.pc.in | 31 ++++ .../projects/ANTLR4Dependency/CMakeLists.txt | 2 +- .../projects/InstalledConsumer/CMakeLists.txt | 42 +++++ .../tests/projects/InstalledConsumer/main.cc | 24 +++ .../projects/PkgConfigConsumer/CMakeLists.txt | 32 ++++ .../tests/projects/PkgConfigConsumer/main.c | 21 +++ .../tests/projects/PkgConfigConsumer/main.cc | 21 +++ cpp/src/CMakeLists.txt | 118 ++++++++++-- cpp/third_party/CMakeLists.txt | 23 +++ cpp/tools/CMakeLists.txt | 17 +- packaging/README.md | 92 ++++++++++ packaging/homebrew/tsfile.rb | 71 ++++++++ 16 files changed, 853 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/cpp-packaging.yml create mode 100644 cpp/cmake/TsFileConfig.cmake.in create mode 100644 cpp/cmake/TsFilePublicHeaders.cmake create mode 100644 cpp/cmake/libtsfile.pc.in create mode 100644 cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt create mode 100644 cpp/cmake/tests/projects/InstalledConsumer/main.cc create mode 100644 cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt create mode 100644 cpp/cmake/tests/projects/PkgConfigConsumer/main.c create mode 100644 cpp/cmake/tests/projects/PkgConfigConsumer/main.cc create mode 100644 packaging/README.md create mode 100644 packaging/homebrew/tsfile.rb diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml new file mode 100644 index 000000000..ef5a457c2 --- /dev/null +++ b/.github/workflows/cpp-packaging.yml @@ -0,0 +1,130 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Cpp-Packaging + +on: + push: + branches: + - develop + - rc/** + paths: + - '.github/workflows/cpp-packaging.yml' + - 'cpp/**' + - 'packaging/**' + - 'LICENSE' + - 'NOTICE' + pull_request: + branches: + - develop + - rc/** + paths: + - '.github/workflows/cpp-packaging.yml' + - 'cpp/**' + - 'packaging/**' + - 'LICENSE' + - 'NOTICE' + workflow_dispatch: + +permissions: + contents: read + +jobs: + deb: + name: Debian package + runs-on: ubuntu-24.04 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install packaging tools + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev + + - name: Configure and build + run: | + cmake -S cpp -B build/deb -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + cmake --build build/deb --parallel + cmake --install build/deb + + - name: Build and inspect DEB packages + run: | + cpack --config build/deb/CPackConfig.cmake -G DEB + ls -lh ./*.deb + for package in ./*.deb; do + dpkg-deb --info "$package" + dpkg-deb --contents "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true + done + + - name: Upload DEB packages + uses: actions/upload-artifact@v4 + with: + name: tsfile-deb + path: '*.deb' + if-no-files-found: error + + rpm: + name: Fedora package + runs-on: ubuntu-24.04 + container: fedora:latest + timeout-minutes: 30 + steps: + - name: Install build and packaging tools + run: | + dnf install -y \ + cmake gcc-c++ make ninja-build pkgconf-pkg-config rpm-build \ + git curl tar xz unzip gzip + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Configure and build + run: | + cmake -S cpp -B build/rpm -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + cmake --build build/rpm --parallel + cmake --install build/rpm + + - name: Build and inspect RPM packages + run: | + cpack --config build/rpm/CPackConfig.cmake -G RPM + ls -lh ./*.rpm + for package in ./*.rpm; do + rpm -qip "$package" + rpm -qlp "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true + done + + - name: Upload RPM packages + uses: actions/upload-artifact@v4 + with: + name: tsfile-rpm + path: '*.rpm' + if-no-files-found: error diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c39bcbc28..567286fa5 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -34,6 +34,25 @@ if (POLICY CMP0074) endif () set(TsFile_CPP_VERSION 2.3.2.dev) +include(GNUInstallDirs) +include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFilePublicHeaders.cmake) + +# The package version identifies the source release. Development suffixes are +# intentionally removed from generated package metadata and never enter the +# SONAME. +string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" TSFILE_PACKAGE_VERSION + "${TsFile_CPP_VERSION}") +if ("${TSFILE_PACKAGE_VERSION}" STREQUAL "") + message(FATAL_ERROR + "TsFile_CPP_VERSION must start with a semantic version: " + "${TsFile_CPP_VERSION}") +endif () +set(TSFILE_ABI_VERSION "1" CACHE STRING + "TsFile shared-library ABI epoch (changes only on ABI breaks)") +if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") + message(FATAL_ERROR "TSFILE_ABI_VERSION must be a positive integer") +endif () + include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/DependencySource.cmake) list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") @@ -64,10 +83,18 @@ endif () message("cmake using: USE_CPP11=${USE_CPP11}") # MSVC has no /std:c++11; CMake maps this to the closest supported standard # (C++14 default on MSVC), which compiles the C++11 codebase fine. -set(CMAKE_CXX_STANDARD 11) +set(TSFILE_CXX_STANDARD "11" CACHE STRING + "C++ language standard used to build TsFile (11, 14, or 17)") +set_property(CACHE TSFILE_CXX_STANDARD PROPERTY STRINGS 11 14 17) +if (NOT TSFILE_CXX_STANDARD MATCHES "^(11|14|17)$") + message(FATAL_ERROR + "TSFILE_CXX_STANDARD must be one of 11, 14, or 17") +endif () +set(CMAKE_CXX_STANDARD ${TSFILE_CXX_STANDARD}) set(CMAKE_CXX_STANDARD_REQUIRED OFF) if (NOT MSVC) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} -std=c++${TSFILE_CXX_STANDARD}") endif () if (DEFINED ENV{CXX}) @@ -336,7 +363,8 @@ if (MSVC) # C++17 extensions), so we pin it explicitly for reproducibility. set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14") else () - set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11") + set(CMAKE_CXX_FLAGS + "$ENV{CXXFLAGS} -Wall -std=c++${TSFILE_CXX_STANDARD}") endif () add_subdirectory(third_party) @@ -418,3 +446,140 @@ endif () unset(_TSFILE_PROJECT_DEPENDENCIES) add_subdirectory(examples) + +# Install the compatibility header closure from the generated staging tree. The +# staging tree is populated by the existing copy_* targets and contains only +# headers selected by the reviewed closure manifest. The closure is retained +# for source compatibility; it is not the final public API boundary. +foreach (header_dir IN LISTS TSFILE_PUBLIC_HEADER_CLOSURE) + install(DIRECTORY "${LIBRARY_INCLUDE_DIR}/${header_dir}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/tsfile/${header_dir}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endforeach() +if (ENABLE_SIMD AND TSFILE_SIMDE_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_SIMDE_INCLUDE_ROOT}/simde/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/simde" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_ANTLR4_INCLUDE_ROOT}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () +if (ENABLE_ANTLR4 AND TSFILE_UTF8CPP_INCLUDE_ROOT) + install(DIRECTORY "${TSFILE_UTF8CPP_INCLUDE_ROOT}/" + DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}" + COMPONENT development + FILES_MATCHING PATTERN "*.h") +endif () + +set(TSFILE_LICENSE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE") +set(TSFILE_NOTICE_FILE "${CMAKE_CURRENT_SOURCE_DIR}/../NOTICE") +if (EXISTS "${TSFILE_LICENSE_FILE}") + install(FILES "${TSFILE_LICENSE_FILE}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/tsfile" + COMPONENT runtime) +endif() +if (EXISTS "${TSFILE_NOTICE_FILE}") + install(FILES "${TSFILE_NOTICE_FILE}" + DESTINATION "${CMAKE_INSTALL_DATADIR}/doc/tsfile" + COMPONENT runtime) +endif() + +include(CMakePackageConfigHelpers) +set(TSFILE_CMAKE_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile") +configure_package_config_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/TsFileConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + INSTALL_DESTINATION "${TSFILE_CMAKE_INSTALL_DIR}") +write_basic_package_version_file( + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfigVersion.cmake" + VERSION "${TSFILE_PACKAGE_VERSION}" + COMPATIBILITY SameMajorVersion) +install(FILES + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfigVersion.cmake" + DESTINATION "${TSFILE_CMAKE_INSTALL_DIR}" + COMPONENT development) + +set(TSFILE_PKGCONFIG_CFLAGS "") +foreach (_TSFILE_PUBLIC_FEATURE + ENABLE_ANTLR4 + ENABLE_MEM_STAT + ENABLE_SNAPPY + ENABLE_LZ4 + ENABLE_LZOKAY + ENABLE_ZLIB + ENABLE_GZIP + ENABLE_ZSTD + ENABLE_LZMA2 + ENABLE_THREADS + ENABLE_SIMD) + if (${_TSFILE_PUBLIC_FEATURE}) + string(APPEND TSFILE_PKGCONFIG_CFLAGS + " -D${_TSFILE_PUBLIC_FEATURE}") + endif () +endforeach () +configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libtsfile.pc.in" + "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" + @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" + COMPONENT development) +unset(_TSFILE_PUBLIC_FEATURE) + +# CPack is an opt-in local/package-builder integration. Native release jobs +# can configure the project with TSFILE_DEPENDENCY_SOURCE=SYSTEM when the +# target image provides every compatible dependency; AUTO remains the +# reproducible fallback for distribution images with older packages. +option(TSFILE_ENABLE_CPACK + "Enable CPack DEB/RPM package metadata generation" OFF) +if (TSFILE_ENABLE_CPACK) + set(CPACK_PACKAGE_NAME "tsfile") + set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") + set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "Apache TsFile C++ library") + set(CPACK_PACKAGE_DESCRIPTION + "Columnar storage library and command-line tools for time series data") + set(CPACK_PACKAGE_HOMEPAGE_URL "https://tsfile.apache.org/") + set(CPACK_PACKAGE_CONTACT "dev@tsfile.apache.org") + set(CPACK_RESOURCE_FILE_LICENSE "${TSFILE_LICENSE_FILE}") + set(CPACK_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}") + string(REPLACE "." ";" _TSFILE_PACKAGE_VERSION_PARTS + "${TSFILE_PACKAGE_VERSION}") + list(GET _TSFILE_PACKAGE_VERSION_PARTS 0 CPACK_PACKAGE_VERSION_MAJOR) + list(GET _TSFILE_PACKAGE_VERSION_PARTS 1 CPACK_PACKAGE_VERSION_MINOR) + list(GET _TSFILE_PACKAGE_VERSION_PARTS 2 CPACK_PACKAGE_VERSION_PATCH) + + set(CPACK_COMPONENTS_ALL runtime development tools) + set(CPACK_COMPONENT_DEVELOPMENT_DEPENDS runtime) + set(CPACK_COMPONENT_TOOLS_DEPENDS runtime) + set(CPACK_DEB_COMPONENT_INSTALL ON) + set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) + set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) + set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "tsfile") + set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_NAME "tsfile-dev") + set(CPACK_DEBIAN_TOOLS_PACKAGE_NAME "tsfile-tools") + set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_DEPENDS + "tsfile (= ${TSFILE_PACKAGE_VERSION})") + set(CPACK_DEBIAN_TOOLS_PACKAGE_DEPENDS + "tsfile (= ${TSFILE_PACKAGE_VERSION})") + + set(CPACK_RPM_COMPONENT_INSTALL ON) + set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") + set(CPACK_RPM_PACKAGE_RELEASE "1") + set(CPACK_RPM_PACKAGE_AUTOREQPROV ON) + set(CPACK_RPM_RUNTIME_PACKAGE_NAME "tsfile") + set(CPACK_RPM_DEVELOPMENT_PACKAGE_NAME "tsfile-devel") + set(CPACK_RPM_TOOLS_PACKAGE_NAME "tsfile-tools") + set(CPACK_RPM_DEVELOPMENT_PACKAGE_REQUIRES + "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + set(CPACK_RPM_TOOLS_PACKAGE_REQUIRES + "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + + include(CPack) + unset(_TSFILE_PACKAGE_VERSION_PARTS) +endif () diff --git a/cpp/cmake/TsFileConfig.cmake.in b/cpp/cmake/TsFileConfig.cmake.in new file mode 100644 index 000000000..eea711a9e --- /dev/null +++ b/cpp/cmake/TsFileConfig.cmake.in @@ -0,0 +1,29 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +@PACKAGE_INIT@ + +include(CMakeFindDependencyMacro) +if (@ENABLE_THREADS@) + find_dependency(Threads) +endif() + +include("${CMAKE_CURRENT_LIST_DIR}/TsFileTargets.cmake") + +check_required_components(TsFile) diff --git a/cpp/cmake/TsFilePublicHeaders.cmake b/cpp/cmake/TsFilePublicHeaders.cmake new file mode 100644 index 000000000..a517d3a9c --- /dev/null +++ b/cpp/cmake/TsFilePublicHeaders.cmake @@ -0,0 +1,47 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Public entry points are intentionally listed separately from the transitive +# header closure. The current public headers include implementation-level +# declarations, so the closure is installed until those dependencies can be +# hidden behind a stable facade. Adding a new public entry point requires an +# explicit review of this list. +set(TSFILE_PUBLIC_ENTRYPOINT_HEADERS + cwrapper/tsfile_cwrapper.h + cwrapper/tsfile_cwrapper_expression.h + reader/tsfile_reader.h + writer/tsfile_writer.h + writer/tsfile_table_writer.h + writer/tsfile_tree_writer.h) + +# These directories form the reviewed transitive closure of the entry points +# above. This is an installation compatibility closure, not a promise that +# every header is a stable public API. Compression, encoding, parser, and +# utility headers are implementation dependencies today, but are required for +# consumers to parse the installed public declarations. +set(TSFILE_PUBLIC_HEADER_CLOSURE + common + compress + cwrapper + encoding + file + parser + reader + utils + writer) diff --git a/cpp/cmake/libtsfile.pc.in b/cpp/cmake/libtsfile.pc.in new file mode 100644 index 000000000..ff820d0d8 --- /dev/null +++ b/cpp/cmake/libtsfile.pc.in @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The .pc file is installed below ${prefix}/@CMAKE_INSTALL_LIBDIR@/pkgconfig. +# Deriving the prefix from its own location keeps DESTDIR and relocations +# working without embedding the build-time install prefix. +prefix=${pcfiledir}/../.. +exec_prefix=${prefix} +libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ +includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ + +Name: libtsfile +Description: Apache TsFile C++ library +Version: @TSFILE_PACKAGE_VERSION@ +URL: https://tsfile.apache.org/ +Libs: -L${libdir} -ltsfile +Cflags: -I${includedir} -I${includedir}/tsfile@TSFILE_PKGCONFIG_CFLAGS@ diff --git a/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt b/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt index 61901e38a..551373afe 100644 --- a/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt +++ b/cpp/cmake/tests/projects/ANTLR4Dependency/CMakeLists.txt @@ -18,7 +18,7 @@ under the License. ]] cmake_minimum_required(VERSION 3.11) -project(TsFileANTLR4DependencyTest NONE) +project(TsFileANTLR4DependencyTest LANGUAGES CXX) get_filename_component(_TSFILE_CMAKE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../.." ABSOLUTE) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt new file mode 100644 index 000000000..bf0f83e34 --- /dev/null +++ b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt @@ -0,0 +1,42 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +cmake_minimum_required(VERSION 3.11) +project(TsFileInstalledConsumer LANGUAGES CXX) + +find_package(TsFile CONFIG REQUIRED) + +get_target_property(_TSFILE_INCLUDE_DIRS TsFile::tsfile + INTERFACE_INCLUDE_DIRECTORIES) +if (NOT _TSFILE_INCLUDE_DIRS MATCHES "/include") + message(FATAL_ERROR "TsFile target does not expose the installed include root") +endif () +if (_TSFILE_INCLUDE_DIRS MATCHES "cpp/src|build/") + message(FATAL_ERROR "TsFile target leaks a source or build include path: ${_TSFILE_INCLUDE_DIRS}") +endif () + +get_target_property(_TSFILE_LINK_LIBRARIES TsFile::tsfile + INTERFACE_LINK_LIBRARIES) +if (_TSFILE_LINK_LIBRARIES MATCHES "(^|;)[^;]*_obj($|;)|cpp/src|build/") + message(FATAL_ERROR + "TsFile target leaks private build link requirements: " + "${_TSFILE_LINK_LIBRARIES}") +endif () + +add_executable(tsfile_installed_consumer main.cc) +target_link_libraries(tsfile_installed_consumer PRIVATE TsFile::tsfile) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/main.cc b/cpp/cmake/tests/projects/InstalledConsumer/main.cc new file mode 100644 index 000000000..8d242f678 --- /dev/null +++ b/cpp/cmake/tests/projects/InstalledConsumer/main.cc @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include +#include +#include +#include + +int main() { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt new file mode 100644 index 000000000..1aaabc396 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/CMakeLists.txt @@ -0,0 +1,32 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] +cmake_minimum_required(VERSION 3.11) +project(TsFilePkgConfigConsumer LANGUAGES C CXX) + +find_package(PkgConfig REQUIRED) +pkg_check_modules(TsFile REQUIRED IMPORTED_TARGET libtsfile) +if (NOT TsFile_INCLUDEDIR MATCHES "/include$") + message(FATAL_ERROR "pkg-config did not report the installed include root: ${TsFile_INCLUDEDIR}") +endif () + +add_executable(tsfile_pkgconfig_consumer main.c) +target_link_libraries(tsfile_pkgconfig_consumer PRIVATE PkgConfig::TsFile) + +add_executable(tsfile_pkgconfig_cpp_consumer main.cc) +target_link_libraries(tsfile_pkgconfig_cpp_consumer PRIVATE PkgConfig::TsFile) diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c new file mode 100644 index 000000000..f32e5a550 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include + +int main(void) { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc new file mode 100644 index 000000000..be94b8543 --- /dev/null +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#include + +int main() { return 0; } diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 2ab2dd5d6..8854e5d6c 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -79,6 +79,18 @@ add_subdirectory(reader) add_subdirectory(utils) add_subdirectory(writer) +# Object libraries need the directory-level project dependencies on older +# CMake versions, but those dependencies are implementation details of the +# shared library and must not become exported consumer requirements. Keep the +# platform thread target in the directory scope for its public link interface, +# then link codecs explicitly as PRIVATE below. +get_property(_TSFILE_SRC_LINK_LIBRARIES DIRECTORY PROPERTY LINK_LIBRARIES) +if (ENABLE_THREADS) + set_property(DIRECTORY PROPERTY LINK_LIBRARIES Threads::Threads) +else () + set_property(DIRECTORY PROPERTY LINK_LIBRARIES "") +endif () + set(_TSFILE_OBJECT_TARGETS common_obj compress_obj @@ -125,35 +137,115 @@ if (${COV_ENABLED}) else() set(COV_LINK_LIB -lgcov) endif() - if (CMAKE_VERSION VERSION_LESS "3.12") - target_link_libraries(tsfile ${COV_LINK_LIB}) - else() - target_link_libraries(tsfile ${_TSFILE_OBJECT_TARGETS} ${COV_LINK_LIB}) - endif() + target_link_libraries(tsfile PRIVATE ${COV_LINK_LIB}) else() message("Disable code cov...") - if (NOT CMAKE_VERSION VERSION_LESS "3.12") - target_link_libraries(tsfile ${_TSFILE_OBJECT_TARGETS}) - endif() endif() +# Linking object libraries directly makes static-library exports depend on +# private, non-exported object targets. Add their object files as sources +# instead; this works on the CMake 3.11 baseline and keeps the install target +# self-contained for both shared and static builds. +foreach (_TSFILE_OBJECT_TARGET IN LISTS _TSFILE_OBJECT_TARGETS) + target_sources(tsfile PRIVATE + $) +endforeach () + +if (NOT "${_TSFILE_PROJECT_DEPENDENCIES}" STREQUAL "") + target_link_libraries(tsfile PRIVATE ${_TSFILE_PROJECT_DEPENDENCIES}) +endif () + unset(_TSFILE_OBJECT_SOURCES) unset(_TSFILE_OBJECT_TARGET) unset(_TSFILE_OBJECT_TARGETS) add_dependencies(tsfile utils_obj encoding_obj) +set_property(DIRECTORY PROPERTY LINK_LIBRARIES + "${_TSFILE_SRC_LINK_LIBRARIES}") +unset(_TSFILE_SRC_LINK_LIBRARIES) + if (TSFILE_BUILD_SHARED) - set(LIBTSFILE_PROJECT_VERSION ${TsFile_CPP_VERSION}) - set(LIBTSFILE_SO_VERSION ${TsFile_CPP_VERSION}) + # Keep development suffixes out of the install name and SONAME. The ABI + # epoch changes only for incompatible binary interface changes. + set(LIBTSFILE_PROJECT_VERSION ${TSFILE_PACKAGE_VERSION}) + set(LIBTSFILE_SO_VERSION ${TSFILE_ABI_VERSION}) set_target_properties(tsfile PROPERTIES VERSION ${LIBTSFILE_PROJECT_VERSION}) set_target_properties(tsfile PROPERTIES SOVERSION ${LIBTSFILE_SO_VERSION}) endif() +add_library(TsFile::tsfile ALIAS tsfile) + +target_include_directories(tsfile PUBLIC + $ + $ + $ + $) +if (ENABLE_SIMD AND TSFILE_SIMDE_INCLUDE_ROOT) + # Public TsFile headers include SIMDe directly. Copy the resolved header + # tree into the install prefix so consumers do not need the build host's + # system or dependency-cache paths. + target_include_directories(tsfile PUBLIC + $ + $) +endif () +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_INCLUDE_ROOT) + target_include_directories(tsfile PUBLIC + $ + $) +endif () +if (ENABLE_ANTLR4 AND TSFILE_UTF8CPP_INCLUDE_ROOT) + target_include_directories(tsfile PUBLIC + $ + $) +endif () +target_compile_features(tsfile PUBLIC cxx_std_11) +set(_TSFILE_PUBLIC_FEATURE_DEFINITIONS "") +foreach (_TSFILE_PUBLIC_FEATURE + ENABLE_ANTLR4 + ENABLE_MEM_STAT + ENABLE_SNAPPY + ENABLE_LZ4 + ENABLE_LZOKAY + ENABLE_ZLIB + ENABLE_GZIP + ENABLE_ZSTD + ENABLE_LZMA2 + ENABLE_THREADS + ENABLE_SIMD) + if (${_TSFILE_PUBLIC_FEATURE}) + list(APPEND _TSFILE_PUBLIC_FEATURE_DEFINITIONS + ${_TSFILE_PUBLIC_FEATURE}) + endif () +endforeach () +if (NOT "${_TSFILE_PUBLIC_FEATURE_DEFINITIONS}" STREQUAL "") + target_compile_definitions(tsfile INTERFACE + ${_TSFILE_PUBLIC_FEATURE_DEFINITIONS}) +endif () +unset(_TSFILE_PUBLIC_FEATURE) +unset(_TSFILE_PUBLIC_FEATURE_DEFINITIONS) + +if (TSFILE_BUILD_SHARED) + # A shared library already resolves its private codec and object-library + # dependencies at link time. Do not export those build-only targets as + # consumer link requirements; they are not installed with the package. + set_property(TARGET tsfile PROPERTY INTERFACE_LINK_LIBRARIES "") +endif () + # A shared library is a RUNTIME plus an import ARCHIVE on Windows and a LIBRARY # on Unix. A static library is an ARCHIVE on every platform. Cover all three so # the install step works for either library type. install(TARGETS tsfile - RUNTIME DESTINATION ${LIBRARY_OUTPUT_PATH} - LIBRARY DESTINATION ${LIBRARY_OUTPUT_PATH} - ARCHIVE DESTINATION ${LIBRARY_OUTPUT_PATH}) + EXPORT TsFileTargets + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT runtime + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT runtime + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + COMPONENT development) + +install(EXPORT TsFileTargets + FILE TsFileTargets.cmake + NAMESPACE TsFile:: + DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/TsFile + COMPONENT development) diff --git a/cpp/third_party/CMakeLists.txt b/cpp/third_party/CMakeLists.txt index f7a8851b5..203b93c34 100755 --- a/cpp/third_party/CMakeLists.txt +++ b/cpp/third_party/CMakeLists.txt @@ -35,6 +35,12 @@ if (ENABLE_ANTLR4) "${TSFILE_UTF8CPP_SOURCE_DIR}/source") add_library(tsfile_antlr4_bundled STATIC ${_TSFILE_ANTLR4_SOURCES}) + set(TSFILE_ANTLR4_INCLUDE_ROOT + "${TSFILE_ANTLR4_SOURCE_DIR}/runtime/Cpp/runtime/src" + CACHE INTERNAL "Resolved ANTLR4 include root" FORCE) + set(TSFILE_UTF8CPP_INCLUDE_ROOT + "${TSFILE_UTF8CPP_SOURCE_DIR}/source" + CACHE INTERNAL "Resolved utf8cpp include root" FORCE) set_target_properties(tsfile_antlr4_bundled PROPERTIES POSITION_INDEPENDENT_CODE ON CXX_STANDARD 11) @@ -93,6 +99,17 @@ if (ENABLE_ANTLR4) target_link_libraries(tsfile_antlr4 INTERFACE tsfile_antlr4_bundled) else () + set(TSFILE_ANTLR4_INCLUDE_ROOT + "${TSFILE_ANTLR4_SYSTEM_INCLUDE_DIR}" + CACHE INTERNAL "Resolved ANTLR4 include root" FORCE) + if (TARGET utf8cpp) + get_target_property(_TSFILE_UTF8CPP_INCLUDE_ROOT utf8cpp + INTERFACE_INCLUDE_DIRECTORIES) + set(TSFILE_UTF8CPP_INCLUDE_ROOT + "${_TSFILE_UTF8CPP_INCLUDE_ROOT}" + CACHE INTERNAL "Resolved utf8cpp include root" FORCE) + unset(_TSFILE_UTF8CPP_INCLUDE_ROOT) + endif () target_link_libraries(tsfile_antlr4 INTERFACE ${TSFILE_ANTLR4_SYSTEM_TARGET}) if (TSFILE_ANTLR4_SYSTEM_INCLUDE_DIR) @@ -449,9 +466,15 @@ if (ENABLE_SIMD) add_library(tsfile_simde INTERFACE) if (TSFILE_SIMDE_SOURCE STREQUAL "BUNDLED") + set(TSFILE_SIMDE_INCLUDE_ROOT "${TSFILE_SIMDE_SOURCE_DIR}" + CACHE INTERNAL "Resolved SIMDe include root" FORCE) target_include_directories(tsfile_simde INTERFACE "${TSFILE_SIMDE_SOURCE_DIR}") else () + get_target_property(TSFILE_SIMDE_INCLUDE_ROOT simde::simde + INTERFACE_INCLUDE_DIRECTORIES) + set(TSFILE_SIMDE_INCLUDE_ROOT "${TSFILE_SIMDE_INCLUDE_ROOT}" + CACHE INTERNAL "Resolved SIMDe include root" FORCE) target_link_libraries(tsfile_simde INTERFACE simde::simde) endif () add_library(TsFile::SIMDe ALIAS tsfile_simde) diff --git a/cpp/tools/CMakeLists.txt b/cpp/tools/CMakeLists.txt index 00bc20ea0..b78d06442 100644 --- a/cpp/tools/CMakeLists.txt +++ b/cpp/tools/CMakeLists.txt @@ -59,4 +59,19 @@ set_target_properties(tsfile_cli PROPERTIES OUTPUT_NAME tsfile-cli RUNTIME_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/bin) -install(TARGETS tsfile_cli RUNTIME DESTINATION bin) +# Keep the installed CLI self-contained with the sibling lib directory. This +# is relative to the executable location and therefore remains valid when the +# installation prefix is relocated. +if (APPLE) + set_target_properties(tsfile_cli PROPERTIES + INSTALL_RPATH "@loader_path/../${CMAKE_INSTALL_LIBDIR}" + BUILD_WITH_INSTALL_RPATH TRUE) +elseif (UNIX) + set_target_properties(tsfile_cli PROPERTIES + INSTALL_RPATH "\$ORIGIN/../${CMAKE_INSTALL_LIBDIR}" + BUILD_WITH_INSTALL_RPATH TRUE) +endif () + +install(TARGETS tsfile_cli + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT tools) diff --git a/packaging/README.md b/packaging/README.md new file mode 100644 index 000000000..ff77cf46b --- /dev/null +++ b/packaging/README.md @@ -0,0 +1,92 @@ + + +# TsFile C++ Packages + +The C++ installation contract is shared by all package formats. It installs +the library, the current compatibility header closure, `TsFileConfig.cmake`, +pkg-config metadata, the CLI, and Apache license files below one prefix. + +The header closure mirrors the include relationships used by the current C++ +implementation. It is intentionally broader than the long-term stable public +API; a separate API cleanup will narrow it in a future version. + +## Portable archive + +Build a relocatable source archive on any host with CMake and CPack: + +```bash +cmake -S cpp -B cpp/build/package \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO +cmake --build cpp/build/package --parallel +cmake --install cpp/build/package +cpack --config cpp/build/package/CPackConfig.cmake -G TGZ +``` + +The resulting `tsfile--.tar.gz` contains a standard +prefix layout and can be unpacked at `/usr/local`, a user directory, or a +relocated application prefix. + +## Native Linux packages + +DEB and RPM packages must be built in the target distribution environment so +CPack can run the native dependency scanner (`dpkg-shlibdeps` or +`rpmbuild`). On Debian/Ubuntu use `-G DEB`; on Fedora/RHEL use `-G RPM`: + +```bash +cmake -S cpp -B cpp/build/package \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO +cmake --build cpp/build/package --parallel +cmake --install cpp/build/package +cpack --config cpp/build/package/CPackConfig.cmake -G DEB +# or: cpack --config cpp/build/package/CPackConfig.cmake -G RPM +``` + +`SYSTEM` can be used instead of `AUTO` when the build image provides every +compatible dependency, including ANTLR4 4.9.x. `AUTO` is the reproducible +release default and uses the verified source fallback for unavailable or +incompatible distro versions. + +The repository workflow `Cpp-Packaging` builds and uploads native DEB and RPM +artifacts on packaging-related changes. The DEB job runs on Ubuntu and the RPM +job runs in Fedora, so each package is generated with its native dependency +metadata tool. + +## macOS + +Homebrew is the native macOS distribution path. The formula is +`packaging/homebrew/tsfile.rb`; it builds the same CMake install layout and +tests both a C++ consumer and `tsfile-cli --version` after installation. + +```bash +brew install ./packaging/homebrew/tsfile.rb +``` + +The stable formula URL and checksum must be updated to the ASF source archive +when the first release containing this packaging work is published. diff --git a/packaging/homebrew/tsfile.rb b/packaging/homebrew/tsfile.rb new file mode 100644 index 000000000..3e96f3dca --- /dev/null +++ b/packaging/homebrew/tsfile.rb @@ -0,0 +1,71 @@ +# typed: strict +# frozen_string_literal: true + +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Formula for the Apache TsFile C++ library. +class Tsfile < Formula + desc "Columnar storage library for time series data" + homepage "https://tsfile.apache.org/" + # Replace this development snapshot with the ASF source archive and its + # release checksum when the first native TsFile release is published. + url "https://github.com/apache/tsfile/archive/d4c3c94690cf9af819bc26bec7114a0de7900359.tar.gz" + version "2.3.2" + sha256 "369cf43742601fe6107b299647d2c01f705bfc9ff111402d7b5ddb9b50f40f16" + license "Apache-2.0" + head "https://github.com/apache/tsfile.git", branch: "develop" + + depends_on "cmake" => :build + depends_on "lz4" + depends_on "simde" + depends_on "snappy" + depends_on "utf8cpp" + depends_on "zstd" + + uses_from_macos "zlib" + + def install + args = %W[ + -DBUILD_TEST=OFF + -DBUILD_TOOLS=ON + -DENABLE_LZMA2=OFF + -DTSFILE_DEPENDENCY_SOURCE=AUTO + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + -DCMAKE_INSTALL_RPATH=#{rpath} + ] + + system "cmake", "-S", "cpp", "-B", "build", *args, *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + (testpath / "test.cpp").write <<~CPP + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + + system ENV.cxx, "-std=c++11", "test.cpp", "-I#{include}", "-L#{lib}", + "-Wl,-rpath,#{lib}", "-ltsfile", "-o", "test" + system "./test" + system bin / "tsfile-cli", "--version" + end +end From 3e4220f1b6894d0f1dc039cd35a0565d3b5bb403 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 17:14:33 +0800 Subject: [PATCH 02/31] fix(ci): remove redundant install step before cpack --- .github/workflows/cpp-packaging.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index ef5a457c2..197cd7b18 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -68,7 +68,6 @@ jobs: -DTSFILE_DEPENDENCY_SOURCE=AUTO \ -DTSFILE_ENABLE_NATIVE_ARCH=OFF cmake --build build/deb --parallel - cmake --install build/deb - name: Build and inspect DEB packages run: | @@ -111,7 +110,6 @@ jobs: -DTSFILE_DEPENDENCY_SOURCE=AUTO \ -DTSFILE_ENABLE_NATIVE_ARCH=OFF cmake --build build/rpm --parallel - cmake --install build/rpm - name: Build and inspect RPM packages run: | From 5fccdc4ddf5577716adc25d840fe69ada5c0a841 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 20 Aug 2026 17:31:10 +0800 Subject: [PATCH 03/31] fix(ci): install uuid headers for bundled ANTLR4 --- .github/workflows/cpp-packaging.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml index 197cd7b18..54fa92211 100644 --- a/.github/workflows/cpp-packaging.yml +++ b/.github/workflows/cpp-packaging.yml @@ -56,7 +56,7 @@ jobs: - name: Install packaging tools run: | sudo apt-get update - sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev + sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev uuid-dev - name: Configure and build run: | @@ -95,7 +95,7 @@ jobs: run: | dnf install -y \ cmake gcc-c++ make ninja-build pkgconf-pkg-config rpm-build \ - git curl tar xz unzip gzip + git curl tar xz unzip gzip libuuid-devel - name: Checkout repository uses: actions/checkout@v7 From 4f5ed026c0a0b36702c2055fa6aa78fcb835bec8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:09:03 +0800 Subject: [PATCH 04/31] feat(ci): add native package development versions --- cpp/CMakeLists.txt | 23 ++- packaging/scripts/native_package_versions.py | 128 ++++++++++++++++ .../tests/test_native_package_versions.py | 144 ++++++++++++++++++ 3 files changed, 289 insertions(+), 6 deletions(-) create mode 100644 packaging/scripts/native_package_versions.py create mode 100644 packaging/tests/test_native_package_versions.py diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 567286fa5..4b6f868b2 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -47,6 +47,14 @@ if ("${TSFILE_PACKAGE_VERSION}" STREQUAL "") "TsFile_CPP_VERSION must start with a semantic version: " "${TsFile_CPP_VERSION}") endif () +set(TSFILE_ARCHIVE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "Version used in portable archive names") +set(TSFILE_DEBIAN_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "Complete Debian package version") +set(TSFILE_RPM_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}" CACHE STRING + "RPM Version value") +set(TSFILE_RPM_PACKAGE_RELEASE "1" CACHE STRING + "RPM Release value") set(TSFILE_ABI_VERSION "1" CACHE STRING "TsFile shared-library ABI epoch (changes only on ABI breaks)") if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") @@ -547,7 +555,7 @@ if (TSFILE_ENABLE_CPACK) set(CPACK_PACKAGE_HOMEPAGE_URL "https://tsfile.apache.org/") set(CPACK_PACKAGE_CONTACT "dev@tsfile.apache.org") set(CPACK_RESOURCE_FILE_LICENSE "${TSFILE_LICENSE_FILE}") - set(CPACK_PACKAGE_VERSION "${TSFILE_PACKAGE_VERSION}") + set(CPACK_PACKAGE_VERSION "${TSFILE_ARCHIVE_VERSION}") string(REPLACE "." ";" _TSFILE_PACKAGE_VERSION_PARTS "${TSFILE_PACKAGE_VERSION}") list(GET _TSFILE_PACKAGE_VERSION_PARTS 0 CPACK_PACKAGE_VERSION_MAJOR) @@ -560,25 +568,28 @@ if (TSFILE_ENABLE_CPACK) set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) + set(CPACK_DEBIAN_PACKAGE_VERSION "${TSFILE_DEBIAN_PACKAGE_VERSION}") set(CPACK_DEBIAN_RUNTIME_PACKAGE_NAME "tsfile") set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_NAME "tsfile-dev") set(CPACK_DEBIAN_TOOLS_PACKAGE_NAME "tsfile-tools") set(CPACK_DEBIAN_DEVELOPMENT_PACKAGE_DEPENDS - "tsfile (= ${TSFILE_PACKAGE_VERSION})") + "tsfile (= ${TSFILE_DEBIAN_PACKAGE_VERSION})") set(CPACK_DEBIAN_TOOLS_PACKAGE_DEPENDS - "tsfile (= ${TSFILE_PACKAGE_VERSION})") + "tsfile (= ${TSFILE_DEBIAN_PACKAGE_VERSION})") set(CPACK_RPM_COMPONENT_INSTALL ON) + set(CPACK_RPM_FILE_NAME RPM-DEFAULT) set(CPACK_RPM_PACKAGE_LICENSE "Apache-2.0") - set(CPACK_RPM_PACKAGE_RELEASE "1") + set(CPACK_RPM_PACKAGE_VERSION "${TSFILE_RPM_PACKAGE_VERSION}") + set(CPACK_RPM_PACKAGE_RELEASE "${TSFILE_RPM_PACKAGE_RELEASE}") set(CPACK_RPM_PACKAGE_AUTOREQPROV ON) set(CPACK_RPM_RUNTIME_PACKAGE_NAME "tsfile") set(CPACK_RPM_DEVELOPMENT_PACKAGE_NAME "tsfile-devel") set(CPACK_RPM_TOOLS_PACKAGE_NAME "tsfile-tools") set(CPACK_RPM_DEVELOPMENT_PACKAGE_REQUIRES - "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + "tsfile = ${TSFILE_RPM_PACKAGE_VERSION}-${TSFILE_RPM_PACKAGE_RELEASE}") set(CPACK_RPM_TOOLS_PACKAGE_REQUIRES - "tsfile = ${TSFILE_PACKAGE_VERSION}-1") + "tsfile = ${TSFILE_RPM_PACKAGE_VERSION}-${TSFILE_RPM_PACKAGE_RELEASE}") include(CPack) unset(_TSFILE_PACKAGE_VERSION_PARTS) diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py new file mode 100644 index 000000000..f9849267f --- /dev/null +++ b/packaging/scripts/native_package_versions.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Build deterministic development versions for native TsFile packages.""" + +from __future__ import annotations + +import argparse +import json +import re +from pathlib import Path +from typing import Sequence + + +_SOURCE_VERSION_RE = re.compile( + r"^(?P(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))\.dev$" +) +_CMAKE_VERSION_RE = re.compile( + r"^\s*set\(\s*TsFile_CPP_VERSION\s+(?P[^\s)]+)\s*\)\s*(?:#.*)?$", + re.MULTILINE, +) +_BUILD_DATE_RE = re.compile(r"^[0-9]{8}$") +_GIT_SHA_RE = re.compile(r"^[0-9a-fA-F]{7,}$") + + +def read_cpp_version(path: Path) -> str: + """Read and validate the development version declared by CMake.""" + match = _CMAKE_VERSION_RE.search(path.read_text(encoding="utf-8")) + if match is None or _SOURCE_VERSION_RE.fullmatch(match.group("version")) is None: + raise ValueError(f"{path} must declare TsFile_CPP_VERSION as MAJOR.MINOR.PATCH.dev") + return match.group("version") + + +def _positive_integer(value: int, name: str) -> str: + if isinstance(value, bool) or not isinstance(value, int) or value < 1: + raise ValueError(f"{name} must be a positive integer") + return str(value) + + +def build_versions( + source_version: str, + build_date: str, + run_number: int, + run_attempt: int, + git_sha: str, +) -> dict[str, str]: + """Create archive, DEB, RPM, and Homebrew versions from one build identity.""" + source_match = _SOURCE_VERSION_RE.fullmatch(source_version) + if source_match is None: + raise ValueError("source_version must be MAJOR.MINOR.PATCH.dev") + if _BUILD_DATE_RE.fullmatch(build_date) is None: + raise ValueError("build_date must be YYYYMMDD") + if _GIT_SHA_RE.fullmatch(git_sha) is None: + raise ValueError("git_sha must contain at least seven hexadecimal characters") + + run_number_text = _positive_integer(run_number, "run_number") + run_attempt_text = _positive_integer(run_attempt, "run_attempt") + base_version = source_match.group("base") + short_sha = git_sha[:7].lower() + build_identity = f"{build_date}.{run_number_text}.{run_attempt_text}.g{short_sha}" + + return { + "base_version": base_version, + "logical_version": f"{base_version}.dev0+{build_identity}", + "deb_version": f"{base_version}~dev0+{build_identity}-1", + "rpm_version": base_version, + "rpm_release": f"0.dev0.{build_identity}.el9", + "archive_version": f"{base_version}-dev0.{build_identity}", + "homebrew_version": f"{base_version}.dev0.{build_identity}", + } + + +def _parse_positive_integer(value: str) -> int: + if not re.fullmatch(r"[1-9][0-9]*", value): + raise argparse.ArgumentTypeError("must be a positive integer") + return int(value) + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Generate deterministic development versions for native TsFile packages." + ) + parser.add_argument("--cmake-file", type=Path, required=True) + parser.add_argument("--build-date", required=True) + parser.add_argument("--run-number", type=_parse_positive_integer, required=True) + parser.add_argument("--run-attempt", type=_parse_positive_integer, required=True) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--json-out", type=Path, required=True) + parser.add_argument("--github-output", type=Path) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Write the generated versions to JSON and, optionally, GitHub outputs.""" + arguments = _argument_parser().parse_args(argv) + versions = build_versions( + read_cpp_version(arguments.cmake_file), + arguments.build_date, + arguments.run_number, + arguments.run_attempt, + arguments.git_sha, + ) + arguments.json_out.write_text(json.dumps(versions, indent=2) + "\n", encoding="utf-8") + if arguments.github_output is not None: + arguments.github_output.write_text( + "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n", + encoding="utf-8", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py new file mode 100644 index 000000000..6b864c44c --- /dev/null +++ b/packaging/tests/test_native_package_versions.py @@ -0,0 +1,144 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/native_package_versions.py" +CPP_CMAKE_FILE = REPOSITORY_ROOT / "cpp/CMakeLists.txt" + + +def load_version_module(): + if not MODULE_PATH.is_file(): + return None + spec = importlib.util.spec_from_file_location("native_package_versions", MODULE_PATH) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load native package version helper from {MODULE_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class NativePackageVersionsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_version_module() + + def require_module(self): + self.assertIsNotNone( + self.module, "native_package_versions helper must exist for native package builds" + ) + return self.module + + def test_reads_development_version_from_cpp_cmake(self): + module = self.require_module() + + self.assertEqual(module.read_cpp_version(CPP_CMAKE_FILE), "2.5.0.dev") + + def test_builds_exact_development_package_versions(self): + module = self.require_module() + + self.assertEqual( + module.build_versions("2.5.0.dev", "20260910", 123, 1, "abcdef123456"), + { + "base_version": "2.5.0", + "logical_version": "2.5.0.dev0+20260910.123.1.gabcdef1", + "deb_version": "2.5.0~dev0+20260910.123.1.gabcdef1-1", + "rpm_version": "2.5.0", + "rpm_release": "0.dev0.20260910.123.1.gabcdef1.el9", + "archive_version": "2.5.0-dev0.20260910.123.1.gabcdef1", + "homebrew_version": "2.5.0.dev0.20260910.123.1.gabcdef1", + }, + ) + + def test_rejects_invalid_source_versions(self): + module = self.require_module() + + for source_version in ("2.5.0", "2.5.dev"): + with self.subTest(source_version=source_version): + with self.assertRaises(ValueError): + module.build_versions(source_version, "20260910", 123, 1, "abcdef123456") + + def test_rejects_invalid_build_identity_fields(self): + module = self.require_module() + + invalid_cases = ( + ("2026-09-10", 123, 1, "abcdef123456"), + ("2026091", 123, 1, "abcdef123456"), + ("20260910", "run", 1, "abcdef123456"), + ("20260910", 123, "attempt", "abcdef123456"), + ("20260910", 123, 1, "abcdef"), + ("20260910", 123, 1, "abcdefg123456"), + ) + for build_date, run_number, run_attempt, git_sha in invalid_cases: + with self.subTest( + build_date=build_date, + run_number=run_number, + run_attempt=run_attempt, + git_sha=git_sha, + ): + with self.assertRaises(ValueError): + module.build_versions( + "2.5.0.dev", build_date, run_number, run_attempt, git_sha + ) + + def test_cli_writes_json_and_github_outputs(self): + module = self.require_module() + expected = module.build_versions("2.5.0.dev", "20260910", 123, 1, "abcdef123456") + + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = Path(temporary_directory) + json_output = temporary_path / "versions.json" + github_output = temporary_path / "github-output.txt" + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(CPP_CMAKE_FILE), + "--build-date", + "20260910", + "--run-number", + "123", + "--run-attempt", + "1", + "--git-sha", + "abcdef123456", + "--json-out", + str(json_output), + "--github-output", + str(github_output), + ], + check=True, + ) + + self.assertEqual(json.loads(json_output.read_text(encoding="utf-8")), expected) + self.assertEqual( + github_output.read_text(encoding="utf-8").splitlines(), + [f"{key}={value}" for key, value in expected.items()], + ) + + +if __name__ == "__main__": + unittest.main() From 15e69f2879e03235cfbdf1b92990ce9dc1d29497 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:16:10 +0800 Subject: [PATCH 05/31] fix(ci): validate native package build identity --- packaging/scripts/native_package_versions.py | 10 ++++++---- packaging/tests/test_native_package_versions.py | 4 +++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py index f9849267f..d4bd653eb 100644 --- a/packaging/scripts/native_package_versions.py +++ b/packaging/scripts/native_package_versions.py @@ -23,6 +23,7 @@ import argparse import json import re +from datetime import datetime from pathlib import Path from typing import Sequence @@ -65,6 +66,7 @@ def build_versions( raise ValueError("source_version must be MAJOR.MINOR.PATCH.dev") if _BUILD_DATE_RE.fullmatch(build_date) is None: raise ValueError("build_date must be YYYYMMDD") + datetime.strptime(build_date, "%Y%m%d") if _GIT_SHA_RE.fullmatch(git_sha) is None: raise ValueError("git_sha must contain at least seven hexadecimal characters") @@ -117,10 +119,10 @@ def main(argv: Sequence[str] | None = None) -> int: ) arguments.json_out.write_text(json.dumps(versions, indent=2) + "\n", encoding="utf-8") if arguments.github_output is not None: - arguments.github_output.write_text( - "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n", - encoding="utf-8", - ) + with arguments.github_output.open("a", encoding="utf-8") as github_output: + github_output.write( + "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n" + ) return 0 diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py index 6b864c44c..a2cc05221 100644 --- a/packaging/tests/test_native_package_versions.py +++ b/packaging/tests/test_native_package_versions.py @@ -86,6 +86,7 @@ def test_rejects_invalid_build_identity_fields(self): invalid_cases = ( ("2026-09-10", 123, 1, "abcdef123456"), ("2026091", 123, 1, "abcdef123456"), + ("20261340", 123, 1, "abcdef123456"), ("20260910", "run", 1, "abcdef123456"), ("20260910", 123, "attempt", "abcdef123456"), ("20260910", 123, 1, "abcdef"), @@ -111,6 +112,7 @@ def test_cli_writes_json_and_github_outputs(self): temporary_path = Path(temporary_directory) json_output = temporary_path / "versions.json" github_output = temporary_path / "github-output.txt" + github_output.write_text("existing=value\n", encoding="utf-8") subprocess.run( [ sys.executable, @@ -136,7 +138,7 @@ def test_cli_writes_json_and_github_outputs(self): self.assertEqual(json.loads(json_output.read_text(encoding="utf-8")), expected) self.assertEqual( github_output.read_text(encoding="utf-8").splitlines(), - [f"{key}={value}" for key, value in expected.items()], + ["existing=value", *[f"{key}={value}" for key, value in expected.items()]], ) From b5a8b30c8ed007f468dddac7e68884cba2b8262b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:26:29 +0800 Subject: [PATCH 06/31] feat(ci): build native Linux package artifacts --- .github/workflows/native-packages.yml | 328 ++++++++++++++++++++++++++ packaging/README.md | 11 +- 2 files changed, 335 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/native-packages.yml diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml new file mode 100644 index 000000000..5c1bafa5e --- /dev/null +++ b/.github/workflows/native-packages.yml @@ -0,0 +1,328 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Build native package artifacts + +on: + workflow_dispatch: + +permissions: + contents: read + +jobs: + prepare: + name: Prepare package versions + runs-on: ubuntu-24.04 + outputs: + base_version: ${{ steps.versions.outputs.base_version }} + logical_version: ${{ steps.versions.outputs.logical_version }} + deb_version: ${{ steps.versions.outputs.deb_version }} + rpm_version: ${{ steps.versions.outputs.rpm_version }} + rpm_release: ${{ steps.versions.outputs.rpm_release }} + archive_version: ${{ steps.versions.outputs.archive_version }} + homebrew_version: ${{ steps.versions.outputs.homebrew_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Generate package versions + id: versions + run: | + python3 packaging/scripts/native_package_versions.py \ + --cmake-file cpp/CMakeLists.txt \ + --build-date "$(date -u +%Y%m%d)" \ + --run-number "${{ github.run_number }}" \ + --run-attempt "${{ github.run_attempt }}" \ + --git-sha "${{ github.sha }}" \ + --json-out versions.json \ + --github-output "$GITHUB_OUTPUT" + + - name: Upload package versions + uses: actions/upload-artifact@v7 + with: + name: native-package-versions + path: versions.json + if-no-files-found: error + retention-days: 14 + + build-deb: + name: Build Ubuntu 22.04 DEB packages + needs: prepare + runs-on: ubuntu-22.04 + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Install DEB build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + cmake \ + ninja-build \ + pkg-config \ + dpkg-dev \ + uuid-dev + + - name: Configure and build + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + TSFILE_DEBIAN_PACKAGE_VERSION: ${{ needs.prepare.outputs.deb_version }} + run: | + cmake -S cpp -B build/deb -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF \ + -DTSFILE_ARCHIVE_VERSION="$TSFILE_ARCHIVE_VERSION" \ + -DTSFILE_DEBIAN_PACKAGE_VERSION="$TSFILE_DEBIAN_PACKAGE_VERSION" + cmake --build build/deb --parallel + + - name: Build and verify DEB packages + env: + EXPECTED_DEB_VERSION: ${{ needs.prepare.outputs.deb_version }} + shell: bash + run: | + mkdir -p packages + cpack -G DEB --config build/deb/CPackConfig.cmake -B packages + + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.deb' -print | sort) + if [[ ${#packages[@]} -ne 3 ]]; then + printf 'Expected exactly 3 DEB packages, found %s\n' "${#packages[@]}" + exit 1 + fi + + actual_names="$({ + for package in "${packages[@]}"; do + dpkg-deb --field "$package" Package + done + } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-dev tsfile-tools | sort)" + if [[ "$actual_names" != "$expected_names" ]]; then + printf 'Unexpected DEB package names:\n%s\n' "$actual_names" + exit 1 + fi + + for package in "${packages[@]}"; do + package_version="$(dpkg-deb --field "$package" Version)" + if [[ "$package_version" != "$EXPECTED_DEB_VERSION" ]]; then + printf '%s has version %s, expected %s\n' \ + "$package" "$package_version" "$EXPECTED_DEB_VERSION" + exit 1 + fi + done + + - name: Upload DEB packages + uses: actions/upload-artifact@v7 + with: + name: native-deb-ubuntu22.04-amd64 + path: packages/*.deb + if-no-files-found: error + retention-days: 14 + + test-deb: + name: Test DEB packages on ${{ matrix.container }} + needs: build-deb + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + container: + - ubuntu:22.04 + - ubuntu:24.04 + container: ${{ matrix.container }} + steps: + - name: Download DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-amd64 + path: packages + + - name: Install DEB packages + shell: bash + run: | + apt-get update + apt-get install -y ./packages/*.deb cmake build-essential + + - name: Verify installed DEB packages + run: | + tsfile-cli --version + dpkg-query -W tsfile tsfile-dev tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + mkdir -p consumer + cat > consumer/CMakeLists.txt <<'CMAKE' + cmake_minimum_required(VERSION 3.11) + project(TsFileInstalledConsumer LANGUAGES CXX) + + find_package(TsFile CONFIG REQUIRED) + add_executable(installed_consumer main.cpp) + target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) + CMAKE + cat > consumer/main.cpp <<'CPP' + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + cmake -S consumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/installed_consumer + + build-rpm: + name: Build AlmaLinux 9 RPM packages + needs: prepare + runs-on: ubuntu-24.04 + container: almalinux:9 + timeout-minutes: 30 + steps: + - name: Install RPM build prerequisites + run: | + dnf install -y \ + cmake \ + gcc-c++ \ + ninja-build \ + pkgconf-pkg-config \ + rpm-build \ + git \ + curl \ + tar \ + gzip \ + libuuid-devel + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Configure and build + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + TSFILE_RPM_PACKAGE_VERSION: ${{ needs.prepare.outputs.rpm_version }} + TSFILE_RPM_PACKAGE_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + cmake -S cpp -B build/rpm -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_TEST=OFF \ + -DBUILD_TOOLS=ON \ + -DTSFILE_ENABLE_CPACK=ON \ + -DTSFILE_DEPENDENCY_SOURCE=AUTO \ + -DTSFILE_ENABLE_NATIVE_ARCH=OFF \ + -DTSFILE_ARCHIVE_VERSION="$TSFILE_ARCHIVE_VERSION" \ + -DTSFILE_RPM_PACKAGE_VERSION="$TSFILE_RPM_PACKAGE_VERSION" \ + -DTSFILE_RPM_PACKAGE_RELEASE="$TSFILE_RPM_PACKAGE_RELEASE" + cmake --build build/rpm --parallel + + - name: Build and verify RPM packages + env: + EXPECTED_RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + EXPECTED_RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + mkdir -p packages + cpack -G RPM --config build/rpm/CPackConfig.cmake -B packages + + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.rpm' -print | sort) + if [[ ${#packages[@]} -ne 3 ]]; then + printf 'Expected exactly 3 RPM packages, found %s\n' "${#packages[@]}" + exit 1 + fi + + actual_names="$({ + for package in "${packages[@]}"; do + rpm -qp --queryformat '%{NAME}\n' "$package" + done + } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-devel tsfile-tools | sort)" + if [[ "$actual_names" != "$expected_names" ]]; then + printf 'Unexpected RPM package names:\n%s\n' "$actual_names" + exit 1 + fi + + for package in "${packages[@]}"; do + package_version="$(rpm -qp --queryformat '%{VERSION}' "$package")" + package_release="$(rpm -qp --queryformat '%{RELEASE}' "$package")" + if [[ "$package_version" != "$EXPECTED_RPM_VERSION" ]]; then + printf '%s has version %s, expected %s\n' \ + "$package" "$package_version" "$EXPECTED_RPM_VERSION" + exit 1 + fi + if [[ "$package_release" != "$EXPECTED_RPM_RELEASE" ]]; then + printf '%s has release %s, expected %s\n' \ + "$package" "$package_release" "$EXPECTED_RPM_RELEASE" + exit 1 + fi + done + + - name: Upload RPM packages + uses: actions/upload-artifact@v7 + with: + name: native-rpm-almalinux9-x86_64 + path: packages/*.rpm + if-no-files-found: error + retention-days: 14 + + test-rpm: + name: Test RPM packages on AlmaLinux 9 + needs: build-rpm + runs-on: ubuntu-24.04 + container: almalinux:9 + steps: + - name: Download RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-x86_64 + path: packages + + - name: Install RPM packages + run: dnf install -y packages/*.rpm cmake gcc-c++ + + - name: Verify installed RPM packages + run: | + tsfile-cli --version + rpm -q tsfile tsfile-devel tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + mkdir -p consumer + cat > consumer/CMakeLists.txt <<'CMAKE' + cmake_minimum_required(VERSION 3.11) + project(TsFileInstalledConsumer LANGUAGES CXX) + + find_package(TsFile CONFIG REQUIRED) + add_executable(installed_consumer main.cpp) + target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) + CMAKE + cat > consumer/main.cpp <<'CPP' + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + cmake -S consumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/installed_consumer diff --git a/packaging/README.md b/packaging/README.md index ff77cf46b..5cb218c78 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -73,10 +73,13 @@ compatible dependency, including ANTLR4 4.9.x. `AUTO` is the reproducible release default and uses the verified source fallback for unavailable or incompatible distro versions. -The repository workflow `Cpp-Packaging` builds and uploads native DEB and RPM -artifacts on packaging-related changes. The DEB job runs on Ubuntu and the RPM -job runs in Fedora, so each package is generated with its native dependency -metadata tool. +The manual `Build native package artifacts` workflow is the authoritative +native package build. Its Linux jobs create `tsfile`, `tsfile-dev`, and +`tsfile-tools` DEBs on Ubuntu 22.04 and `tsfile`, `tsfile-devel`, and +`tsfile-tools` RPMs on AlmaLinux 9. Fresh Ubuntu 22.04, Ubuntu 24.04, and +AlmaLinux 9 containers install the packages and verify both `tsfile-cli` and an +external CMake consumer. The workflow uploads intermediate artifacts for 14 +days and does not publish packages. ## macOS From 0958b56fed6e7941bd145f0d8fee4c623d552d2d Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:33:54 +0800 Subject: [PATCH 07/31] fix(ci): verify native CLI package ownership --- .github/workflows/native-packages.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 5c1bafa5e..79adbb5c3 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -165,7 +165,16 @@ jobs: apt-get install -y ./packages/*.deb cmake build-essential - name: Verify installed DEB packages + shell: bash run: | + cli_path="$(command -v tsfile-cli)" + cli_path="$(readlink -f -- "$cli_path")" + cli_owner="$(dpkg-query -S "$cli_path")" + cli_owner="${cli_owner%%:*}" + if [[ "$cli_owner" != "tsfile-tools" ]]; then + printf '%s is owned by %s, expected tsfile-tools\n' "$cli_path" "$cli_owner" + exit 1 + fi tsfile-cli --version dpkg-query -W tsfile tsfile-dev tsfile-tools @@ -300,7 +309,15 @@ jobs: run: dnf install -y packages/*.rpm cmake gcc-c++ - name: Verify installed RPM packages + shell: bash run: | + cli_path="$(command -v tsfile-cli)" + cli_path="$(readlink -f -- "$cli_path")" + cli_owner="$(rpm -qf --queryformat '%{NAME}' "$cli_path")" + if [[ "$cli_owner" != "tsfile-tools" ]]; then + printf '%s is owned by %s, expected tsfile-tools\n' "$cli_path" "$cli_owner" + exit 1 + fi tsfile-cli --version rpm -q tsfile tsfile-devel tsfile-tools From 8519ed5b8fe4001784268837fd73883194a85aa5 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 18:57:43 +0800 Subject: [PATCH 08/31] feat(ci): build Windows native SDK archive --- .github/workflows/native-packages.yml | 133 ++++++++++++++++++++++++++ cpp/CMakeLists.txt | 9 +- packaging/README.md | 16 ++++ 3 files changed, 157 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 79adbb5c3..8b6bd3e4e 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -343,3 +343,136 @@ jobs: cmake -S consumer -B consumer/build cmake --build consumer/build --parallel ./consumer/build/installed_consumer + + build-windows: + name: Build Windows MSVC x86_64 ZIP + needs: prepare + runs-on: windows-2022 + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Configure the MSVC environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: x64 + + - name: Configure, build, and stage + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + cmake -S cpp -B build/windows -G "Visual Studio 17 2022" -A x64 ` + -DBUILD_TEST=OFF ` + -DBUILD_TOOLS=ON ` + -DTSFILE_ENABLE_CPACK=ON ` + -DTSFILE_DEPENDENCY_SOURCE=BUNDLED ` + -DTSFILE_ENABLE_NATIVE_ARCH=OFF ` + -DTSFILE_ARCHIVE_VERSION="$env:TSFILE_ARCHIVE_VERSION" + cmake --build build/windows --config Release --parallel + cmake --install build/windows --config Release --prefix stage/windows + + - name: Test staged CLI and SDK + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $stage = (Resolve-Path stage/windows).Path + $requiredPaths = @( + "$stage/bin/tsfile-cli.exe" + "$stage/bin/tsfile.dll" + "$stage/lib/tsfile.lib" + "$stage/include/tsfile/cwrapper/tsfile_cwrapper.h" + "$stage/lib/cmake/TsFile/TsFileConfig.cmake" + "$stage/share/doc/tsfile/LICENSE" + "$stage/share/doc/tsfile/NOTICE" + ) + foreach ($path in $requiredPaths) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required staged file is absent: $path" + } + } + + $env:PATH = "$stage/bin;$env:PATH" + tsfile-cli.exe --version + + New-Item -ItemType Directory -Path consumer -Force | Out-Null + @' + cmake_minimum_required(VERSION 3.11) + project(TsFileInstalledConsumer LANGUAGES CXX) + + find_package(TsFile CONFIG REQUIRED) + add_executable(installed_consumer main.cpp) + target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) + '@ | Set-Content -Path consumer/CMakeLists.txt + @' + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + '@ | Set-Content -Path consumer/main.cpp + cmake -S consumer -B consumer/build -G "Visual Studio 17 2022" -A x64 ` + -DCMAKE_PREFIX_PATH="$stage" + cmake --build consumer/build --config Release --parallel + & consumer/build/Release/installed_consumer.exe + + - name: Build and inspect combined ZIP + env: + TSFILE_ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $cpackConfig = Get-Content build/windows/CPackConfig.cmake -Raw + if ($cpackConfig -notmatch 'set\(CPACK_ARCHIVE_COMPONENT_INSTALL "OFF"\)') { + throw 'CPACK_ARCHIVE_COMPONENT_INSTALL must remain disabled' + } + + New-Item -ItemType Directory -Path packages -Force | Out-Null + cpack -G ZIP --config build/windows/CPackConfig.cmake -B packages ` + -D "CPACK_PACKAGE_VERSION=$env:TSFILE_ARCHIVE_VERSION" + + $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) + if ($archives.Count -ne 1) { + throw "Expected exactly one combined ZIP, found $($archives.Count)" + } + $expectedName = "tsfile-$env:TSFILE_ARCHIVE_VERSION-windows-x86_64.zip" + if ($archives[0].Name -cne $expectedName) { + throw "Unexpected ZIP name '$($archives[0].Name)'; expected '$expectedName'" + } + + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead($archives[0].FullName) + try { + $entryPaths = @($archive.Entries | ForEach-Object { + $_.FullName.Replace('\', '/') + }) + $requiredSuffixes = @( + 'bin/tsfile-cli.exe' + 'bin/tsfile.dll' + 'lib/tsfile.lib' + 'include/tsfile/cwrapper/tsfile_cwrapper.h' + ) + foreach ($suffix in $requiredSuffixes) { + $match = $entryPaths | Where-Object { + $_.EndsWith($suffix, [System.StringComparison]::Ordinal) + } + if (-not $match) { + throw "ZIP is missing a path ending in '$suffix'" + } + } + } finally { + $archive.Dispose() + } + + - name: Upload Windows ZIP + uses: actions/upload-artifact@v7 + with: + name: native-windows-msvc-x86_64 + path: packages/*.zip + if-no-files-found: error + retention-days: 14 diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4b6f868b2..c7b486e00 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -545,7 +545,7 @@ unset(_TSFILE_PUBLIC_FEATURE) # target image provides every compatible dependency; AUTO remains the # reproducible fallback for distribution images with older packages. option(TSFILE_ENABLE_CPACK - "Enable CPack DEB/RPM package metadata generation" OFF) + "Enable CPack native package metadata generation" OFF) if (TSFILE_ENABLE_CPACK) set(CPACK_PACKAGE_NAME "tsfile") set(CPACK_PACKAGE_VENDOR "Apache Software Foundation") @@ -565,6 +565,13 @@ if (TSFILE_ENABLE_CPACK) set(CPACK_COMPONENTS_ALL runtime development tools) set(CPACK_COMPONENT_DEVELOPMENT_DEPENDS runtime) set(CPACK_COMPONENT_TOOLS_DEPENDS runtime) + # Portable archives contain the complete runtime, SDK, and tools install + # tree in one file. Native DEB and RPM generators remain componentized. + set(CPACK_ARCHIVE_COMPONENT_INSTALL OFF) + if (WIN32) + set(CPACK_PACKAGE_FILE_NAME + "tsfile-${TSFILE_ARCHIVE_VERSION}-windows-x86_64") + endif () set(CPACK_DEB_COMPONENT_INSTALL ON) set(CPACK_DEBIAN_FILE_NAME DEB-DEFAULT) set(CPACK_DEBIAN_PACKAGE_SHLIBDEPS ON) diff --git a/packaging/README.md b/packaging/README.md index 5cb218c78..8045a4f11 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -93,3 +93,19 @@ brew install ./packaging/homebrew/tsfile.rb The stable formula URL and checksum must be updated to the ASF source archive when the first release containing this packaging work is published. + +## Windows + +The manual workflow builds a 64-bit Release package with Visual Studio 2022 and +bundled dependencies. It stages and tests the CLI plus an external CMake SDK +consumer before producing +`tsfile--windows-x86_64.zip`. The archive is deliberately one +combined ZIP rather than one archive per CPack component, and contains the +runtime, development files, and tools under a relocatable prefix. In +particular, the MSVC outputs are installed as `bin/tsfile.dll`, +`lib/tsfile.lib`, and `bin/tsfile-cli.exe`. + +The workflow uploads that ZIP as the intermediate artifact +`native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the +license files, CMake package config, staged executable, library, import library, +and public headers without publishing the artifact. From ba595ab023b1ba3ae2d872446b743977a1abb13e Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 19:07:24 +0800 Subject: [PATCH 09/31] fix(ci): package Windows Release artifacts --- .github/workflows/native-packages.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 8b6bd3e4e..fdc560e3a 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -433,7 +433,7 @@ jobs: } New-Item -ItemType Directory -Path packages -Force | Out-Null - cpack -G ZIP --config build/windows/CPackConfig.cmake -B packages ` + cpack -G ZIP -C Release --config build/windows/CPackConfig.cmake -B packages ` -D "CPACK_PACKAGE_VERSION=$env:TSFILE_ARCHIVE_VERSION" $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) From 19192b660d6471d1e56e4114de9a9b508f857ee2 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 19:19:17 +0800 Subject: [PATCH 10/31] feat(ci): build Homebrew development bottles --- .github/workflows/native-packages.yml | 183 ++++++++++++++++++ packaging/README.md | 24 +++ packaging/homebrew/tsfile-dev.rb.in | 66 +++++++ packaging/scripts/render_homebrew_formula.py | 83 ++++++++ .../tests/test_render_homebrew_formula.py | 177 +++++++++++++++++ 5 files changed, 533 insertions(+) create mode 100644 packaging/homebrew/tsfile-dev.rb.in create mode 100644 packaging/scripts/render_homebrew_formula.py create mode 100644 packaging/tests/test_render_homebrew_formula.py diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index fdc560e3a..019918a70 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -476,3 +476,186 @@ jobs: path: packages/*.zip if-no-files-found: error retention-days: 14 + + build-homebrew: + name: Build Homebrew bottle on ${{ matrix.name }} + needs: prepare + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - name: macos-arm64 + os: macos-latest + - name: macos-x86_64 + os: macos-15-intel + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_CLEANUP: "1" + TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} + SOURCE_REPOSITORY: ColinLeeo/tsfile + SOURCE_GIT_SHA: ${{ github.sha }} + BOTTLE_ROOT_URL: https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Render immutable source Formula in a temporary local tap + shell: bash + run: | + mkdir -p homebrew/Formula homebrew/bottles + curl --fail --location --retry 3 \ + "https://github.com/$SOURCE_REPOSITORY/archive/$SOURCE_GIT_SHA.tar.gz" \ + --output "$RUNNER_TEMP/tsfile-source.tar.gz" + source_sha256="$(shasum -a 256 "$RUNNER_TEMP/tsfile-source.tar.gz" | awk '{print $1}')" + python3 packaging/scripts/render_homebrew_formula.py \ + --template packaging/homebrew/tsfile-dev.rb.in \ + --repository "$SOURCE_REPOSITORY" \ + --git-sha "$SOURCE_GIT_SHA" \ + --version "$TSFILE_HOMEBREW_VERSION" \ + --source-sha256 "$source_sha256" \ + --output homebrew/Formula/tsfile-dev.rb + ruby -c homebrew/Formula/tsfile-dev.rb + brew tap-new apache/tsfile-dev + tap_path="$(brew --repository apache/tsfile-dev)" + cp homebrew/Formula/tsfile-dev.rb "$tap_path/Formula/tsfile-dev.rb" + + - name: Build, test, and bottle the development Formula + shell: bash + run: | + brew install --build-bottle apache/tsfile-dev/tsfile-dev + brew test apache/tsfile-dev/tsfile-dev + cd homebrew/bottles + brew bottle --json --root-url "$BOTTLE_ROOT_URL" apache/tsfile-dev/tsfile-dev + shopt -s nullglob + bottles=(*.bottle*.tar.gz) + metadata=(*.bottle.json) + [[ ${#bottles[@]} -eq 1 && ${#metadata[@]} -eq 1 ]] + + - name: Upload platform Homebrew bottle and source Formula + uses: actions/upload-artifact@v7 + with: + name: native-homebrew-${{ matrix.name }} + path: homebrew/ + if-no-files-found: error + retention-days: 14 + + merge-homebrew: + name: Merge Homebrew development bottles + needs: [prepare, build-homebrew] + runs-on: macos-latest + timeout-minutes: 15 + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} + BOTTLE_ROOT_URL: https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions/${{ needs.prepare.outputs.homebrew_version }}/bottles + steps: + - name: Download both platform Homebrew artifacts separately + uses: actions/download-artifact@v8 + with: + pattern: native-homebrew-macos-* + path: homebrew-intermediate + merge-multiple: false + + - name: Recreate the same pre-bottle Formula and collect metadata + shell: bash + run: | + arm=homebrew-intermediate/native-homebrew-macos-arm64 + intel=homebrew-intermediate/native-homebrew-macos-x86_64 + cmp "$arm/Formula/tsfile-dev.rb" "$intel/Formula/tsfile-dev.rb" + brew tap-new apache/tsfile-dev + tap_path="$(brew --repository apache/tsfile-dev)" + cp "$arm/Formula/tsfile-dev.rb" "$tap_path/Formula/tsfile-dev.rb" + mkdir -p homebrew/Formula homebrew/bottles + python3 - <<'PY' + import hashlib + import json + import os + import shutil + from pathlib import Path + + destination = Path("homebrew/bottles") + tags = set() + for platform in ("macos-arm64", "macos-x86_64"): + directory = Path("homebrew-intermediate") / f"native-homebrew-{platform}" / "bottles" + metadata = list(directory.glob("*.bottle.json")) + assert len(metadata) == 1, f"Expected one JSON for {platform}" + data = json.loads(metadata[0].read_text()) + assert list(data) == ["apache/tsfile-dev/tsfile-dev"] + entry = data["apache/tsfile-dev/tsfile-dev"] + assert entry["formula"]["pkg_version"] == os.environ["TSFILE_HOMEBREW_VERSION"] + assert entry["formula"]["path"] == "Library/Taps/apache/homebrew-tsfile-dev/Formula/tsfile-dev.rb" + assert entry["bottle"]["root_url"] == os.environ["BOTTLE_ROOT_URL"] + assert len(entry["bottle"]["tags"]) == 1 + tag, bottle = next(iter(entry["bottle"]["tags"].items())) + assert (platform == "macos-arm64") == tag.startswith("arm64_") + assert tag not in tags, f"Duplicate bottle tag: {tag}" + tags.add(tag) + local_name = bottle["local_filename"] + assert Path(local_name).name == local_name + source = directory / local_name + assert hashlib.sha256(source.read_bytes()).hexdigest() == bottle["sha256"] + for source in (source, metadata[0]): + target = destination / source.name + assert not target.exists(), f"Duplicate artifact: {target}" + shutil.copy2(source, target) + assert len(tags) == 2 + PY + + - name: Merge both bottle tags into the Formula + shell: bash + run: | + brew bottle --merge --write --no-commit homebrew/bottles/*.json + tap_path="$(brew --repository apache/tsfile-dev)" + cp "$tap_path/Formula/tsfile-dev.rb" homebrew/Formula/tsfile-dev.rb + ruby -c homebrew/Formula/tsfile-dev.rb + + - name: Verify merged Formula and normalize HTTP bottle filenames + shell: bash + run: | + python3 - <<'PY' + import hashlib + import json + import os + import re + from pathlib import Path + from urllib.parse import unquote + + formula = Path("homebrew/Formula/tsfile-dev.rb").read_text() + assert not re.search(r"@[A-Z0-9_]+@", formula), "Unresolved Formula token" + assert len(re.findall(r"^\s*bottle\s+do\b", formula, re.M)) == 1 + assert f'root_url "{os.environ["BOTTLE_ROOT_URL"]}"' in formula + directory = Path("homebrew/bottles") + metadata = list(directory.glob("*.bottle.json")) + assert len(metadata) == 2 + tags = set() + for path in metadata: + data = json.loads(path.read_text()) + entry = data["apache/tsfile-dev/tsfile-dev"] + for tag, bottle in entry["bottle"]["tags"].items(): + assert tag not in tags, f"Duplicate bottle tag: {tag}" + tags.add(tag) + assert re.search(rf'\b{re.escape(tag)}:\s*"{bottle["sha256"]}"', formula) + # Homebrew emits a double-dash local name but serves a single-dash URL. + filename = unquote(bottle["filename"]) + assert Path(filename).name == filename and filename.endswith(".tar.gz") + source = directory / bottle["local_filename"] + target = directory / filename + assert hashlib.sha256(source.read_bytes()).hexdigest() == bottle["sha256"] + if source != target: + assert not target.exists(), f"Duplicate HTTP bottle name: {filename}" + source.rename(target) + bottle["local_filename"] = filename + path.write_text(json.dumps(data, indent=2) + "\n") + assert len(tags) == 2 + assert len(list(directory.glob("*.tar.gz"))) == 2 + PY + + - name: Upload merged Homebrew Formula and bottles + uses: actions/upload-artifact@v7 + with: + name: native-homebrew + path: homebrew/ + if-no-files-found: error + retention-days: 14 diff --git a/packaging/README.md b/packaging/README.md index 8045a4f11..bf4afaee5 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -94,6 +94,30 @@ brew install ./packaging/homebrew/tsfile.rb The stable formula URL and checksum must be updated to the ASF source archive when the first release containing this packaging work is published. +The manual native-package workflow also builds the self-hosted development +Formula `tsfile-dev`, using `packaging/homebrew/tsfile-dev.rb.in`. This is not a +Homebrew/core submission. Each build pins the full workflow commit from +`ColinLeeo/tsfile`, hashes that source archive, and uses the generated immutable +Homebrew development version. The Formula retains the CMake install behavior +and tests the installed C++ consumer and CLI before bottling. + +The ARM64 job runs on `macos-latest` and the Intel job on `macos-15-intel`. +Their `native-homebrew-macos-arm64` and `native-homebrew-macos-x86_64` +intermediate artifacts remain separate until merge. The merge job checks that +both source Formula files match, merges both platform JSON files with Homebrew, +and checks both generated tags and checksums in the resulting Formula. + +The `native-homebrew` artifact contains `Formula/tsfile-dev.rb` and `bottles/` +with both bottle tarballs and both JSON metadata files. When downloaded into +`homebrew/`, this gives the final `homebrew/Formula/` and `homebrew/bottles/` +layout. The configured future bottle root is +`https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions//bottles`. +Homebrew generates local tarballs with a double dash before the version but +requests a single dash in HTTP URLs. After merge, the workflow renames each +tarball to the JSON `filename` (URL-decoded) and updates its `local_filename` +to match, preserving its checksum. The workflow only uploads GitHub Actions +artifacts for 14 days; it does not publish to that root or update `latest`. + ## Windows The manual workflow builds a 64-bit Release package with Visual Studio 2022 and diff --git a/packaging/homebrew/tsfile-dev.rb.in b/packaging/homebrew/tsfile-dev.rb.in new file mode 100644 index 000000000..cc5b60bb2 --- /dev/null +++ b/packaging/homebrew/tsfile-dev.rb.in @@ -0,0 +1,66 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class TsfileDev < Formula + desc "Columnar storage library for time series data (development snapshot)" + homepage "https://tsfile.apache.org/" + url "https://github.com/@SOURCE_REPOSITORY@/archive/@GIT_SHA@.tar.gz" + version "@HOMEBREW_VERSION@" + sha256 "@SOURCE_SHA256@" + license "Apache-2.0" + +@BOTTLE_BLOCK@ + + depends_on "cmake" => :build + depends_on "lz4" + depends_on "simde" + depends_on "snappy" + depends_on "utf8cpp" + depends_on "zstd" + + uses_from_macos "zlib" + + def install + args = %W[ + -DBUILD_TEST=OFF + -DBUILD_TOOLS=ON + -DENABLE_LZMA2=OFF + -DTSFILE_DEPENDENCY_SOURCE=AUTO + -DTSFILE_ENABLE_NATIVE_ARCH=OFF + -DCMAKE_INSTALL_RPATH=#{rpath} + ] + + system "cmake", "-S", "cpp", "-B", "build", *args, *std_cmake_args + system "cmake", "--build", "build" + system "cmake", "--install", "build" + end + + test do + (testpath / "test.cpp").write <<~CPP + #include + + int main() { + return TS_DATATYPE_INT32 == 1 ? 0 : 1; + } + CPP + + system ENV.cxx, "-std=c++11", "test.cpp", "-I#{include}", "-L#{lib}", + "-Wl,-rpath,#{lib}", "-ltsfile", "-o", "test" + system "./test" + system bin / "tsfile-cli", "--version" + end +end diff --git a/packaging/scripts/render_homebrew_formula.py b/packaging/scripts/render_homebrew_formula.py new file mode 100644 index 000000000..a39ba31ee --- /dev/null +++ b/packaging/scripts/render_homebrew_formula.py @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Render the development Formula using an immutable source identity.""" + +import argparse +import re +from pathlib import Path + + +VALUE_PATTERNS = { + "SOURCE_REPOSITORY": r"[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+", + "GIT_SHA": r"[0-9a-f]{40}", + "HOMEBREW_VERSION": r"[0-9][A-Za-z0-9.]*", + "SOURCE_SHA256": r"[0-9a-f]{64}", +} +TOKEN = re.compile(r"@([A-Z0-9_]+)@") + + +def render_formula( + template: str, values: dict[str, str], bottle_block: str = "" +) -> str: + """Replace declared tokens without interpreting Ruby braces or at-signs.""" + if values.keys() != VALUE_PATTERNS.keys(): + raise ValueError( + "Formula values must contain exactly the declared source fields" + ) + for name, pattern in VALUE_PATTERNS.items(): + if not re.fullmatch(pattern, values[name]): + raise ValueError(f"Invalid Formula value: {name}") + replacements = {**values, "BOTTLE_BLOCK": bottle_block} + unknown = set(TOKEN.findall(template)) - replacements.keys() + if unknown: + raise ValueError(f"Unknown Formula placeholders: {sorted(unknown)}") + rendered = TOKEN.sub(lambda match: replacements[match.group(1)], template) + if TOKEN.search(rendered): + raise ValueError("Unresolved Formula placeholder") + if len(re.findall(r"^\s*bottle\s+do\b", rendered, re.MULTILINE)) > 1: + raise ValueError("Duplicate bottle blocks") + return rendered + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--template", type=Path, required=True) + parser.add_argument("--repository", required=True) + parser.add_argument("--git-sha", required=True) + parser.add_argument("--version", required=True) + parser.add_argument("--source-sha256", required=True) + parser.add_argument("--bottle-block", type=Path) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + values = { + "SOURCE_REPOSITORY": args.repository, + "GIT_SHA": args.git_sha, + "HOMEBREW_VERSION": args.version, + "SOURCE_SHA256": args.source_sha256, + } + bottle_block = ( + args.bottle_block.read_text(encoding="utf-8") if args.bottle_block else "" + ) + args.output.write_text( + render_formula(args.template.read_text(encoding="utf-8"), values, bottle_block), + encoding="utf-8", + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py new file mode 100644 index 000000000..0c0047aed --- /dev/null +++ b/packaging/tests/test_render_homebrew_formula.py @@ -0,0 +1,177 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = ROOT / "packaging/scripts/render_homebrew_formula.py" +TEMPLATE_PATH = ROOT / "packaging/homebrew/tsfile-dev.rb.in" +VALUES = { + "SOURCE_REPOSITORY": "ColinLeeo/tsfile", + "GIT_SHA": "abcdef1234567890abcdef1234567890abcdef12", + "HOMEBREW_VERSION": "2.5.0.dev0.20260910.123.1.gabcdef1", + "SOURCE_SHA256": "0123456789abcdef" * 4, +} +TEMPLATE = """class TsfileDev < Formula + url "https://github.com/@SOURCE_REPOSITORY@/archive/@GIT_SHA@.tar.gz" + version "@HOMEBREW_VERSION@" + sha256 "@SOURCE_SHA256@" +@BOTTLE_BLOCK@ +end +""" +BOTTLE_BLOCK = """ bottle do + root_url "https://packages.apache.org/artifactory/tsfile/homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles" + sha256 cellar: :any, arm64_sequoia: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + sha256 cellar: :any, sequoia: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + end""" + + +class RenderHomebrewFormulaTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = None + if MODULE_PATH.is_file(): + spec = importlib.util.spec_from_file_location( + "render_homebrew_formula", MODULE_PATH + ) + cls.module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(cls.module) + + def renderer(self): + self.assertIsNotNone(self.module, "Homebrew Formula renderer must exist") + return self.module.render_formula + + def test_renders_exact_immutable_source_and_both_bottle_tags(self): + rendered = self.renderer()(TEMPLATE, VALUES, BOTTLE_BLOCK) + self.assertEqual( + rendered, + """class TsfileDev < Formula + url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz" + version "2.5.0.dev0.20260910.123.1.gabcdef1" + sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" +""" + + BOTTLE_BLOCK + + "\nend\n", + ) + + def test_pre_bottle_formula_has_no_bottle_block(self): + rendered = self.renderer()(TEMPLATE, VALUES) + self.assertNotIn("bottle do", rendered) + self.assertNotRegex(rendered, r"@[A-Z0-9_]+@") + + def test_rejects_missing_and_undeclared_placeholders(self): + render = self.renderer() + with self.assertRaises(ValueError): + render( + TEMPLATE, + {key: value for key, value in VALUES.items() if key != "GIT_SHA"}, + ) + with self.assertRaises(ValueError): + render(TEMPLATE + "@UNKNOWN@", {**VALUES, "UNKNOWN": "anything"}) + + def test_rejects_template_tokens_in_bottle_block(self): + with self.assertRaises(ValueError): + self.renderer()(TEMPLATE, VALUES, BOTTLE_BLOCK.replace("sequoia", "@TAG@")) + + def test_rejects_duplicate_bottle_blocks(self): + render = self.renderer() + for template, block in ( + (TEMPLATE, BOTTLE_BLOCK + "\n" + BOTTLE_BLOCK), + (TEMPLATE + "\nbottle do\nend\n", BOTTLE_BLOCK), + ( + TEMPLATE.replace("@BOTTLE_BLOCK@", "@BOTTLE_BLOCK@\n@BOTTLE_BLOCK@"), + BOTTLE_BLOCK, + ), + ): + with self.subTest(template=template, block=block): + with self.assertRaises(ValueError): + render(template, VALUES, block) + + def test_rejects_empty_or_non_immutable_source_identity(self): + render = self.renderer() + for key, value in ( + ("SOURCE_SHA256", ""), + ("SOURCE_SHA256", "bad"), + ("GIT_SHA", ""), + ("GIT_SHA", "abcdef1"), + ("GIT_SHA", "develop"), + ("SOURCE_REPOSITORY", 'ColinLeeo/tsfile"'), + ("HOMEBREW_VERSION", 'bad"\nversion "injected'), + ): + with self.subTest(key=key, value=value): + with self.assertRaises(ValueError): + render(TEMPLATE, {**VALUES, key: value}) + + def test_preserves_ruby_interpolation_and_other_at_signs(self): + extra = "\n# maintainer@example.org\n# #{rpath}\n" + self.assertTrue(self.renderer()(TEMPLATE + extra, VALUES).endswith(extra)) + + def test_cli_renders_real_template_with_optional_bottle_file(self): + self.renderer() + with tempfile.TemporaryDirectory() as temporary_directory: + directory = Path(temporary_directory) + output = directory / "tsfile-dev.rb" + block = directory / "bottle.rb" + block.write_text(BOTTLE_BLOCK, encoding="utf-8") + command = [ + sys.executable, + str(MODULE_PATH), + "--template", + str(TEMPLATE_PATH), + "--repository", + VALUES["SOURCE_REPOSITORY"], + "--git-sha", + VALUES["GIT_SHA"], + "--version", + VALUES["HOMEBREW_VERSION"], + "--source-sha256", + VALUES["SOURCE_SHA256"], + "--output", + str(output), + ] + for bottle_args, expected_block in ( + ([], ""), + (["--bottle-block", str(block)], BOTTLE_BLOCK), + ): + subprocess.run(command + bottle_args, check=True) + rendered = output.read_text(encoding="utf-8") + self.assertIn("class TsfileDev < Formula", rendered) + self.assertIn( + 'url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz"', + rendered, + ) + self.assertIn('version "2.5.0.dev0.20260910.123.1.gabcdef1"', rendered) + self.assertIn( + 'sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"', + rendered, + ) + self.assertIn("#{rpath}", rendered) + self.assertNotRegex(rendered, r"@[A-Z0-9_]+@") + if expected_block: + self.assertIn(BOTTLE_BLOCK, rendered) + else: + self.assertNotIn("bottle do", rendered) + + +if __name__ == "__main__": + unittest.main() From 340d0cd2a717d59492b0a4248fb39f8fe3af1007 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 20:51:37 +0800 Subject: [PATCH 11/31] feat(ci): assemble native package artifact bundle --- .github/workflows/native-packages.yml | 96 ++++++ packaging/README.md | 17 + packaging/scripts/assemble_native_packages.py | 244 ++++++++++++++ .../tests/test_assemble_native_packages.py | 298 ++++++++++++++++++ 4 files changed, 655 insertions(+) create mode 100644 packaging/scripts/assemble_native_packages.py create mode 100644 packaging/tests/test_assemble_native_packages.py diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 019918a70..dc01c872d 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -659,3 +659,99 @@ jobs: path: homebrew/ if-no-files-found: error retention-days: 14 + + assemble: + name: Assemble checksummed native package bundle + needs: [prepare, test-deb, test-rpm, merge-homebrew, build-windows] + runs-on: ubuntu-24.04 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Download package versions + uses: actions/download-artifact@v8 + with: + name: native-package-versions + path: versions + + - name: Download DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-amd64 + path: input/deb + + - name: Download RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-x86_64 + path: input/rpm + + - name: Download Homebrew Formula and bottles + uses: actions/download-artifact@v8 + with: + name: native-homebrew + path: input/homebrew + + - name: Download Windows ZIP + uses: actions/download-artifact@v8 + with: + name: native-windows-msvc-x86_64 + path: input/windows + + - name: Assemble the publishable bundle + env: + LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} + shell: bash + run: | + python3 packaging/scripts/assemble_native_packages.py \ + --input-dir input \ + --output-dir "tsfile-native-packages-$LOGICAL_VERSION" \ + --versions-json versions/versions.json \ + --source-repository "${{ github.repository }}" \ + --source-commit "${{ github.sha }}" + + - name: Verify assembled manifest checksums + env: + LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} + shell: bash + run: | + python3 - <<'PY' + import hashlib + import json + import os + import re + from pathlib import Path + + output = Path(f"tsfile-native-packages-{os.environ['LOGICAL_VERSION']}") + manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) + checksums = (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines() + expected_checksums = [] + paths = set() + for artifact in manifest["artifacts"]: + relative = Path(artifact["path"]) + assert relative.as_posix() == artifact["path"] + assert relative not in paths, f"Duplicate manifest path: {relative}" + paths.add(relative) + path = output / relative + assert path.is_file() and not path.is_symlink(), f"Missing artifact: {relative}" + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"]) + assert digest == artifact["sha256"], f"Hash mismatch: {relative}" + assert path.stat().st_size == artifact["size"], f"Size mismatch: {relative}" + expected_checksums.append(f"{digest} {relative.as_posix()}") + assert checksums == sorted(expected_checksums) + actual_paths = { + path.relative_to(output) + for path in output.rglob("*") + if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"} + } + assert actual_paths == paths + PY + + - name: Upload checksummed native package bundle + uses: actions/upload-artifact@v7 + with: + name: tsfile-native-packages-${{ needs.prepare.outputs.archive_version }} + path: tsfile-native-packages-${{ needs.prepare.outputs.logical_version }}/ + if-no-files-found: error + retention-days: 14 diff --git a/packaging/README.md b/packaging/README.md index bf4afaee5..c693cd1f6 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -133,3 +133,20 @@ The workflow uploads that ZIP as the intermediate artifact `native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the license files, CMake package config, staged executable, library, import library, and public headers without publishing the artifact. + +## Final native package bundle + +Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM +installation test, Homebrew bottle merge, and Windows SDK/CLI build all succeed, +the workflow assembles `tsfile-native-packages-`. The final +GitHub Actions artifact retains the DEBs, RPMs, merged Formula and bottles, and +Windows ZIP in their package-family layouts for 14 days. It also contains a +sorted `SHA256SUMS` and `manifest.json` with the source identity, generated +versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target +path, and properties required for later manual publication. + +The final job has no publishing credentials and does not upload to JFrog. A +maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the +recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, and Homebrew +and Windows files to their immutable `tsfile/homebrew/dev/versions/` +and `tsfile/windows/dev/versions/` paths. diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py new file mode 100644 index 000000000..d9c6a356c --- /dev/null +++ b/packaging/scripts/assemble_native_packages.py @@ -0,0 +1,244 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Assemble native package artifacts into a checksummed publishable bundle.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +from pathlib import Path +from typing import Sequence + +DEB_PROPERTIES = { + "deb.distribution": ["jammy", "noble"], + "deb.component": ["dev"], + "deb.architecture": ["amd64"], +} +REQUIRED_VERSIONS = {"archive_version", "homebrew_version"} +REQUIRED_SOURCE = {"commit", "repository"} + + +def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> None: + if not isinstance(values, dict) or not required.issubset(values): + raise ValueError(f"{name} must contain {sorted(required)}") + if any( + not isinstance(key, str) or not isinstance(value, str) + for key, value in values.items() + ): + raise ValueError(f"{name} must map strings to strings") + + +def _input_files(input_dir: Path) -> list[Path]: + if not input_dir.is_dir() or input_dir.is_symlink(): + raise ValueError(f"input directory is not a real directory: {input_dir}") + files: list[Path] = [] + for directory, directories, names in os.walk(input_dir, followlinks=False): + directory_path = Path(directory) + for child in directories: + path = directory_path / child + if path.is_symlink(): + raise ValueError(f"symbolic link input is not allowed: {path}") + for name in names: + path = directory_path / name + if path.is_symlink(): + raise ValueError(f"symbolic link input is not allowed: {path}") + if not path.is_file(): + raise ValueError(f"undeclared file type: {path}") + if path.stat().st_size == 0: + raise ValueError(f"empty file input is not allowed: {path}") + files.append(path) + return sorted(files, key=lambda path: path.relative_to(input_dir).as_posix()) + + +def _artifact_description( + input_dir: Path, source_path: Path, versions: dict[str, str] +) -> tuple[Path, dict[str, object]]: + relative_path = source_path.relative_to(input_dir) + parts = relative_path.parts + filename = source_path.name + if len(parts) >= 2 and parts[0] == "deb" and filename.endswith(".deb"): + target = Path("deb/ubuntu22.04-amd64") / filename + return target, { + "family": "deb", + "platform": "ubuntu22.04-amd64", + "targetRepository": "tsfile-debian", + "targetPath": f"pool/dev/ubuntu22.04-amd64/{filename}", + "properties": DEB_PROPERTIES, + } + if len(parts) >= 2 and parts[0] == "rpm" and filename.endswith(".rpm"): + target = Path("rpm/almalinux9-x86_64") / filename + return target, { + "family": "rpm", + "platform": "almalinux9-x86_64", + "targetRepository": "tsfile-rpm", + "targetPath": f"dev/el9/x86_64/{filename}", + "properties": {}, + } + if ( + len(parts) == 3 + and parts[:2] == ("homebrew", "Formula") + and filename == "tsfile-dev.rb" + ): + target = Path(*parts) + return target, { + "family": "homebrew", + "platform": "homebrew", + "targetRepository": "tsfile", + "targetPath": f"homebrew/dev/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", + "properties": {}, + } + if ( + len(parts) == 3 + and parts[:2] == ("homebrew", "bottles") + and (filename.endswith(".tar.gz") or filename.endswith(".bottle.json")) + ): + target = Path(*parts) + return target, { + "family": "homebrew", + "platform": "homebrew", + "targetRepository": "tsfile", + "targetPath": f"homebrew/dev/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", + "properties": {}, + } + if len(parts) >= 2 and parts[0] == "windows" and filename.endswith(".zip"): + target = Path("windows") / filename + return target, { + "family": "windows", + "platform": "windows-msvc-x86_64", + "targetRepository": "tsfile", + "targetPath": f"windows/dev/versions/{versions['archive_version']}/{filename}", + "properties": {}, + } + raise ValueError(f"undeclared file type: {relative_path.as_posix()}") + + +def _require_complete_families( + planned: list[tuple[Path, Path, dict[str, object]]], +) -> None: + families = [metadata["family"] for _, _, metadata in planned] + if not any(family == "deb" for family in families): + raise ValueError("missing DEB package input") + if not any(family == "rpm" for family in families): + raise ValueError("missing RPM package input") + if not any(family == "windows" for family in families): + raise ValueError("missing Windows ZIP input") + homebrew_paths = { + target.as_posix() + for _, target, metadata in planned + if metadata["family"] == "homebrew" + } + if "homebrew/Formula/tsfile-dev.rb" not in homebrew_paths: + raise ValueError("missing Homebrew Formula input") + if not any(path.endswith(".tar.gz") for path in homebrew_paths): + raise ValueError("missing Homebrew Bottle input") + if not any(path.endswith(".bottle.json") for path in homebrew_paths): + raise ValueError("missing Homebrew Bottle metadata input") + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def assemble( + input_dir: Path, output_dir: Path, versions: dict[str, str], source: dict[str, str] +) -> dict[str, object]: + """Copy approved artifacts and return their deterministic publication manifest.""" + _validate_metadata(versions, REQUIRED_VERSIONS, "versions") + _validate_metadata(source, REQUIRED_SOURCE, "source") + input_dir = Path(input_dir) + output_dir = Path(output_dir) + files = _input_files(input_dir) + planned: list[tuple[Path, Path, dict[str, object]]] = [] + target_paths: set[Path] = set() + for source_path in files: + target, metadata = _artifact_description(input_dir, source_path, versions) + if target in target_paths: + raise ValueError(f"duplicate target path: {target.as_posix()}") + target_paths.add(target) + planned.append((source_path, target, metadata)) + _require_complete_families(planned) + if output_dir.is_symlink(): + raise ValueError(f"output directory must not be a symbolic link: {output_dir}") + if output_dir.exists() and any(output_dir.iterdir()): + raise ValueError(f"output directory must be empty: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + + artifacts: list[dict[str, object]] = [] + for source_path, target, metadata in sorted( + planned, key=lambda item: item[1].as_posix() + ): + destination = output_dir / target + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source_path, destination) + artifact = { + **metadata, + "filename": destination.name, + "path": target.as_posix(), + "size": destination.stat().st_size, + "sha256": _sha256(destination), + } + artifacts.append(artifact) + + checksums = "".join( + f"{artifact['sha256']} {artifact['path']}\n" for artifact in artifacts + ) + (output_dir / "SHA256SUMS").write_text(checksums, encoding="utf-8") + manifest: dict[str, object] = { + "artifacts": artifacts, + "source": source, + "versions": versions, + } + (output_dir / "manifest.json").write_text( + json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + return manifest + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--versions-json", type=Path, required=True) + parser.add_argument("--source-repository", required=True) + parser.add_argument("--source-commit", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + """Assemble a bundle from an Actions artifact download directory.""" + arguments = _argument_parser().parse_args(argv) + versions = json.loads(arguments.versions_json.read_text(encoding="utf-8")) + assemble( + arguments.input_dir, + arguments.output_dir, + versions, + {"repository": arguments.source_repository, "commit": arguments.source_commit}, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py new file mode 100644 index 000000000..ab2342378 --- /dev/null +++ b/packaging/tests/test_assemble_native_packages.py @@ -0,0 +1,298 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/assemble_native_packages.py" +VERSIONS = { + "archive_version": "2.5.0-dev0.20260910.123.1.gabcdef1", + "homebrew_version": "2.5.0.dev0.20260910.123.1.gabcdef1", + "logical_version": "2.5.0.dev0+20260910.123.1.gabcdef1", +} +SOURCE = { + "commit": "abcdef1234567890abcdef1234567890abcdef12", + "repository": "ColinLeeo/tsfile", +} + + +def load_assembler_module(): + if not MODULE_PATH.is_file(): + return None + spec = importlib.util.spec_from_file_location( + "assemble_native_packages", MODULE_PATH + ) + if spec is None or spec.loader is None: + raise ImportError(f"cannot load native package assembler from {MODULE_PATH}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class AssembleNativePackagesTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.module = load_assembler_module() + + def require_module(self): + self.assertIsNotNone( + self.module, "native package assembler must exist for final bundle assembly" + ) + return self.module + + def write_fixture(self, directory): + files = { + "deb/tsfile_2.5.0-dev_amd64.deb": b"deb artifact\n", + "rpm/tsfile-2.5.0-dev.x86_64.rpm": b"rpm artifact\n", + "homebrew/Formula/tsfile-dev.rb": b"formula artifact\n", + "homebrew/bottles/tsfile-dev.bottle.tar.gz": b"bottle artifact\n", + "homebrew/bottles/tsfile-dev.bottle.json": b"bottle metadata\n", + "windows/tsfile-2.5.0-dev-windows-x86_64.zip": b"windows artifact\n", + } + for relative_path, contents in files.items(): + path = directory / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(contents) + return files + + def test_assembles_publishable_bundle_with_literal_metadata(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + output_directory = Path(temporary_directory) / "bundle" + self.write_fixture(input_directory) + + manifest = module.assemble( + input_directory, output_directory, VERSIONS, SOURCE + ) + + expected_paths = [ + "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "homebrew/Formula/tsfile-dev.rb", + "homebrew/bottles/tsfile-dev.bottle.json", + "homebrew/bottles/tsfile-dev.bottle.tar.gz", + "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "windows/tsfile-2.5.0-dev-windows-x86_64.zip", + ] + self.assertEqual( + sorted( + path.relative_to(output_directory).as_posix() + for path in output_directory.rglob("*") + if path.is_file() + and path.name not in {"SHA256SUMS", "manifest.json"} + ), + expected_paths, + ) + self.assertEqual( + (output_directory / "SHA256SUMS").read_text(encoding="utf-8"), + "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137 deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb\n" + "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224 homebrew/Formula/tsfile-dev.rb\n" + "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967 homebrew/bottles/tsfile-dev.bottle.json\n" + "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82 homebrew/bottles/tsfile-dev.bottle.tar.gz\n" + "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm\n" + "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f windows/tsfile-2.5.0-dev-windows-x86_64.zip\n", + ) + self.assertEqual( + json.loads((output_directory / "manifest.json").read_text()), manifest + ) + self.assertEqual(manifest["source"], SOURCE) + self.assertEqual(manifest["versions"], VERSIONS) + self.assertEqual( + manifest["artifacts"], + [ + { + "family": "deb", + "filename": "tsfile_2.5.0-dev_amd64.deb", + "path": "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "platform": "ubuntu22.04-amd64", + "properties": { + "deb.architecture": ["amd64"], + "deb.component": ["dev"], + "deb.distribution": ["jammy", "noble"], + }, + "sha256": "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137", + "size": 13, + "targetPath": "pool/dev/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "targetRepository": "tsfile-debian", + }, + { + "family": "homebrew", + "filename": "tsfile-dev.rb", + "path": "homebrew/Formula/tsfile-dev.rb", + "platform": "homebrew", + "properties": {}, + "sha256": "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224", + "size": 17, + "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/Formula/tsfile-dev.rb", + "targetRepository": "tsfile", + }, + { + "family": "homebrew", + "filename": "tsfile-dev.bottle.json", + "path": "homebrew/bottles/tsfile-dev.bottle.json", + "platform": "homebrew", + "properties": {}, + "sha256": "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967", + "size": 16, + "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.json", + "targetRepository": "tsfile", + }, + { + "family": "homebrew", + "filename": "tsfile-dev.bottle.tar.gz", + "path": "homebrew/bottles/tsfile-dev.bottle.tar.gz", + "platform": "homebrew", + "properties": {}, + "sha256": "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82", + "size": 16, + "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.tar.gz", + "targetRepository": "tsfile", + }, + { + "family": "rpm", + "filename": "tsfile-2.5.0-dev.x86_64.rpm", + "path": "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "platform": "almalinux9-x86_64", + "properties": {}, + "sha256": "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f", + "size": 13, + "targetPath": "dev/el9/x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "targetRepository": "tsfile-rpm", + }, + { + "family": "windows", + "filename": "tsfile-2.5.0-dev-windows-x86_64.zip", + "path": "windows/tsfile-2.5.0-dev-windows-x86_64.zip", + "platform": "windows-msvc-x86_64", + "properties": {}, + "sha256": "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f", + "size": 17, + "targetPath": "windows/dev/versions/2.5.0-dev0.20260910.123.1.gabcdef1/tsfile-2.5.0-dev-windows-x86_64.zip", + "targetRepository": "tsfile", + }, + ], + ) + + def test_rejects_unknown_files(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + (input_directory / "deb/README.txt").write_text( + "not a package\n", encoding="utf-8" + ) + + with self.assertRaisesRegex(ValueError, "undeclared file type"): + module.assemble( + input_directory, + Path(temporary_directory) / "bundle", + VERSIONS, + SOURCE, + ) + + def test_rejects_empty_and_symbolic_link_inputs(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + (input_directory / "rpm/tsfile-2.5.0-dev.x86_64.rpm").write_bytes(b"") + with self.assertRaisesRegex(ValueError, "empty file"): + module.assemble( + input_directory, + Path(temporary_directory) / "empty", + VERSIONS, + SOURCE, + ) + + self.write_fixture(input_directory) + os.symlink( + input_directory / "deb/tsfile_2.5.0-dev_amd64.deb", + input_directory / "deb/linked.deb", + ) + with self.assertRaisesRegex(ValueError, "symbolic link"): + module.assemble( + input_directory, + Path(temporary_directory) / "linked", + VERSIONS, + SOURCE, + ) + + def test_rejects_duplicate_destination_names(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + duplicate = input_directory / "deb/duplicate/tsfile_2.5.0-dev_amd64.deb" + duplicate.parent.mkdir() + duplicate.write_bytes(b"another deb artifact\n") + + with self.assertRaisesRegex(ValueError, "duplicate target path"): + module.assemble( + input_directory, + Path(temporary_directory) / "bundle", + VERSIONS, + SOURCE, + ) + + def test_cli_assembles_from_versions_json(self): + self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + temporary_path = Path(temporary_directory) + input_directory = temporary_path / "input" + self.write_fixture(input_directory) + versions_path = temporary_path / "versions.json" + versions_path.write_text(json.dumps(VERSIONS), encoding="utf-8") + output_directory = temporary_path / "bundle" + + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--input-dir", + str(input_directory), + "--output-dir", + str(output_directory), + "--versions-json", + str(versions_path), + "--source-repository", + SOURCE["repository"], + "--source-commit", + SOURCE["commit"], + ], + check=True, + ) + manifest_path = output_directory / "manifest.json" + self.assertTrue(manifest_path.is_file(), "CLI must write manifest.json") + self.assertEqual( + json.loads(manifest_path.read_text(encoding="utf-8"))["source"], SOURCE + ) + + +if __name__ == "__main__": + unittest.main() From 6ffc6c0da3be3acbf95e0934bd7ae3063d1db8da Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 20:58:36 +0800 Subject: [PATCH 12/31] fix(ci): verify native package bundle integrity --- .github/workflows/native-packages.yml | 34 +---- packaging/scripts/assemble_native_packages.py | 134 +++++++++++++++++- .../tests/test_assemble_native_packages.py | 57 ++++++++ 3 files changed, 186 insertions(+), 39 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index dc01c872d..3c326d2e2 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -715,38 +715,8 @@ jobs: LOGICAL_VERSION: ${{ needs.prepare.outputs.logical_version }} shell: bash run: | - python3 - <<'PY' - import hashlib - import json - import os - import re - from pathlib import Path - - output = Path(f"tsfile-native-packages-{os.environ['LOGICAL_VERSION']}") - manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) - checksums = (output / "SHA256SUMS").read_text(encoding="utf-8").splitlines() - expected_checksums = [] - paths = set() - for artifact in manifest["artifacts"]: - relative = Path(artifact["path"]) - assert relative.as_posix() == artifact["path"] - assert relative not in paths, f"Duplicate manifest path: {relative}" - paths.add(relative) - path = output / relative - assert path.is_file() and not path.is_symlink(), f"Missing artifact: {relative}" - digest = hashlib.sha256(path.read_bytes()).hexdigest() - assert re.fullmatch(r"[0-9a-f]{64}", artifact["sha256"]) - assert digest == artifact["sha256"], f"Hash mismatch: {relative}" - assert path.stat().st_size == artifact["size"], f"Size mismatch: {relative}" - expected_checksums.append(f"{digest} {relative.as_posix()}") - assert checksums == sorted(expected_checksums) - actual_paths = { - path.relative_to(output) - for path in output.rglob("*") - if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"} - } - assert actual_paths == paths - PY + python3 packaging/scripts/assemble_native_packages.py \ + --verify-bundle "tsfile-native-packages-$LOGICAL_VERSION" - name: Upload checksummed native package bundle uses: actions/upload-artifact@v7 diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py index d9c6a356c..51cec3769 100644 --- a/packaging/scripts/assemble_native_packages.py +++ b/packaging/scripts/assemble_native_packages.py @@ -24,6 +24,7 @@ import hashlib import json import os +import re import shutil from pathlib import Path from typing import Sequence @@ -47,6 +48,27 @@ def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> raise ValueError(f"{name} must map strings to strings") +def _validate_versions(versions: dict[str, str]) -> None: + _validate_metadata(versions, REQUIRED_VERSIONS, "versions") + for name in REQUIRED_VERSIONS: + value = versions[name] + if ( + not value.strip() + or value in {".", ".."} + or "/" in value + or "\\" in value + or "\x00" in value + ): + raise ValueError(f"version {name} must be a safe path component") + + +def _validate_source(source: dict[str, str]) -> None: + _validate_metadata(source, REQUIRED_SOURCE, "source") + for name in REQUIRED_SOURCE: + if not source[name].strip(): + raise ValueError(f"source {name} must be nonempty") + + def _input_files(input_dir: Path) -> list[Path]: if not input_dir.is_dir() or input_dir.is_symlink(): raise ValueError(f"input directory is not a real directory: {input_dir}") @@ -162,12 +184,80 @@ def _sha256(path: Path) -> str: return digest.hexdigest() +def _artifact_paths(output_dir: Path) -> set[Path]: + paths: set[Path] = set() + for path in output_dir.rglob("*"): + if path.is_symlink(): + raise ValueError(f"symbolic link output is not allowed: {path}") + if path.is_file() and path.name not in {"SHA256SUMS", "manifest.json"}: + paths.add(path.relative_to(output_dir)) + return paths + + +def verify_bundle(output_dir: Path) -> None: + """Verify manifest hashes, sizes, and path-sorted checksums in a bundle.""" + output_dir = Path(output_dir) + if not output_dir.is_dir() or output_dir.is_symlink(): + raise ValueError(f"bundle directory is not a real directory: {output_dir}") + manifest_path = output_dir / "manifest.json" + checksums_path = output_dir / "SHA256SUMS" + if ( + not manifest_path.is_file() + or manifest_path.is_symlink() + or not checksums_path.is_file() + or checksums_path.is_symlink() + ): + raise ValueError("bundle must contain real manifest.json and SHA256SUMS files") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + artifacts = manifest.get("artifacts") if isinstance(manifest, dict) else None + if not isinstance(artifacts, list): + raise ValueError("manifest artifacts must be a list") + + paths: set[Path] = set() + for artifact in artifacts: + if not isinstance(artifact, dict): + raise ValueError("manifest artifacts must be objects") + relative_text = artifact.get("path") + if not isinstance(relative_text, str): + raise ValueError("manifest artifact path must be a string") + relative = Path(relative_text) + if ( + relative.is_absolute() + or relative.as_posix() != relative_text + or ".." in relative.parts + ): + raise ValueError(f"unsafe manifest path: {relative_text}") + if relative in paths: + raise ValueError(f"duplicate manifest path: {relative_text}") + paths.add(relative) + path = output_dir / relative + if not path.is_file() or path.is_symlink(): + raise ValueError(f"missing artifact: {relative_text}") + sha256 = artifact.get("sha256") + if not isinstance(sha256, str) or re.fullmatch(r"[0-9a-f]{64}", sha256) is None: + raise ValueError(f"invalid artifact SHA-256: {relative_text}") + if _sha256(path) != sha256: + raise ValueError(f"hash mismatch: {relative_text}") + if path.stat().st_size != artifact.get("size"): + raise ValueError(f"size mismatch: {relative_text}") + + actual_checksums = checksums_path.read_text(encoding="utf-8").splitlines() + expected_checksums = [ + f"{artifact['sha256']} {artifact['path']}" + for artifact in sorted(artifacts, key=lambda artifact: artifact["path"]) + ] + if actual_checksums != expected_checksums: + raise ValueError("SHA256SUMS does not match path-sorted manifest artifacts") + if _artifact_paths(output_dir) != paths: + raise ValueError("bundle files do not match manifest artifacts") + + def assemble( input_dir: Path, output_dir: Path, versions: dict[str, str], source: dict[str, str] ) -> dict[str, object]: """Copy approved artifacts and return their deterministic publication manifest.""" - _validate_metadata(versions, REQUIRED_VERSIONS, "versions") - _validate_metadata(source, REQUIRED_SOURCE, "source") + _validate_versions(versions) + _validate_source(source) input_dir = Path(input_dir) output_dir = Path(output_dir) files = _input_files(input_dir) @@ -219,17 +309,47 @@ def assemble( def _argument_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--input-dir", type=Path, required=True) - parser.add_argument("--output-dir", type=Path, required=True) - parser.add_argument("--versions-json", type=Path, required=True) - parser.add_argument("--source-repository", required=True) - parser.add_argument("--source-commit", required=True) + parser.add_argument("--input-dir", type=Path) + parser.add_argument("--output-dir", type=Path) + parser.add_argument("--versions-json", type=Path) + parser.add_argument("--source-repository") + parser.add_argument("--source-commit") + parser.add_argument("--verify-bundle", type=Path) return parser def main(argv: Sequence[str] | None = None) -> int: """Assemble a bundle from an Actions artifact download directory.""" arguments = _argument_parser().parse_args(argv) + if arguments.verify_bundle is not None: + if any( + value is not None + for value in ( + arguments.input_dir, + arguments.output_dir, + arguments.versions_json, + arguments.source_repository, + arguments.source_commit, + ) + ): + raise ValueError( + "--verify-bundle cannot be combined with assembly arguments" + ) + verify_bundle(arguments.verify_bundle) + return 0 + missing = [ + name + for name, value in ( + ("--input-dir", arguments.input_dir), + ("--output-dir", arguments.output_dir), + ("--versions-json", arguments.versions_json), + ("--source-repository", arguments.source_repository), + ("--source-commit", arguments.source_commit), + ) + if value is None + ] + if missing: + raise ValueError(f"missing assembly arguments: {', '.join(missing)}") versions = json.loads(arguments.versions_json.read_text(encoding="utf-8")) assemble( arguments.input_dir, diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py index ab2342378..54da32c66 100644 --- a/packaging/tests/test_assemble_native_packages.py +++ b/packaging/tests/test_assemble_native_packages.py @@ -195,6 +195,63 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): ], ) + def test_verifies_assembler_checksum_lines_sorted_by_path(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + output_directory = Path(temporary_directory) / "bundle" + self.write_fixture(input_directory) + module.assemble(input_directory, output_directory, VERSIONS, SOURCE) + + checksum_lines = ( + (output_directory / "SHA256SUMS") + .read_text(encoding="utf-8") + .splitlines() + ) + self.assertNotEqual( + checksum_lines, + sorted(checksum_lines), + "fixture must distinguish path ordering from hash ordering", + ) + self.assertIsNone(module.verify_bundle(output_directory)) + + def test_rejects_unsafe_versions_and_empty_source_identity(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + input_directory = Path(temporary_directory) / "input" + self.write_fixture(input_directory) + for index, (field, value) in enumerate( + ( + ("archive_version", "../../latest"), + ("homebrew_version", "../latest"), + ("archive_version", ""), + ("homebrew_version", "version\\latest"), + ) + ): + with self.subTest(field=field, value=value): + versions = {**VERSIONS, field: value} + with self.assertRaisesRegex(ValueError, "safe path component"): + module.assemble( + input_directory, + Path(temporary_directory) / f"invalid-{index}", + versions, + SOURCE, + ) + for index, (field, value) in enumerate( + (("commit", ""), ("repository", " ")) + ): + with self.subTest(field=field, value=value): + source = {**SOURCE, field: value} + with self.assertRaisesRegex(ValueError, "nonempty"): + module.assemble( + input_directory, + Path(temporary_directory) / f"empty-{index}", + VERSIONS, + source, + ) + def test_rejects_unknown_files(self): module = self.require_module() From 205a813d53b3b79df8de1eb018b1e19d26cf6472 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 21:12:20 +0800 Subject: [PATCH 13/31] fix(packaging): omit empty Homebrew bottle spacing --- packaging/scripts/render_homebrew_formula.py | 3 +++ packaging/tests/test_render_homebrew_formula.py | 1 + 2 files changed, 4 insertions(+) diff --git a/packaging/scripts/render_homebrew_formula.py b/packaging/scripts/render_homebrew_formula.py index a39ba31ee..60217917e 100644 --- a/packaging/scripts/render_homebrew_formula.py +++ b/packaging/scripts/render_homebrew_formula.py @@ -46,6 +46,9 @@ def render_formula( unknown = set(TOKEN.findall(template)) - replacements.keys() if unknown: raise ValueError(f"Unknown Formula placeholders: {sorted(unknown)}") + if not bottle_block: + # Omit an empty standalone block and its following separator line. + template = re.sub(r"(?m)^[ \t]*@BOTTLE_BLOCK@\n(?:[ \t]*\n)?", "", template) rendered = TOKEN.sub(lambda match: replacements[match.group(1)], template) if TOKEN.search(rendered): raise ValueError("Unresolved Formula placeholder") diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py index 0c0047aed..a702c40d9 100644 --- a/packaging/tests/test_render_homebrew_formula.py +++ b/packaging/tests/test_render_homebrew_formula.py @@ -167,6 +167,7 @@ def test_cli_renders_real_template_with_optional_bottle_file(self): ) self.assertIn("#{rpath}", rendered) self.assertNotRegex(rendered, r"@[A-Z0-9_]+@") + self.assertNotIn("\n\n\n", rendered) if expected_block: self.assertIn(BOTTLE_BLOCK, rendered) else: From b8874a02869573c30ec669ad5d7c6f9b245e2e8b Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 21:12:34 +0800 Subject: [PATCH 14/31] docs: document native package artifact workflow --- .github/workflows/cpp-packaging.yml | 128 ---------------------------- packaging/README.md | 30 +++++-- 2 files changed, 24 insertions(+), 134 deletions(-) delete mode 100644 .github/workflows/cpp-packaging.yml diff --git a/.github/workflows/cpp-packaging.yml b/.github/workflows/cpp-packaging.yml deleted file mode 100644 index 54fa92211..000000000 --- a/.github/workflows/cpp-packaging.yml +++ /dev/null @@ -1,128 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -name: Cpp-Packaging - -on: - push: - branches: - - develop - - rc/** - paths: - - '.github/workflows/cpp-packaging.yml' - - 'cpp/**' - - 'packaging/**' - - 'LICENSE' - - 'NOTICE' - pull_request: - branches: - - develop - - rc/** - paths: - - '.github/workflows/cpp-packaging.yml' - - 'cpp/**' - - 'packaging/**' - - 'LICENSE' - - 'NOTICE' - workflow_dispatch: - -permissions: - contents: read - -jobs: - deb: - name: Debian package - runs-on: ubuntu-24.04 - timeout-minutes: 30 - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Install packaging tools - run: | - sudo apt-get update - sudo apt-get install -y build-essential cmake ninja-build pkg-config dpkg-dev uuid-dev - - - name: Configure and build - run: | - cmake -S cpp -B build/deb -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO \ - -DTSFILE_ENABLE_NATIVE_ARCH=OFF - cmake --build build/deb --parallel - - - name: Build and inspect DEB packages - run: | - cpack --config build/deb/CPackConfig.cmake -G DEB - ls -lh ./*.deb - for package in ./*.deb; do - dpkg-deb --info "$package" - dpkg-deb --contents "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true - done - - - name: Upload DEB packages - uses: actions/upload-artifact@v4 - with: - name: tsfile-deb - path: '*.deb' - if-no-files-found: error - - rpm: - name: Fedora package - runs-on: ubuntu-24.04 - container: fedora:latest - timeout-minutes: 30 - steps: - - name: Install build and packaging tools - run: | - dnf install -y \ - cmake gcc-c++ make ninja-build pkgconf-pkg-config rpm-build \ - git curl tar xz unzip gzip libuuid-devel - - - name: Checkout repository - uses: actions/checkout@v7 - - - name: Configure and build - run: | - cmake -S cpp -B build/rpm -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO \ - -DTSFILE_ENABLE_NATIVE_ARCH=OFF - cmake --build build/rpm --parallel - - - name: Build and inspect RPM packages - run: | - cpack --config build/rpm/CPackConfig.cmake -G RPM - ls -lh ./*.rpm - for package in ./*.rpm; do - rpm -qip "$package" - rpm -qlp "$package" | grep -E '(/libtsfile|/tsfile-cli|TsFileConfig|libtsfile.pc)' || true - done - - - name: Upload RPM packages - uses: actions/upload-artifact@v4 - with: - name: tsfile-rpm - path: '*.rpm' - if-no-files-found: error diff --git a/packaging/README.md b/packaging/README.md index c693cd1f6..910026236 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -29,9 +29,29 @@ The header closure mirrors the include relationships used by the current C++ implementation. It is intentionally broader than the long-term stable public API; a separate API cleanup will narrow it in a future version. +## Manual artifact workflow + +`Build native package artifacts` (`.github/workflows/native-packages.yml`) is +the authoritative native packaging workflow. In the fork's GitHub Actions tab, +select this workflow, choose **Run workflow**, select the branch containing the +commit to build, and dispatch it manually. It runs only on `workflow_dispatch`, +with read-only repository permissions; pushes and pull requests do not trigger +native packaging. + +A successful run produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and Intel +Homebrew development bottle with a merged Formula, and a Windows x86_64 SDK/CLI +ZIP. It combines these into `tsfile-native-packages-` with +`manifest.json` and `SHA256SUMS`. Download this final artifact from the workflow +run; both intermediate and final artifacts are retained for 14 days. + +Publishing is a separate, manual step. This workflow only builds, tests, and +uploads GitHub Actions artifacts. It has no publishing credentials and does +not upload to JFrog, sign packages, update `latest`, create tags or releases, or +implement RC/final release behavior. + ## Portable archive -Build a relocatable source archive on any host with CMake and CPack: +Build a relocatable binary archive on any host with CMake and CPack: ```bash cmake -S cpp -B cpp/build/package \ @@ -41,7 +61,6 @@ cmake -S cpp -B cpp/build/package \ -DTSFILE_ENABLE_CPACK=ON \ -DTSFILE_DEPENDENCY_SOURCE=AUTO cmake --build cpp/build/package --parallel -cmake --install cpp/build/package cpack --config cpp/build/package/CPackConfig.cmake -G TGZ ``` @@ -63,15 +82,14 @@ cmake -S cpp -B cpp/build/package \ -DTSFILE_ENABLE_CPACK=ON \ -DTSFILE_DEPENDENCY_SOURCE=AUTO cmake --build cpp/build/package --parallel -cmake --install cpp/build/package cpack --config cpp/build/package/CPackConfig.cmake -G DEB # or: cpack --config cpp/build/package/CPackConfig.cmake -G RPM ``` `SYSTEM` can be used instead of `AUTO` when the build image provides every -compatible dependency, including ANTLR4 4.9.x. `AUTO` is the reproducible -release default and uses the verified source fallback for unavailable or -incompatible distro versions. +compatible dependency, including ANTLR4 >=4.9.3 and <4.13.0. `AUTO` is the +reproducible release default and uses the verified source fallback for +unavailable or incompatible distro versions. The manual `Build native package artifacts` workflow is the authoritative native package build. Its Linux jobs create `tsfile`, `tsfile-dev`, and From 868114503797387c3a468599897043d0050247cb Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 22:08:17 +0800 Subject: [PATCH 15/31] fix(packaging): complete native runtime and install contracts --- .github/workflows/native-packages.yml | 85 +++----- cpp/CMakeLists.txt | 27 +++ cpp/cmake/TsFileConfig.cmake.in | 14 ++ cpp/cmake/TsFileStaticDependencies.cmake | 104 +++++++++ cpp/cmake/TsFileStaticDependencies.cmake.in | 20 ++ cpp/cmake/libtsfile.pc.in | 2 +- cpp/cmake/tests/CheckStaticMSVCRuntime.cmake | 38 ++++ .../projects/InstalledConsumer/CMakeLists.txt | 3 + .../tests/projects/InstalledConsumer/main.cc | 5 +- .../tests/projects/PkgConfigConsumer/main.c | 5 +- .../tests/projects/PkgConfigConsumer/main.cc | 6 +- cpp/src/CMakeLists.txt | 13 +- packaging/README.md | 41 ++++ packaging/homebrew/tsfile-dev.rb.in | 3 +- packaging/homebrew/tsfile.rb | 3 +- packaging/scripts/native_package_versions.py | 11 +- packaging/scripts/verify_windows_runtime.py | 101 +++++++++ packaging/tests/check_native_workflow.rb | 53 +++++ packaging/tests/test_native_install.py | 206 ++++++++++++++++++ .../tests/test_native_package_versions.py | 41 ++++ packaging/tests/test_windows_runtime.py | 73 +++++++ 21 files changed, 788 insertions(+), 66 deletions(-) create mode 100644 cpp/cmake/TsFileStaticDependencies.cmake create mode 100644 cpp/cmake/TsFileStaticDependencies.cmake.in create mode 100644 cpp/cmake/tests/CheckStaticMSVCRuntime.cmake create mode 100644 packaging/scripts/verify_windows_runtime.py create mode 100644 packaging/tests/check_native_workflow.rb create mode 100644 packaging/tests/test_native_install.py create mode 100644 packaging/tests/test_windows_runtime.py diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 3c326d2e2..f6f349315 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -42,6 +42,11 @@ jobs: with: fetch-depth: 0 + - name: Check packaging helpers and workflow contracts + run: | + python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v + ruby packaging/tests/check_native_workflow.rb + - name: Generate package versions id: versions run: | @@ -152,6 +157,9 @@ jobs: - ubuntu:24.04 container: ${{ matrix.container }} steps: + - name: Checkout installed-consumer fixture + uses: actions/checkout@v7 + - name: Download DEB packages uses: actions/download-artifact@v8 with: @@ -181,25 +189,9 @@ jobs: - name: Build and run an installed-package consumer shell: bash run: | - mkdir -p consumer - cat > consumer/CMakeLists.txt <<'CMAKE' - cmake_minimum_required(VERSION 3.11) - project(TsFileInstalledConsumer LANGUAGES CXX) - - find_package(TsFile CONFIG REQUIRED) - add_executable(installed_consumer main.cpp) - target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) - CMAKE - cat > consumer/main.cpp <<'CPP' - #include - - int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; - } - CPP - cmake -S consumer -B consumer/build + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build cmake --build consumer/build --parallel - ./consumer/build/installed_consumer + ./consumer/build/tsfile_installed_consumer build-rpm: name: Build AlmaLinux 9 RPM packages @@ -217,7 +209,7 @@ jobs: pkgconf-pkg-config \ rpm-build \ git \ - curl \ + curl-minimal \ tar \ gzip \ libuuid-devel @@ -299,6 +291,9 @@ jobs: runs-on: ubuntu-24.04 container: almalinux:9 steps: + - name: Checkout installed-consumer fixture + uses: actions/checkout@v7 + - name: Download RPM packages uses: actions/download-artifact@v8 with: @@ -324,25 +319,9 @@ jobs: - name: Build and run an installed-package consumer shell: bash run: | - mkdir -p consumer - cat > consumer/CMakeLists.txt <<'CMAKE' - cmake_minimum_required(VERSION 3.11) - project(TsFileInstalledConsumer LANGUAGES CXX) - - find_package(TsFile CONFIG REQUIRED) - add_executable(installed_consumer main.cpp) - target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) - CMAKE - cat > consumer/main.cpp <<'CPP' - #include - - int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; - } - CPP - cmake -S consumer -B consumer/build + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build cmake --build consumer/build --parallel - ./consumer/build/installed_consumer + ./consumer/build/tsfile_installed_consumer build-windows: name: Build Windows MSVC x86_64 ZIP @@ -370,6 +349,8 @@ jobs: -DBUILD_TOOLS=ON ` -DTSFILE_ENABLE_CPACK=ON ` -DTSFILE_DEPENDENCY_SOURCE=BUNDLED ` + -DTSFILE_MSVC_STATIC_RUNTIME=ON ` + -DCMAKE_PROJECT_TsFile_CPP_INCLUDE="$((Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path)" ` -DTSFILE_ENABLE_NATIVE_ARCH=OFF ` -DTSFILE_ARCHIVE_VERSION="$env:TSFILE_ARCHIVE_VERSION" cmake --build build/windows --config Release --parallel @@ -396,29 +377,16 @@ jobs: } } + python packaging/scripts/verify_windows_runtime.py "$stage" $env:PATH = "$stage/bin;$env:PATH" tsfile-cli.exe --version - New-Item -ItemType Directory -Path consumer -Force | Out-Null - @' - cmake_minimum_required(VERSION 3.11) - project(TsFileInstalledConsumer LANGUAGES CXX) - - find_package(TsFile CONFIG REQUIRED) - add_executable(installed_consumer main.cpp) - target_link_libraries(installed_consumer PRIVATE TsFile::tsfile) - '@ | Set-Content -Path consumer/CMakeLists.txt - @' - #include - - int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; - } - '@ | Set-Content -Path consumer/main.cpp - cmake -S consumer -B consumer/build -G "Visual Studio 17 2022" -A x64 ` + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build ` + -G "Visual Studio 17 2022" -A x64 ` + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ` -DCMAKE_PREFIX_PATH="$stage" cmake --build consumer/build --config Release --parallel - & consumer/build/Release/installed_consumer.exe + & consumer/build/Release/tsfile_installed_consumer.exe - name: Build and inspect combined ZIP env: @@ -469,6 +437,13 @@ jobs: $archive.Dispose() } + Expand-Archive -LiteralPath $archives[0].FullName -DestinationPath extracted/windows + python packaging/scripts/verify_windows_runtime.py extracted/windows + $zipCli = @(Get-ChildItem extracted/windows -Recurse -Filter tsfile-cli.exe -File) + if ($zipCli.Count -ne 1) { throw 'Expected exactly one extracted CLI' } + $env:PATH = "$($zipCli[0].DirectoryName);$env:SystemRoot/System32" + & $zipCli[0].FullName --version + - name: Upload Windows ZIP uses: actions/upload-artifact@v7 with: diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c7b486e00..d7863ba88 100755 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -17,6 +17,18 @@ specific language governing permissions and limitations under the License. ]] cmake_minimum_required(VERSION 3.11) +option(TSFILE_MSVC_STATIC_RUNTIME + "Use the static MSVC runtime throughout a portable bundled build" OFF) +if (TSFILE_MSVC_STATIC_RUNTIME) + if (NOT POLICY CMP0091) + message(FATAL_ERROR "TSFILE_MSVC_STATIC_RUNTIME requires CMake 3.15+") + endif () + # The policy must be selected before the first project(), and inherited by + # upstream projects that reset their policy version in add_subdirectory(). + cmake_policy(SET CMP0091 NEW) + set(CMAKE_POLICY_DEFAULT_CMP0091 NEW) + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$:Debug>") +endif () project(TsFile_CPP) if (DEFINED ToolChain) @@ -62,6 +74,12 @@ if (NOT TSFILE_ABI_VERSION MATCHES "^[0-9]+$") endif () include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/DependencySource.cmake) +if (TSFILE_MSVC_STATIC_RUNTIME AND + NOT TSFILE_DEPENDENCY_SOURCE STREQUAL "BUNDLED") + message(FATAL_ERROR + "TSFILE_MSVC_STATIC_RUNTIME requires TSFILE_DEPENDENCY_SOURCE=BUNDLED " + "so prebuilt dependencies cannot introduce a different MSVC runtime") +endif () list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake") option(TSFILE_BUILD_SHARED "Build libtsfile as a shared library" ON) @@ -514,6 +532,13 @@ install(FILES COMPONENT development) set(TSFILE_PKGCONFIG_CFLAGS "") +if (NOT TSFILE_BUILD_SHARED) + string(APPEND TSFILE_PKGCONFIG_CFLAGS " -DTSFILE_STATIC") + if (ENABLE_ANTLR4 AND (TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED" OR + TSFILE_ANTLR4_SYSTEM_STATIC)) + string(APPEND TSFILE_PKGCONFIG_CFLAGS " -DANTLR4CPP_STATIC") + endif () +endif () foreach (_TSFILE_PUBLIC_FEATURE ENABLE_ANTLR4 ENABLE_MEM_STAT @@ -531,6 +556,8 @@ foreach (_TSFILE_PUBLIC_FEATURE " -D${_TSFILE_PUBLIC_FEATURE}") endif () endforeach () +file(RELATIVE_PATH TSFILE_PKGCONFIG_PREFIX_RELATIVE + "${CMAKE_INSTALL_FULL_LIBDIR}/pkgconfig" "${CMAKE_INSTALL_PREFIX}") configure_file( "${CMAKE_CURRENT_SOURCE_DIR}/cmake/libtsfile.pc.in" "${CMAKE_CURRENT_BINARY_DIR}/libtsfile.pc" diff --git a/cpp/cmake/TsFileConfig.cmake.in b/cpp/cmake/TsFileConfig.cmake.in index eea711a9e..1357f4898 100644 --- a/cpp/cmake/TsFileConfig.cmake.in +++ b/cpp/cmake/TsFileConfig.cmake.in @@ -24,6 +24,20 @@ if (@ENABLE_THREADS@) find_dependency(Threads) endif() +if (NOT @TSFILE_BUILD_SHARED@) + set(_TSFILE_SAVED_MODULE_PATH "${CMAKE_MODULE_PATH}") + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}") + file(GLOB _TSFILE_STATIC_CONFIGS + "${CMAKE_CURRENT_LIST_DIR}/TsFileStaticDependencies-*.cmake") + foreach (_TSFILE_STATIC_CONFIG IN LISTS _TSFILE_STATIC_CONFIGS) + include("${_TSFILE_STATIC_CONFIG}") + endforeach () + set(CMAKE_MODULE_PATH "${_TSFILE_SAVED_MODULE_PATH}") + unset(_TSFILE_STATIC_CONFIG) + unset(_TSFILE_STATIC_CONFIGS) + unset(_TSFILE_SAVED_MODULE_PATH) +endif () + include("${CMAKE_CURRENT_LIST_DIR}/TsFileTargets.cmake") check_required_components(TsFile) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake b/cpp/cmake/TsFileStaticDependencies.cmake new file mode 100644 index 000000000..a4942ca77 --- /dev/null +++ b/cpp/cmake/TsFileStaticDependencies.cmake @@ -0,0 +1,104 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# Static archives do not absorb their link dependencies. Keep the private +# build wrappers out of the export: install bundled archives and describe +# imported link-only targets, or rediscover system packages on the consumer. +# install(FILES) also works on CMake 3.11 for targets in another directory. +function(tsfile_install_static_dependency NAME SOURCE SYSTEM_TARGET FIND_CODE) + if (NOT TARGET TsFile::${NAME}) + return() + endif () + target_link_libraries(tsfile INTERFACE + $>) + set(_CODE "${TSFILE_STATIC_DEPENDENCY_CODE}") + if (SOURCE STREQUAL "BUNDLED") + get_target_property(_TARGET TsFile::${NAME} INTERFACE_LINK_LIBRARIES) + install(FILES $ + DESTINATION "${CMAKE_INSTALL_LIBDIR}/tsfile/$" + COMPONENT development) + string(APPEND _CODE + "if (NOT TARGET TsFile::Static_${NAME})\n" + " add_library(TsFile::Static_${NAME} STATIC IMPORTED)\n" + "endif ()\n" + "set_property(TARGET TsFile::Static_${NAME} APPEND PROPERTY IMPORTED_CONFIGURATIONS $>)\n" + "set_target_properties(TsFile::Static_${NAME} PROPERTIES IMPORTED_LOCATION_$> \"\${CMAKE_CURRENT_LIST_DIR}/../../tsfile/$/$\")\n") + else () + string(APPEND _CODE "${FIND_CODE}\n" + "if (NOT TARGET TsFile::Static_${NAME})\n" + " add_library(TsFile::Static_${NAME} INTERFACE IMPORTED)\n" + " set_target_properties(TsFile::Static_${NAME} PROPERTIES INTERFACE_LINK_LIBRARIES ${SYSTEM_TARGET})\n" + "endif ()\n") + endif () + set(TSFILE_STATIC_DEPENDENCY_CODE "${_CODE}" PARENT_SCOPE) +endfunction() + +set(TSFILE_STATIC_DEPENDENCY_CODE "") +tsfile_install_static_dependency(ANTLR4 "${TSFILE_ANTLR4_SOURCE}" + "${TSFILE_ANTLR4_SYSTEM_TARGET}" + "find_package(utf8cpp CONFIG QUIET)\nfind_dependency(antlr4-runtime CONFIG)") +if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED") + if (WIN32) + string(APPEND TSFILE_STATIC_DEPENDENCY_CODE + "set_property(TARGET TsFile::Static_ANTLR4 PROPERTY INTERFACE_LINK_LIBRARIES ole32)\n") + else () + if (APPLE) + set(_PLATFORM_LIBRARY CoreFoundation) + else () + set(_PLATFORM_LIBRARY uuid) + endif () + string(APPEND TSFILE_STATIC_DEPENDENCY_CODE + "find_library(_TSFILE_ANTLR4_PLATFORM_LIBRARY ${_PLATFORM_LIBRARY})\n" + "if (NOT _TSFILE_ANTLR4_PLATFORM_LIBRARY)\n" + " set(TsFile_FOUND FALSE)\n" + " set(TsFile_NOT_FOUND_MESSAGE \"Static TsFile needs ${_PLATFORM_LIBRARY}\")\n" + " return()\n" + "endif ()\n" + "set_property(TARGET TsFile::Static_ANTLR4 PROPERTY INTERFACE_LINK_LIBRARIES \"\${_TSFILE_ANTLR4_PLATFORM_LIBRARY}\")\n") + unset(_PLATFORM_LIBRARY) + endif () +endif () +tsfile_install_static_dependency(Snappy "${TSFILE_SNAPPY_SOURCE}" Snappy::snappy + "find_dependency(Snappy ${TSFILE_SNAPPY_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(LZ4 "${TSFILE_LZ4_SOURCE}" LZ4::LZ4 + "find_dependency(LZ4 ${TSFILE_LZ4_MIN_VERSION})") +if (ENABLE_LZ4 AND TSFILE_LZ4_SOURCE STREQUAL "SYSTEM") + install(FILES "${CMAKE_SOURCE_DIR}/cmake/FindLZ4.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile" + COMPONENT development) +endif () +tsfile_install_static_dependency(LZOKAY "${TSFILE_LZOKAY_SOURCE}" lzokay::lzokay + "find_dependency(lzokay ${TSFILE_LZOKAY_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(ZLIB "${TSFILE_ZLIB_SOURCE}" ZLIB::ZLIB + "find_dependency(ZLIB ${TSFILE_ZLIB_MIN_VERSION})") +tsfile_install_static_dependency(ZSTD "${TSFILE_ZSTD_SOURCE}" + "${TSFILE_ZSTD_SYSTEM_TARGET}" + "find_dependency(zstd ${TSFILE_ZSTD_MIN_VERSION} CONFIG)") +tsfile_install_static_dependency(LibLZMA "${TSFILE_LIBLZMA_SOURCE}" LibLZMA::LibLZMA + "find_dependency(LibLZMA ${TSFILE_LIBLZMA_MIN_VERSION})\nif (NOT TARGET LibLZMA::LibLZMA)\n add_library(LibLZMA::LibLZMA INTERFACE IMPORTED)\n set_target_properties(LibLZMA::LibLZMA PROPERTIES INTERFACE_LINK_LIBRARIES \"\${LIBLZMA_LIBRARIES}\")\nendif ()") + +configure_file("${CMAKE_SOURCE_DIR}/cmake/TsFileStaticDependencies.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies.cmake.in" @ONLY) +file(GENERATE + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies-$.cmake" + INPUT "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies.cmake.in") +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/TsFileStaticDependencies-$.cmake" + DESTINATION "${CMAKE_INSTALL_LIBDIR}/cmake/TsFile" + COMPONENT development) +unset(TSFILE_STATIC_DEPENDENCY_CODE) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake.in b/cpp/cmake/TsFileStaticDependencies.cmake.in new file mode 100644 index 000000000..877bc21df --- /dev/null +++ b/cpp/cmake/TsFileStaticDependencies.cmake.in @@ -0,0 +1,20 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +@TSFILE_STATIC_DEPENDENCY_CODE@ diff --git a/cpp/cmake/libtsfile.pc.in b/cpp/cmake/libtsfile.pc.in index ff820d0d8..f7b3db9bf 100644 --- a/cpp/cmake/libtsfile.pc.in +++ b/cpp/cmake/libtsfile.pc.in @@ -18,7 +18,7 @@ # The .pc file is installed below ${prefix}/@CMAKE_INSTALL_LIBDIR@/pkgconfig. # Deriving the prefix from its own location keeps DESTDIR and relocations # working without embedding the build-time install prefix. -prefix=${pcfiledir}/../.. +prefix=${pcfiledir}/@TSFILE_PKGCONFIG_PREFIX_RELATIVE@ exec_prefix=${prefix} libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@ diff --git a/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake b/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake new file mode 100644 index 000000000..a30ddfcab --- /dev/null +++ b/cpp/cmake/tests/CheckStaticMSVCRuntime.cmake @@ -0,0 +1,38 @@ +#[[ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +]] + +# A configure-time contract test also runnable on non-MSVC hosts. Runtime +# properties initialize on every platform; Windows CI checks actual PE imports. +function(tsfile_check_static_runtime DIRECTORY) + get_property(_TARGETS DIRECTORY "${DIRECTORY}" PROPERTY BUILDSYSTEM_TARGETS) + foreach (_TARGET IN LISTS _TARGETS) + get_target_property(_TYPE ${_TARGET} TYPE) + if (_TYPE MATCHES "^(STATIC_LIBRARY|SHARED_LIBRARY|OBJECT_LIBRARY|EXECUTABLE)$") + get_target_property(_RUNTIME ${_TARGET} MSVC_RUNTIME_LIBRARY) + if (NOT _RUNTIME STREQUAL "MultiThreaded$<$:Debug>") + message(FATAL_ERROR "${_TARGET} has inconsistent MSVC runtime: ${_RUNTIME}") + endif () + endif () + endforeach () + get_property(_SUBDIRECTORIES DIRECTORY "${DIRECTORY}" PROPERTY SUBDIRECTORIES) + foreach (_SUBDIRECTORY IN LISTS _SUBDIRECTORIES) + tsfile_check_static_runtime("${_SUBDIRECTORY}") + endforeach () +endfunction() +cmake_language(DEFER CALL tsfile_check_static_runtime "${CMAKE_SOURCE_DIR}") diff --git a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt index bf0f83e34..40ea9481d 100644 --- a/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt +++ b/cpp/cmake/tests/projects/InstalledConsumer/CMakeLists.txt @@ -17,6 +17,9 @@ specific language governing permissions and limitations under the License. ]] cmake_minimum_required(VERSION 3.11) +if (POLICY CMP0091) + cmake_policy(SET CMP0091 NEW) +endif () project(TsFileInstalledConsumer LANGUAGES CXX) find_package(TsFile CONFIG REQUIRED) diff --git a/cpp/cmake/tests/projects/InstalledConsumer/main.cc b/cpp/cmake/tests/projects/InstalledConsumer/main.cc index 8d242f678..2d265d429 100644 --- a/cpp/cmake/tests/projects/InstalledConsumer/main.cc +++ b/cpp/cmake/tests/projects/InstalledConsumer/main.cc @@ -21,4 +21,7 @@ #include #include -int main() { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } +int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c index f32e5a550..05a1566f7 100644 --- a/cpp/cmake/tests/projects/PkgConfigConsumer/main.c +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.c @@ -18,4 +18,7 @@ */ #include -int main(void) { return TS_DATATYPE_INT32 == 1 ? 0 : 1; } +int main(void) { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc index be94b8543..daa807236 100644 --- a/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc +++ b/cpp/cmake/tests/projects/PkgConfigConsumer/main.cc @@ -16,6 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +#include #include -int main() { return 0; } +int main() { + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; +} diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 8854e5d6c..96c7d9f47 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -126,6 +126,10 @@ else() # Consumers of the CMake target must see TSFILE_API without DLL import # decoration when linking the static library on MSVC. target_compile_definitions(tsfile INTERFACE TSFILE_STATIC) + if (ENABLE_ANTLR4 AND (TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED" OR + TSFILE_ANTLR4_SYSTEM_STATIC)) + target_compile_definitions(tsfile INTERFACE ANTLR4CPP_STATIC) + endif () endif() if (${COV_ENABLED}) @@ -151,9 +155,10 @@ foreach (_TSFILE_OBJECT_TARGET IN LISTS _TSFILE_OBJECT_TARGETS) $) endforeach () -if (NOT "${_TSFILE_PROJECT_DEPENDENCIES}" STREQUAL "") - target_link_libraries(tsfile PRIVATE ${_TSFILE_PROJECT_DEPENDENCIES}) -endif () +foreach (_TSFILE_DEPENDENCY IN LISTS _TSFILE_PROJECT_DEPENDENCIES) + target_link_libraries(tsfile PRIVATE $) +endforeach () +unset(_TSFILE_DEPENDENCY) unset(_TSFILE_OBJECT_SOURCES) unset(_TSFILE_OBJECT_TARGET) @@ -230,6 +235,8 @@ if (TSFILE_BUILD_SHARED) # dependencies at link time. Do not export those build-only targets as # consumer link requirements; they are not installed with the package. set_property(TARGET tsfile PROPERTY INTERFACE_LINK_LIBRARIES "") +else () + include(${CMAKE_SOURCE_DIR}/cmake/TsFileStaticDependencies.cmake) endif () # A shared library is a RUNTIME plus an import ARCHIVE on Windows and a LIBRARY diff --git a/packaging/README.md b/packaging/README.md index 910026236..feffc2cb2 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -147,11 +147,52 @@ runtime, development files, and tools under a relocatable prefix. In particular, the MSVC outputs are installed as `bin/tsfile.dll`, `lib/tsfile.lib`, and `bin/tsfile-cli.exe`. +The portable ZIP targets Windows 10 / Windows Server 2022 or newer and uses +the static MSVC runtime (`/MT`) in Release. `TSFILE_MSVC_STATIC_RUNTIME=ON` +requires CMake 3.15+ and `TSFILE_DEPENDENCY_SOURCE=BUNDLED`: all codec archives, +TsFile, and the CLI are compiled with the same runtime selection. The workflow +checks the runtime property on every native target and inspects every shipped +EXE/DLL with `dumpbin`, rejecting dynamic Visual C++ runtime imports and missing +non-system DLLs in both the staged tree and the extracted ZIP. Running the CLI +does not require a separate Visual C++ Redistributable installation. Microsoft +redistributable DLLs are not copied from the runner or included in the ZIP; +runtime security updates require rebuilding the static-runtime package. + +C++ SDK consumers should use the matching MSVC v143 Release toolset and `/MT` +(`CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded`, with CMake policy CMP0091 set to +NEW before `project()`). Keep allocation and release paired through the public +APIs; CRT-owned objects such as `FILE*` must not cross DLL boundaries. The +installed-consumer fixture demonstrates a public setting/getter round trip +that requires real symbols from `tsfile.dll`. + The workflow uploads that ZIP as the intermediate artifact `native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the license files, CMake package config, staged executable, library, import library, and public headers without publishing the artifact. +## Static SDK and install regressions + +`TSFILE_BUILD_SHARED=OFF` remains installable. Its CMake package rediscovers +system codec packages on the consuming machine and installs bundled codec +archives under `/tsfile/`. Link with `TsFile::tsfile` +to receive the complete static dependency list. Bundled archives and CMake +metadata move with the install prefix; system dependency development packages +must be available on the consuming machine. The shared package retains its +existing runtime/development/tools layout. + +The pkg-config prefix is derived from the configured pkg-config installation +directory, including multiarch library directories. The optional Unix native +integration suite builds, installs, relocates, and links CMake and pkg-config +consumers. It requires CMake 3.19+ for the runtime-property test, pkg-config, +system LZ4, and a verified dependency cache containing ANTLR4, utf8cpp, zlib, +and LZOKAY archives: + +```bash +TSFILE_RUN_INSTALL_TESTS=1 \ +TSFILE_TEST_DEPENDENCY_CACHE=/path/to/dependency-cache \ +python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v +``` + ## Final native package bundle Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM diff --git a/packaging/homebrew/tsfile-dev.rb.in b/packaging/homebrew/tsfile-dev.rb.in index cc5b60bb2..567a2fcfc 100644 --- a/packaging/homebrew/tsfile-dev.rb.in +++ b/packaging/homebrew/tsfile-dev.rb.in @@ -54,7 +54,8 @@ class TsfileDev < Formula #include int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; } CPP diff --git a/packaging/homebrew/tsfile.rb b/packaging/homebrew/tsfile.rb index 3e96f3dca..1beaac75b 100644 --- a/packaging/homebrew/tsfile.rb +++ b/packaging/homebrew/tsfile.rb @@ -59,7 +59,8 @@ def install #include int main() { - return TS_DATATYPE_INT32 == 1 ? 0 : 1; + if (set_global_time_encoding(TS_ENCODING_TS_2DIFF) != 0) return 1; + return get_global_time_encoding() == TS_ENCODING_TS_2DIFF ? 0 : 1; } CPP diff --git a/packaging/scripts/native_package_versions.py b/packaging/scripts/native_package_versions.py index d4bd653eb..19981e377 100644 --- a/packaging/scripts/native_package_versions.py +++ b/packaging/scripts/native_package_versions.py @@ -119,9 +119,16 @@ def main(argv: Sequence[str] | None = None) -> int: ) arguments.json_out.write_text(json.dumps(versions, indent=2) + "\n", encoding="utf-8") if arguments.github_output is not None: - with arguments.github_output.open("a", encoding="utf-8") as github_output: + with arguments.github_output.open("a+b") as github_output: + if github_output.tell(): + github_output.seek(-1, 2) + if github_output.read(1) != b"\n": + github_output.write(b"\n") github_output.write( - "\n".join(f"{key}={value}" for key, value in versions.items()) + "\n" + ( + "\n".join(f"{key}={value}" for key, value in versions.items()) + + "\n" + ).encode("utf-8") ) return 0 diff --git a/packaging/scripts/verify_windows_runtime.py b/packaging/scripts/verify_windows_runtime.py new file mode 100644 index 000000000..79a7c90b4 --- /dev/null +++ b/packaging/scripts/verify_windows_runtime.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reject portable Windows binaries that need an unshipped DLL or dynamic CRT.""" + +import argparse +import re +import subprocess +from pathlib import Path + +# Windows 10 / Server 2022 inbox DLLs used by TsFile and the static CRT. This +# explicit list deliberately does not trust whatever happens to be on PATH or +# in the development runner's System32 (which also contains the VC redist). +SYSTEM_DLLS = { + "advapi32.dll", + "bcrypt.dll", + "crypt32.dll", + "kernel32.dll", + "ntdll.dll", + "ole32.dll", + "oleaut32.dll", + "rpcrt4.dll", + "secur32.dll", + "shell32.dll", + "shlwapi.dll", + "user32.dll", + "version.dll", + "ws2_32.dll", +} +DYNAMIC_CRT = re.compile(r"^(msvcp|msvcr|vcruntime|ucrtbase|api-ms-win-crt-)", re.I) + + +def parse_dependents(output: str) -> set[str]: + imports = set(re.findall(r"^\s+([A-Za-z0-9_.-]+\.dll)\s*$", output, re.M | re.I)) + if not imports: + raise ValueError("dumpbin did not report any DLL imports") + return {name.lower() for name in imports} + + +def validate_dependents(binary: str, imports: set[str], colocated: set[str]) -> None: + for name in sorted(imports): + if DYNAMIC_CRT.match(name): + raise ValueError( + f"{binary} imports {name}; the ZIP requires a static MSVC runtime" + ) + if ( + name not in SYSTEM_DLLS + and not name.startswith("api-ms-win-") + and name not in colocated + ): + raise ValueError(f"{binary} needs an unshipped non-system DLL: {name}") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("prefix", type=Path) + args = parser.parse_args() + binaries = sorted( + path + for path in args.prefix.rglob("*") + if path.suffix.lower() in (".exe", ".dll") + ) + if not binaries: + parser.error("the prefix contains no Windows binaries") + for binary in binaries: + output = subprocess.run( + ["dumpbin", "/nologo", "/dependents", str(binary)], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + ).stdout + imports = parse_dependents(output) + colocated = { + path.name.lower() + for path in binary.parent.iterdir() + if path.suffix.lower() == ".dll" + } + validate_dependents(binary.name, imports, colocated) + print( + f"{binary.name}: static CRT; all {len(imports)} DLL dependencies are inbox or colocated" + ) + + +if __name__ == "__main__": + main() diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb new file mode 100644 index 000000000..82e5e506f --- /dev/null +++ b/packaging/tests/check_native_workflow.rb @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require "yaml" +require "shellwords" +require "open3" + +path = ARGV.fetch(0, File.expand_path("../../.github/workflows/native-packages.yml", __dir__)) +workflow = YAML.load_file(path) +triggers = workflow["on"] || workflow[true] # Psych's YAML 1.1 boolean parsing +raise "Workflow must remain manual-only" unless triggers.keys == ["workflow_dispatch"] +raise "Workflow permissions must remain read-only" unless workflow["permissions"] == { "contents" => "read" } + +jobs = workflow.fetch("jobs") +bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } +packages = Shellwords.split(bootstrap.fetch("run").gsub("\\\n", " ")) +raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") +raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") + +windows = jobs.fetch("build-windows").fetch("steps").filter_map { |step| step["run"] }.join("\n") +raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-DTSFILE_MSVC_STATIC_RUNTIME=ON") && windows.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") +raise "All native target runtimes must be checked" unless windows.include?("CheckStaticMSVCRuntime.cmake") +raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 + +%w[test-deb test-rpm build-windows].each do |name| + steps = jobs.fetch(name).fetch("steps") + raise "#{name} must check out the consumer fixture" unless steps.any? { |step| step["uses"].to_s.start_with?("actions/checkout@") } + runs = steps.filter_map { |step| step["run"] }.join("\n") + raise "#{name} must use the installed CMake consumer" unless runs.include?("-S cpp/cmake/tests/projects/InstalledConsumer") +end + +jobs.each_value do |job| + job.fetch("steps").each do |step| + next unless step["run"] && step.fetch("shell", "bash") == "bash" + output, status = Open3.capture2e("bash", "-n", stdin_data: step["run"]) + raise "Invalid shell in #{step['name']}: #{output}" unless status.success? + end +end +puts "PASS: manual/read-only workflow, AlmaLinux bootstrap, Windows CRT checks, shared consumers, and Bash syntax" diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py new file mode 100644 index 000000000..ea0574834 --- /dev/null +++ b/packaging/tests/test_native_install.py @@ -0,0 +1,206 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Opt-in native configure/install/link regressions (requires CMake and LZ4).""" + +import os +import re +import shlex +import subprocess +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +FIXTURES = ROOT / "cpp/cmake/tests/projects" + + +@unittest.skipUnless( + os.environ.get("TSFILE_RUN_INSTALL_TESTS") == "1", "opt-in native builds" +) +class NativeInstallTest(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory(prefix="tsfile-install-") + self.addCleanup(self.temporary.cleanup) + self.directory = Path(self.temporary.name) + self.build = self.directory / "build" + self.stage = self.directory / "stage" + + def run_command(self, *arguments, env=None): + result = subprocess.run( + [str(argument) for argument in arguments], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + env=env, + ) + self.assertEqual(result.returncode, 0, result.stdout[-14000:]) + return result.stdout.strip() + + def configure(self, *options): + arguments = [ + "cmake", + "-S", + ROOT / "cpp", + "-B", + self.build, + "-DCMAKE_BUILD_TYPE=Release", + "-DBUILD_TEST=OFF", + "-DBUILD_TOOLS=OFF", + "-DTSFILE_DEPENDENCY_SOURCE=SYSTEM", + "-DTSFILE_ENABLE_NATIVE_ARCH=OFF", + f"-DCMAKE_INSTALL_PREFIX={self.stage}", + ] + arguments.extend( + f"-DENABLE_{name}=OFF" + for name in ( + "ANTLR4", + "SNAPPY", + "LZ4", + "LZOKAY", + "ZLIB", + "ZSTD", + "LZMA2", + "SIMD", + ) + ) + self.run_command(*arguments, *options) + + def install(self): + self.run_command("cmake", "--build", self.build, "--parallel", "8") + self.run_command("cmake", "--install", self.build, "--config", "Release") + relocated = self.directory / "relocated" + self.stage.rename(relocated) + self.stage = relocated + for path in self.stage.rglob("*.cmake"): + self.assertNotIn(str(self.build), path.read_text(), str(path)) + self.assertNotIn(str(ROOT / "cpp"), path.read_text(), str(path)) + + def consumer(self, fixture, target, env=None, *options): + build = self.directory / fixture + self.run_command( + "cmake", + "-S", + FIXTURES / fixture, + "-B", + build, + f"-DCMAKE_PREFIX_PATH={self.stage}", + *options, + env=env, + ) + self.run_command("cmake", "--build", build, "--parallel", "8", env=env) + self.run_command(build / target, env=env) + + def test_static_system_lz4_install_and_relocated_consumer(self): + self.configure("-DTSFILE_BUILD_SHARED=OFF", "-DENABLE_LZ4=ON") + self.install() + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + + def test_static_bundled_install_and_relocated_consumer(self): + self.configure( + "-DTSFILE_BUILD_SHARED=OFF", + "-DTSFILE_DEPENDENCY_SOURCE=BUNDLED", + "-DENABLE_ANTLR4=ON", + "-DENABLE_ZLIB=ON", + "-DENABLE_LZOKAY=ON", + "-DTSFILE_DEPENDENCY_OFFLINE=ON", + f"-DTSFILE_DEPENDENCY_CACHE={os.environ['TSFILE_TEST_DEPENDENCY_CACHE']}", + ) + self.install() + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + + def test_multiarch_pkgconfig_install_and_relocated_consumers(self): + self.configure("-DCMAKE_INSTALL_LIBDIR=lib/x86_64-linux-gnu") + self.install() + env = dict( + os.environ, + PKG_CONFIG_PATH=str(self.stage / "lib/x86_64-linux-gnu/pkgconfig"), + ) + prefix = self.run_command( + "pkg-config", "--variable=prefix", "libtsfile", env=env + ) + self.assertEqual(Path(prefix).resolve(), self.stage.resolve()) + self.consumer("PkgConfigConsumer", "tsfile_pkgconfig_consumer", env) + self.run_command( + self.directory / "PkgConfigConsumer/tsfile_pkgconfig_cpp_consumer", env=env + ) + + def test_installed_smoke_requires_a_library_symbol(self): + self.configure() + self.install() + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-std=c++11", + f"-I{self.stage / 'include'}", + f"-I{self.stage / 'include/tsfile'}", + str(FIXTURES / "InstalledConsumer/main.cc"), + "-o", + str(self.directory / "unlinked"), + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertNotEqual( + result.returncode, 0, "Smoke consumer linked successfully without libtsfile" + ) + self.assertIn("get_global_time_encoding", result.stdout) + self.consumer("InstalledConsumer", "tsfile_installed_consumer") + for formula in ("tsfile.rb", "tsfile-dev.rb.in"): + source = re.search( + r"write <<~CPP\n(.*?)\n\s+CPP", + (ROOT / "packaging/homebrew" / formula).read_text(), + re.S, + ).group(1) + source_path = self.directory / f"{formula}.cc" + source_path.write_text(textwrap.dedent(source)) + command = [ + *shlex.split(os.environ.get("CXX", "c++")), + "-std=c++11", + f"-I{self.stage / 'include'}", + str(source_path), + "-o", + str(self.directory / "brew-consumer"), + ] + unlinked = subprocess.run( + command, text=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + ) + self.assertNotEqual(unlinked.returncode, 0, formula) + self.assertIn("get_global_time_encoding", unlinked.stdout) + self.run_command( + *command, + f"-L{self.stage / 'lib'}", + f"-Wl,-rpath,{self.stage / 'lib'}", + "-ltsfile", + ) + self.run_command(self.directory / "brew-consumer") + + def test_static_msvc_runtime_reaches_dependency_and_project_targets(self): + self.configure( + "-DTSFILE_MSVC_STATIC_RUNTIME=ON", + "-DTSFILE_DEPENDENCY_SOURCE=BUNDLED", + "-DENABLE_ZLIB=ON", + "-DTSFILE_DEPENDENCY_OFFLINE=ON", + f"-DTSFILE_DEPENDENCY_CACHE={os.environ['TSFILE_TEST_DEPENDENCY_CACHE']}", + f"-DCMAKE_PROJECT_TsFile_CPP_INCLUDE={ROOT / 'cpp/cmake/tests/CheckStaticMSVCRuntime.cmake'}", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py index a2cc05221..b5a6b80c9 100644 --- a/packaging/tests/test_native_package_versions.py +++ b/packaging/tests/test_native_package_versions.py @@ -141,6 +141,47 @@ def test_cli_writes_json_and_github_outputs(self): ["existing=value", *[f"{key}={value}" for key, value in expected.items()]], ) + def test_cli_separates_unterminated_github_output_without_changing_bytes(self): + for existing in ( + b"", + b"existing=value", + b"existing=value\n", + b"existing=value\r\n", + ): + with self.subTest( + existing=existing + ), tempfile.TemporaryDirectory() as directory: + output = Path(directory) / "github-output.txt" + output.write_bytes(existing) + subprocess.run( + [ + sys.executable, + str(MODULE_PATH), + "--cmake-file", + str(CPP_CMAKE_FILE), + "--build-date", + "20260910", + "--run-number", + "123", + "--run-attempt", + "1", + "--git-sha", + "abcdef123456", + "--json-out", + str(Path(directory) / "versions.json"), + "--github-output", + str(output), + ], + check=True, + ) + separator = b"\n" if existing and not existing.endswith(b"\n") else b"" + self.assertTrue( + output.read_bytes().startswith( + existing + separator + b"base_version=2.5.0\n" + ), + output.read_bytes(), + ) + if __name__ == "__main__": unittest.main() diff --git a/packaging/tests/test_windows_runtime.py b/packaging/tests/test_windows_runtime.py new file mode 100644 index 000000000..8258f00a0 --- /dev/null +++ b/packaging/tests/test_windows_runtime.py @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import importlib.util +import unittest +from pathlib import Path + +MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts/verify_windows_runtime.py" + + +class WindowsRuntimeTest(unittest.TestCase): + def checker(self): + self.assertTrue( + MODULE_PATH.is_file(), "Windows artifacts need a runtime dependency check" + ) + spec = importlib.util.spec_from_file_location( + "verify_windows_runtime", MODULE_PATH + ) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_allows_only_inbox_or_colocated_package_dlls(self): + module = self.checker() + imports = module.parse_dependents( + "File Type: EXECUTABLE IMAGE\n\n Image has the following dependencies:\n\n" + " tsfile.dll\n KERNEL32.dll\n api-ms-win-core-synch-l1-2-0.dll\n\n" + " Summary\n 1000 .data\n" + ) + module.validate_dependents("tsfile-cli.exe", imports, {"tsfile.dll"}) + self.assertEqual(len(imports), 3) + + def test_rejects_dynamic_crt_even_when_runner_or_package_provides_it(self): + module = self.checker() + for dll in ( + "VCRUNTIME140.dll", + "MSVCP140.dll", + "ucrtbase.dll", + "api-ms-win-crt-runtime-l1-1-0.dll", + ): + with self.subTest(dll=dll), self.assertRaisesRegex( + ValueError, "static MSVC runtime" + ): + module.validate_dependents("tsfile.dll", {dll.lower()}, {dll.lower()}) + + def test_rejects_unshipped_codec_dll(self): + module = self.checker() + with self.assertRaisesRegex(ValueError, "snappy.dll"): + module.validate_dependents("tsfile.dll", {"snappy.dll"}, {"tsfile.dll"}) + + def test_rejects_unrecognized_dumpbin_output(self): + with self.assertRaisesRegex(ValueError, "DLL imports"): + self.checker().parse_dependents( + "fatal error LNK1107: invalid or corrupt file" + ) + + +if __name__ == "__main__": + unittest.main() From d3bd297b2b7116314b26986a2806d7945a730e37 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 22:35:05 +0800 Subject: [PATCH 16/31] fix(cmake): preserve static SDK ANTLR compatibility bounds --- cpp/cmake/TsFileStaticDependencies.cmake | 26 +++- packaging/tests/test_native_install.py | 163 +++++++++++++++++++++++ 2 files changed, 188 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/TsFileStaticDependencies.cmake b/cpp/cmake/TsFileStaticDependencies.cmake index a4942ca77..8233487f3 100644 --- a/cpp/cmake/TsFileStaticDependencies.cmake +++ b/cpp/cmake/TsFileStaticDependencies.cmake @@ -50,9 +50,33 @@ function(tsfile_install_static_dependency NAME SOURCE SYSTEM_TARGET FIND_CODE) endfunction() set(TSFILE_STATIC_DEPENDENCY_CODE "") +# Keep the build-side compatibility interval when a static consumer rediscovers +# ANTLR4. Explicit comparisons also support CMake 3.11 and packages that report +# ANTLR_VERSION instead of antlr4-runtime_VERSION. +string(CONFIGURE [=[ +find_package(utf8cpp CONFIG QUIET) +find_package(antlr4-runtime CONFIG QUIET) +set(_TSFILE_ANTLR4_VERSION "${antlr4-runtime_VERSION}") +if ("${_TSFILE_ANTLR4_VERSION}" STREQUAL "" AND DEFINED ANTLR_VERSION) + set(_TSFILE_ANTLR4_VERSION "${ANTLR_VERSION}") +endif () +if (NOT antlr4-runtime_FOUND OR + "${_TSFILE_ANTLR4_VERSION}" STREQUAL "" OR + _TSFILE_ANTLR4_VERSION VERSION_LESS "@TSFILE_ANTLR4_MIN_VERSION@" OR + NOT _TSFILE_ANTLR4_VERSION VERSION_LESS "@TSFILE_ANTLR4_NEXT_INCOMPATIBLE_VERSION@" OR + NOT TARGET @TSFILE_ANTLR4_SYSTEM_TARGET@) + set(TsFile_FOUND FALSE) + set(TsFile_NOT_FOUND_MESSAGE + "Static TsFile requires a compatible system ANTLR4 >=@TSFILE_ANTLR4_MIN_VERSION@ and <@TSFILE_ANTLR4_NEXT_INCOMPATIBLE_VERSION@ with target @TSFILE_ANTLR4_SYSTEM_TARGET@; reported version '${_TSFILE_ANTLR4_VERSION}'.") + unset(_TSFILE_ANTLR4_VERSION) + return() +endif () +unset(_TSFILE_ANTLR4_VERSION) +]=] _TSFILE_FIND_STATIC_ANTLR4 @ONLY) tsfile_install_static_dependency(ANTLR4 "${TSFILE_ANTLR4_SOURCE}" "${TSFILE_ANTLR4_SYSTEM_TARGET}" - "find_package(utf8cpp CONFIG QUIET)\nfind_dependency(antlr4-runtime CONFIG)") + "${_TSFILE_FIND_STATIC_ANTLR4}") +unset(_TSFILE_FIND_STATIC_ANTLR4) if (ENABLE_ANTLR4 AND TSFILE_ANTLR4_SOURCE STREQUAL "BUNDLED") if (WIN32) string(APPEND TSFILE_STATIC_DEPENDENCY_CODE diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py index ea0574834..76128fcf9 100644 --- a/packaging/tests/test_native_install.py +++ b/packaging/tests/test_native_install.py @@ -20,6 +20,7 @@ import os import re import shlex +import shutil import subprocess import tempfile import textwrap @@ -30,6 +31,168 @@ FIXTURES = ROOT / "cpp/cmake/tests/projects" +class StaticANTLRPackageTest(unittest.TestCase): + """Exercise installed metadata without compiling the ANTLR runtime.""" + + def test_installed_system_antlr_requires_compatible_runtime(self): + with tempfile.TemporaryDirectory(prefix="tsfile-antlr-config-") as temporary: + directory = Path(temporary) + producer = directory / "producer" + (producer / "cmake").mkdir(parents=True) + shutil.copyfile( + ROOT / "cpp/cmake/TsFileStaticDependencies.cmake.in", + producer / "cmake/TsFileStaticDependencies.cmake.in", + ) + (producer / "CMakeLists.txt").write_text(textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.11) + project(StaticANTLRPackage NONE) + include(CMakePackageConfigHelpers) + include("${TSFILE_CMAKE_DIR}/DependencySource.cmake") + include("${TSFILE_CMAKE_DIR}/ANTLR4Dependency.cmake") + set(CMAKE_INSTALL_LIBDIR lib) + set(ENABLE_ANTLR4 ON) + set(ENABLE_THREADS OFF) + set(TSFILE_BUILD_SHARED OFF) + add_library(tsfile INTERFACE) + add_library(TsFile::ANTLR4 INTERFACE IMPORTED) + include("${TSFILE_CMAKE_DIR}/TsFileStaticDependencies.cmake") + configure_package_config_file( + "${TSFILE_CMAKE_DIR}/TsFileConfig.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + INSTALL_DESTINATION lib/cmake/TsFile) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/TsFileConfig.cmake" + DESTINATION lib/cmake/TsFile) + install(TARGETS tsfile EXPORT TsFileTargets) + install(EXPORT TsFileTargets NAMESPACE TsFile:: + DESTINATION lib/cmake/TsFile) + """)) + consumer = directory / "consumer" + consumer.mkdir() + (consumer / "CMakeLists.txt").write_text(textwrap.dedent("""\ + cmake_minimum_required(VERSION 3.11) + project(StaticANTLRConsumer NONE) + find_package(TsFile CONFIG REQUIRED) + get_target_property(runtime TsFile::Static_ANTLR4 + INTERFACE_LINK_LIBRARIES) + if (NOT runtime STREQUAL EXPECTED_TARGET OR NOT TARGET ${runtime}) + message(FATAL_ERROR "Static TsFile lost its system target") + endif () + if (NOT TARGET utf8cpp) + message(FATAL_ERROR "Static TsFile lost utf8cpp compatibility") + endif () + """)) + + def configure(source, build, *options): + return subprocess.run( + [ + "cmake", + "-S", + str(source), + "-B", + str(build), + "-DCMAKE_FIND_USE_PACKAGE_REGISTRY=FALSE", + "-DCMAKE_FIND_USE_SYSTEM_PACKAGE_REGISTRY=FALSE", + "-DCMAKE_FIND_USE_CMAKE_SYSTEM_PATH=FALSE", + "-DCMAKE_FIND_USE_SYSTEM_ENVIRONMENT_PATH=FALSE", + f"-DCMAKE_MAKE_PROGRAM={shutil.which('make')}", + "-G", + "Unix Makefiles", + *options, + ], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + + def package(prefix, version, target, version_field): + prefix.mkdir(parents=True) + (prefix / "utf8cppConfig.cmake").write_text( + "add_library(utf8cpp INTERFACE IMPORTED)\n" + ) + if version == "missing-package": + return + content = "" + if version is not None: + content += f'set({version_field} "{version}")\n' + if target is not None: + content += ( + f"add_library({target} INTERFACE IMPORTED)\n" + f"set_property(TARGET {target} PROPERTY " + "INTERFACE_LINK_LIBRARIES utf8cpp)\n" + ) + (prefix / "antlr4-runtime-config.cmake").write_text(content) + + targets = ( + "antlr4_static", + "antlr4-runtime::antlr4_static", + "antlr4_shared", + "antlr4-runtime::antlr4_shared", + ) + for index, target in enumerate(targets): + build = directory / f"producer-build-{index}" + stage = directory / f"stage-{index}" + build_package = directory / f"build-package-{index}" + package(build_package, "4.9.3", target, "ANTLR_VERSION") + result = configure( + producer, + build, + f"-DTSFILE_CMAKE_DIR={ROOT / 'cpp/cmake'}", + "-DTSFILE_DEPENDENCY_SOURCE=SYSTEM", + f"-DCMAKE_PREFIX_PATH={build_package}", + f"-DCMAKE_INSTALL_PREFIX={stage}", + ) + self.assertEqual(result.returncode, 0, result.stdout) + result = subprocess.run( + ["cmake", "--install", str(build)], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + relocated = directory / f"relocated-{index}" + stage.rename(relocated) + cases = ( + ("4.9.3", target, True), + ("4.12.0", target, True), + ("4.13.0", target, False), + ("4.13.2", target, False), + ("4.9.2", target, False), + (None, target, False), + ("4.9.3", None, False), + ("missing-package", None, False), + ) + for field in ("antlr4-runtime_VERSION", "ANTLR_VERSION"): + for case, (version, consumer_target, succeeds) in enumerate(cases): + with self.subTest( + target=target, + field=field, + version=version, + consumer_target=consumer_target, + ): + suffix = f"{index}-{field}-{case}" + runtime = directory / f"runtime-{suffix}" + package(runtime, version, consumer_target, field) + result = configure( + consumer, + directory / f"consumer-{suffix}", + f"-DCMAKE_PREFIX_PATH={relocated};{runtime}", + f"-DEXPECTED_TARGET={target}", + ) + if succeeds: + self.assertEqual(result.returncode, 0, result.stdout) + else: + self.assertNotEqual( + result.returncode, + 0, + f"Installed static TsFile accepted {version}:\n" + + result.stdout, + ) + diagnostic = " ".join(result.stdout.split()) + self.assertIn("compatible system ANTLR4", diagnostic) + self.assertIn(">=4.9.3", diagnostic) + self.assertIn("<4.13.0", diagnostic) + + @unittest.skipUnless( os.environ.get("TSFILE_RUN_INSTALL_TESTS") == "1", "opt-in native builds" ) From 485b99f43c4e1aa4303f2e7630f5f4182d6becb8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 23:18:44 +0800 Subject: [PATCH 17/31] fix(ci): address native packaging runner failures --- .github/workflows/native-packages.yml | 4 +-- cpp/src/utils/util_define.h | 4 +-- packaging/README.md | 7 ++++- packaging/homebrew/tsfile-dev.rb.in | 1 + packaging/tests/check_native_workflow.rb | 5 ++++ packaging/tests/test_native_install.py | 30 +++++++++++++++++++ .../tests/test_render_homebrew_formula.py | 5 ++++ 7 files changed, 51 insertions(+), 5 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index f6f349315..99bf2d8bb 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -205,7 +205,7 @@ jobs: dnf install -y \ cmake \ gcc-c++ \ - ninja-build \ + make \ pkgconf-pkg-config \ rpm-build \ git \ @@ -224,7 +224,7 @@ jobs: TSFILE_RPM_PACKAGE_RELEASE: ${{ needs.prepare.outputs.rpm_release }} shell: bash run: | - cmake -S cpp -B build/rpm -G Ninja \ + cmake -S cpp -B build/rpm \ -DCMAKE_BUILD_TYPE=Release \ -DBUILD_TEST=OFF \ -DBUILD_TOOLS=ON \ diff --git a/cpp/src/utils/util_define.h b/cpp/src/utils/util_define.h index 564581a11..f7f638e5b 100644 --- a/cpp/src/utils/util_define.h +++ b/cpp/src/utils/util_define.h @@ -147,13 +147,13 @@ typedef int mode_t; * @msg should be a single word (use -/_ to concat) * such as This_should_be_TRUE */ -#if __cplusplus < 201103L +#if __cplusplus < 201103L && !defined(_MSC_VER) // TODO only define this when DEBUG #define STATIC_ASSERT(cond, msg) \ typedef char static_assertion_##msg[(cond) ? 1 : -1] __attribute__((unused)) #else #define STATIC_ASSERT(cond, msg) static_assert((cond), #msg) -#endif // __cplusplus < 201103L +#endif // __cplusplus < 201103L && !defined(_MSC_VER) /* ======== atomic operation ======== * diff --git a/packaging/README.md b/packaging/README.md index feffc2cb2..32b8106d4 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -117,7 +117,12 @@ Formula `tsfile-dev`, using `packaging/homebrew/tsfile-dev.rb.in`. This is not a Homebrew/core submission. Each build pins the full workflow commit from `ColinLeeo/tsfile`, hashes that source archive, and uses the generated immutable Homebrew development version. The Formula retains the CMake install behavior -and tests the installed C++ consumer and CLI before bottling. +and tests the installed C++ consumer and CLI before bottling. `tsfile-dev` is +keg-only because the current SDK intentionally ships its dependency header +closure; linking that closure into Homebrew's shared prefix would collide with +headers owned by dependencies such as `simde`. After installation, invoke the +CLI as `$(brew --prefix tsfile-dev)/bin/tsfile-cli`, or add that Formula's +`bin` directory to `PATH`. The ARM64 job runs on `macos-latest` and the Intel job on `macos-15-intel`. Their `native-homebrew-macos-arm64` and `native-homebrew-macos-x86_64` diff --git a/packaging/homebrew/tsfile-dev.rb.in b/packaging/homebrew/tsfile-dev.rb.in index 567a2fcfc..b5eac6431 100644 --- a/packaging/homebrew/tsfile-dev.rb.in +++ b/packaging/homebrew/tsfile-dev.rb.in @@ -22,6 +22,7 @@ class TsfileDev < Formula version "@HOMEBREW_VERSION@" sha256 "@SOURCE_SHA256@" license "Apache-2.0" + keg_only "development snapshots install their dependency header closure" @BOTTLE_BLOCK@ diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 82e5e506f..053e25174 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -30,6 +30,11 @@ packages = Shellwords.split(bootstrap.fetch("run").gsub("\\\n", " ")) raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") +raise "AlmaLinux 9 default repositories do not provide ninja-build" if packages.include?("ninja-build") +raise "RPM bootstrap must install make" unless packages.include?("make") + +rpm_build = jobs.fetch("build-rpm").fetch("steps").filter_map { |step| step["run"] }.join("\n") +raise "RPM build must not require Ninja" if rpm_build.include?("-G Ninja") windows = jobs.fetch("build-windows").fetch("steps").filter_map { |step| step["run"] }.join("\n") raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-DTSFILE_MSVC_STATIC_RUNTIME=ON") && windows.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py index 76128fcf9..ed3eb5a10 100644 --- a/packaging/tests/test_native_install.py +++ b/packaging/tests/test_native_install.py @@ -31,6 +31,36 @@ FIXTURES = ROOT / "cpp/cmake/tests/projects" +class PublicHeaderCompatibilityTest(unittest.TestCase): + def test_static_assert_uses_the_native_keyword_for_msvc(self): + source = textwrap.dedent("""\ + #include "utils/util_define.h" + class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-E", + "-P", + "-x", + "c++", + "-std=c++11", + "-D_MSC_VER=1930", + "-DTSFILE_STATIC", + "-D__cplusplus=199711L", + f"-I{ROOT / 'cpp/src'}", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout[-4000:]) + self.assertRegex(result.stdout, r"class Probe \{\s*(?:static_assert|_Static_assert)") + self.assertNotIn("static_assertion_MSVC_old_cplusplus", result.stdout) + + class StaticANTLRPackageTest(unittest.TestCase): """Exercise installed metadata without compiling the ANTLR runtime.""" diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py index a702c40d9..a60f9a6c0 100644 --- a/packaging/tests/test_render_homebrew_formula.py +++ b/packaging/tests/test_render_homebrew_formula.py @@ -128,6 +128,11 @@ def test_preserves_ruby_interpolation_and_other_at_signs(self): def test_cli_renders_real_template_with_optional_bottle_file(self): self.renderer() + template = TEMPLATE_PATH.read_text(encoding="utf-8") + self.assertIn( + 'keg_only "development snapshots install their dependency header closure"', + template, + ) with tempfile.TemporaryDirectory() as temporary_directory: directory = Path(temporary_directory) output = directory / "tsfile-dev.rb" From 5094e29c8a50d2e8bb4d7f47e8120009bfe3a972 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 10 Sep 2026 23:55:27 +0800 Subject: [PATCH 18/31] fix(ci): resolve native package portability failures --- .github/workflows/native-packages.yml | 1 + cpp/src/utils/util_define.h | 4 +- packaging/tests/check_native_workflow.rb | 5 ++ packaging/tests/test_native_install.py | 76 +++++++++++++++++++++++- 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 99bf2d8bb..187bb91fd 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -581,6 +581,7 @@ jobs: - name: Merge both bottle tags into the Formula shell: bash run: | + brew trust apache/tsfile-dev brew bottle --merge --write --no-commit homebrew/bottles/*.json tap_path="$(brew --repository apache/tsfile-dev)" cp "$tap_path/Formula/tsfile-dev.rb" homebrew/Formula/tsfile-dev.rb diff --git a/cpp/src/utils/util_define.h b/cpp/src/utils/util_define.h index f7f638e5b..33784e152 100644 --- a/cpp/src/utils/util_define.h +++ b/cpp/src/utils/util_define.h @@ -120,14 +120,14 @@ typedef int mode_t; #endif // __GNUC__ >= 4 /* ======== nullptr ======== */ -#if __cplusplus < 201103L +#if __cplusplus < 201103L && !defined(_MSC_VER) #ifndef nullptr #define nullptr NULL #endif #define OVERRIDE #else #define OVERRIDE override -#endif // __cplusplus < 201103L +#endif // __cplusplus < 201103L && !defined(_MSC_VER) /* ======== cache line ======== */ #ifndef CACHE_LINE_SIZE diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 053e25174..828c6630a 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -41,6 +41,11 @@ raise "All native target runtimes must be checked" unless windows.include?("CheckStaticMSVCRuntime.cmake") raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 +homebrew_merge = jobs.fetch("merge-homebrew").fetch("steps").filter_map { |step| step["run"] }.join("\n") +trust = homebrew_merge.index("brew trust apache/tsfile-dev") +merge = homebrew_merge.index("brew bottle --merge") +raise "Homebrew merge must trust its temporary tap before loading the Formula" unless trust && merge && trust < merge + %w[test-deb test-rpm build-windows].each do |name| steps = jobs.fetch(name).fetch("steps") raise "#{name} must check out the consumer fixture" unless steps.any? { |step| step["uses"].to_s.start_with?("actions/checkout@") } diff --git a/packaging/tests/test_native_install.py b/packaging/tests/test_native_install.py index ed3eb5a10..0ce6d8b42 100644 --- a/packaging/tests/test_native_install.py +++ b/packaging/tests/test_native_install.py @@ -32,10 +32,44 @@ class PublicHeaderCompatibilityTest(unittest.TestCase): - def test_static_assert_uses_the_native_keyword_for_msvc(self): + def test_simd_int64_reduction_normalizes_lane_types(self): + statistic = (ROOT / "cpp/src/common/statistic.h").read_text() + self.assertIn("std::min(simde_mm_cvtsi128_si64(vmin)", statistic) + self.assertIn("std::max(simde_mm_cvtsi128_si64(vmax)", statistic) + source = textwrap.dedent("""\ + #include + #include + + long simde_mm_cvtsi128_si64(int); + long long simde_mm_extract_epi64(int, int); + + std::int64_t reduce_min(int value) { + return std::min(simde_mm_cvtsi128_si64(value), + simde_mm_extract_epi64(value, 1)); + } + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-fsyntax-only", + "-x", + "c++", + "-std=c++11", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + + def test_msvc_old_cplusplus_uses_native_cxx11_features(self): source = textwrap.dedent("""\ #include "utils/util_define.h" class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; + class Base { virtual void reset(); }; + class Derived : public Base { void reset() OVERRIDE; }; """) result = subprocess.run( [ @@ -59,6 +93,46 @@ class Probe { STATIC_ASSERT(true, MSVC_old_cplusplus); }; self.assertEqual(result.returncode, 0, result.stdout[-4000:]) self.assertRegex(result.stdout, r"class Probe \{\s*(?:static_assert|_Static_assert)") self.assertNotIn("static_assertion_MSVC_old_cplusplus", result.stdout) + self.assertIn("void reset() override", result.stdout) + + def test_msvc_old_cplusplus_does_not_define_nullptr(self): + compatibility = (ROOT / "cpp/src/utils/util_define.h").read_text() + section = re.search( + r"/\* ======== nullptr ======== \*/\n(.*?)\n/\* ======== cache line", + compatibility, + re.S, + ) + self.assertIsNotNone(section) + source = section.group(1) + textwrap.dedent("""\ + + #ifdef nullptr + #error nullptr must remain a native MSVC keyword + #endif + class Base { virtual void reset(); }; + class Derived : public Base { void reset() OVERRIDE; }; + int* pointer = nullptr; + """) + result = subprocess.run( + [ + *shlex.split(os.environ.get("CXX", "c++")), + "-E", + "-P", + "-nostdinc", + "-x", + "c++", + "-std=c++11", + "-D_MSC_VER=1930", + "-D__cplusplus=199711L", + "-", + ], + input=source, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + self.assertEqual(result.returncode, 0, result.stdout) + self.assertIn("void reset() override", result.stdout) + self.assertRegex(result.stdout, r"int\s*\*\s*pointer\s*=\s*nullptr") class StaticANTLRPackageTest(unittest.TestCase): From 72abe301c86a27be693093a556b6a17bd5282dd9 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Thu, 17 Sep 2026 15:35:19 +0800 Subject: [PATCH 19/31] fix(packaging): derive package test versions from the checkout The native package version tests hardcoded the fork's 2.5.0 development version, so they failed on any checkout that declares a different cpp/CMakeLists.txt version. Read the version under test instead and keep only the unit cases that intentionally pin an input version. --- .gitignore | 2 ++ .../tests/test_native_package_versions.py | 26 ++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index cf943dae7..a447d8353 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,8 @@ python/tsfile/**/*so* python/data python/venv/* python/tests/__pycache__/* +packaging/**/__pycache__/ +packaging/**/*.py[cod] python/tests/*.tsfile python/tsfile/include diff --git a/packaging/tests/test_native_package_versions.py b/packaging/tests/test_native_package_versions.py index b5a6b80c9..15264486a 100644 --- a/packaging/tests/test_native_package_versions.py +++ b/packaging/tests/test_native_package_versions.py @@ -17,6 +17,7 @@ import importlib.util import json +import re import subprocess import sys import tempfile @@ -27,6 +28,16 @@ REPOSITORY_ROOT = Path(__file__).resolve().parents[2] MODULE_PATH = REPOSITORY_ROOT / "packaging/scripts/native_package_versions.py" CPP_CMAKE_FILE = REPOSITORY_ROOT / "cpp/CMakeLists.txt" +CPP_VERSION_PATTERN = re.compile(r"^set\(TsFile_CPP_VERSION\s+(\S+)\)", re.MULTILINE) + + +def repository_cpp_version(): + """Return the C++ version declared by the checkout under test.""" + + match = CPP_VERSION_PATTERN.search(CPP_CMAKE_FILE.read_text(encoding="utf-8")) + if match is None: + raise AssertionError(f"cannot find TsFile_CPP_VERSION in {CPP_CMAKE_FILE}") + return match.group(1) def load_version_module(): @@ -54,7 +65,7 @@ def require_module(self): def test_reads_development_version_from_cpp_cmake(self): module = self.require_module() - self.assertEqual(module.read_cpp_version(CPP_CMAKE_FILE), "2.5.0.dev") + self.assertEqual(module.read_cpp_version(CPP_CMAKE_FILE), repository_cpp_version()) def test_builds_exact_development_package_versions(self): module = self.require_module() @@ -106,7 +117,9 @@ def test_rejects_invalid_build_identity_fields(self): def test_cli_writes_json_and_github_outputs(self): module = self.require_module() - expected = module.build_versions("2.5.0.dev", "20260910", 123, 1, "abcdef123456") + expected = module.build_versions( + repository_cpp_version(), "20260910", 123, 1, "abcdef123456" + ) with tempfile.TemporaryDirectory() as temporary_directory: temporary_path = Path(temporary_directory) @@ -142,6 +155,11 @@ def test_cli_writes_json_and_github_outputs(self): ) def test_cli_separates_unterminated_github_output_without_changing_bytes(self): + module = self.require_module() + base_version = module.build_versions( + repository_cpp_version(), "20260910", 123, 1, "abcdef123456" + )["base_version"] + for existing in ( b"", b"existing=value", @@ -177,7 +195,9 @@ def test_cli_separates_unterminated_github_output_without_changing_bytes(self): separator = b"\n" if existing and not existing.endswith(b"\n") else b"" self.assertTrue( output.read_bytes().startswith( - existing + separator + b"base_version=2.5.0\n" + existing + + separator + + f"base_version={base_version}\n".encode("utf-8") ), output.read_bytes(), ) From 46a7a86ba71fc3b26c3f265c03864ba451bb42db Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 16:01:38 +0800 Subject: [PATCH 20/31] build(python): add -Pwith-python-only to reuse the C++ build - pom.xml: add the with-python-only profile plus the shared check default - python/pom.xml: resolve tsfile.cpp.build, pass it to setup.py via TSFILE_CPP_BUILD, and validate the C++ artifacts at the validate phase - python/check_cpp_build.py: fail fast with an actionable message - wheels.yml: drop the explicit -Denable.lzma2=ON so Python wheels follow the single LZMA2 default owned by cpp/pom.xml --- .github/workflows/wheels.yml | 3 -- pom.xml | 24 +++++++++++++ python/check_cpp_build.py | 70 ++++++++++++++++++++++++++++++++++++ python/pom.xml | 36 +++++++++++++++++++ 4 files changed, 130 insertions(+), 3 deletions(-) create mode 100755 python/check_cpp_build.py diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml index 8161cda0c..53ac76607 100644 --- a/.github/workflows/wheels.yml +++ b/.github/workflows/wheels.yml @@ -97,7 +97,6 @@ jobs: -DskipTests -Dspotless.check.skip=true -Dspotless.apply.skip=true \ -Dbuild.test=OFF \ -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ -Dcmake.args="-DCMAKE_OSX_DEPLOYMENT_TARGET=12.0" otool -l cpp/target/build/lib/libtsfile*.dylib | grep -A2 LC_VERSION_MIN_MACOSX || true @@ -140,7 +139,6 @@ jobs: ./mvnw -Pwith-cpp clean package \ -DskipTests -Dbuild.test=OFF \ -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ -Dspotless.check.skip=true -Dspotless.apply.skip=true test -d cpp/target/build/lib && test -d cpp/target/build/include @@ -208,7 +206,6 @@ jobs: ./mvnw -Pwith-cpp clean package \ -DskipTests -Dbuild.test=OFF \ -Dtsfile.dependency.source=BUNDLED \ - -Denable.lzma2=ON \ -Dspotless.check.skip=true -Dspotless.apply.skip=true test -d cpp/target/build/lib test -d cpp/target/build/include diff --git a/pom.xml b/pom.xml index c89304eab..524dedc88 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,14 @@ pom Apache TsFile Project Parent POM + + true 17 17 17 @@ -620,6 +628,22 @@ python + + + with-python-only + + python + + + ${maven.multiModuleProjectDirectory}/cpp/target/build + false + + .java-17-and-above diff --git a/python/check_cpp_build.py b/python/check_cpp_build.py new file mode 100755 index 000000000..b8e6f61a8 --- /dev/null +++ b/python/check_cpp_build.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +"""Validate that the C++ build tree required by the Python module exists. + +The Python module links against the native library produced by the ``cpp`` +module. When the two are built together (``-Pwith-python``) the artifacts are +always present. When the Python module is built on its own +(``-Pwith-python-only``) they must already exist, so this check turns a late and +opaque failure inside ``setup.py`` into an early, actionable one. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +def missing_parts(build_root: Path) -> list[Path]: + """Return the required subdirectories of *build_root* that are absent.""" + return [ + part + for part in (build_root / "include", build_root / "lib") + if not part.is_dir() + ] + + +def main(argv: list[str]) -> int: + if len(argv) != 2: + print(f"usage: {Path(argv[0]).name} ", file=sys.stderr) + return 2 + + build_root = Path(argv[1]).expanduser().resolve() + missing = missing_parts(build_root) + if not missing: + return 0 + + print( + f"error: TsFile C++ build not found at {build_root}", + file=sys.stderr, + ) + for part in missing: + print(f" missing directory: {part}", file=sys.stderr) + print( + " Run './mvnw -Pwith-cpp package' first, " + "or set -Dtsfile.cpp.build=.", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv)) diff --git a/python/pom.xml b/python/pom.xml index 8de3d2fbb..ca5777941 100644 --- a/python/pom.xml +++ b/python/pom.xml @@ -29,6 +29,13 @@ TsFile: Python 2.6.0 + + ${project.basedir}/../cpp/target/build tsfile ${project.build.directory}/build-wrapper-output @@ -69,7 +76,36 @@ org.codehaus.mojo exec-maven-plugin + + + + ${tsfile.cpp.build} + + + + + check-cpp-build + validate + + exec + + + ${tsfile.cpp.check.skip} + ${python.exe.bin} + + ${project.basedir}/check_cpp_build.py + ${tsfile.cpp.build} + + + python-venv From 89fbeda713ecc8b96357a9fba6561d0cc793750a Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 16:09:37 +0800 Subject: [PATCH 21/31] build(cpp): expose CPack generation via -Denable.cpack=ON Let the build tree produced by '-Pwith-cpp package' also generate the CPack configuration, so native packages can be produced from that same tree without reconfiguring or rebuilding it. --- cpp/pom.xml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/pom.xml b/cpp/pom.xml index 30c70b858..10035aa5a 100644 --- a/cpp/pom.xml +++ b/cpp/pom.xml @@ -52,6 +52,13 @@ OFF ON ON + + OFF ${project.basedir} @@ -105,6 +112,7 @@ + OFF + + OFF + + ${project.basedir} @@ -113,6 +124,8 @@ + + + + native-package-versions + + + tsfile.archive.version + + + + + + com.googlecode.cmake-maven-project + cmake-maven-plugin + + + cmake-generate-test-compile + + + + + + + + + + + + + + with-code-coverage diff --git a/packaging/README.md b/packaging/README.md index 32b8106d4..f8ca30309 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -32,16 +32,20 @@ API; a separate API cleanup will narrow it in a future version. ## Manual artifact workflow `Build native package artifacts` (`.github/workflows/native-packages.yml`) is -the authoritative native packaging workflow. In the fork's GitHub Actions tab, +the authoritative native packaging workflow. In the Apache TsFile repository's GitHub Actions tab, select this workflow, choose **Run workflow**, select the branch containing the commit to build, and dispatch it manually. It runs only on `workflow_dispatch`, with read-only repository permissions; pushes and pull requests do not trigger native packaging. -A successful run produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and Intel -Homebrew development bottle with a merged Formula, and a Windows x86_64 SDK/CLI -ZIP. It combines these into `tsfile-native-packages-` with -`manifest.json` and `SHA256SUMS`. Download this final artifact from the workflow +A successful run builds the C++ core through Maven, stages one SDK per +release platform, and then produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and +Intel Homebrew development bottle with a merged Formula, a Windows x86_64 +SDK/CLI ZIP, and a Linux Python wheel built from the staged Ubuntu SDK. It also +runs the Go and Python consumers against the SDK rather than rebuilding the C++ +core downstream. The job combines these into +`tsfile-native-packages-` with `manifest.json` and +`SHA256SUMS`. Download this final artifact from the workflow run; both intermediate and final artifacts are retained for 14 days. Publishing is a separate, manual step. This workflow only builds, tests, and @@ -51,39 +55,34 @@ implement RC/final release behavior. ## Portable archive -Build a relocatable binary archive on any host with CMake and CPack: +Build a relocatable SDK and portable archive through the Maven entry point: ```bash -cmake -S cpp -B cpp/build/package \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO -cmake --build cpp/build/package --parallel -cpack --config cpp/build/package/CPackConfig.cmake -G TGZ +./mvnw -Pwith-cpp package \ + -Dbuild.test=OFF \ + -Dtsfile.dependency.source=AUTO \ + -Denable.cpack=ON +cmake --install cpp/target/build --prefix cpp/build/sdk-root +cpack --config cpp/target/build/CPackConfig.cmake -G TGZ -B cpp/build/packages ``` -The resulting `tsfile--.tar.gz` contains a standard -prefix layout and can be unpacked at `/usr/local`, a user directory, or a -relocated application prefix. +The staged `cpp/build/sdk-root` contains a standard prefix layout and can be +unpacked at `/usr/local`, a user directory, or a relocated application prefix. ## Native Linux packages DEB and RPM packages must be built in the target distribution environment so CPack can run the native dependency scanner (`dpkg-shlibdeps` or -`rpmbuild`). On Debian/Ubuntu use `-G DEB`; on Fedora/RHEL use `-G RPM`: +`rpmbuild`). Build the same Maven-driven tree once, then select the native +generator in the same job: ```bash -cmake -S cpp -B cpp/build/package \ - -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TEST=OFF \ - -DBUILD_TOOLS=ON \ - -DTSFILE_ENABLE_CPACK=ON \ - -DTSFILE_DEPENDENCY_SOURCE=AUTO -cmake --build cpp/build/package --parallel -cpack --config cpp/build/package/CPackConfig.cmake -G DEB -# or: cpack --config cpp/build/package/CPackConfig.cmake -G RPM +./mvnw -Pwith-cpp package \ + -Dbuild.test=OFF \ + -Dtsfile.dependency.source=AUTO \ + -Denable.cpack=ON +cpack -G DEB --config cpp/target/build/CPackConfig.cmake -B cpp/build/packages +# or: cpack -G RPM --config cpp/target/build/CPackConfig.cmake -B cpp/build/packages ``` `SYSTEM` can be used instead of `AUTO` when the build image provides every @@ -115,7 +114,7 @@ when the first release containing this packaging work is published. The manual native-package workflow also builds the self-hosted development Formula `tsfile-dev`, using `packaging/homebrew/tsfile-dev.rb.in`. This is not a Homebrew/core submission. Each build pins the full workflow commit from -`ColinLeeo/tsfile`, hashes that source archive, and uses the generated immutable +the Apache TsFile source repository, hashes that source archive, and uses the generated immutable Homebrew development version. The Formula retains the CMake install behavior and tests the installed C++ consumer and CLI before bottling. `tsfile-dev` is keg-only because the current SDK intentionally ships its dependency header @@ -203,8 +202,9 @@ python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM installation test, Homebrew bottle merge, and Windows SDK/CLI build all succeed, the workflow assembles `tsfile-native-packages-`. The final -GitHub Actions artifact retains the DEBs, RPMs, merged Formula and bottles, and -Windows ZIP in their package-family layouts for 14 days. It also contains a +GitHub Actions artifact retains the platform SDKs, the Linux Python wheel, the +DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family +layouts for 14 days. It also contains a sorted `SHA256SUMS` and `manifest.json` with the source identity, generated versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target path, and properties required for later manual publication. @@ -212,5 +212,6 @@ path, and properties required for later manual publication. The final job has no publishing credentials and does not upload to JFrog. A maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, and Homebrew -and Windows files to their immutable `tsfile/homebrew/dev/versions/` -and `tsfile/windows/dev/versions/` paths. +SDKs, and Windows files to their immutable +`tsfile/homebrew/dev/versions/`, +`tsfile/windows/dev/versions/`, and `sdk/dev/versions/` paths. diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py index 51cec3769..963915113 100644 --- a/packaging/scripts/assemble_native_packages.py +++ b/packaging/scripts/assemble_native_packages.py @@ -36,6 +36,12 @@ } REQUIRED_VERSIONS = {"archive_version", "homebrew_version"} REQUIRED_SOURCE = {"commit", "repository"} +SDK_PLATFORMS = { + "ubuntu22.04-amd64", + "almalinux9-x86_64", + "windows-msvc-x86_64", +} +PYTHON_PLATFORMS = {"ubuntu22.04-x86_64"} def _validate_metadata(values: dict[str, str], required: set[str], name: str) -> None: @@ -141,6 +147,34 @@ def _artifact_description( "targetPath": f"homebrew/dev/versions/{versions['homebrew_version']}/{target.as_posix()[len('homebrew/'):]}", "properties": {}, } + if len(parts) == 3 and parts[0] == "sdk" and filename.endswith(".tar.gz"): + platform = parts[1] + if platform not in SDK_PLATFORMS: + raise ValueError(f"unsupported SDK platform: {platform}") + target = Path("sdk") / platform / filename + return target, { + "family": "sdk", + "platform": platform, + "targetRepository": "tsfile", + "targetPath": ( + f"sdk/dev/versions/{versions['archive_version']}/{platform}/{filename}" + ), + "properties": {}, + } + if len(parts) == 3 and parts[0] == "python" and filename.endswith(".whl"): + platform = parts[1] + if platform not in PYTHON_PLATFORMS: + raise ValueError(f"unsupported Python wheel platform: {platform}") + target = Path("python/wheels") / platform / filename + return target, { + "family": "python", + "platform": platform, + "targetRepository": "tsfile-python", + "targetPath": ( + f"python/dev/versions/{versions['archive_version']}/{platform}/{filename}" + ), + "properties": {}, + } if len(parts) >= 2 and parts[0] == "windows" and filename.endswith(".zip"): target = Path("windows") / filename return target, { @@ -163,6 +197,27 @@ def _require_complete_families( raise ValueError("missing RPM package input") if not any(family == "windows" for family in families): raise ValueError("missing Windows ZIP input") + sdk_platforms = { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == "sdk" + } + missing_sdk_platforms = SDK_PLATFORMS - sdk_platforms + if missing_sdk_platforms: + raise ValueError( + "missing SDK inputs for: " + ", ".join(sorted(missing_sdk_platforms)) + ) + python_platforms = { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == "python" + } + missing_python_platforms = PYTHON_PLATFORMS - python_platforms + if missing_python_platforms: + raise ValueError( + "missing Python wheel inputs for: " + + ", ".join(sorted(missing_python_platforms)) + ) homebrew_paths = { target.as_posix() for _, target, metadata in planned diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 828c6630a..506d8a593 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -6,7 +6,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# https://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -26,22 +26,51 @@ raise "Workflow permissions must remain read-only" unless workflow["permissions"] == { "contents" => "read" } jobs = workflow.fetch("jobs") -bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } -packages = Shellwords.split(bootstrap.fetch("run").gsub("\\\n", " ")) + +def run_text(job) + job.fetch("steps").filter_map { |step| step["run"] }.join("\n") +end + +%w[build-deb build-rpm build-windows].each do |name| + text = run_text(jobs.fetch(name)) + raise "#{name} must build C++ through Maven" unless text.include?("-Pwith-cpp package") + raise "#{name} must enable CPack through Maven" unless text.include?("-Denable.cpack=ON") + raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") + raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") + raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") + raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") +end + +rpm_bootstrap = jobs.fetch("build-rpm").fetch("steps").find { |step| step["name"] == "Install RPM build prerequisites" } +packages = Shellwords.split(rpm_bootstrap.fetch("run").gsub("\\\n", " ")) raise "AlmaLinux 9 curl-minimal conflicts with full curl" if packages.include?("curl") raise "RPM bootstrap must retain curl-minimal" unless packages.include?("curl-minimal") raise "AlmaLinux 9 default repositories do not provide ninja-build" if packages.include?("ninja-build") raise "RPM bootstrap must install make" unless packages.include?("make") +raise "RPM bootstrap must install Java" unless packages.include?("java-17-openjdk-devel") -rpm_build = jobs.fetch("build-rpm").fetch("steps").filter_map { |step| step["run"] }.join("\n") -raise "RPM build must not require Ninja" if rpm_build.include?("-G Ninja") - -windows = jobs.fetch("build-windows").fetch("steps").filter_map { |step| step["run"] }.join("\n") -raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-DTSFILE_MSVC_STATIC_RUNTIME=ON") && windows.include?("-DTSFILE_DEPENDENCY_SOURCE=BUNDLED") -raise "All native target runtimes must be checked" unless windows.include?("CheckStaticMSVCRuntime.cmake") +windows = run_text(jobs.fetch("build-windows")) +raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-Dtsfile.msvc.static.runtime=ON") && windows.include?("-Dtsfile.dependency.source=BUNDLED") +raise "Windows runtime check must be wired through Maven" unless windows.include?("CheckStaticMSVCRuntime.cmake") && windows.include?("-Dtsfile.project.include") raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 -homebrew_merge = jobs.fetch("merge-homebrew").fetch("steps").filter_map { |step| step["run"] }.join("\n") +go_job = jobs.fetch("test-go-linux") +raise "Go test must consume the Ubuntu SDK" unless go_job.fetch("needs") == "build-deb" +go_text = run_text(go_job) +raise "Go test must download the SDK artifact" unless go_job.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-ubuntu22.04-amd64" } +raise "Go test must run go test" unless go_text.include?("go test ./...") + +python_job = jobs.fetch("build-python-linux") +raise "Python wheel must consume the Ubuntu SDK" unless python_job.fetch("needs") == "build-deb" +python_text = run_text(python_job) +raise "Python wheel must use with-python-only" unless python_text.include?("-Pwith-python-only package") +raise "Python wheel must point at the downloaded SDK" unless python_text.include?("-Dtsfile.cpp.build") +raise "Python wheel must freeze source versions" unless python_text.include?("-Dtsfile.version.sync.skip=true") + +homebrew = jobs.fetch("build-homebrew") +raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" + +homebrew_merge = run_text(jobs.fetch("merge-homebrew")) trust = homebrew_merge.index("brew trust apache/tsfile-dev") merge = homebrew_merge.index("brew bottle --merge") raise "Homebrew merge must trust its temporary tap before loading the Formula" unless trust && merge && trust < merge @@ -53,6 +82,18 @@ raise "#{name} must use the installed CMake consumer" unless runs.include?("-S cpp/cmake/tests/projects/InstalledConsumer") end +assemble = jobs.fetch("assemble") +needs = Array(assemble.fetch("needs")) +%w[test-deb test-rpm test-go-linux build-python-linux merge-homebrew build-windows].each do |name| + raise "assemble must depend on #{name}" unless needs.include?(name) +end +assemble_text = run_text(assemble) +%w[native-sdk-ubuntu22.04-amd64 native-sdk-almalinux9-x86_64 native-sdk-windows-msvc-x86_64 native-python-wheel-ubuntu22.04-x86_64].each do |name| + raise "assemble must download #{name}" unless assemble.fetch("steps").any? { |step| step["with"].to_h["name"] == name } +end +raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle") +raise "workflow must not hard-code a fork repository" if File.read(path).include?("ColinLeeo/tsfile") + jobs.each_value do |job| job.fetch("steps").each do |step| next unless step["run"] && step.fetch("shell", "bash") == "bash" @@ -60,4 +101,4 @@ raise "Invalid shell in #{step['name']}: #{output}" unless status.success? end end -puts "PASS: manual/read-only workflow, AlmaLinux bootstrap, Windows CRT checks, shared consumers, and Bash syntax" +puts "PASS: manual/read-only Maven build hierarchy, artifact consumers, workflow contracts, and shell syntax" diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py index 54da32c66..034a6386c 100644 --- a/packaging/tests/test_assemble_native_packages.py +++ b/packaging/tests/test_assemble_native_packages.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import hashlib import importlib.util import json import os @@ -33,7 +34,7 @@ } SOURCE = { "commit": "abcdef1234567890abcdef1234567890abcdef12", - "repository": "ColinLeeo/tsfile", + "repository": "apache/tsfile", } @@ -69,6 +70,10 @@ def write_fixture(self, directory): "homebrew/bottles/tsfile-dev.bottle.tar.gz": b"bottle artifact\n", "homebrew/bottles/tsfile-dev.bottle.json": b"bottle metadata\n", "windows/tsfile-2.5.0-dev-windows-x86_64.zip": b"windows artifact\n", + "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk ubuntu\n", + "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk almalinux\n", + "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk windows\n", + "python/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl": b"python wheel\n", } for relative_path, contents in files.items(): path = directory / relative_path @@ -93,7 +98,11 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): "homebrew/Formula/tsfile-dev.rb", "homebrew/bottles/tsfile-dev.bottle.json", "homebrew/bottles/tsfile-dev.bottle.tar.gz", + "python/wheels/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl", "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "windows/tsfile-2.5.0-dev-windows-x86_64.zip", ] self.assertEqual( @@ -105,95 +114,37 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): ), expected_paths, ) - self.assertEqual( - (output_directory / "SHA256SUMS").read_text(encoding="utf-8"), - "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137 deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb\n" - "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224 homebrew/Formula/tsfile-dev.rb\n" - "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967 homebrew/bottles/tsfile-dev.bottle.json\n" - "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82 homebrew/bottles/tsfile-dev.bottle.tar.gz\n" - "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm\n" - "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f windows/tsfile-2.5.0-dev-windows-x86_64.zip\n", - ) self.assertEqual( json.loads((output_directory / "manifest.json").read_text()), manifest ) self.assertEqual(manifest["source"], SOURCE) self.assertEqual(manifest["versions"], VERSIONS) self.assertEqual( - manifest["artifacts"], - [ - { - "family": "deb", - "filename": "tsfile_2.5.0-dev_amd64.deb", - "path": "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", - "platform": "ubuntu22.04-amd64", - "properties": { - "deb.architecture": ["amd64"], - "deb.component": ["dev"], - "deb.distribution": ["jammy", "noble"], - }, - "sha256": "ef5ca1431457b6dec117aff4aafcfd14edec3fe628d3fefe334867e502b8c137", - "size": 13, - "targetPath": "pool/dev/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", - "targetRepository": "tsfile-debian", - }, - { - "family": "homebrew", - "filename": "tsfile-dev.rb", - "path": "homebrew/Formula/tsfile-dev.rb", - "platform": "homebrew", - "properties": {}, - "sha256": "b045d33311f553503342e1b061df60161e414503d68dc14af90537f254dfc224", - "size": 17, - "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/Formula/tsfile-dev.rb", - "targetRepository": "tsfile", - }, - { - "family": "homebrew", - "filename": "tsfile-dev.bottle.json", - "path": "homebrew/bottles/tsfile-dev.bottle.json", - "platform": "homebrew", - "properties": {}, - "sha256": "ae98b99fb84c92fb10b00d75e0c51035006067659043b9dc0cb08d9902633967", - "size": 16, - "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.json", - "targetRepository": "tsfile", - }, - { - "family": "homebrew", - "filename": "tsfile-dev.bottle.tar.gz", - "path": "homebrew/bottles/tsfile-dev.bottle.tar.gz", - "platform": "homebrew", - "properties": {}, - "sha256": "95f3ddb71b1a0c5c95c5aa0352321075cc465d6a583dff8bf601e888abec4b82", - "size": 16, - "targetPath": "homebrew/dev/versions/2.5.0.dev0.20260910.123.1.gabcdef1/bottles/tsfile-dev.bottle.tar.gz", - "targetRepository": "tsfile", - }, - { - "family": "rpm", - "filename": "tsfile-2.5.0-dev.x86_64.rpm", - "path": "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", - "platform": "almalinux9-x86_64", - "properties": {}, - "sha256": "f557fbfdaf15a34e59adaf4fa487062419dd8fe39eb057f6b581b4469cd3a25f", - "size": 13, - "targetPath": "dev/el9/x86_64/tsfile-2.5.0-dev.x86_64.rpm", - "targetRepository": "tsfile-rpm", - }, - { - "family": "windows", - "filename": "tsfile-2.5.0-dev-windows-x86_64.zip", - "path": "windows/tsfile-2.5.0-dev-windows-x86_64.zip", - "platform": "windows-msvc-x86_64", - "properties": {}, - "sha256": "f59fc30fd2aaa4a4594eb006fce33041e2702b621abf936af5619b3eabf23c8f", - "size": 17, - "targetPath": "windows/dev/versions/2.5.0-dev0.20260910.123.1.gabcdef1/tsfile-2.5.0-dev-windows-x86_64.zip", - "targetRepository": "tsfile", - }, - ], + {artifact["family"] for artifact in manifest["artifacts"]}, + {"deb", "homebrew", "python", "rpm", "sdk", "windows"}, ) + self.assertEqual( + {artifact["platform"] for artifact in manifest["artifacts"]}, + { + "almalinux9-x86_64", + "homebrew", + "ubuntu22.04-amd64", + "ubuntu22.04-x86_64", + "windows-msvc-x86_64", + }, + ) + for artifact in manifest["artifacts"]: + output_path = output_directory / artifact["path"] + input_path = next( + path + for path in input_directory.rglob(artifact["filename"]) + if path.is_file() + ) + self.assertEqual(artifact["size"], input_path.stat().st_size) + self.assertEqual( + artifact["sha256"], + hashlib.sha256(input_path.read_bytes()).hexdigest(), + ) def test_verifies_assembler_checksum_lines_sorted_by_path(self): module = self.require_module() diff --git a/packaging/tests/test_render_homebrew_formula.py b/packaging/tests/test_render_homebrew_formula.py index a60f9a6c0..f3e500369 100644 --- a/packaging/tests/test_render_homebrew_formula.py +++ b/packaging/tests/test_render_homebrew_formula.py @@ -27,7 +27,7 @@ MODULE_PATH = ROOT / "packaging/scripts/render_homebrew_formula.py" TEMPLATE_PATH = ROOT / "packaging/homebrew/tsfile-dev.rb.in" VALUES = { - "SOURCE_REPOSITORY": "ColinLeeo/tsfile", + "SOURCE_REPOSITORY": "apache/tsfile", "GIT_SHA": "abcdef1234567890abcdef1234567890abcdef12", "HOMEBREW_VERSION": "2.5.0.dev0.20260910.123.1.gabcdef1", "SOURCE_SHA256": "0123456789abcdef" * 4, @@ -66,7 +66,7 @@ def test_renders_exact_immutable_source_and_both_bottle_tags(self): self.assertEqual( rendered, """class TsfileDev < Formula - url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz" + url "https://github.com/apache/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz" version "2.5.0.dev0.20260910.123.1.gabcdef1" sha256 "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" """ @@ -115,7 +115,7 @@ def test_rejects_empty_or_non_immutable_source_identity(self): ("GIT_SHA", ""), ("GIT_SHA", "abcdef1"), ("GIT_SHA", "develop"), - ("SOURCE_REPOSITORY", 'ColinLeeo/tsfile"'), + ("SOURCE_REPOSITORY", 'apache/tsfile"'), ("HOMEBREW_VERSION", 'bad"\nversion "injected'), ): with self.subTest(key=key, value=value): @@ -162,7 +162,7 @@ def test_cli_renders_real_template_with_optional_bottle_file(self): rendered = output.read_text(encoding="utf-8") self.assertIn("class TsfileDev < Formula", rendered) self.assertIn( - 'url "https://github.com/ColinLeeo/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz"', + 'url "https://github.com/apache/tsfile/archive/abcdef1234567890abcdef1234567890abcdef12.tar.gz"', rendered, ) self.assertIn('version "2.5.0.dev0.20260910.123.1.gabcdef1"', rendered) diff --git a/python/VersionUpdater.groovy b/python/VersionUpdater.groovy index a586fe936..fa2c5c970 100644 --- a/python/VersionUpdater.groovy +++ b/python/VersionUpdater.groovy @@ -19,6 +19,17 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Synchronize the version in setup.py and the one used in the maven pom. + +// Native package jobs derive their identity from native_package_versions.py. +// Keep that identity stable by leaving source versions untouched in those jobs. +def skipVersionSync = project.properties.getProperty("tsfile.version.sync.skip") +if (skipVersionSync == null) { + skipVersionSync = System.getProperty("tsfile.version.sync.skip") +} +if (skipVersionSync != null && skipVersionSync.equalsIgnoreCase("true")) { + println "Skipping source version synchronization for native packaging" + return +} //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// def currentMavenVersion = project.version as String From c4405fda163900c48d086def9aae85234f8af667 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 16:28:01 +0800 Subject: [PATCH 23/31] docs(packaging): describe SDK and wheel consumers --- packaging/README.md | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packaging/README.md b/packaging/README.md index f8ca30309..a7bf7092c 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -200,8 +200,9 @@ python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v ## Final native package bundle Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM -installation test, Homebrew bottle merge, and Windows SDK/CLI build all succeed, -the workflow assembles `tsfile-native-packages-`. The final +installation test, Go SDK tests, Linux Python wheel smoke test, Homebrew bottle +merge, and Windows SDK/CLI build all succeed, the workflow assembles +`tsfile-native-packages-`. The final GitHub Actions artifact retains the platform SDKs, the Linux Python wheel, the DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family layouts for 14 days. It also contains a @@ -211,7 +212,8 @@ path, and properties required for later manual publication. The final job has no publishing credentials and does not upload to JFrog. A maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the -recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, and Homebrew -SDKs, and Windows files to their immutable -`tsfile/homebrew/dev/versions/`, -`tsfile/windows/dev/versions/`, and `sdk/dev/versions/` paths. +recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, Homebrew +formula/bottles to `tsfile/homebrew/dev/versions/`, SDKs to +`tsfile/sdk/dev/versions/`, Windows ZIPs to +`tsfile/windows/dev/versions/`, and Python wheels to the +`tsfile-python` repository recorded in the manifest. From 1583130e83f79bc8feac86aea3a914bc01b38f92 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 17:11:30 +0800 Subject: [PATCH 24/31] fix(ci): make native SDK consumers path-safe --- .github/workflows/native-packages.yml | 24 ++++++++++++------------ packaging/tests/check_native_workflow.rb | 2 ++ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 64e98bd5f..0633c4ffc 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -120,7 +120,7 @@ jobs: shell: bash run: | sdk_name="tsfile-sdk-ubuntu22.04-amd64-$ARCHIVE_VERSION" - sdk_root="sdk/$sdk_name" + sdk_root="$PWD/sdk/$sdk_name" cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" @@ -285,7 +285,7 @@ jobs: shell: bash run: | sdk_name="tsfile-sdk-almalinux9-x86_64-$ARCHIVE_VERSION" - sdk_root="sdk/$sdk_name" + sdk_root="$PWD/sdk/$sdk_name" cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" @@ -426,16 +426,16 @@ jobs: $PSNativeCommandUseErrorActionPreference = $true $projectInclude = (Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path .\mvnw.cmd -Pwith-cpp package ` - -Dcpp.toolchain=msvc ` - -Dbuild.type=Release ` - -Dbuild.test=OFF ` - -DskipTests ` - -Dspotless.skip=true ` - -Dtsfile.dependency.source=BUNDLED ` - -Dtsfile.msvc.static.runtime=ON ` + "-Dcpp.toolchain=msvc" ` + "-Dbuild.type=Release" ` + "-Dbuild.test=OFF" ` + "-DskipTests" ` + "-Dspotless.skip=true" ` + "-Dtsfile.dependency.source=BUNDLED" ` + "-Dtsfile.msvc.static.runtime=ON" ` "-Dtsfile.project.include=$projectInclude" ` - -Dtsfile.version.sync.skip=true ` - -Denable.cpack=ON ` + "-Dtsfile.version.sync.skip=true" ` + "-Denable.cpack=ON" ` "-Dtsfile.archive.version=$env:ARCHIVE_VERSION" ` "-Dtsfile.debian.package.version=$env:DEBIAN_VERSION" ` "-Dtsfile.rpm.package.version=$env:RPM_VERSION" ` @@ -452,7 +452,7 @@ jobs: $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true $sdkName = "tsfile-sdk-windows-msvc-x86_64-$env:ARCHIVE_VERSION" - $sdkRoot = "sdk/$sdkName" + $sdkRoot = Join-Path $PWD "sdk/$sdkName" cmake --install cpp/target/build --config Release --prefix $sdkRoot $requiredPaths = @( "$sdkRoot/bin/tsfile-cli.exe" diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 506d8a593..f6e2996fb 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -37,6 +37,7 @@ def run_text(job) raise "#{name} must enable CPack through Maven" unless text.include?("-Denable.cpack=ON") raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") + raise "#{name} must use an absolute SDK prefix" unless text.include?("$PWD/sdk/") || text.include?('Join-Path $PWD "sdk/') raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") end @@ -52,6 +53,7 @@ def run_text(job) windows = run_text(jobs.fetch("build-windows")) raise "Windows ZIP needs a consistent static CRT" unless windows.include?("-Dtsfile.msvc.static.runtime=ON") && windows.include?("-Dtsfile.dependency.source=BUNDLED") raise "Windows runtime check must be wired through Maven" unless windows.include?("CheckStaticMSVCRuntime.cmake") && windows.include?("-Dtsfile.project.include") +raise "Windows Maven properties must be quoted for PowerShell" unless windows.include?('"-Dcpp.toolchain=msvc"') && windows.include?('"-Dbuild.type=Release"') raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 go_job = jobs.fetch("test-go-linux") From a46bf8148c08be59183565417d4c621bbcd16de8 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 17:30:21 +0800 Subject: [PATCH 25/31] fix(ci): consume platform SDK layouts correctly --- .github/workflows/native-packages.yml | 16 ++++++++++------ packaging/tests/check_native_workflow.rb | 5 +++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 0633c4ffc..05c96f541 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -124,9 +124,11 @@ jobs: cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" - test -d "$sdk_root/lib" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" "$sdk_root/bin/tsfile-cli" --version - PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/deb \ -DCMAKE_PREFIX_PATH="$sdk_root" cmake --build consumer/deb --parallel @@ -289,9 +291,11 @@ jobs: cmake --install cpp/target/build --prefix "$sdk_root" test -f "$sdk_root/bin/tsfile-cli" test -d "$sdk_root/include" - test -d "$sdk_root/lib" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" "$sdk_root/bin/tsfile-cli" --version - PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/rpm \ -DCMAKE_PREFIX_PATH="$sdk_root" cmake --build consumer/rpm --parallel @@ -579,7 +583,7 @@ jobs: mkdir -p sdk-download sdk cpp/target/build sdk_tar="$(find sdk-download -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" test -n "$sdk_tar" - sdk_dir="$(tar -tzf "$sdk_tar" | head -n 1 | cut -d/ -f1)" + sdk_dir="$(tar -tzf "$sdk_tar" | sed -n '1s#/.*##p')" tar -xzf "$sdk_tar" -C sdk cp -a "sdk/$sdk_dir/." cpp/target/build/ test -f cpp/target/build/include/tsfile/cwrapper/tsfile_cwrapper.h @@ -624,7 +628,7 @@ jobs: run: | sdk_tar="$(find sdk-download -maxdepth 1 -type f -name '*.tar.gz' -print -quit)" test -n "$sdk_tar" - sdk_dir="$(tar -tzf "$sdk_tar" | head -n 1 | cut -d/ -f1)" + sdk_dir="$(tar -tzf "$sdk_tar" | sed -n '1s#/.*##p')" mkdir -p sdk tar -xzf "$sdk_tar" -C sdk sdk_root="$PWD/sdk/$sdk_dir" diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index f6e2996fb..81dcb1efe 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -38,6 +38,9 @@ def run_text(job) raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") raise "#{name} must use an absolute SDK prefix" unless text.include?("$PWD/sdk/") || text.include?('Join-Path $PWD "sdk/') + if name != "build-windows" + raise "#{name} must locate pkg-config without assuming lib" unless text.include?('dirname "$sdk_pkgconfig"') + end raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") raise "#{name} must freeze source versions" unless text.include?("-Dtsfile.version.sync.skip=true") end @@ -61,6 +64,7 @@ def run_text(job) go_text = run_text(go_job) raise "Go test must download the SDK artifact" unless go_job.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-ubuntu22.04-amd64" } raise "Go test must run go test" unless go_text.include?("go test ./...") +raise "Go SDK extraction must not fail on SIGPIPE" if go_text.include?("tar -tzf") && go_text.include?("head -n 1") python_job = jobs.fetch("build-python-linux") raise "Python wheel must consume the Ubuntu SDK" unless python_job.fetch("needs") == "build-deb" @@ -68,6 +72,7 @@ def run_text(job) raise "Python wheel must use with-python-only" unless python_text.include?("-Pwith-python-only package") raise "Python wheel must point at the downloaded SDK" unless python_text.include?("-Dtsfile.cpp.build") raise "Python wheel must freeze source versions" unless python_text.include?("-Dtsfile.version.sync.skip=true") +raise "Python SDK extraction must not fail on SIGPIPE" if python_text.include?("tar -tzf") && python_text.include?("head -n 1") homebrew = jobs.fetch("build-homebrew") raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" From 2fb44640216d8fce81a245f47553d2759712b475 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 17:50:12 +0800 Subject: [PATCH 26/31] fix(python): resolve staged SDK headers --- python/setup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/setup.py b/python/setup.py index fe19fc91c..83d0b0b8b 100644 --- a/python/setup.py +++ b/python/setup.py @@ -207,6 +207,9 @@ def finalize_options(self): libraries = [] library_dirs = [str(PKG)] include_dirs = [str(PKG), np.get_include(), str(PKG / "include")] +sdk_include = PKG / "include" / "tsfile" +if sdk_include.is_dir(): + include_dirs.append(str(sdk_include)) if sys.platform.startswith("linux"): libraries = ["tsfile"] From dafad98e1a58ee0715c9fdb5f83ec5ff83f72999 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Fri, 18 Sep 2026 18:11:02 +0800 Subject: [PATCH 27/31] fix(python): ship versioned shared libraries --- python/setup.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/setup.py b/python/setup.py index 83d0b0b8b..a6a26fa42 100644 --- a/python/setup.py +++ b/python/setup.py @@ -92,10 +92,9 @@ def _find_lib(root, patterns): if not candidates: raise FileNotFoundError("missing libtsfile.so* in build output") src = candidates[0] - dst = PKG / src.name - shutil.copy2(src, dst) - link_name = PKG / "libtsfile.so" - shutil.copy2(src, link_name) + for candidate in candidates: + shutil.copy2(candidate, PKG / candidate.name) + shutil.copy2(src, PKG / "libtsfile.so") elif sys.platform == "darwin": candidates = sorted(CPP_LIB.rglob("libtsfile.*.dylib")) or list( From 6fa9bc583e249ab53d839085ed5f51e21ca7face Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 21 Sep 2026 13:45:44 +0800 Subject: [PATCH 28/31] feat(ci): add macOS SDK artifacts --- .github/workflows/native-packages.yml | 41 +++++++++++++++++++ packaging/README.md | 14 ++++--- packaging/scripts/assemble_native_packages.py | 2 + packaging/tests/check_native_workflow.rb | 6 ++- .../tests/test_assemble_native_packages.py | 6 +++ 5 files changed, 63 insertions(+), 6 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 05c96f541..3006fb08a 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -672,6 +672,7 @@ jobs: env: HOMEBREW_NO_AUTO_UPDATE: "1" HOMEBREW_NO_INSTALL_CLEANUP: "1" + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} TSFILE_HOMEBREW_VERSION: ${{ needs.prepare.outputs.homebrew_version }} SOURCE_REPOSITORY: ${{ github.repository }} SOURCE_GIT_SHA: ${{ github.sha }} @@ -712,6 +713,26 @@ jobs: metadata=(*.bottle.json) [[ ${#bottles[@]} -eq 1 && ${#metadata[@]} -eq 1 ]] + - name: Stage and verify the macOS SDK + shell: bash + run: | + sdk_name="tsfile-sdk-${{ matrix.name }}-$ARCHIVE_VERSION" + sdk_root="$(brew --prefix apache/tsfile-dev/tsfile-dev)" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile*.dylib' -print -quit)" + test -x "$sdk_root/bin/tsfile-cli" + test -f "$sdk_root/lib/cmake/TsFile/TsFileConfig.cmake" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/macos \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/macos --parallel + ./consumer/macos/tsfile_installed_consumer + mkdir -p sdk + tar -C "$(dirname "$sdk_root")" -czf "sdk/$sdk_name.tar.gz" "$(basename "$sdk_root")" + - name: Upload platform Homebrew bottle and source Formula uses: actions/upload-artifact@v7 with: @@ -720,6 +741,14 @@ jobs: if-no-files-found: error retention-days: 14 + - name: Upload macOS SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-macos-${{ matrix.name }} + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + merge-homebrew: name: Merge Homebrew development bottles needs: [prepare, build-homebrew] @@ -903,6 +932,18 @@ jobs: name: native-sdk-windows-msvc-x86_64 path: input/sdk/windows-msvc-x86_64 + - name: Download macOS arm64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-macos-arm64 + path: input/sdk/macos-arm64 + + - name: Download macOS x86_64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-macos-x86_64 + path: input/sdk/macos-x86_64 + - name: Download Linux Python wheel uses: actions/download-artifact@v8 with: diff --git a/packaging/README.md b/packaging/README.md index a7bf7092c..21cd02771 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -125,7 +125,11 @@ CLI as `$(brew --prefix tsfile-dev)/bin/tsfile-cli`, or add that Formula's The ARM64 job runs on `macos-latest` and the Intel job on `macos-15-intel`. Their `native-homebrew-macos-arm64` and `native-homebrew-macos-x86_64` -intermediate artifacts remain separate until merge. The merge job checks that +intermediate artifacts remain separate until merge. Each job also archives its +installed Formula prefix as a standalone SDK artifact, +`native-sdk-macos-arm64` or `native-sdk-macos-x86_64`, so macOS C++ consumers +do not have to depend on Homebrew. The SDK archive is checked with the CLI, +pkg-config, and an installed CMake consumer before upload. The merge job checks that both source Formula files match, merges both platform JSON files with Homebrew, and checks both generated tags and checksums in the resulting Formula. @@ -200,11 +204,11 @@ python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v ## Final native package bundle Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM -installation test, Go SDK tests, Linux Python wheel smoke test, Homebrew bottle -merge, and Windows SDK/CLI build all succeed, the workflow assembles +installation test, Go SDK tests, Linux Python wheel smoke test, macOS SDK and +Homebrew bottle merge, and Windows SDK/CLI build all succeed, the workflow assembles `tsfile-native-packages-`. The final -GitHub Actions artifact retains the platform SDKs, the Linux Python wheel, the -DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family +GitHub Actions artifact retains the platform SDKs, including both macOS archives, +the Linux Python wheel, the DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family layouts for 14 days. It also contains a sorted `SHA256SUMS` and `manifest.json` with the source identity, generated versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py index 963915113..6fb6abf84 100644 --- a/packaging/scripts/assemble_native_packages.py +++ b/packaging/scripts/assemble_native_packages.py @@ -39,6 +39,8 @@ SDK_PLATFORMS = { "ubuntu22.04-amd64", "almalinux9-x86_64", + "macos-arm64", + "macos-x86_64", "windows-msvc-x86_64", } PYTHON_PLATFORMS = {"ubuntu22.04-x86_64"} diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 81dcb1efe..7bfe1951d 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -76,6 +76,10 @@ def run_text(job) homebrew = jobs.fetch("build-homebrew") raise "Homebrew must use the current repository" unless homebrew.fetch("env").fetch("SOURCE_REPOSITORY") == "${{ github.repository }}" +homebrew_text = run_text(homebrew) +raise "Homebrew job must stage a macOS SDK from the installed formula" unless homebrew_text.include?("brew --prefix apache/tsfile-dev/tsfile-dev") +raise "Homebrew job must archive the staged macOS SDK" unless homebrew_text.include?("tsfile-sdk-${{ matrix.name }}-$ARCHIVE_VERSION") +raise "Homebrew job must upload a macOS SDK artifact" unless homebrew.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-macos-${{ matrix.name }}" } homebrew_merge = run_text(jobs.fetch("merge-homebrew")) trust = homebrew_merge.index("brew trust apache/tsfile-dev") @@ -95,7 +99,7 @@ def run_text(job) raise "assemble must depend on #{name}" unless needs.include?(name) end assemble_text = run_text(assemble) -%w[native-sdk-ubuntu22.04-amd64 native-sdk-almalinux9-x86_64 native-sdk-windows-msvc-x86_64 native-python-wheel-ubuntu22.04-x86_64].each do |name| +%w[native-sdk-ubuntu22.04-amd64 native-sdk-almalinux9-x86_64 native-sdk-windows-msvc-x86_64 native-sdk-macos-arm64 native-sdk-macos-x86_64 native-python-wheel-ubuntu22.04-x86_64].each do |name| raise "assemble must download #{name}" unless assemble.fetch("steps").any? { |step| step["with"].to_h["name"] == name } end raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle") diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py index 034a6386c..9c3475a1b 100644 --- a/packaging/tests/test_assemble_native_packages.py +++ b/packaging/tests/test_assemble_native_packages.py @@ -72,6 +72,8 @@ def write_fixture(self, directory): "windows/tsfile-2.5.0-dev-windows-x86_64.zip": b"windows artifact\n", "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk ubuntu\n", "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk almalinux\n", + "sdk/macos-arm64/tsfile-sdk-macos-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk macos arm64\n", + "sdk/macos-x86_64/tsfile-sdk-macos-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk macos x86_64\n", "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk windows\n", "python/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl": b"python wheel\n", } @@ -101,6 +103,8 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): "python/wheels/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl", "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/macos-arm64/tsfile-sdk-macos-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/macos-x86_64/tsfile-sdk-macos-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "windows/tsfile-2.5.0-dev-windows-x86_64.zip", @@ -128,6 +132,8 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): { "almalinux9-x86_64", "homebrew", + "macos-arm64", + "macos-x86_64", "ubuntu22.04-amd64", "ubuntu22.04-x86_64", "windows-msvc-x86_64", From e27a2d7f6f8abf8e3439b4af837c994eaedee358 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 21 Sep 2026 17:18:52 +0800 Subject: [PATCH 29/31] fix(ci): stage and verify macOS SDK outside the Homebrew keg --- .github/workflows/native-packages.yml | 41 ++++++++++++++++++------ packaging/tests/check_native_workflow.rb | 5 +++ 2 files changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 3006fb08a..a5057bf5b 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -716,22 +716,43 @@ jobs: - name: Stage and verify the macOS SDK shell: bash run: | + # The runner's /bin/bash is 3.2, where "set -u" rejects empty arrays. + set -eo pipefail + fail() { printf '::error::%s\n' "$1"; exit 1; } sdk_name="tsfile-sdk-${{ matrix.name }}-$ARCHIVE_VERSION" - sdk_root="$(brew --prefix apache/tsfile-dev/tsfile-dev)" - test -d "$sdk_root/include" - test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile*.dylib' -print -quit)" - test -x "$sdk_root/bin/tsfile-cli" - test -f "$sdk_root/lib/cmake/TsFile/TsFileConfig.cmake" - sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" - test -n "$sdk_pkgconfig" + # Homebrew may print advisory lines before the path, so keep the last line. + formula_prefix="$(brew --prefix apache/tsfile-dev/tsfile-dev | tail -n 1)" + [ -n "$formula_prefix" ] || fail "brew --prefix returned no formula path" + printf 'formula prefix: <%s>\n' "$formula_prefix" + keg_root="$(cd "$formula_prefix" && pwd -P)" + printf 'resolved keg: <%s>\n' "$keg_root" + command -v pkg-config > /dev/null 2>&1 || brew install pkgconf + command -v pkg-config > /dev/null 2>&1 || fail "pkg-config is unavailable on the runner" + shopt -s nullglob + dylibs=("$keg_root"/lib/libtsfile*.dylib) + pkgconfigs=("$keg_root"/lib/pkgconfig/libtsfile.pc) + [ -d "$keg_root/include" ] || fail "missing include/ under $keg_root" + [ ${#dylibs[@]} -gt 0 ] || fail "missing libtsfile*.dylib under $keg_root/lib" + [ -x "$keg_root/bin/tsfile-cli" ] || fail "missing bin/tsfile-cli under $keg_root" + [ -f "$keg_root/lib/cmake/TsFile/TsFileConfig.cmake" ] || + fail "missing TsFileConfig.cmake under $keg_root/lib/cmake/TsFile" + [ ${#pkgconfigs[@]} -gt 0 ] || fail "missing libtsfile.pc under $keg_root/lib/pkgconfig" + + # Stage a real tree: the keg lives at a symlinked opt path and must not + # be archived as a single link. + sdk_root="$PWD/sdk/$sdk_name" + rm -rf "$sdk_root" + mkdir -p "$sdk_root" + ditto "$keg_root" "$sdk_root" + "$sdk_root/bin/tsfile-cli" --version - PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/macos \ -DCMAKE_PREFIX_PATH="$sdk_root" cmake --build consumer/macos --parallel ./consumer/macos/tsfile_installed_consumer - mkdir -p sdk - tar -C "$(dirname "$sdk_root")" -czf "sdk/$sdk_name.tar.gz" "$(basename "$sdk_root")" + tar -C sdk -czf "sdk/$sdk_name.tar.gz" "$sdk_name" + rm -rf "$sdk_root" - name: Upload platform Homebrew bottle and source Formula uses: actions/upload-artifact@v7 diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 7bfe1951d..c41223fda 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -79,6 +79,11 @@ def run_text(job) homebrew_text = run_text(homebrew) raise "Homebrew job must stage a macOS SDK from the installed formula" unless homebrew_text.include?("brew --prefix apache/tsfile-dev/tsfile-dev") raise "Homebrew job must archive the staged macOS SDK" unless homebrew_text.include?("tsfile-sdk-${{ matrix.name }}-$ARCHIVE_VERSION") +raise "Homebrew SDK must be verified from its own staged tree" unless homebrew_text.include?('sdk_root="$PWD/sdk/$sdk_name"') +raise "Homebrew SDK must be archived from the staging root" unless homebrew_text.include?('tar -C sdk -czf "sdk/$sdk_name.tar.gz" "$sdk_name"') +raise "Homebrew prefix capture must tolerate extra stdout lines" unless homebrew_text.include?("brew --prefix apache/tsfile-dev/tsfile-dev | tail -n 1") +raise "Homebrew SDK verification must provide pkg-config" unless homebrew_text.include?("brew install pkgconf") +raise "Homebrew SDK step must keep bash 3.2 compatibility" if homebrew_text.include?("set -euo pipefail") raise "Homebrew job must upload a macOS SDK artifact" unless homebrew.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-macos-${{ matrix.name }}" } homebrew_merge = run_text(jobs.fetch("merge-homebrew")) From d693f560b952597337c2485b1abbe8f7e79a1bb0 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Mon, 21 Sep 2026 17:48:07 +0800 Subject: [PATCH 30/31] fix(ci): correct macOS SDK artifact names and relocatable dylib identity --- .github/workflows/native-packages.yml | 20 +++++++++++++++++++- packaging/tests/check_native_workflow.rb | 7 ++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index a5057bf5b..67ff7c71e 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -745,6 +745,24 @@ jobs: mkdir -p "$sdk_root" ditto "$keg_root" "$sdk_root" + # Drop Homebrew keg bookkeeping so the archive holds the same SDK + # layout as the Linux and Windows archives. + rm -rf "$sdk_root/.brew" "$sdk_root/INSTALL_RECEIPT.json" "$sdk_root/sbom.spdx.json" + + # A keg records its own absolute path as the dylib identity, which + # would pin every downstream consumer to this runner's keg. + real_dylib="" + for candidate in "$sdk_root"/lib/libtsfile*.dylib; do + [ -L "$candidate" ] || real_dylib="$candidate" + done + [ -n "$real_dylib" ] || fail "missing real libtsfile dylib under $sdk_root/lib" + dylib_id="$(basename "$(otool -D "$real_dylib" | sed -n '2p')")" + [ -n "$dylib_id" ] || fail "cannot read the dylib identity from $real_dylib" + install_name_tool -id "@rpath/$dylib_id" "$real_dylib" + codesign --force --sign - "$real_dylib" + [ "$(otool -D "$real_dylib" | sed -n '2p')" = "@rpath/$dylib_id" ] || + fail "dylib identity was not normalized to @rpath" + "$sdk_root/bin/tsfile-cli" --version PKG_CONFIG_PATH="$sdk_root/lib/pkgconfig" pkg-config --cflags --libs libtsfile cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/macos \ @@ -765,7 +783,7 @@ jobs: - name: Upload macOS SDK uses: actions/upload-artifact@v7 with: - name: native-sdk-macos-${{ matrix.name }} + name: native-sdk-${{ matrix.name }} path: sdk/*.tar.gz if-no-files-found: error retention-days: 14 diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index c41223fda..6461972d4 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -84,7 +84,10 @@ def run_text(job) raise "Homebrew prefix capture must tolerate extra stdout lines" unless homebrew_text.include?("brew --prefix apache/tsfile-dev/tsfile-dev | tail -n 1") raise "Homebrew SDK verification must provide pkg-config" unless homebrew_text.include?("brew install pkgconf") raise "Homebrew SDK step must keep bash 3.2 compatibility" if homebrew_text.include?("set -euo pipefail") -raise "Homebrew job must upload a macOS SDK artifact" unless homebrew.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-macos-${{ matrix.name }}" } +raise "Homebrew job must upload a macOS SDK artifact" unless homebrew.fetch("steps").any? { |step| step["with"].to_h["name"] == "native-sdk-${{ matrix.name }}" } +raise "Homebrew SDK must drop keg bookkeeping" unless homebrew_text.include?("INSTALL_RECEIPT.json") && homebrew_text.include?("sbom.spdx.json") +raise "Homebrew SDK must normalize the dylib identity" unless homebrew_text.include?("install_name_tool -id") +raise "Homebrew SDK must re-sign the normalized dylib" unless homebrew_text.include?("codesign --force --sign -") homebrew_merge = run_text(jobs.fetch("merge-homebrew")) trust = homebrew_merge.index("brew trust apache/tsfile-dev") @@ -112,6 +115,8 @@ def run_text(job) jobs.each_value do |job| job.fetch("steps").each do |step| + artifact = step["with"].to_h["name"].to_s + raise "artifact names must not repeat the macOS matrix prefix" if artifact.include?("macos-macos") next unless step["run"] && step.fetch("shell", "bash") == "bash" output, status = Open3.capture2e("bash", "-n", stdin_data: step["run"]) raise "Invalid shell in #{step['name']}: #{output}" unless status.success? From 6a955cec705974a8474b738da9049047de4c8828 Mon Sep 17 00:00:00 2001 From: ColinLee Date: Wed, 23 Sep 2026 17:29:08 +0800 Subject: [PATCH 31/31] feat(ci): add native Linux and Windows ARM packages --- .github/workflows/native-packages.yml | 494 +++++++++++++++++- packaging/README.md | 63 ++- packaging/scripts/assemble_native_packages.py | 66 ++- packaging/tests/check_native_workflow.rb | 29 +- .../tests/test_assemble_native_packages.py | 90 ++++ pom.xml | 19 + 6 files changed, 709 insertions(+), 52 deletions(-) diff --git a/.github/workflows/native-packages.yml b/.github/workflows/native-packages.yml index 67ff7c71e..20b6208f7 100644 --- a/.github/workflows/native-packages.yml +++ b/.github/workflows/native-packages.yml @@ -234,6 +234,156 @@ jobs: cmake --build consumer/build --parallel ./consumer/build/tsfile_installed_consumer + build-deb-arm64: + name: Build Ubuntu 22.04 ARM64 C++ SDK and DEB packages + needs: prepare + runs-on: ubuntu-22.04-arm + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Set up Java 17 + uses: actions/setup-java@v6.0.0 + with: + distribution: temurin + java-version: "17" + + - name: Install DEB build prerequisites + run: | + sudo apt-get update + sudo apt-get install -y \ + build-essential \ + cmake \ + pkg-config \ + dpkg-dev \ + uuid-dev + + - name: Verify native ARM64 runner + run: test "$(dpkg --print-architecture)" = arm64 + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + ./mvnw -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.dependency.source=AUTO \ + -Dtsfile.version.sync.skip=true \ + -Denable.cpack=ON \ + -Dtsfile.archive.version="$ARCHIVE_VERSION" \ + -Dtsfile.debian.package.version="$DEBIAN_VERSION" \ + -Dtsfile.rpm.package.version="$RPM_VERSION" \ + -Dtsfile.rpm.package.release="$RPM_RELEASE" + test -f cpp/target/build/CPackConfig.cmake + + - name: Stage and verify the Ubuntu ARM64 SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: bash + run: | + sdk_name="tsfile-sdk-ubuntu22.04-arm64-$ARCHIVE_VERSION" + sdk_root="$PWD/sdk/$sdk_name" + cmake --install cpp/target/build --prefix "$sdk_root" + test -f "$sdk_root/bin/tsfile-cli" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/deb-arm64 \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/deb-arm64 --parallel + ./consumer/deb-arm64/tsfile_installed_consumer + tar -C sdk -czf "$sdk_root.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Build and verify ARM64 DEB packages + env: + EXPECTED_DEB_VERSION: ${{ needs.prepare.outputs.deb_version }} + shell: bash + run: | + mkdir -p packages + cpack -G DEB --config cpp/target/build/CPackConfig.cmake -B packages + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.deb' -print | sort) + [[ ${#packages[@]} -eq 3 ]] + actual_names="$({ for package in "${packages[@]}"; do dpkg-deb --field "$package" Package; done; } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-dev tsfile-tools | sort)" + [[ "$actual_names" == "$expected_names" ]] + for package in "${packages[@]}"; do + [[ "$(dpkg-deb --field "$package" Architecture)" == arm64 ]] + [[ "$(dpkg-deb --field "$package" Version)" == "$EXPECTED_DEB_VERSION" ]] + done + + - name: Upload Ubuntu ARM64 SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-ubuntu22.04-arm64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload ARM64 DEB packages + uses: actions/upload-artifact@v7 + with: + name: native-deb-ubuntu22.04-arm64 + path: packages/*.deb + if-no-files-found: error + retention-days: 14 + + test-deb-arm64: + name: Test ARM64 DEB packages on ${{ matrix.container }} + needs: build-deb-arm64 + runs-on: ubuntu-24.04-arm + strategy: + fail-fast: false + matrix: + container: + - ubuntu:22.04 + - ubuntu:24.04 + container: ${{ matrix.container }} + steps: + - name: Checkout installed-consumer fixture + uses: actions/checkout@v7 + + - name: Download ARM64 DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-arm64 + path: packages + + - name: Install ARM64 DEB packages + shell: bash + run: | + test "$(dpkg --print-architecture)" = arm64 + apt-get update + apt-get install -y ./packages/*.deb cmake build-essential + + - name: Verify installed ARM64 DEB packages + shell: bash + run: | + cli_path="$(readlink -f -- "$(command -v tsfile-cli)")" + cli_owner="$(dpkg-query -S "$cli_path")" + cli_owner="${cli_owner%%:*}" + [[ "$cli_owner" == tsfile-tools ]] + tsfile-cli --version + dpkg-query -W tsfile tsfile-dev tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/tsfile_installed_consumer + build-rpm: name: Build AlmaLinux 9 C++ SDK and RPM packages needs: prepare @@ -398,6 +548,147 @@ jobs: cmake --build consumer/build --parallel ./consumer/build/tsfile_installed_consumer + build-rpm-aarch64: + name: Build AlmaLinux 9 aarch64 C++ SDK and RPM packages + needs: prepare + runs-on: ubuntu-24.04-arm + container: almalinux:9 + timeout-minutes: 45 + steps: + - name: Install RPM build prerequisites + run: | + test "$(uname -m)" = aarch64 + dnf install -y \ + java-17-openjdk-devel \ + cmake \ + gcc-c++ \ + make \ + pkgconf-pkg-config \ + rpm-build \ + git \ + curl-minimal \ + tar \ + gzip \ + libuuid-devel + + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + ./mvnw -Pwith-cpp package \ + -Dbuild.type=Release \ + -Dbuild.test=OFF \ + -DskipTests \ + -Dspotless.skip=true \ + -Dtsfile.dependency.source=AUTO \ + -Dtsfile.version.sync.skip=true \ + -Denable.cpack=ON \ + -Dtsfile.archive.version="$ARCHIVE_VERSION" \ + -Dtsfile.debian.package.version="$DEBIAN_VERSION" \ + -Dtsfile.rpm.package.version="$RPM_VERSION" \ + -Dtsfile.rpm.package.release="$RPM_RELEASE" + test -f cpp/target/build/CPackConfig.cmake + + - name: Stage and verify the AlmaLinux aarch64 SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: bash + run: | + sdk_name="tsfile-sdk-almalinux9-aarch64-$ARCHIVE_VERSION" + sdk_root="$PWD/sdk/$sdk_name" + cmake --install cpp/target/build --prefix "$sdk_root" + test -f "$sdk_root/bin/tsfile-cli" + test -d "$sdk_root/include" + test -n "$(find "$sdk_root" -maxdepth 2 -name 'libtsfile.so*' -print -quit)" + sdk_pkgconfig="$(find "$sdk_root" -type f -name libtsfile.pc -print -quit)" + test -n "$sdk_pkgconfig" + "$sdk_root/bin/tsfile-cli" --version + PKG_CONFIG_PATH="$(dirname "$sdk_pkgconfig")" pkg-config --cflags --libs libtsfile + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/rpm-aarch64 \ + -DCMAKE_PREFIX_PATH="$sdk_root" + cmake --build consumer/rpm-aarch64 --parallel + ./consumer/rpm-aarch64/tsfile_installed_consumer + tar -C sdk -czf "$sdk_root.tar.gz" "$sdk_name" + rm -rf "$sdk_root" + + - name: Build and verify aarch64 RPM packages + env: + EXPECTED_RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + EXPECTED_RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: bash + run: | + mkdir -p packages + cpack -G RPM --config cpp/target/build/CPackConfig.cmake -B packages + mapfile -t packages < <(find packages -maxdepth 1 -type f -name '*.rpm' -print | sort) + [[ ${#packages[@]} -eq 3 ]] + actual_names="$({ for package in "${packages[@]}"; do rpm -qp --queryformat '%{NAME}\n' "$package"; done; } | sort)" + expected_names="$(printf '%s\n' tsfile tsfile-devel tsfile-tools | sort)" + [[ "$actual_names" == "$expected_names" ]] + for package in "${packages[@]}"; do + [[ "$(rpm -qp --queryformat '%{ARCH}' "$package")" == aarch64 ]] + [[ "$(rpm -qp --queryformat '%{VERSION}' "$package")" == "$EXPECTED_RPM_VERSION" ]] + [[ "$(rpm -qp --queryformat '%{RELEASE}' "$package")" == "$EXPECTED_RPM_RELEASE" ]] + done + + - name: Upload AlmaLinux aarch64 SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-almalinux9-aarch64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload aarch64 RPM packages + uses: actions/upload-artifact@v7 + with: + name: native-rpm-almalinux9-aarch64 + path: packages/*.rpm + if-no-files-found: error + retention-days: 14 + + test-rpm-aarch64: + name: Test RPM packages on AlmaLinux 9 aarch64 + needs: build-rpm-aarch64 + runs-on: ubuntu-24.04-arm + container: almalinux:9 + steps: + - name: Checkout installed-consumer fixture + uses: actions/checkout@v7 + + - name: Download aarch64 RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-aarch64 + path: packages + + - name: Install aarch64 RPM packages + run: | + test "$(uname -m)" = aarch64 + dnf install -y packages/*.rpm cmake gcc-c++ + + - name: Verify installed aarch64 RPM packages + shell: bash + run: | + cli_path="$(readlink -f -- "$(command -v tsfile-cli)")" + cli_owner="$(rpm -qf --queryformat '%{NAME}' "$cli_path")" + [[ "$cli_owner" == tsfile-tools ]] + tsfile-cli --version + rpm -q tsfile tsfile-devel tsfile-tools + + - name: Build and run an installed-package consumer + shell: bash + run: | + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/build + cmake --build consumer/build --parallel + ./consumer/build/tsfile_installed_consumer + build-windows: name: Build Windows MSVC C++ SDK and ZIP needs: prepare @@ -556,6 +847,164 @@ jobs: if-no-files-found: error retention-days: 14 + build-windows-arm64: + name: Build Windows MSVC ARM64 C++ SDK and ZIP + needs: prepare + runs-on: windows-11-vs2026-arm + timeout-minutes: 50 + steps: + - name: Checkout repository + uses: actions/checkout@v7 + + - name: Select the native ARM64 Java runtime + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + if (-not $env:JAVA_HOME_21_AARCH64) { + throw 'The native ARM64 Java runtime is missing from the runner image' + } + "JAVA_HOME=$env:JAVA_HOME_21_AARCH64" >> $env:GITHUB_ENV + "$env:JAVA_HOME_21_AARCH64\bin" >> $env:GITHUB_PATH + java -version + + - name: Configure the native ARM64 MSVC environment + uses: ilammy/msvc-dev-cmd@v1 + with: + arch: arm64 + + - name: Verify native ARM64 MSVC compiler + shell: pwsh + run: | + $compiler = (Get-Command cl.exe).Source + Write-Host "Native compiler: $compiler" + if ($compiler -notmatch 'HostARM64\\ARM64\\cl\.exe$') { + throw "Expected the ARM64-hosted ARM64 compiler, got $compiler" + } + + - name: Build C++ core and CPack metadata with Maven + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + DEBIAN_VERSION: ${{ needs.prepare.outputs.deb_version }} + RPM_VERSION: ${{ needs.prepare.outputs.rpm_version }} + RPM_RELEASE: ${{ needs.prepare.outputs.rpm_release }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $projectInclude = (Resolve-Path cpp/cmake/tests/CheckStaticMSVCRuntime.cmake).Path + .\mvnw.cmd -Pwith-cpp package ` + "-Dcpp.toolchain=msvc-arm64-native" ` + "-Dbuild.type=Release" ` + "-Dbuild.test=OFF" ` + "-DskipTests" ` + "-Dspotless.skip=true" ` + "-Dtsfile.dependency.source=BUNDLED" ` + "-Dtsfile.msvc.static.runtime=ON" ` + "-Dtsfile.project.include=$projectInclude" ` + "-Dtsfile.version.sync.skip=true" ` + "-Denable.cpack=ON" ` + "-Dtsfile.archive.version=$env:ARCHIVE_VERSION" ` + "-Dtsfile.debian.package.version=$env:DEBIAN_VERSION" ` + "-Dtsfile.rpm.package.version=$env:RPM_VERSION" ` + "-Dtsfile.rpm.package.release=$env:RPM_RELEASE" + if (-not (Test-Path -LiteralPath cpp/target/build/CPackConfig.cmake -PathType Leaf)) { + throw 'Maven did not generate cpp/target/build/CPackConfig.cmake' + } + + - name: Stage and verify the Windows ARM64 SDK + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $sdkName = "tsfile-sdk-windows-msvc-arm64-$env:ARCHIVE_VERSION" + $sdkRoot = Join-Path $PWD "sdk/$sdkName" + cmake --install cpp/target/build --config Release --prefix $sdkRoot + $requiredPaths = @( + "$sdkRoot/bin/tsfile-cli.exe" + "$sdkRoot/bin/tsfile.dll" + "$sdkRoot/lib/tsfile.lib" + "$sdkRoot/include/tsfile/cwrapper/tsfile_cwrapper.h" + "$sdkRoot/lib/cmake/TsFile/TsFileConfig.cmake" + "$sdkRoot/share/doc/tsfile/LICENSE" + "$sdkRoot/share/doc/tsfile/NOTICE" + ) + foreach ($path in $requiredPaths) { + if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { + throw "Required ARM64 SDK file is absent: $path" + } + } + python packaging/scripts/verify_windows_runtime.py $sdkRoot + $env:PATH = "$sdkRoot/bin;$env:PATH" + & "$sdkRoot/bin/tsfile-cli.exe" --version + cmake -S cpp/cmake/tests/projects/InstalledConsumer -B consumer/windows-arm64 ` + -G "Ninja Multi-Config" ` + -DCMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded ` + "-DCMAKE_PREFIX_PATH=$sdkRoot" + cmake --build consumer/windows-arm64 --config Release --parallel + & consumer/windows-arm64/Release/tsfile_installed_consumer.exe + tar.exe -czf "$sdkRoot.tar.gz" -C sdk $sdkName + Remove-Item -LiteralPath $sdkRoot -Recurse -Force + + - name: Build and inspect the Windows ARM64 ZIP + env: + ARCHIVE_VERSION: ${{ needs.prepare.outputs.archive_version }} + shell: pwsh + run: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $cpackConfig = Get-Content cpp/target/build/CPackConfig.cmake -Raw + if ($cpackConfig -notmatch 'set\(CPACK_ARCHIVE_COMPONENT_INSTALL "OFF"\)') { + throw 'CPACK_ARCHIVE_COMPONENT_INSTALL must remain disabled' + } + New-Item -ItemType Directory -Path packages -Force | Out-Null + cpack -G ZIP -C Release --config cpp/target/build/CPackConfig.cmake -B packages ` + -D "CPACK_PACKAGE_VERSION=$env:ARCHIVE_VERSION" ` + -D "CPACK_PACKAGE_FILE_NAME=tsfile-$env:ARCHIVE_VERSION-windows-arm64" + $archives = @(Get-ChildItem -Path packages -Filter *.zip -File) + if ($archives.Count -ne 1) { throw "Expected exactly one ARM64 ZIP, found $($archives.Count)" } + $expectedName = "tsfile-$env:ARCHIVE_VERSION-windows-arm64.zip" + if ($archives[0].Name -cne $expectedName) { + throw "Unexpected ZIP name '$($archives[0].Name)'; expected '$expectedName'" + } + Add-Type -AssemblyName System.IO.Compression.FileSystem + $archive = [System.IO.Compression.ZipFile]::OpenRead($archives[0].FullName) + try { + $entryPaths = @($archive.Entries | ForEach-Object { $_.FullName.Replace('\', '/') }) + foreach ($suffix in @( + 'bin/tsfile-cli.exe' + 'bin/tsfile.dll' + 'lib/tsfile.lib' + 'include/tsfile/cwrapper/tsfile_cwrapper.h' + )) { + if (-not ($entryPaths | Where-Object { $_.EndsWith($suffix, [System.StringComparison]::Ordinal) })) { + throw "ARM64 ZIP is missing a path ending in '$suffix'" + } + } + } finally { $archive.Dispose() } + Expand-Archive -LiteralPath $archives[0].FullName -DestinationPath extracted/windows-arm64 + python packaging/scripts/verify_windows_runtime.py extracted/windows-arm64 + $zipCli = @(Get-ChildItem extracted/windows-arm64 -Recurse -Filter tsfile-cli.exe -File) + if ($zipCli.Count -ne 1) { throw 'Expected exactly one ARM64 CLI' } + & $zipCli[0].FullName --version + + - name: Upload Windows ARM64 SDK + uses: actions/upload-artifact@v7 + with: + name: native-sdk-windows-msvc-arm64 + path: sdk/*.tar.gz + if-no-files-found: error + retention-days: 14 + + - name: Upload Windows ARM64 ZIP + uses: actions/upload-artifact@v7 + with: + name: native-windows-msvc-arm64 + path: packages/*.zip + if-no-files-found: error + retention-days: 14 + test-go-linux: name: Test Go binding against the Ubuntu SDK needs: build-deb @@ -913,11 +1362,14 @@ jobs: needs: - prepare - test-deb + - test-deb-arm64 - test-rpm + - test-rpm-aarch64 - test-go-linux - build-python-linux - merge-homebrew - build-windows + - build-windows-arm64 runs-on: ubuntu-24.04 steps: - name: Checkout repository @@ -933,13 +1385,25 @@ jobs: uses: actions/download-artifact@v8 with: name: native-deb-ubuntu22.04-amd64 - path: input/deb + path: input/deb/amd64 + + - name: Download ARM64 DEB packages + uses: actions/download-artifact@v8 + with: + name: native-deb-ubuntu22.04-arm64 + path: input/deb/arm64 - name: Download RPM packages uses: actions/download-artifact@v8 with: name: native-rpm-almalinux9-x86_64 - path: input/rpm + path: input/rpm/x86_64 + + - name: Download aarch64 RPM packages + uses: actions/download-artifact@v8 + with: + name: native-rpm-almalinux9-aarch64 + path: input/rpm/aarch64 - name: Download Homebrew Formula and bottles uses: actions/download-artifact@v8 @@ -951,7 +1415,13 @@ jobs: uses: actions/download-artifact@v8 with: name: native-windows-msvc-x86_64 - path: input/windows + path: input/windows/x86_64 + + - name: Download Windows ARM64 ZIP + uses: actions/download-artifact@v8 + with: + name: native-windows-msvc-arm64 + path: input/windows/arm64 - name: Download Ubuntu SDK uses: actions/download-artifact@v8 @@ -959,18 +1429,36 @@ jobs: name: native-sdk-ubuntu22.04-amd64 path: input/sdk/ubuntu22.04-amd64 + - name: Download Ubuntu ARM64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-ubuntu22.04-arm64 + path: input/sdk/ubuntu22.04-arm64 + - name: Download AlmaLinux SDK uses: actions/download-artifact@v8 with: name: native-sdk-almalinux9-x86_64 path: input/sdk/almalinux9-x86_64 + - name: Download AlmaLinux aarch64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-almalinux9-aarch64 + path: input/sdk/almalinux9-aarch64 + - name: Download Windows SDK uses: actions/download-artifact@v8 with: name: native-sdk-windows-msvc-x86_64 path: input/sdk/windows-msvc-x86_64 + - name: Download Windows ARM64 SDK + uses: actions/download-artifact@v8 + with: + name: native-sdk-windows-msvc-arm64 + path: input/sdk/windows-msvc-arm64 + - name: Download macOS arm64 SDK uses: actions/download-artifact@v8 with: diff --git a/packaging/README.md b/packaging/README.md index 21cd02771..0035f835e 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -39,11 +39,12 @@ with read-only repository permissions; pushes and pull requests do not trigger native packaging. A successful run builds the C++ core through Maven, stages one SDK per -release platform, and then produces Ubuntu DEBs, AlmaLinux RPMs, an ARM64 and -Intel Homebrew development bottle with a merged Formula, a Windows x86_64 -SDK/CLI ZIP, and a Linux Python wheel built from the staged Ubuntu SDK. It also -runs the Go and Python consumers against the SDK rather than rebuilding the C++ -core downstream. The job combines these into +release platform, and then produces Ubuntu amd64 and arm64 DEBs, AlmaLinux +x86_64 and aarch64 RPMs, ARM64 and Intel Homebrew development bottles with a +merged Formula, Windows x86_64 and ARM64 SDK/CLI ZIPs, and a Linux Python wheel +built from the staged Ubuntu SDK. It also runs the Go and Python consumers +against the SDK rather than rebuilding the C++ core downstream. The job combines +these into `tsfile-native-packages-` with `manifest.json` and `SHA256SUMS`. Download this final artifact from the workflow run; both intermediate and final artifacts are retained for 14 days. @@ -92,9 +93,10 @@ unavailable or incompatible distro versions. The manual `Build native package artifacts` workflow is the authoritative native package build. Its Linux jobs create `tsfile`, `tsfile-dev`, and -`tsfile-tools` DEBs on Ubuntu 22.04 and `tsfile`, `tsfile-devel`, and -`tsfile-tools` RPMs on AlmaLinux 9. Fresh Ubuntu 22.04, Ubuntu 24.04, and -AlmaLinux 9 containers install the packages and verify both `tsfile-cli` and an +`tsfile-tools` DEBs on native Ubuntu 22.04 amd64 and arm64 runners, and +`tsfile`, `tsfile-devel`, and `tsfile-tools` RPMs on native AlmaLinux 9 +x86_64 and aarch64 runners. Fresh Ubuntu 22.04, Ubuntu 24.04, and AlmaLinux 9 +containers install the matching packages and verify both `tsfile-cli` and an external CMake consumer. The workflow uploads intermediate artifacts for 14 days and does not publish packages. @@ -146,14 +148,14 @@ artifacts for 14 days; it does not publish to that root or update `latest`. ## Windows -The manual workflow builds a 64-bit Release package with Visual Studio 2022 and -bundled dependencies. It stages and tests the CLI plus an external CMake SDK -consumer before producing -`tsfile--windows-x86_64.zip`. The archive is deliberately one -combined ZIP rather than one archive per CPack component, and contains the -runtime, development files, and tools under a relocatable prefix. In -particular, the MSVC outputs are installed as `bin/tsfile.dll`, -`lib/tsfile.lib`, and `bin/tsfile-cli.exe`. +The manual workflow builds native x86_64 and ARM64 Release packages with +Visual Studio 2022 and bundled dependencies. Each job stages and tests the CLI +plus an external CMake SDK consumer before producing +`tsfile--windows-x86_64.zip` or +`tsfile--windows-arm64.zip`. Each archive is one combined ZIP +rather than one archive per CPack component, and contains the runtime, +development files, and tools under a relocatable prefix. The MSVC outputs are +installed as `bin/tsfile.dll`, `lib/tsfile.lib`, and `bin/tsfile-cli.exe`. The portable ZIP targets Windows 10 / Windows Server 2022 or newer and uses the static MSVC runtime (`/MT`) in Release. `TSFILE_MSVC_STATIC_RUNTIME=ON` @@ -166,17 +168,20 @@ does not require a separate Visual C++ Redistributable installation. Microsoft redistributable DLLs are not copied from the runner or included in the ZIP; runtime security updates require rebuilding the static-runtime package. -C++ SDK consumers should use the matching MSVC v143 Release toolset and `/MT` +C++ SDK consumers should use an MSVC Release toolset compatible with the +compiler used to build that architecture and `/MT` (`CMAKE_MSVC_RUNTIME_LIBRARY=MultiThreaded`, with CMake policy CMP0091 set to NEW before `project()`). Keep allocation and release paired through the public APIs; CRT-owned objects such as `FILE*` must not cross DLL boundaries. The installed-consumer fixture demonstrates a public setting/getter round trip that requires real symbols from `tsfile.dll`. -The workflow uploads that ZIP as the intermediate artifact -`native-windows-msvc-x86_64` for 14 days. Like the Linux jobs, it validates the +The workflow uploads the ZIPs as `native-windows-msvc-x86_64` and +`native-windows-msvc-arm64` for 14 days. Like the Linux jobs, it validates the license files, CMake package config, staged executable, library, import library, -and public headers without publishing the artifact. +and public headers without publishing the artifacts. Windows ARM uses the +native ARM64 compiler and produces unsigned binary archives; no MSIX/AppX +signing step is part of this workflow. ## Static SDK and install regressions @@ -203,21 +208,23 @@ python3 -m unittest discover -s packaging/tests -p 'test_*.py' -v ## Final native package bundle -Only after the Ubuntu 22.04 and 24.04 DEB installation tests, AlmaLinux 9 RPM -installation test, Go SDK tests, Linux Python wheel smoke test, macOS SDK and -Homebrew bottle merge, and Windows SDK/CLI build all succeed, the workflow assembles +Only after the Ubuntu amd64 and arm64 DEB installation tests, AlmaLinux x86_64 +and aarch64 RPM installation tests, Go SDK tests, Linux Python wheel smoke test, +macOS SDK and Homebrew bottle merge, and Windows x86_64 and ARM64 SDK/CLI builds +all succeed, the workflow assembles `tsfile-native-packages-`. The final -GitHub Actions artifact retains the platform SDKs, including both macOS archives, -the Linux Python wheel, the DEBs, RPMs, merged Formula and bottles, and Windows ZIP in their package-family -layouts for 14 days. It also contains a +GitHub Actions artifact retains the platform SDKs, including both Linux ARM +archives, both macOS archives, the Windows ARM64 archive, the Linux Python +wheel, the DEBs, RPMs, merged Formula and bottles, and Windows ZIPs in their +package-family layouts for 14 days. It also contains a sorted `SHA256SUMS` and `manifest.json` with the source identity, generated versions, byte sizes, SHA-256 values, and the JFrog repository, immutable target path, and properties required for later manual publication. The final job has no publishing credentials and does not upload to JFrog. A maintainer can later use the manifest to upload DEBs to `tsfile-debian` with the -recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/x86_64`, Homebrew -formula/bottles to `tsfile/homebrew/dev/versions/`, SDKs to +recorded Debian coordinates, RPMs to `tsfile-rpm/dev/el9/`, +Homebrew formula/bottles to `tsfile/homebrew/dev/versions/`, SDKs to `tsfile/sdk/dev/versions/`, Windows ZIPs to `tsfile/windows/dev/versions/`, and Python wheels to the `tsfile-python` repository recorded in the manifest. diff --git a/packaging/scripts/assemble_native_packages.py b/packaging/scripts/assemble_native_packages.py index 6fb6abf84..aef2c9dab 100644 --- a/packaging/scripts/assemble_native_packages.py +++ b/packaging/scripts/assemble_native_packages.py @@ -32,16 +32,21 @@ DEB_PROPERTIES = { "deb.distribution": ["jammy", "noble"], "deb.component": ["dev"], - "deb.architecture": ["amd64"], } +DEB_PLATFORMS = {"ubuntu22.04-amd64", "ubuntu22.04-arm64"} +RPM_PLATFORMS = {"almalinux9-x86_64", "almalinux9-aarch64"} +WINDOWS_PLATFORMS = {"windows-msvc-x86_64", "windows-msvc-arm64"} REQUIRED_VERSIONS = {"archive_version", "homebrew_version"} REQUIRED_SOURCE = {"commit", "repository"} SDK_PLATFORMS = { "ubuntu22.04-amd64", + "ubuntu22.04-arm64", "almalinux9-x86_64", + "almalinux9-aarch64", "macos-arm64", "macos-x86_64", "windows-msvc-x86_64", + "windows-msvc-arm64", } PYTHON_PLATFORMS = {"ubuntu22.04-x86_64"} @@ -106,21 +111,34 @@ def _artifact_description( parts = relative_path.parts filename = source_path.name if len(parts) >= 2 and parts[0] == "deb" and filename.endswith(".deb"): - target = Path("deb/ubuntu22.04-amd64") / filename + match = re.search(r"_(amd64|arm64)\.deb$", filename) + if not match: + raise ValueError(f"unsupported DEB architecture: {filename}") + architecture = match.group(1) + platform = f"ubuntu22.04-{architecture}" + target = Path("deb") / platform / filename return target, { "family": "deb", - "platform": "ubuntu22.04-amd64", + "platform": platform, "targetRepository": "tsfile-debian", - "targetPath": f"pool/dev/ubuntu22.04-amd64/{filename}", - "properties": DEB_PROPERTIES, + "targetPath": f"pool/dev/{platform}/{filename}", + "properties": { + **DEB_PROPERTIES, + "deb.architecture": [architecture], + }, } if len(parts) >= 2 and parts[0] == "rpm" and filename.endswith(".rpm"): - target = Path("rpm/almalinux9-x86_64") / filename + match = re.search(r"\.(x86_64|aarch64)\.rpm$", filename) + if not match: + raise ValueError(f"unsupported RPM architecture: {filename}") + architecture = match.group(1) + platform = f"almalinux9-{architecture}" + target = Path("rpm") / platform / filename return target, { "family": "rpm", - "platform": "almalinux9-x86_64", + "platform": platform, "targetRepository": "tsfile-rpm", - "targetPath": f"dev/el9/x86_64/{filename}", + "targetPath": f"dev/el9/{architecture}/{filename}", "properties": {}, } if ( @@ -178,10 +196,14 @@ def _artifact_description( "properties": {}, } if len(parts) >= 2 and parts[0] == "windows" and filename.endswith(".zip"): + match = re.search(r"-windows-(x86_64|arm64)\.zip$", filename) + if not match: + raise ValueError(f"unsupported Windows ZIP architecture: {filename}") + platform = f"windows-msvc-{match.group(1)}" target = Path("windows") / filename return target, { "family": "windows", - "platform": "windows-msvc-x86_64", + "platform": platform, "targetRepository": "tsfile", "targetPath": f"windows/dev/versions/{versions['archive_version']}/{filename}", "properties": {}, @@ -192,13 +214,25 @@ def _artifact_description( def _require_complete_families( planned: list[tuple[Path, Path, dict[str, object]]], ) -> None: - families = [metadata["family"] for _, _, metadata in planned] - if not any(family == "deb" for family in families): - raise ValueError("missing DEB package input") - if not any(family == "rpm" for family in families): - raise ValueError("missing RPM package input") - if not any(family == "windows" for family in families): - raise ValueError("missing Windows ZIP input") + platforms_by_family = { + family: { + metadata["platform"] + for _, _, metadata in planned + if metadata["family"] == family + } + for family in ("deb", "rpm", "windows") + } + for family, required_platforms in ( + ("deb", DEB_PLATFORMS), + ("rpm", RPM_PLATFORMS), + ("windows", WINDOWS_PLATFORMS), + ): + missing_platforms = required_platforms - platforms_by_family[family] + if missing_platforms: + raise ValueError( + f"missing {family.upper()} inputs for: " + + ", ".join(sorted(missing_platforms)) + ) sdk_platforms = { metadata["platform"] for _, _, metadata in planned diff --git a/packaging/tests/check_native_workflow.rb b/packaging/tests/check_native_workflow.rb index 6461972d4..aeb476371 100644 --- a/packaging/tests/check_native_workflow.rb +++ b/packaging/tests/check_native_workflow.rb @@ -31,14 +31,14 @@ def run_text(job) job.fetch("steps").filter_map { |step| step["run"] }.join("\n") end -%w[build-deb build-rpm build-windows].each do |name| +%w[build-deb build-deb-arm64 build-rpm build-rpm-aarch64 build-windows build-windows-arm64].each do |name| text = run_text(jobs.fetch(name)) raise "#{name} must build C++ through Maven" unless text.include?("-Pwith-cpp package") raise "#{name} must enable CPack through Maven" unless text.include?("-Denable.cpack=ON") raise "#{name} must use the generated CPack config" unless text.include?("cpp/target/build/CPackConfig.cmake") raise "#{name} must stage an SDK" unless text.include?("cmake --install cpp/target/build") raise "#{name} must use an absolute SDK prefix" unless text.include?("$PWD/sdk/") || text.include?('Join-Path $PWD "sdk/') - if name != "build-windows" + unless name.start_with?("build-windows") raise "#{name} must locate pkg-config without assuming lib" unless text.include?('dirname "$sdk_pkgconfig"') end raise "#{name} must pass the generated archive version" unless text.include?("-Dtsfile.archive.version") @@ -59,6 +59,25 @@ def run_text(job) raise "Windows Maven properties must be quoted for PowerShell" unless windows.include?('"-Dcpp.toolchain=msvc"') && windows.include?('"-Dbuild.type=Release"') raise "Both staged and extracted PE imports must be checked" unless windows.scan("python packaging/scripts/verify_windows_runtime.py").size == 2 +raise "Ubuntu ARM64 packages must use a native ARM runner" unless jobs.fetch("build-deb-arm64").fetch("runs-on") == "ubuntu-22.04-arm" +raise "AlmaLinux ARM64 packages must use a native ARM runner" unless jobs.fetch("build-rpm-aarch64").fetch("runs-on") == "ubuntu-24.04-arm" +raise "Windows ARM64 binaries must use the pinned native ARM runner" unless jobs.fetch("build-windows-arm64").fetch("runs-on") == "windows-11-vs2026-arm" +windows_arm = run_text(jobs.fetch("build-windows-arm64")) +raise "Windows ARM64 build must use its Maven toolchain profile" unless windows_arm.include?("-Dcpp.toolchain=msvc-arm64-native") +raise "Windows ARM64 build must put the native compiler first on PATH" unless windows_arm.include?("HostARM64") && windows_arm.include?("Get-Command cl.exe") +raise "Windows ARM64 consumer must use the same Ninja Multi-Config toolchain" unless windows_arm.include?('-G "Ninja Multi-Config"') +raise "Windows ARM64 ZIP must have an architecture-specific name" unless windows_arm.include?("windows-arm64") +raise "Windows ARM64 must not add signed MSIX/AppX packaging" if windows_arm.match?(/msix|appx|signtool/i) +root_pom = File.read(File.expand_path("../../pom.xml", __dir__)) +raise "Windows ARM64 Maven profile must use Ninja Multi-Config" unless root_pom.include?("cpp-msvc-arm64-native") && root_pom.include?("Ninja Multi-Config") + +%w[test-deb-arm64 test-rpm-aarch64].each do |name| + job = jobs.fetch(name) + raise "#{name} must run on a native ARM runner" unless job.fetch("runs-on").end_with?("-arm") + runs = run_text(job) + raise "#{name} must build and run an installed CMake consumer" unless runs.include?("cpp/cmake/tests/projects/InstalledConsumer") +end + go_job = jobs.fetch("test-go-linux") raise "Go test must consume the Ubuntu SDK" unless go_job.fetch("needs") == "build-deb" go_text = run_text(go_job) @@ -94,7 +113,7 @@ def run_text(job) merge = homebrew_merge.index("brew bottle --merge") raise "Homebrew merge must trust its temporary tap before loading the Formula" unless trust && merge && trust < merge -%w[test-deb test-rpm build-windows].each do |name| +%w[test-deb test-deb-arm64 test-rpm test-rpm-aarch64 build-windows build-windows-arm64].each do |name| steps = jobs.fetch(name).fetch("steps") raise "#{name} must check out the consumer fixture" unless steps.any? { |step| step["uses"].to_s.start_with?("actions/checkout@") } runs = steps.filter_map { |step| step["run"] }.join("\n") @@ -103,11 +122,11 @@ def run_text(job) assemble = jobs.fetch("assemble") needs = Array(assemble.fetch("needs")) -%w[test-deb test-rpm test-go-linux build-python-linux merge-homebrew build-windows].each do |name| +%w[test-deb test-deb-arm64 test-rpm test-rpm-aarch64 test-go-linux build-python-linux merge-homebrew build-windows build-windows-arm64].each do |name| raise "assemble must depend on #{name}" unless needs.include?(name) end assemble_text = run_text(assemble) -%w[native-sdk-ubuntu22.04-amd64 native-sdk-almalinux9-x86_64 native-sdk-windows-msvc-x86_64 native-sdk-macos-arm64 native-sdk-macos-x86_64 native-python-wheel-ubuntu22.04-x86_64].each do |name| +%w[native-deb-ubuntu22.04-arm64 native-rpm-almalinux9-aarch64 native-windows-msvc-arm64 native-sdk-ubuntu22.04-amd64 native-sdk-ubuntu22.04-arm64 native-sdk-almalinux9-x86_64 native-sdk-almalinux9-aarch64 native-sdk-windows-msvc-x86_64 native-sdk-windows-msvc-arm64 native-sdk-macos-arm64 native-sdk-macos-x86_64 native-python-wheel-ubuntu22.04-x86_64].each do |name| raise "assemble must download #{name}" unless assemble.fetch("steps").any? { |step| step["with"].to_h["name"] == name } end raise "assemble must verify the bundle" unless assemble_text.include?("--verify-bundle") diff --git a/packaging/tests/test_assemble_native_packages.py b/packaging/tests/test_assemble_native_packages.py index 9c3475a1b..7f62a43cf 100644 --- a/packaging/tests/test_assemble_native_packages.py +++ b/packaging/tests/test_assemble_native_packages.py @@ -65,16 +65,22 @@ def require_module(self): def write_fixture(self, directory): files = { "deb/tsfile_2.5.0-dev_amd64.deb": b"deb artifact\n", + "deb/tsfile_2.5.0-dev_arm64.deb": b"arm64 deb artifact\n", "rpm/tsfile-2.5.0-dev.x86_64.rpm": b"rpm artifact\n", + "rpm/tsfile-2.5.0-dev.aarch64.rpm": b"aarch64 rpm artifact\n", "homebrew/Formula/tsfile-dev.rb": b"formula artifact\n", "homebrew/bottles/tsfile-dev.bottle.tar.gz": b"bottle artifact\n", "homebrew/bottles/tsfile-dev.bottle.json": b"bottle metadata\n", "windows/tsfile-2.5.0-dev-windows-x86_64.zip": b"windows artifact\n", + "windows/tsfile-2.5.0-dev-windows-arm64.zip": b"windows arm64 artifact\n", "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk ubuntu\n", + "sdk/ubuntu22.04-arm64/tsfile-sdk-ubuntu22.04-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk ubuntu arm64\n", "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk almalinux\n", + "sdk/almalinux9-aarch64/tsfile-sdk-almalinux9-aarch64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk almalinux arm64\n", "sdk/macos-arm64/tsfile-sdk-macos-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk macos arm64\n", "sdk/macos-x86_64/tsfile-sdk-macos-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk macos x86_64\n", "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk windows\n", + "sdk/windows-msvc-arm64/tsfile-sdk-windows-msvc-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz": b"sdk windows arm64\n", "python/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl": b"python wheel\n", } for relative_path, contents in files.items(): @@ -97,16 +103,22 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): expected_paths = [ "deb/ubuntu22.04-amd64/tsfile_2.5.0-dev_amd64.deb", + "deb/ubuntu22.04-arm64/tsfile_2.5.0-dev_arm64.deb", "homebrew/Formula/tsfile-dev.rb", "homebrew/bottles/tsfile-dev.bottle.json", "homebrew/bottles/tsfile-dev.bottle.tar.gz", "python/wheels/ubuntu22.04-x86_64/tsfile-2.5.0.dev0.20260910.123.1.gabcdef1-cp311-cp311-linux_x86_64.whl", + "rpm/almalinux9-aarch64/tsfile-2.5.0-dev.aarch64.rpm", "rpm/almalinux9-x86_64/tsfile-2.5.0-dev.x86_64.rpm", + "sdk/almalinux9-aarch64/tsfile-sdk-almalinux9-aarch64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/almalinux9-x86_64/tsfile-sdk-almalinux9-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/macos-arm64/tsfile-sdk-macos-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/macos-x86_64/tsfile-sdk-macos-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/ubuntu22.04-amd64/tsfile-sdk-ubuntu22.04-amd64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/ubuntu22.04-arm64/tsfile-sdk-ubuntu22.04-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "sdk/windows-msvc-arm64/tsfile-sdk-windows-msvc-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", "sdk/windows-msvc-x86_64/tsfile-sdk-windows-msvc-x86_64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "windows/tsfile-2.5.0-dev-windows-arm64.zip", "windows/tsfile-2.5.0-dev-windows-x86_64.zip", ] self.assertEqual( @@ -130,15 +142,49 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): self.assertEqual( {artifact["platform"] for artifact in manifest["artifacts"]}, { + "almalinux9-aarch64", "almalinux9-x86_64", "homebrew", "macos-arm64", "macos-x86_64", "ubuntu22.04-amd64", + "ubuntu22.04-arm64", "ubuntu22.04-x86_64", + "windows-msvc-arm64", "windows-msvc-x86_64", }, ) + artifact_platforms = { + artifact["filename"]: artifact["platform"] + for artifact in manifest["artifacts"] + } + self.assertEqual( + artifact_platforms["tsfile_2.5.0-dev_arm64.deb"], + "ubuntu22.04-arm64", + ) + self.assertEqual( + artifact_platforms["tsfile-2.5.0-dev.aarch64.rpm"], + "almalinux9-aarch64", + ) + self.assertEqual( + artifact_platforms["tsfile-2.5.0-dev-windows-arm64.zip"], + "windows-msvc-arm64", + ) + deb_arm64 = next( + artifact + for artifact in manifest["artifacts"] + if artifact["filename"] == "tsfile_2.5.0-dev_arm64.deb" + ) + self.assertEqual(deb_arm64["properties"]["deb.architecture"], ["arm64"]) + rpm_arm64 = next( + artifact + for artifact in manifest["artifacts"] + if artifact["filename"] == "tsfile-2.5.0-dev.aarch64.rpm" + ) + self.assertEqual( + rpm_arm64["targetPath"], + "dev/el9/aarch64/tsfile-2.5.0-dev.aarch64.rpm", + ) for artifact in manifest["artifacts"]: output_path = output_directory / artifact["path"] input_path = next( @@ -152,6 +198,50 @@ def test_assembles_publishable_bundle_with_literal_metadata(self): hashlib.sha256(input_path.read_bytes()).hexdigest(), ) + def test_rejects_bundles_missing_arm_architectures(self): + module = self.require_module() + + with tempfile.TemporaryDirectory() as temporary_directory: + missing_artifacts = ( + ( + "deb/tsfile_2.5.0-dev_arm64.deb", + "missing DEB inputs for: ubuntu22.04-arm64", + ), + ( + "rpm/tsfile-2.5.0-dev.aarch64.rpm", + "missing RPM inputs for: almalinux9-aarch64", + ), + ( + "windows/tsfile-2.5.0-dev-windows-arm64.zip", + "missing WINDOWS inputs for: windows-msvc-arm64", + ), + ( + "sdk/ubuntu22.04-arm64/tsfile-sdk-ubuntu22.04-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "missing SDK inputs for: ubuntu22.04-arm64", + ), + ( + "sdk/almalinux9-aarch64/tsfile-sdk-almalinux9-aarch64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "missing SDK inputs for: almalinux9-aarch64", + ), + ( + "sdk/windows-msvc-arm64/tsfile-sdk-windows-msvc-arm64-2.5.0-dev0.20260910.123.1.gabcdef1.tar.gz", + "missing SDK inputs for: windows-msvc-arm64", + ), + ) + for index, (missing_path, error_message) in enumerate(missing_artifacts): + with self.subTest(missing=missing_path): + input_directory = Path(temporary_directory) / f"input-{index}" + self.write_fixture(input_directory) + (input_directory / missing_path).unlink() + + with self.assertRaisesRegex(ValueError, error_message): + module.assemble( + input_directory, + Path(temporary_directory) / f"incomplete-{index}", + VERSIONS, + SOURCE, + ) + def test_verifies_assembler_checksum_lines_sorted_by_path(self): module = self.require_module() diff --git a/pom.xml b/pom.xml index 524dedc88..46785bb15 100644 --- a/pom.xml +++ b/pom.xml @@ -847,6 +847,25 @@ ${msvc.generator} + + + cpp-msvc-arm64-native + + + cpp.toolchain + msvc-arm64-native + + + + Ninja Multi-Config + + .skipTests