From 8e5a02df8c9f664cee84ad53307e5cf9d5818de1 Mon Sep 17 00:00:00 2001 From: aybuke Date: Wed, 9 Sep 2026 21:07:35 +0000 Subject: [PATCH 1/5] Add focal tdigest build pipeline (PG12-16) Rebuild tdigest .deb packages for Ubuntu 20.04 (focal) from upstream source, mirroring the focal PostGIS pipeline. PGDG dropped focal (focal-pgdg 404s, frozen archive stops at tdigest 1.4.3), so newer tdigest (default 1.4.5, memory-safety fixes) is rebuilt from upstream source + the frozen focal-era debian/ packaging. One multi-version source build emits postgresql--tdigest for PG12-16, signed via the debsigner image and smoke-tested in a stock ubuntu:20.04. --- .github/workflows/build-tdigest-focal.yml | 171 ++++++++++++++++++ dockerfiles/focal-tdigest-builder/Dockerfile | 69 ++++++++ scripts/build_tdigest_focal | 173 +++++++++++++++++++ scripts/smoke_test_focal_tdigest_debs | 138 +++++++++++++++ 4 files changed, 551 insertions(+) create mode 100644 .github/workflows/build-tdigest-focal.yml create mode 100644 dockerfiles/focal-tdigest-builder/Dockerfile create mode 100755 scripts/build_tdigest_focal create mode 100755 scripts/smoke_test_focal_tdigest_debs diff --git a/.github/workflows/build-tdigest-focal.yml b/.github/workflows/build-tdigest-focal.yml new file mode 100644 index 00000000..57f68671 --- /dev/null +++ b/.github/workflows/build-tdigest-focal.yml @@ -0,0 +1,171 @@ +name: Build tdigest (focal) + +# Builds tdigest .deb packages for Ubuntu 20.04 (focal) from upstream source +# (PGDG removed focal entirely -- focal-pgdg 404s and the frozen archive mirror +# stops at tdigest 1.4.3), then signs them with debsigs (--sign=maint) using the +# existing packaging key. +# +# Like build-postgis-focal.yml there is no per-major matrix: the tdigest Debian +# packaging is multi-version by design (debian/pgversions + pg_buildext), so one +# source build emits postgresql--tdigest for every requested major in a +# single pass. That also means no assemble/de-duplicate job is needed -- each +# runtime package is self-contained and produced exactly once. + +on: + workflow_dispatch: + inputs: + tdigest_version: + description: "tdigest upstream version to build (e.g. 1.4.5)" + required: true + default: "1.4.5" + pg_versions: + description: "Space-separated PostgreSQL majors (tdigest supports 10+)" + required: false + default: "12 13 14 15 16" + tdigest_sha256: + description: "sha256 of tdigest-.tar.gz. Blank = use the pinned default in scripts/build_tdigest_focal (only valid for that version)." + required: false + default: "" + run_tests: + description: "Run upstream regression suite (1=yes, much slower)" + required: false + default: "0" + push: + branches: + - tdigest-focal + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-sign: + name: Build & sign tdigest ${{ github.event.inputs.tdigest_version || '1.4.5' }} (focal) + runs-on: ubuntu-latest + env: + PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} + PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} + TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.5' }} + PG_VERSIONS: ${{ github.event.inputs.pg_versions || '12 13 14 15 16' }} + TDIGEST_SHA256: ${{ github.event.inputs.tdigest_sha256 || '' }} + RUN_TESTS: ${{ github.event.inputs.run_tests || '0' }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USER_NAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build focal tdigest builder image + run: | + docker build -t focal-tdigest-builder \ + -f dockerfiles/focal-tdigest-builder/Dockerfile . + + - name: Build tdigest packages + run: | + mkdir -p packages + docker run --rm \ + -e TDIGEST_VERSION="${TDIGEST_VERSION}" \ + -e PG_VERSIONS="${PG_VERSIONS}" \ + -e TDIGEST_SHA256="${TDIGEST_SHA256}" \ + -e RUN_TESTS="${RUN_TESTS}" \ + -v "${PWD}/packages:/packages" \ + focal-tdigest-builder + echo "Built packages:" + ls -1 packages/focal/tdigest/*.deb + + - name: Sign packages (debsigs --sign=maint) + # Use the prebuilt, deployed debsigner image (the one all Citus signing + # uses). Its entrypoint signs exactly "/packages/*/*.deb" (one dir level + # deep), so mount the parent of the output dir: with + # "${PWD}/packages/focal:/packages" the debs land at + # /packages/tdigest/*.deb, which is what that glob expects. + run: | + if [ -z "${PACKAGING_SECRET_KEY}" ] || [ -z "${PACKAGING_PASSPHRASE}" ]; then + echo "::error::PACKAGING_SECRET_KEY / PACKAGING_PASSPHRASE secrets are not set" >&2 + exit 1 + fi + printf '%s' "${PACKAGING_PASSPHRASE}" | docker run --rm -i \ + -e PACKAGING_SECRET_KEY \ + -e PACKAGING_PASSPHRASE \ + -v "${PWD}/packages/focal:/packages" \ + citusdata/packaging:debsigner + + - name: Verify signatures are embedded + run: | + rc=0 + for deb in packages/focal/tdigest/*.deb; do + if ar t "$deb" | grep -q '^_gpgmaint$'; then + echo "signed: $deb" + else + echo "::error::missing _gpgmaint signature in $deb" >&2 + rc=1 + fi + done + exit $rc + + - name: Verify the set is self-contained + # Every runtime must ship its own extension control file and SQL, so a + # package can never be paired with another version's SQL and there is no + # separate -scripts package to depend on. + run: | + rc=0 + for v in ${PG_VERSIONS}; do + deb="$(ls packages/focal/tdigest/postgresql-${v}-tdigest_*.deb)" + if ! dpkg-deb -c "$deb" | grep -q 'extension/tdigest\.control'; then + echo "::error::PG${v} runtime ships no extension control file" >&2; rc=1 + fi + if ! dpkg-deb -c "$deb" | grep -q "extension/tdigest--${TDIGEST_VERSION}\.sql"; then + echo "::error::PG${v} runtime ships no tdigest--${TDIGEST_VERSION}.sql" >&2; rc=1 + fi + done + exit $rc + + - name: Upload signed packages + uses: actions/upload-artifact@v4 + with: + name: tdigest-focal-deb + path: | + packages/focal/tdigest/*.deb + packages/focal/tdigest/*.changes + packages/focal/tdigest/*.buildinfo + if-no-files-found: error + + install-smoke-test: + needs: build-and-sign + name: Install smoke test (focal) + runs-on: ubuntu-latest + env: + PG_VERSIONS: ${{ github.event.inputs.pg_versions || '12 13 14 15 16' }} + TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.5' }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download built packages + uses: actions/download-artifact@v4 + with: + name: tdigest-focal-deb + path: debs + + - name: Install and verify in a clean focal container + # The jobs above only prove the packages exist, are signed and are + # self-contained -- not that the extension can actually be created. This + # installs the set into a stock ubuntu:20.04 twice: clean, and over + # PGDG's tdigest 1.4.3 (the in-place upgrade path), then runs + # CREATE EXTENSION / ALTER EXTENSION UPDATE on every major. + run: | + docker run --rm \ + -v "${PWD}/debs:/debs:ro" \ + -v "${PWD}/scripts/smoke_test_focal_tdigest_debs:/usr/local/bin/smoke_test_focal_tdigest_debs:ro" \ + -e DEBS_DIR=/debs \ + -e PG_VERSIONS="${PG_VERSIONS}" \ + -e EXPECTED_TDIGEST="${TDIGEST_VERSION}" \ + ubuntu:20.04 \ + /usr/local/bin/smoke_test_focal_tdigest_debs diff --git a/dockerfiles/focal-tdigest-builder/Dockerfile b/dockerfiles/focal-tdigest-builder/Dockerfile new file mode 100644 index 00000000..84ba137b --- /dev/null +++ b/dockerfiles/focal-tdigest-builder/Dockerfile @@ -0,0 +1,69 @@ +# vim:set ft=dockerfile: +# +# Builder image for tdigest packages targeting Ubuntu 20.04 (focal). One image +# builds every focal-buildable PostgreSQL major at once (PG 12..16 by default); +# like PostGIS -- and unlike PostgreSQL core -- the tdigest Debian packaging is +# multi-version by design (debian/pgversions + the pgxs_loop debhelper addon +# drive pg_buildext), so a single source build emits postgresql--tdigest +# for each major in one pass. +# +# Why this exists: +# apt.postgresql.org (PGDG) no longer ships focal binaries -- focal-pgdg 404s +# and the frozen apt-archive mirror tops out at tdigest 1.4.3. Anything newer +# (e.g. 1.4.5 for the memory-safety fixes) has to be rebuilt from upstream +# source. +# +# Strategy (mirrors dockerfiles/focal-postgis-builder/Dockerfile): +# - Upstream source: tdigest-.tar.gz from GitHub, sha256-pinned. +# - Debian packaging: the frozen focal-era debian/ from PGDG's last focal +# tdigest source package (1.4.3-1.pgdg20.04+1), fetched with `apt-get source` +# so it is authenticated by the archive's signed Release. +# - Build tooling restored from the PGDG *archive*, which keeps the removed +# focal-pgdg suite. +# +# The heavy lifting lives in scripts/build_tdigest_focal (the entrypoint). +FROM ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + +# PGDG repository signing key fingerprint: +# B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl gnupg; \ + install -d /usr/share/keyrings; \ + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg; \ + # 'main' carries the build tooling and the tdigest source package; each + # PostgreSQL major is a separate component. tdigest supports PG12+ here and + # Marlin still ships PG12, so component 12 is listed even though + # focal-pg-builder omits it. + echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 12 13 14 15 16" \ + > /etc/apt/sources.list.d/pgdg-archive.list; \ + echo "deb-src [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 12 13 14 15 16" \ + >> /etc/apt/sources.list.d/pgdg-archive.list; \ + apt-get update; \ + # base build tooling; per-build Build-Depends are resolved at run time by + # scripts/build_tdigest_focal via mk-build-deps against debian/control. + apt-get install -y --no-install-recommends \ + build-essential \ + devscripts \ + equivs \ + fakeroot \ + quilt \ + dpkg-dev \ + debhelper \ + dh-exec \ + postgresql-common-dev \ + postgresql-server-dev-all \ + xz-utils; \ + rm -rf /var/lib/apt/lists/* + +# Fail the image build early if the archived focal-pgdg debhelper (>= 13) is not +# what we picked up (debhelper-compat (= 13) is required by the packaging). +RUN dpkg-query -W -f='${Package} ${Version}\n' debhelper postgresql-common-dev dh-exec + +COPY scripts/build_tdigest_focal /usr/local/bin/build_tdigest_focal +RUN chmod +x /usr/local/bin/build_tdigest_focal + +VOLUME /packages +ENTRYPOINT ["/usr/local/bin/build_tdigest_focal"] diff --git a/scripts/build_tdigest_focal b/scripts/build_tdigest_focal new file mode 100755 index 00000000..7050ec91 --- /dev/null +++ b/scripts/build_tdigest_focal @@ -0,0 +1,173 @@ +#!/bin/bash +# +# build_tdigest_focal -- rebuild tdigest Debian packages for Ubuntu 20.04 (focal). +# +# PGDG removed focal entirely (focal-pgdg 404s; the frozen apt-archive mirror +# stops at tdigest 1.4.3), so a newer tdigest on focal has to be rebuilt from +# upstream source. +# +# Approach (see dockerfiles/focal-tdigest-builder/Dockerfile for the rationale): +# upstream tdigest-.tar.gz + the frozen focal-era debian/ packaging +# from PGDG's last focal tdigest source package +# -> dpkg-buildpackage inside a focal environment with the archived focal-pgdg +# build tooling available. +# +# Like PostGIS -- and unlike PostgreSQL core -- one source build emits packages +# for *every* PostgreSQL major at once. tdigest is a plain PGXS extension whose +# Debian packaging is multi-version by design (debian/pgversions + the +# pgxs_loop debhelper addon drive pg_buildext), so a single source build emits +# postgresql--tdigest for each requested major in one pass. There is no +# per-major matrix here. +# +# tdigest's packaging is much simpler than PostGIS's: there is no arch-independent +# -scripts package and no update-alternatives dance -- each runtime package ships +# its own extension control file and SQL, so none of the PostGIS control-suffix / +# alternatives workarounds are needed. +# +# Output: unsigned *.deb in ${OUTPUT_DIR} (default /packages/focal/tdigest). +# Signing is a separate step performed by the debsigner image (debsigs +# --sign=maint), so the build and the signing key never live in the same +# container. +# +# Examples: +# build_tdigest_focal # 1.4.5 for PG12..16 +# TDIGEST_VERSION=1.4.6 build_tdigest_focal +# PG_VERSIONS="15 16" RUN_TESTS=1 build_tdigest_focal + +set -euo pipefail + +# ---------------------------------------------------------------------------- +# Inputs (override via env) +# ---------------------------------------------------------------------------- +TDIGEST_VERSION="${TDIGEST_VERSION:-1.4.5}" + +# tdigest supports PostgreSQL 10+, but focal-relevant Citus/Marlin ships PG12+, +# so the default set matches the focal PostGIS pipeline. Add "11" here if a +# PG11 build is ever needed. +PG_VERSIONS="${PG_VERSIONS:-12 13 14 15 16}" + +# The last focal tdigest source package PGDG published. Its debian/ is frozen +# (focal is EOL); override only if the archive is ever re-touched. +PACKAGING_SRC_VERSION="${PACKAGING_SRC_VERSION:-1.4.3-1.pgdg20.04+1}" + +# sha256 of the upstream tarball. Pinned by default: the GitHub tag archive +# serves no detached signature, so this is the only integrity check available +# and it must not silently degrade to "whatever was served". +TDIGEST_SHA256="${TDIGEST_SHA256:-4b84834e88a87f2b3538fbd2ac56033642b7e14b2744e78344b16d6e609733af}" + +DEB_REVISION="${DEB_REVISION:-1.citus20.04+1}" +TARGET_VERSION="${TARGET_VERSION:-${TDIGEST_VERSION}-${DEB_REVISION}}" + +# Set RUN_TESTS=1 to run the upstream regression suite (much slower, and pulls +# in postgresql-all for a full test matrix). +RUN_TESTS="${RUN_TESTS:-0}" + +OUTPUT_DIR="${OUTPUT_DIR:-/packages/focal/tdigest}" +WORK="${WORK_DIR:-/build}" +UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/tvondra/tdigest/archive/refs/tags/v${TDIGEST_VERSION}.tar.gz}" + +export DEBEMAIL="${DEBEMAIL:-packaging@citusdata.com}" +export DEBFULLNAME="${DEBFULLNAME:-Citus Data}" +export DEBIAN_FRONTEND=noninteractive + +echo "==> Building tdigest ${TDIGEST_VERSION} as ${TARGET_VERSION} for PG${PG_VERSIONS// /,} (focal)" + +mkdir -p "${WORK}" "${OUTPUT_DIR}" +cd "${WORK}" + +echo "==> [1/6] Fetch frozen focal debian/ packaging (tdigest ${PACKAGING_SRC_VERSION})" +apt-get update +# apt-get source validates against the archive's signed Release file. +apt-get source "tdigest=${PACKAGING_SRC_VERSION}" +PKG_DIR="$(find "${WORK}" -maxdepth 1 -type d -name 'tdigest-*' -exec test -d '{}/debian' \; -print | head -1)" +[ -n "${PKG_DIR}" ] || { echo "ERROR: could not locate unpacked packaging source" >&2; exit 1; } + +echo "==> [2/6] Fetch and verify upstream tdigest-${TDIGEST_VERSION}.tar.gz" +curl -4 -fsSL -o "tdigest-${TDIGEST_VERSION}.tar.gz" "${UPSTREAM_URL}" +echo "${TDIGEST_SHA256} tdigest-${TDIGEST_VERSION}.tar.gz" | sha256sum -c - + +SRCDIR="${WORK}/tdigest-${TDIGEST_VERSION}" +rm -rf "${SRCDIR}" +tar xzf "tdigest-${TDIGEST_VERSION}.tar.gz" +cp -a "${PKG_DIR}/debian" "${SRCDIR}/debian" +cd "${SRCDIR}" + +echo "==> [3/6] Triage quilt patches against the new upstream" +if [ -f debian/patches/series ]; then + KEPT="" + while read -r patch; do + [ -z "${patch}" ] && continue + case "${patch}" in \#*) continue ;; esac + if patch -p1 --dry-run --silent < "debian/patches/${patch}" >/dev/null 2>&1; then + echo " keep ${patch}"; KEPT="${KEPT}${patch}\n" + else + echo " drop ${patch} (does not apply to ${TDIGEST_VERSION})" + fi + done < debian/patches/series + printf "%b" "${KEPT}" > debian/patches/series +else + echo " no debian/patches/series -- nothing to triage" +fi + +echo "==> [4/6] Target PostgreSQL majors: ${PG_VERSIONS}" +# debian/control ships with every major PGDG built (10..17); regenerate it from +# debian/control.in so only the requested majors are declared and built. +printf '%s\n' ${PG_VERSIONS} > debian/pgversions +pg_buildext updatecontrol +echo " binary packages now declared:" +grep '^Package:' debian/control | sed 's/^/ /' + +echo "==> [5/6] Set version to ${TARGET_VERSION} and install Build-Depends" +dch --newversion "${TARGET_VERSION}" --distribution focal --force-distribution \ + "Rebuild of tdigest ${TDIGEST_VERSION} for focal (upstream PGDG focal-pgdg discontinued)." + +BUILD_OPTIONS="parallel=$(nproc)" +BUILD_PROFILES="" +if [ "${RUN_TESTS}" != "1" ]; then + # nocheck drops the postgresql-all Build-Depends (a full server + # matrix pulled in only for the upstream test suite) and skips dh_auto_test. + BUILD_OPTIONS="${BUILD_OPTIONS} nocheck" + BUILD_PROFILES="nocheck" +fi +export DEB_BUILD_OPTIONS="${BUILD_OPTIONS}" +export DEB_BUILD_PROFILES="${BUILD_PROFILES}" + +mk-build-deps --install --remove \ + --tool 'apt-get -o Debug::pkgProblemResolver=yes --yes --no-install-recommends' \ + debian/control + +echo "==> [6/6] Build binary packages (DEB_BUILD_OPTIONS='${BUILD_OPTIONS}')" +dpkg-buildpackage -b -uc -us + +echo "==> Collect artifacts into ${OUTPUT_DIR}" +for v in ${PG_VERSIONS}; do + cp -v "${WORK}"/postgresql-${v}-tdigest_*.deb "${OUTPUT_DIR}/" + # Debug-symbol packages are emitted as .ddeb on Ubuntu. Ship them as .deb (the + # on-disk format is identical) so they flow through the same signing, + # verification and publishing as everything else. Consumers commonly install + # with an `*.deb` glob, which does not match `.ddeb`; because a dbgsym depends + # on its runtime with an exact `=` version, silently dropping it strands the + # previously installed dbgsym and breaks `apt --fix-broken install`. + for ddeb in "${WORK}"/postgresql-${v}-tdigest-dbgsym_*.ddeb; do + [ -e "${ddeb}" ] || continue + cp -v "${ddeb}" "${OUTPUT_DIR}/$(basename "${ddeb}" .ddeb).deb" + done +done +cp -v "${WORK}"/*.buildinfo "${WORK}"/*.changes "${OUTPUT_DIR}/" 2>/dev/null || true + +echo "==> Verify runtime <-> dbgsym pairing" +for v in ${PG_VERSIONS}; do + rt="$(dpkg-deb -f "${OUTPUT_DIR}"/postgresql-${v}-tdigest_*.deb Version)" + for dbg in "${OUTPUT_DIR}"/postgresql-${v}-tdigest-dbgsym_*.deb; do + [ -e "${dbg}" ] || continue + dep="$(dpkg-deb -f "${dbg}" Depends)" + if [ "${dep}" != "postgresql-${v}-tdigest (= ${rt})" ]; then + echo "ERROR: PG${v} dbgsym wants '${dep}' but runtime is '${rt}'" >&2 + exit 1 + fi + done + echo " PG${v} OK (${rt})" +done + +echo "==> DONE. Packages:" +ls -1 "${OUTPUT_DIR}"/*.deb diff --git a/scripts/smoke_test_focal_tdigest_debs b/scripts/smoke_test_focal_tdigest_debs new file mode 100755 index 00000000..ffc4c8b2 --- /dev/null +++ b/scripts/smoke_test_focal_tdigest_debs @@ -0,0 +1,138 @@ +#!/bin/bash +# +# smoke_test_focal_tdigest_debs -- install the built tdigest .deb set inside a +# clean Ubuntu 20.04 (focal) container and prove it actually works. +# +# The build pipeline already checks the packages exist, are signed, and pair +# correctly with their dbgsym. None of that proves the extension is usable, so +# this installs the set into a stock ubuntu:20.04 and runs CREATE EXTENSION plus +# a real aggregate query on every major. +# +# It tests both paths that matter: +# A. clean install on a stock focal + PGDG PostgreSQL +# B. install *over* PGDG's tdigest 1.4.3 and ALTER EXTENSION ... UPDATE, which +# is what an in-place upgrade on an existing host actually does +# +# Usage (inside a stock ubuntu:20.04 container, as root): +# DEBS_DIR=/debs smoke_test_focal_tdigest_debs +# +# Overridable via environment: +# DEBS_DIR directory holding the *.deb set (default /debs) +# PG_VERSIONS majors to verify (default "12 13 14 15 16") +# EXPECTED_TDIGEST expected extension version (default 1.4.5) +# OLD_TDIGEST PGDG version to upgrade from in test B (default 1.4.3-1.pgdg20.04+1) +# PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) + +set -uo pipefail + +DEBS_DIR="${DEBS_DIR:-/debs}" +PG_VERSIONS="${PG_VERSIONS:-12 13 14 15 16}" +EXPECTED_TDIGEST="${EXPECTED_TDIGEST:-1.4.5}" +OLD_TDIGEST="${OLD_TDIGEST:-1.4.3-1.pgdg20.04+1}" +PGDG_ARCHIVE_SUITE="${PGDG_ARCHIVE_SUITE:-focal-pgdg}" +PGDG_ARCHIVE_URL="${PGDG_ARCHIVE_URL:-https://apt-archive.postgresql.org/pub/repos/apt}" + +export DEBIAN_FRONTEND=noninteractive +export LANG=C.UTF-8 + +fail=0 +note() { echo "==> $*"; } +err() { echo "::error::$*" >&2; fail=1; } + +shopt -s nullglob +debs=("${DEBS_DIR}"/*.deb) +shopt -u nullglob +[ ${#debs[@]} -gt 0 ] || { echo "ERROR: no .deb files found in ${DEBS_DIR}" >&2; exit 1; } + +note "[1/6] Preparing container environment (${#debs[@]} packages)" +# postgresql-NN.postinst ends in `invoke-rc.d postgresql start $VERSION` under +# `set -e`, and there is no systemd in a container. A policy-rc.d denying the +# action makes invoke-rc.d return 0 so the postinst still creates the cluster; +# clusters are started explicitly below. +printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d +chmod +x /usr/sbin/policy-rc.d + +apt-get update -qq +apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg >/dev/null + +note "[2/6] Enabling the PGDG archive" +install -d /usr/share/keyrings +curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg +echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} ${PGDG_ARCHIVE_SUITE} main 12 13 14 15 16" \ + > /etc/apt/sources.list.d/pgdg-archive.list +apt-get update -qq + +psql_v() { su postgres -c "psql --cluster $1/main -qtAX -c \"$2\"" 2>&1; } + +ensure_cluster() { + pg_lsclusters -h | awk '{print $1"/"$2}' | grep -qx "$1/main" \ + || pg_createcluster "$1" main >/dev/null 2>&1 + pg_ctlcluster "$1" main start >/dev/null 2>&1 +} + +# Median of 1..1000 is ~500. t-digest is approximate, so accept a small window. +check_percentile() { + local v="$1" p + p="$(psql_v "${v}" "SELECT round(tdigest_percentile(i, 100, 0.5)) FROM generate_series(1,1000) i;")" + awk -v p="${p}" 'BEGIN{exit !(p+0>=480 && p+0<=520)}' +} + +note "[3/6] Scenario A: clean install" +for v in ${PG_VERSIONS}; do + apt-get install -y -qq postgresql-${v} >/dev/null 2>&1 +done +dpkg -i --force-all "${DEBS_DIR}"/*.deb >/dev/null 2>&1 +if ! apt-get --fix-broken install -y >/dev/null 2>&1; then + err "apt --fix-broken install failed after clean install" +fi + +for v in ${PG_VERSIONS}; do + ensure_cluster "${v}" + psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 + ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" + if [ "${ext}" = "${EXPECTED_TDIGEST}" ] && check_percentile "${v}"; then + echo " PG${v} OK ext=${ext}" + else + err "PG${v} clean install: ext='${ext}' (expected ${EXPECTED_TDIGEST}) or percentile check failed" + fi + pg_ctlcluster "${v}" main stop >/dev/null 2>&1 +done + +note "[4/6] Scenario B: upgrade over PGDG tdigest ${OLD_TDIGEST}" +for v in ${PG_VERSIONS}; do + apt-get remove -y -qq postgresql-${v}-tdigest >/dev/null 2>&1 + pg_dropcluster "${v}" main >/dev/null 2>&1 + pg_createcluster "${v}" main >/dev/null 2>&1 + apt-get install -y -qq --allow-downgrades \ + postgresql-${v}-tdigest=${OLD_TDIGEST} >/dev/null 2>&1 + ensure_cluster "${v}" + psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 + pg_ctlcluster "${v}" main stop >/dev/null 2>&1 +done + +# Mirrors how consumers apply the set: force-all dpkg, then let apt settle. +dpkg -i --force-all "${DEBS_DIR}"/*.deb >/dev/null 2>&1 +if ! apt-get --fix-broken install -y >/dev/null 2>&1; then + err "apt --fix-broken install failed after upgrade over ${OLD_TDIGEST}" +fi + +note "[5/6] Verifying the extension upgrades and works after the package upgrade" +for v in ${PG_VERSIONS}; do + ensure_cluster "${v}" + out="$(psql_v "${v}" "ALTER EXTENSION tdigest UPDATE;")" + ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" + if [ "${ext}" = "${EXPECTED_TDIGEST}" ] && check_percentile "${v}"; then + echo " PG${v} OK ext=${ext}" + else + err "PG${v} after upgrade: ext='${ext}' (expected ${EXPECTED_TDIGEST}) ${out}" + fi + pg_ctlcluster "${v}" main stop >/dev/null 2>&1 +done + +note "[6/6] Result" +if [ "${fail}" -ne 0 ]; then + echo "SMOKE TEST FAILED" >&2 + exit 1 +fi +echo "SMOKE TEST PASSED" From d6d31a57dbc80563afe0c3bfe19624dff320c5bd Mon Sep 17 00:00:00 2001 From: aybuke Date: Wed, 9 Sep 2026 21:10:43 +0000 Subject: [PATCH 2/5] focal tdigest: include PG11 in the default build set --- .github/workflows/build-tdigest-focal.yml | 6 +++--- dockerfiles/focal-tdigest-builder/Dockerfile | 12 +++++------- scripts/build_tdigest_focal | 9 ++++----- scripts/smoke_test_focal_tdigest_debs | 6 +++--- 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build-tdigest-focal.yml b/.github/workflows/build-tdigest-focal.yml index 57f68671..67fd7d5f 100644 --- a/.github/workflows/build-tdigest-focal.yml +++ b/.github/workflows/build-tdigest-focal.yml @@ -21,7 +21,7 @@ on: pg_versions: description: "Space-separated PostgreSQL majors (tdigest supports 10+)" required: false - default: "12 13 14 15 16" + default: "11 12 13 14 15 16" tdigest_sha256: description: "sha256 of tdigest-.tar.gz. Blank = use the pinned default in scripts/build_tdigest_focal (only valid for that version)." required: false @@ -49,7 +49,7 @@ jobs: PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.5' }} - PG_VERSIONS: ${{ github.event.inputs.pg_versions || '12 13 14 15 16' }} + PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} TDIGEST_SHA256: ${{ github.event.inputs.tdigest_sha256 || '' }} RUN_TESTS: ${{ github.event.inputs.run_tests || '0' }} steps: @@ -142,7 +142,7 @@ jobs: name: Install smoke test (focal) runs-on: ubuntu-latest env: - PG_VERSIONS: ${{ github.event.inputs.pg_versions || '12 13 14 15 16' }} + PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.5' }} steps: - name: Checkout repository diff --git a/dockerfiles/focal-tdigest-builder/Dockerfile b/dockerfiles/focal-tdigest-builder/Dockerfile index 84ba137b..778c647a 100644 --- a/dockerfiles/focal-tdigest-builder/Dockerfile +++ b/dockerfiles/focal-tdigest-builder/Dockerfile @@ -1,7 +1,7 @@ # vim:set ft=dockerfile: # # Builder image for tdigest packages targeting Ubuntu 20.04 (focal). One image -# builds every focal-buildable PostgreSQL major at once (PG 12..16 by default); +# builds every focal-buildable PostgreSQL major at once (PG 11..16 by default); # like PostGIS -- and unlike PostgreSQL core -- the tdigest Debian packaging is # multi-version by design (debian/pgversions + the pgxs_loop debhelper addon # drive pg_buildext), so a single source build emits postgresql--tdigest @@ -33,13 +33,11 @@ RUN set -ex; \ install -d /usr/share/keyrings; \ curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg; \ - # 'main' carries the build tooling and the tdigest source package; each - # PostgreSQL major is a separate component. tdigest supports PG12+ here and - # Marlin still ships PG12, so component 12 is listed even though - # focal-pg-builder omits it. - echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 12 13 14 15 16" \ + # 'main' carries the build tooling, the tdigest source package and the PG11 + # binaries; PG12..16 are each a separate component. + echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 11 12 13 14 15 16" \ > /etc/apt/sources.list.d/pgdg-archive.list; \ - echo "deb-src [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 12 13 14 15 16" \ + echo "deb-src [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 11 12 13 14 15 16" \ >> /etc/apt/sources.list.d/pgdg-archive.list; \ apt-get update; \ # base build tooling; per-build Build-Depends are resolved at run time by diff --git a/scripts/build_tdigest_focal b/scripts/build_tdigest_focal index 7050ec91..f6284700 100755 --- a/scripts/build_tdigest_focal +++ b/scripts/build_tdigest_focal @@ -30,7 +30,7 @@ # container. # # Examples: -# build_tdigest_focal # 1.4.5 for PG12..16 +# build_tdigest_focal # 1.4.5 for PG11..16 # TDIGEST_VERSION=1.4.6 build_tdigest_focal # PG_VERSIONS="15 16" RUN_TESTS=1 build_tdigest_focal @@ -41,10 +41,9 @@ set -euo pipefail # ---------------------------------------------------------------------------- TDIGEST_VERSION="${TDIGEST_VERSION:-1.4.5}" -# tdigest supports PostgreSQL 10+, but focal-relevant Citus/Marlin ships PG12+, -# so the default set matches the focal PostGIS pipeline. Add "11" here if a -# PG11 build is ever needed. -PG_VERSIONS="${PG_VERSIONS:-12 13 14 15 16}" +# tdigest supports PostgreSQL 10+. focal shipped PG11..16, so the default set +# covers every focal-buildable major; drop "11" if only PG12+ is needed. +PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" # The last focal tdigest source package PGDG published. Its debian/ is frozen # (focal is EOL); override only if the archive is ever re-touched. diff --git a/scripts/smoke_test_focal_tdigest_debs b/scripts/smoke_test_focal_tdigest_debs index ffc4c8b2..db903c76 100755 --- a/scripts/smoke_test_focal_tdigest_debs +++ b/scripts/smoke_test_focal_tdigest_debs @@ -18,7 +18,7 @@ # # Overridable via environment: # DEBS_DIR directory holding the *.deb set (default /debs) -# PG_VERSIONS majors to verify (default "12 13 14 15 16") +# PG_VERSIONS majors to verify (default "11 12 13 14 15 16") # EXPECTED_TDIGEST expected extension version (default 1.4.5) # OLD_TDIGEST PGDG version to upgrade from in test B (default 1.4.3-1.pgdg20.04+1) # PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) @@ -26,7 +26,7 @@ set -uo pipefail DEBS_DIR="${DEBS_DIR:-/debs}" -PG_VERSIONS="${PG_VERSIONS:-12 13 14 15 16}" +PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" EXPECTED_TDIGEST="${EXPECTED_TDIGEST:-1.4.5}" OLD_TDIGEST="${OLD_TDIGEST:-1.4.3-1.pgdg20.04+1}" PGDG_ARCHIVE_SUITE="${PGDG_ARCHIVE_SUITE:-focal-pgdg}" @@ -59,7 +59,7 @@ note "[2/6] Enabling the PGDG archive" install -d /usr/share/keyrings curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg -echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} ${PGDG_ARCHIVE_SUITE} main 12 13 14 15 16" \ +echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} ${PGDG_ARCHIVE_SUITE} main 11 12 13 14 15 16" \ > /etc/apt/sources.list.d/pgdg-archive.list apt-get update -qq From 439675c5a0fe5dca3d22f6b3406e7b7f09880f62 Mon Sep 17 00:00:00 2001 From: aybuke Date: Thu, 10 Sep 2026 17:43:32 +0000 Subject: [PATCH 3/5] focal tdigest: bump to 1.4.6 --- .github/workflows/build-tdigest-focal.yml | 37 +++++++++++--------- dockerfiles/focal-tdigest-builder/Dockerfile | 12 +++---- scripts/build_tdigest_focal | 25 ++++++------- scripts/smoke_test_focal_tdigest_debs | 4 +-- 4 files changed, 39 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-tdigest-focal.yml b/.github/workflows/build-tdigest-focal.yml index 67fd7d5f..7c271de6 100644 --- a/.github/workflows/build-tdigest-focal.yml +++ b/.github/workflows/build-tdigest-focal.yml @@ -5,19 +5,19 @@ name: Build tdigest (focal) # stops at tdigest 1.4.3), then signs them with debsigs (--sign=maint) using the # existing packaging key. # -# Like build-postgis-focal.yml there is no per-major matrix: the tdigest Debian -# packaging is multi-version by design (debian/pgversions + pg_buildext), so one -# source build emits postgresql--tdigest for every requested major in a -# single pass. That also means no assemble/de-duplicate job is needed -- each -# runtime package is self-contained and produced exactly once. +# There is no per-major matrix: the tdigest Debian packaging is multi-version by +# design (debian/pgversions + pg_buildext), so one source build emits +# postgresql--tdigest for every requested major in a single pass. That +# also means no assemble/de-duplicate job is needed -- each runtime package is +# self-contained and produced exactly once. on: workflow_dispatch: inputs: tdigest_version: - description: "tdigest upstream version to build (e.g. 1.4.5)" + description: "tdigest upstream version to build (e.g. 1.4.6)" required: true - default: "1.4.5" + default: "1.4.6" pg_versions: description: "Space-separated PostgreSQL majors (tdigest supports 10+)" required: false @@ -43,12 +43,12 @@ concurrency: jobs: build-and-sign: - name: Build & sign tdigest ${{ github.event.inputs.tdigest_version || '1.4.5' }} (focal) + name: Build & sign tdigest ${{ github.event.inputs.tdigest_version || '1.4.6' }} (focal) runs-on: ubuntu-latest env: PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} - TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.5' }} + TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.6' }} PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} TDIGEST_SHA256: ${{ github.event.inputs.tdigest_sha256 || '' }} RUN_TESTS: ${{ github.event.inputs.run_tests || '0' }} @@ -111,18 +111,23 @@ jobs: exit $rc - name: Verify the set is self-contained - # Every runtime must ship its own extension control file and SQL, so a - # package can never be paired with another version's SQL and there is no - # separate -scripts package to depend on. + # Every runtime must ship its own extension control file and the full SQL + # tree, so a package can never be paired with another version's SQL and + # there is no separate -scripts package to depend on. tdigest is installed + # from the base script and reaches the built version through the shipped + # upgrade chain, whose terminal script names that version (e.g. + # tdigest--1.4.5--1.4.6.sql), so match either that or a direct + # tdigest--.sql. run: | rc=0 for v in ${PG_VERSIONS}; do deb="$(ls packages/focal/tdigest/postgresql-${v}-tdigest_*.deb)" - if ! dpkg-deb -c "$deb" | grep -q 'extension/tdigest\.control'; then + contents="$(dpkg-deb -c "$deb")" + if ! grep -q 'extension/tdigest\.control' <<<"$contents"; then echo "::error::PG${v} runtime ships no extension control file" >&2; rc=1 fi - if ! dpkg-deb -c "$deb" | grep -q "extension/tdigest--${TDIGEST_VERSION}\.sql"; then - echo "::error::PG${v} runtime ships no tdigest--${TDIGEST_VERSION}.sql" >&2; rc=1 + if ! grep -qE "extension/tdigest--([0-9.]+--)?${TDIGEST_VERSION}\.sql" <<<"$contents"; then + echo "::error::PG${v} runtime ships no SQL producing tdigest ${TDIGEST_VERSION}" >&2; rc=1 fi done exit $rc @@ -143,7 +148,7 @@ jobs: runs-on: ubuntu-latest env: PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} - TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.5' }} + TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.6' }} steps: - name: Checkout repository uses: actions/checkout@v6 diff --git a/dockerfiles/focal-tdigest-builder/Dockerfile b/dockerfiles/focal-tdigest-builder/Dockerfile index 778c647a..9da80ce1 100644 --- a/dockerfiles/focal-tdigest-builder/Dockerfile +++ b/dockerfiles/focal-tdigest-builder/Dockerfile @@ -2,18 +2,16 @@ # # Builder image for tdigest packages targeting Ubuntu 20.04 (focal). One image # builds every focal-buildable PostgreSQL major at once (PG 11..16 by default); -# like PostGIS -- and unlike PostgreSQL core -- the tdigest Debian packaging is -# multi-version by design (debian/pgversions + the pgxs_loop debhelper addon -# drive pg_buildext), so a single source build emits postgresql--tdigest -# for each major in one pass. +# the tdigest Debian packaging is multi-version by design (debian/pgversions + +# the pgxs_loop debhelper addon drive pg_buildext), so a single source build +# emits postgresql--tdigest for each major in one pass. # # Why this exists: # apt.postgresql.org (PGDG) no longer ships focal binaries -- focal-pgdg 404s # and the frozen apt-archive mirror tops out at tdigest 1.4.3. Anything newer -# (e.g. 1.4.5 for the memory-safety fixes) has to be rebuilt from upstream -# source. +# (e.g. 1.4.6) has to be rebuilt from upstream source. # -# Strategy (mirrors dockerfiles/focal-postgis-builder/Dockerfile): +# Strategy: # - Upstream source: tdigest-.tar.gz from GitHub, sha256-pinned. # - Debian packaging: the frozen focal-era debian/ from PGDG's last focal # tdigest source package (1.4.3-1.pgdg20.04+1), fetched with `apt-get source` diff --git a/scripts/build_tdigest_focal b/scripts/build_tdigest_focal index f6284700..d40d8dcd 100755 --- a/scripts/build_tdigest_focal +++ b/scripts/build_tdigest_focal @@ -12,17 +12,14 @@ # -> dpkg-buildpackage inside a focal environment with the archived focal-pgdg # build tooling available. # -# Like PostGIS -- and unlike PostgreSQL core -- one source build emits packages -# for *every* PostgreSQL major at once. tdigest is a plain PGXS extension whose -# Debian packaging is multi-version by design (debian/pgversions + the -# pgxs_loop debhelper addon drive pg_buildext), so a single source build emits -# postgresql--tdigest for each requested major in one pass. There is no -# per-major matrix here. +# One source build emits packages for *every* PostgreSQL major at once. tdigest +# is a plain PGXS extension whose Debian packaging is multi-version by design +# (debian/pgversions + the pgxs_loop debhelper addon drive pg_buildext), so a +# single source build emits postgresql--tdigest for each requested major +# in one pass. There is no per-major matrix here. # -# tdigest's packaging is much simpler than PostGIS's: there is no arch-independent -# -scripts package and no update-alternatives dance -- each runtime package ships -# its own extension control file and SQL, so none of the PostGIS control-suffix / -# alternatives workarounds are needed. +# Each runtime package ships its own extension control file and SQL; there is no +# separate -scripts package and no update-alternatives handling to worry about. # # Output: unsigned *.deb in ${OUTPUT_DIR} (default /packages/focal/tdigest). # Signing is a separate step performed by the debsigner image (debsigs @@ -30,7 +27,7 @@ # container. # # Examples: -# build_tdigest_focal # 1.4.5 for PG11..16 +# build_tdigest_focal # 1.4.6 for PG11..16 # TDIGEST_VERSION=1.4.6 build_tdigest_focal # PG_VERSIONS="15 16" RUN_TESTS=1 build_tdigest_focal @@ -39,10 +36,10 @@ set -euo pipefail # ---------------------------------------------------------------------------- # Inputs (override via env) # ---------------------------------------------------------------------------- -TDIGEST_VERSION="${TDIGEST_VERSION:-1.4.5}" +TDIGEST_VERSION="${TDIGEST_VERSION:-1.4.6}" # tdigest supports PostgreSQL 10+. focal shipped PG11..16, so the default set -# covers every focal-buildable major; drop "11" if only PG12+ is needed. +# covers every focal-buildable major PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" # The last focal tdigest source package PGDG published. Its debian/ is frozen @@ -52,7 +49,7 @@ PACKAGING_SRC_VERSION="${PACKAGING_SRC_VERSION:-1.4.3-1.pgdg20.04+1}" # sha256 of the upstream tarball. Pinned by default: the GitHub tag archive # serves no detached signature, so this is the only integrity check available # and it must not silently degrade to "whatever was served". -TDIGEST_SHA256="${TDIGEST_SHA256:-4b84834e88a87f2b3538fbd2ac56033642b7e14b2744e78344b16d6e609733af}" +TDIGEST_SHA256="${TDIGEST_SHA256:-5158d3a57e96883262a299d559ffb18da80616825b13f29f1c8b3113dd467a95}" DEB_REVISION="${DEB_REVISION:-1.citus20.04+1}" TARGET_VERSION="${TARGET_VERSION:-${TDIGEST_VERSION}-${DEB_REVISION}}" diff --git a/scripts/smoke_test_focal_tdigest_debs b/scripts/smoke_test_focal_tdigest_debs index db903c76..ac2dcbee 100755 --- a/scripts/smoke_test_focal_tdigest_debs +++ b/scripts/smoke_test_focal_tdigest_debs @@ -19,7 +19,7 @@ # Overridable via environment: # DEBS_DIR directory holding the *.deb set (default /debs) # PG_VERSIONS majors to verify (default "11 12 13 14 15 16") -# EXPECTED_TDIGEST expected extension version (default 1.4.5) +# EXPECTED_TDIGEST expected extension version (default 1.4.6) # OLD_TDIGEST PGDG version to upgrade from in test B (default 1.4.3-1.pgdg20.04+1) # PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) @@ -27,7 +27,7 @@ set -uo pipefail DEBS_DIR="${DEBS_DIR:-/debs}" PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" -EXPECTED_TDIGEST="${EXPECTED_TDIGEST:-1.4.5}" +EXPECTED_TDIGEST="${EXPECTED_TDIGEST:-1.4.6}" OLD_TDIGEST="${OLD_TDIGEST:-1.4.3-1.pgdg20.04+1}" PGDG_ARCHIVE_SUITE="${PGDG_ARCHIVE_SUITE:-focal-pgdg}" PGDG_ARCHIVE_URL="${PGDG_ARCHIVE_URL:-https://apt-archive.postgresql.org/pub/repos/apt}" From afdadd81fa00b37b47dac27ad32fb2966fe82d22 Mon Sep 17 00:00:00 2001 From: Kemal Buyukkaya Date: Fri, 18 Sep 2026 13:10:02 +0000 Subject: [PATCH 4/5] focal tdigest: discover latest release and require debug symbols Resolve the upstream release and checksum once per workflow, and discover archived packaging and upgrade baselines through APT. Validate PG11-16 inputs, isolate packaging sources, require matching DWARF symbol packages, and strengthen clean-install and upgrade checks. --- .github/workflows/build-tdigest-focal.yml | 40 ++++--- dockerfiles/focal-tdigest-builder/Dockerfile | 12 +- scripts/build_tdigest_focal | 113 ++++++++++++------- scripts/resolve_tdigest_focal | 71 ++++++++++++ scripts/smoke_test_focal_tdigest_debs | 69 ++++++++--- 5 files changed, 234 insertions(+), 71 deletions(-) create mode 100644 scripts/resolve_tdigest_focal diff --git a/.github/workflows/build-tdigest-focal.yml b/.github/workflows/build-tdigest-focal.yml index 7c271de6..efc0ca9a 100644 --- a/.github/workflows/build-tdigest-focal.yml +++ b/.github/workflows/build-tdigest-focal.yml @@ -1,8 +1,8 @@ name: Build tdigest (focal) # Builds tdigest .deb packages for Ubuntu 20.04 (focal) from upstream source -# (PGDG removed focal entirely -- focal-pgdg 404s and the frozen archive mirror -# stops at tdigest 1.4.3), then signs them with debsigs (--sign=maint) using the +# (PGDG removed focal entirely -- focal-pgdg 404s), then signs them with +# debsigs (--sign=maint) using the # existing packaging key. # # There is no per-major matrix: the tdigest Debian packaging is multi-version by @@ -15,15 +15,15 @@ on: workflow_dispatch: inputs: tdigest_version: - description: "tdigest upstream version to build (e.g. 1.4.6)" - required: true - default: "1.4.6" + description: "tdigest upstream version; blank or latest discovers the latest stable release" + required: false + default: "latest" pg_versions: - description: "Space-separated PostgreSQL majors (tdigest supports 10+)" + description: "Space-separated PostgreSQL majors (supported: 11–16)" required: false default: "11 12 13 14 15 16" tdigest_sha256: - description: "sha256 of tdigest-.tar.gz. Blank = use the pinned default in scripts/build_tdigest_focal (only valid for that version)." + description: "Optional SHA-256 for the GitHub tag tarball. Blank = use the release ZIP and its GitHub-published digest." required: false default: "" run_tests: @@ -43,12 +43,14 @@ concurrency: jobs: build-and-sign: - name: Build & sign tdigest ${{ github.event.inputs.tdigest_version || '1.4.6' }} (focal) + name: Build & sign tdigest ${{ github.event.inputs.tdigest_version || 'latest' }} (focal) runs-on: ubuntu-latest + outputs: + tdigest_version: ${{ steps.source.outputs.TDIGEST_VERSION }} env: PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} - TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.6' }} + TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || 'latest' }} PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} TDIGEST_SHA256: ${{ github.event.inputs.tdigest_sha256 || '' }} RUN_TESTS: ${{ github.event.inputs.run_tests || '0' }} @@ -56,6 +58,13 @@ jobs: - name: Checkout repository uses: actions/checkout@v6 + - name: Validate majors and resolve tdigest source + id: source + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: bash scripts/resolve_tdigest_focal | tee -a "$GITHUB_ENV" "$GITHUB_OUTPUT" + - name: Login to Docker Hub uses: docker/login-action@v4 with: @@ -74,6 +83,8 @@ jobs: -e TDIGEST_VERSION="${TDIGEST_VERSION}" \ -e PG_VERSIONS="${PG_VERSIONS}" \ -e TDIGEST_SHA256="${TDIGEST_SHA256}" \ + -e UPSTREAM_URL="${UPSTREAM_URL}" \ + -e UPSTREAM_FORMAT="${UPSTREAM_FORMAT}" \ -e RUN_TESTS="${RUN_TESTS}" \ -v "${PWD}/packages:/packages" \ focal-tdigest-builder @@ -115,13 +126,13 @@ jobs: # tree, so a package can never be paired with another version's SQL and # there is no separate -scripts package to depend on. tdigest is installed # from the base script and reaches the built version through the shipped - # upgrade chain, whose terminal script names that version (e.g. - # tdigest--1.4.5--1.4.6.sql), so match either that or a direct + # upgrade chain, whose terminal script names that version, so match + # either tdigest----.sql or a direct # tdigest--.sql. run: | rc=0 for v in ${PG_VERSIONS}; do - deb="$(ls packages/focal/tdigest/postgresql-${v}-tdigest_*.deb)" + deb="$(ls "packages/focal/tdigest/postgresql-${v}-tdigest_"*.deb)" contents="$(dpkg-deb -c "$deb")" if ! grep -q 'extension/tdigest\.control' <<<"$contents"; then echo "::error::PG${v} runtime ships no extension control file" >&2; rc=1 @@ -140,6 +151,7 @@ jobs: packages/focal/tdigest/*.deb packages/focal/tdigest/*.changes packages/focal/tdigest/*.buildinfo + packages/focal/tdigest/tdigest-source.env if-no-files-found: error install-smoke-test: @@ -148,7 +160,7 @@ jobs: runs-on: ubuntu-latest env: PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} - TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || '1.4.6' }} + TDIGEST_VERSION: ${{ needs.build-and-sign.outputs.tdigest_version }} steps: - name: Checkout repository uses: actions/checkout@v6 @@ -163,7 +175,7 @@ jobs: # The jobs above only prove the packages exist, are signed and are # self-contained -- not that the extension can actually be created. This # installs the set into a stock ubuntu:20.04 twice: clean, and over - # PGDG's tdigest 1.4.3 (the in-place upgrade path), then runs + # PGDG's archived tdigest (the in-place upgrade path), then runs # CREATE EXTENSION / ALTER EXTENSION UPDATE on every major. run: | docker run --rm \ diff --git a/dockerfiles/focal-tdigest-builder/Dockerfile b/dockerfiles/focal-tdigest-builder/Dockerfile index 9da80ce1..5f9daf5c 100644 --- a/dockerfiles/focal-tdigest-builder/Dockerfile +++ b/dockerfiles/focal-tdigest-builder/Dockerfile @@ -8,13 +8,14 @@ # # Why this exists: # apt.postgresql.org (PGDG) no longer ships focal binaries -- focal-pgdg 404s -# and the frozen apt-archive mirror tops out at tdigest 1.4.3. Anything newer -# (e.g. 1.4.6) has to be rebuilt from upstream source. +# and the apt-archive mirror is frozen. New releases have to be rebuilt +# from upstream source. # # Strategy: -# - Upstream source: tdigest-.tar.gz from GitHub, sha256-pinned. +# - Upstream source: latest GitHub release asset and its published SHA-256, +# or a specific tag tarball with an explicitly supplied checksum. # - Debian packaging: the frozen focal-era debian/ from PGDG's last focal -# tdigest source package (1.4.3-1.pgdg20.04+1), fetched with `apt-get source` +# tdigest source package, selected and fetched with `apt-get source` # so it is authenticated by the archive's signed Release. # - Build tooling restored from the PGDG *archive*, which keeps the removed # focal-pgdg suite. @@ -49,6 +50,8 @@ RUN set -ex; \ dpkg-dev \ debhelper \ dh-exec \ + jq \ + unzip \ postgresql-common-dev \ postgresql-server-dev-all \ xz-utils; \ @@ -59,6 +62,7 @@ RUN set -ex; \ RUN dpkg-query -W -f='${Package} ${Version}\n' debhelper postgresql-common-dev dh-exec COPY scripts/build_tdigest_focal /usr/local/bin/build_tdigest_focal +COPY scripts/resolve_tdigest_focal /usr/local/bin/resolve_tdigest_focal RUN chmod +x /usr/local/bin/build_tdigest_focal VOLUME /packages diff --git a/scripts/build_tdigest_focal b/scripts/build_tdigest_focal index d40d8dcd..54ea9fd7 100755 --- a/scripts/build_tdigest_focal +++ b/scripts/build_tdigest_focal @@ -2,12 +2,11 @@ # # build_tdigest_focal -- rebuild tdigest Debian packages for Ubuntu 20.04 (focal). # -# PGDG removed focal entirely (focal-pgdg 404s; the frozen apt-archive mirror -# stops at tdigest 1.4.3), so a newer tdigest on focal has to be rebuilt from -# upstream source. +# PGDG removed focal entirely (focal-pgdg 404s), so current tdigest releases +# on focal have to be rebuilt from upstream source using archived packaging. # # Approach (see dockerfiles/focal-tdigest-builder/Dockerfile for the rationale): -# upstream tdigest-.tar.gz + the frozen focal-era debian/ packaging +# upstream release archive + the frozen focal-era debian/ packaging # from PGDG's last focal tdigest source package # -> dpkg-buildpackage inside a focal environment with the archived focal-pgdg # build tooling available. @@ -27,8 +26,8 @@ # container. # # Examples: -# build_tdigest_focal # 1.4.6 for PG11..16 -# TDIGEST_VERSION=1.4.6 build_tdigest_focal +# build_tdigest_focal # latest for PG11..16 +# TDIGEST_VERSION= build_tdigest_focal # specific release # PG_VERSIONS="15 16" RUN_TESTS=1 build_tdigest_focal set -euo pipefail @@ -36,20 +35,17 @@ set -euo pipefail # ---------------------------------------------------------------------------- # Inputs (override via env) # ---------------------------------------------------------------------------- -TDIGEST_VERSION="${TDIGEST_VERSION:-1.4.6}" +# Validate inputs and resolve latest before doing any package installation. +resolved="$(bash "$(dirname "${BASH_SOURCE[0]}")/resolve_tdigest_focal")" +while IFS='=' read -r key value; do + export "${key}=${value}" +done <<< "${resolved}" -# tdigest supports PostgreSQL 10+. focal shipped PG11..16, so the default set -# covers every focal-buildable major +# This builder supports PG11..16 only. PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" -# The last focal tdigest source package PGDG published. Its debian/ is frozen -# (focal is EOL); override only if the archive is ever re-touched. -PACKAGING_SRC_VERSION="${PACKAGING_SRC_VERSION:-1.4.3-1.pgdg20.04+1}" - -# sha256 of the upstream tarball. Pinned by default: the GitHub tag archive -# serves no detached signature, so this is the only integrity check available -# and it must not silently degrade to "whatever was served". -TDIGEST_SHA256="${TDIGEST_SHA256:-5158d3a57e96883262a299d559ffb18da80616825b13f29f1c8b3113dd467a95}" +# APT selects the newest source package in the focal archive unless overridden. +PACKAGING_SRC_VERSION="${PACKAGING_SRC_VERSION:-}" DEB_REVISION="${DEB_REVISION:-1.citus20.04+1}" TARGET_VERSION="${TARGET_VERSION:-${TDIGEST_VERSION}-${DEB_REVISION}}" @@ -60,7 +56,6 @@ RUN_TESTS="${RUN_TESTS:-0}" OUTPUT_DIR="${OUTPUT_DIR:-/packages/focal/tdigest}" WORK="${WORK_DIR:-/build}" -UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/tvondra/tdigest/archive/refs/tags/v${TDIGEST_VERSION}.tar.gz}" export DEBEMAIL="${DEBEMAIL:-packaging@citusdata.com}" export DEBFULLNAME="${DEBFULLNAME:-Citus Data}" @@ -69,22 +64,37 @@ export DEBIAN_FRONTEND=noninteractive echo "==> Building tdigest ${TDIGEST_VERSION} as ${TARGET_VERSION} for PG${PG_VERSIONS// /,} (focal)" mkdir -p "${WORK}" "${OUTPUT_DIR}" -cd "${WORK}" - -echo "==> [1/6] Fetch frozen focal debian/ packaging (tdigest ${PACKAGING_SRC_VERSION})" +# Isolate each invocation, including the frozen packaging, from upstream source +# extraction and from stale artifacts left by earlier builds. +WORK="$(mktemp -d "${WORK}/tdigest.XXXXXX")" +# pg_virtualenv runs installcheck as postgres and loads staged libraries here. +chmod 755 "${WORK}" +mkdir "${WORK}/packaging" +cd "${WORK}/packaging" + +echo "==> [1/6] Fetch archived focal debian/ packaging" apt-get update # apt-get source validates against the archive's signed Release file. -apt-get source "tdigest=${PACKAGING_SRC_VERSION}" -PKG_DIR="$(find "${WORK}" -maxdepth 1 -type d -name 'tdigest-*' -exec test -d '{}/debian' \; -print | head -1)" +apt-get source "tdigest${PACKAGING_SRC_VERSION:+=$PACKAGING_SRC_VERSION}" +PKG_DIR="$(find "${WORK}/packaging" -maxdepth 1 -type d -name 'tdigest-*' -exec test -d '{}/debian' \; -print | head -1)" [ -n "${PKG_DIR}" ] || { echo "ERROR: could not locate unpacked packaging source" >&2; exit 1; } +PACKAGING_SRC_VERSION="$(dpkg-parsechangelog -l "${PKG_DIR}/debian/changelog" -S Version)" +echo " selected packaging source: ${PACKAGING_SRC_VERSION}" -echo "==> [2/6] Fetch and verify upstream tdigest-${TDIGEST_VERSION}.tar.gz" -curl -4 -fsSL -o "tdigest-${TDIGEST_VERSION}.tar.gz" "${UPSTREAM_URL}" -echo "${TDIGEST_SHA256} tdigest-${TDIGEST_VERSION}.tar.gz" | sha256sum -c - +echo "==> [2/6] Fetch and verify upstream tdigest-${TDIGEST_VERSION}.${UPSTREAM_FORMAT}" +cd "${WORK}" +archive="tdigest-${TDIGEST_VERSION}.${UPSTREAM_FORMAT}" +curl -4 -fsSL --retry 3 -o "${archive}" "${UPSTREAM_URL}" +echo "${TDIGEST_SHA256} ${archive}" | sha256sum -c - SRCDIR="${WORK}/tdigest-${TDIGEST_VERSION}" -rm -rf "${SRCDIR}" -tar xzf "tdigest-${TDIGEST_VERSION}.tar.gz" +if [ "${UPSTREAM_FORMAT}" = zip ]; then + unzip -q "${archive}" + mv "t-digest-${TDIGEST_VERSION}" "${SRCDIR}" +else + mkdir "${SRCDIR}" + tar xzf "${archive}" --strip-components=1 -C "${SRCDIR}" +fi cp -a "${PKG_DIR}/debian" "${SRCDIR}/debian" cd "${SRCDIR}" @@ -108,13 +118,15 @@ fi echo "==> [4/6] Target PostgreSQL majors: ${PG_VERSIONS}" # debian/control ships with every major PGDG built (10..17); regenerate it from # debian/control.in so only the requested majors are declared and built. -printf '%s\n' ${PG_VERSIONS} > debian/pgversions +read -r -a majors <<< "${PG_VERSIONS}" +printf '%s\n' "${majors[@]}" > debian/pgversions pg_buildext updatecontrol echo " binary packages now declared:" grep '^Package:' debian/control | sed 's/^/ /' echo "==> [5/6] Set version to ${TARGET_VERSION} and install Build-Depends" -dch --newversion "${TARGET_VERSION}" --distribution focal --force-distribution \ +# Explicit historical rebuilds may sort below the archived packaging version. +dch --force-bad-version --newversion "${TARGET_VERSION}" --distribution focal --force-distribution \ "Rebuild of tdigest ${TDIGEST_VERSION} for focal (upstream PGDG focal-pgdg discontinued)." BUILD_OPTIONS="parallel=$(nproc)" @@ -127,6 +139,11 @@ if [ "${RUN_TESTS}" != "1" ]; then fi export DEB_BUILD_OPTIONS="${BUILD_OPTIONS}" export DEB_BUILD_PROFILES="${BUILD_PROFILES}" +# Keep debug information during compilation and let dh_strip split it into +# installable dbgsym packages. Never inherit noautodbgsym/nostrip settings. +export DEB_CFLAGS_MAINT_APPEND="${DEB_CFLAGS_MAINT_APPEND:-} -g" +export DH_STRIP_ENABLE=1 +export DH_BUILD_DDEBS=1 mk-build-deps --install --remove \ --tool 'apt-get -o Debug::pkgProblemResolver=yes --yes --no-install-recommends' \ @@ -137,33 +154,53 @@ dpkg-buildpackage -b -uc -us echo "==> Collect artifacts into ${OUTPUT_DIR}" for v in ${PG_VERSIONS}; do - cp -v "${WORK}"/postgresql-${v}-tdigest_*.deb "${OUTPUT_DIR}/" + cp -v "${WORK}/postgresql-${v}-tdigest_"*.deb "${OUTPUT_DIR}/" # Debug-symbol packages are emitted as .ddeb on Ubuntu. Ship them as .deb (the # on-disk format is identical) so they flow through the same signing, # verification and publishing as everything else. Consumers commonly install # with an `*.deb` glob, which does not match `.ddeb`; because a dbgsym depends # on its runtime with an exact `=` version, silently dropping it strands the # previously installed dbgsym and breaks `apt --fix-broken install`. - for ddeb in "${WORK}"/postgresql-${v}-tdigest-dbgsym_*.ddeb; do - [ -e "${ddeb}" ] || continue - cp -v "${ddeb}" "${OUTPUT_DIR}/$(basename "${ddeb}" .ddeb).deb" - done + shopt -s nullglob + debug_packages=("${WORK}/postgresql-${v}-tdigest-dbgsym_"*.ddeb "${WORK}/postgresql-${v}-tdigest-dbgsym_"*.deb) + shopt -u nullglob + [ "${#debug_packages[@]}" -eq 1 ] || { + echo "ERROR: expected exactly one PG${v} dbgsym package, found ${#debug_packages[@]}" >&2; exit 1; + } + dbg="${debug_packages[0]}" + cp -v "${dbg}" "${OUTPUT_DIR}/$(basename "${dbg%.*}").deb" done cp -v "${WORK}"/*.buildinfo "${WORK}"/*.changes "${OUTPUT_DIR}/" 2>/dev/null || true echo "==> Verify runtime <-> dbgsym pairing" for v in ${PG_VERSIONS}; do - rt="$(dpkg-deb -f "${OUTPUT_DIR}"/postgresql-${v}-tdigest_*.deb Version)" - for dbg in "${OUTPUT_DIR}"/postgresql-${v}-tdigest-dbgsym_*.deb; do - [ -e "${dbg}" ] || continue + rt="$(dpkg-deb -f "${OUTPUT_DIR}/postgresql-${v}-tdigest_"*.deb Version)" + for dbg in "${OUTPUT_DIR}/postgresql-${v}-tdigest-dbgsym_"*.deb; do dep="$(dpkg-deb -f "${dbg}" Depends)" if [ "${dep}" != "postgresql-${v}-tdigest (= ${rt})" ]; then echo "ERROR: PG${v} dbgsym wants '${dep}' but runtime is '${rt}'" >&2 exit 1 fi + # Verify real DWARF symbols for this runtime's ELF build ID, not just an + # empty package with the right dependency metadata. + verify_dir="$(mktemp -d "${WORK}/verify.XXXXXX")" + dpkg-deb -x "${OUTPUT_DIR}/postgresql-${v}-tdigest_"*.deb "${verify_dir}" + dpkg-deb -x "${dbg}" "${verify_dir}" + build_id="$(readelf -n "${verify_dir}/usr/lib/postgresql/${v}/lib/tdigest.so" | awk '/Build ID:/ {print $3}')" + [ -n "${build_id}" ] || { echo "ERROR: PG${v} runtime has no build ID" >&2; exit 1; } + symbols="${verify_dir}/usr/lib/debug/.build-id/${build_id:0:2}/${build_id:2}.debug" + sections="$(readelf -S "${symbols}")" + if ! grep -q '\.debug_info' <<< "${sections}"; then + echo "ERROR: PG${v} dbgsym contains no DWARF debug information" >&2; exit 1 + fi + rm -rf "${verify_dir}" done echo " PG${v} OK (${rt})" done +# Record exactly which source was built alongside the package artifacts. +printf '%s\n' "${resolved}" > "${OUTPUT_DIR}/tdigest-source.env" +printf '%s\n' "PACKAGING_SRC_VERSION=${PACKAGING_SRC_VERSION}" >> "${OUTPUT_DIR}/tdigest-source.env" + echo "==> DONE. Packages:" ls -1 "${OUTPUT_DIR}"/*.deb diff --git a/scripts/resolve_tdigest_focal b/scripts/resolve_tdigest_focal new file mode 100644 index 00000000..70f66848 --- /dev/null +++ b/scripts/resolve_tdigest_focal @@ -0,0 +1,71 @@ +#!/bin/bash +# Resolve once per build; stdout is also suitable for GITHUB_ENV/GITHUB_OUTPUT. +# Automatic builds use the release asset's GitHub-published SHA-256 digest. +# Explicit checksums retain support for historical GitHub tag tarballs. +set -euo pipefail + +PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" +read -r -a majors <<< "${PG_VERSIONS}" +[[ "${PG_VERSIONS}" != *$'\n'* && ${#majors[@]} -gt 0 ]] || { + echo "ERROR: PG_VERSIONS must be a space-separated list of majors 11–16" >&2; exit 1; +} +for v in "${majors[@]}"; do + case "${v}" in + 11|12|13|14|15|16) ;; + *) echo "ERROR: unsupported PostgreSQL major '${v}'; supported range is 11–16" >&2; exit 1 ;; + esac +done + +TDIGEST_VERSION="${TDIGEST_VERSION:-latest}" +TDIGEST_SHA256="${TDIGEST_SHA256:-}" +UPSTREAM_URL="${UPSTREAM_URL:-}" +UPSTREAM_FORMAT="${UPSTREAM_FORMAT:-tar.gz}" +api=https://api.github.com/repos/tvondra/tdigest/releases +headers=(-H 'Accept: application/vnd.github+json') +if [ -n "${GITHUB_TOKEN:-}" ]; then + headers+=(-H "Authorization: Bearer ${GITHUB_TOKEN}") +fi + +if [ "${TDIGEST_VERSION}" = latest ]; then + release="$(curl -4 -fsSL --retry 3 "${headers[@]}" "${api}/latest")" + TDIGEST_VERSION="$(jq -er '.tag_name | sub("^v"; "")' <<< "${release}")" +fi +[[ "${TDIGEST_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "ERROR: expected a stable tdigest version (major.minor.patch) or latest" >&2; exit 1; +} + +if [ -z "${TDIGEST_SHA256}" ]; then + [ -z "${UPSTREAM_URL}" ] || { + echo "ERROR: UPSTREAM_URL requires an explicit TDIGEST_SHA256" >&2; exit 1; + } + if [ -z "${release:-}" ]; then + release="$(curl -4 -fsSL --retry 3 "${headers[@]}" "${api}/tags/v${TDIGEST_VERSION}")" + fi + asset="$(jq -er --arg name "t-digest-${TDIGEST_VERSION}.zip" ' + .assets[] | select(.name == $name and .state == "uploaded") + ' <<< "${release}")" || { + echo "ERROR: release has no source asset; provide TDIGEST_SHA256 for the tag tarball" >&2; exit 1; + } + TDIGEST_SHA256="$(jq -er '.digest | select(startswith("sha256:")) | ltrimstr("sha256:")' <<< "${asset}")" || { + echo "ERROR: release asset has no SHA-256 digest; provide TDIGEST_SHA256 for the tag tarball" >&2; exit 1; + } + UPSTREAM_URL="$(jq -er '.browser_download_url' <<< "${asset}")" + UPSTREAM_FORMAT=zip +else + UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/tvondra/tdigest/archive/refs/tags/v${TDIGEST_VERSION}.tar.gz}" +fi + +[[ "${TDIGEST_SHA256}" =~ ^[[:xdigit:]]{64}$ ]] || { + echo "ERROR: TDIGEST_SHA256 must contain exactly 64 hexadecimal digits" >&2; exit 1; +} +[[ "${UPSTREAM_URL}" == https://* && "${UPSTREAM_URL}" != *[[:space:]]* ]] || { + echo "ERROR: UPSTREAM_URL must be an HTTPS URL without whitespace" >&2; exit 1; +} +case "${UPSTREAM_FORMAT}" in + tar.gz|zip) ;; + *) echo "ERROR: unsupported UPSTREAM_FORMAT '${UPSTREAM_FORMAT}'" >&2; exit 1 ;; +esac + +echo "Resolved tdigest ${TDIGEST_VERSION} (${UPSTREAM_FORMAT}, SHA-256 ${TDIGEST_SHA256})" >&2 +printf '%s\n' "TDIGEST_VERSION=${TDIGEST_VERSION}" "TDIGEST_SHA256=${TDIGEST_SHA256}" \ + "UPSTREAM_URL=${UPSTREAM_URL}" "UPSTREAM_FORMAT=${UPSTREAM_FORMAT}" diff --git a/scripts/smoke_test_focal_tdigest_debs b/scripts/smoke_test_focal_tdigest_debs index ac2dcbee..be565517 100755 --- a/scripts/smoke_test_focal_tdigest_debs +++ b/scripts/smoke_test_focal_tdigest_debs @@ -10,27 +10,33 @@ # # It tests both paths that matter: # A. clean install on a stock focal + PGDG PostgreSQL -# B. install *over* PGDG's tdigest 1.4.3 and ALTER EXTENSION ... UPDATE, which +# B. install *over* PGDG's archived tdigest and ALTER EXTENSION ... UPDATE, which # is what an in-place upgrade on an existing host actually does # # Usage (inside a stock ubuntu:20.04 container, as root): -# DEBS_DIR=/debs smoke_test_focal_tdigest_debs +# DEBS_DIR=/debs EXPECTED_TDIGEST= smoke_test_focal_tdigest_debs # # Overridable via environment: # DEBS_DIR directory holding the *.deb set (default /debs) # PG_VERSIONS majors to verify (default "11 12 13 14 15 16") -# EXPECTED_TDIGEST expected extension version (default 1.4.6) -# OLD_TDIGEST PGDG version to upgrade from in test B (default 1.4.3-1.pgdg20.04+1) +# EXPECTED_TDIGEST expected extension version (required) +# OLD_TDIGEST PGDG version to upgrade from in test B (default archive candidate per major) # PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) -set -uo pipefail +set -euo pipefail DEBS_DIR="${DEBS_DIR:-/debs}" PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" -EXPECTED_TDIGEST="${EXPECTED_TDIGEST:-1.4.6}" -OLD_TDIGEST="${OLD_TDIGEST:-1.4.3-1.pgdg20.04+1}" +EXPECTED_TDIGEST="${EXPECTED_TDIGEST:?Set EXPECTED_TDIGEST to the resolved build version}" +OLD_TDIGEST="${OLD_TDIGEST:-}" PGDG_ARCHIVE_SUITE="${PGDG_ARCHIVE_SUITE:-focal-pgdg}" PGDG_ARCHIVE_URL="${PGDG_ARCHIVE_URL:-https://apt-archive.postgresql.org/pub/repos/apt}" +for v in ${PG_VERSIONS}; do + case "${v}" in + 11|12|13|14|15|16) ;; + *) echo "ERROR: unsupported PostgreSQL major '${v}'; supported range is 11–16" >&2; exit 1 ;; + esac +done export DEBIAN_FRONTEND=noninteractive export LANG=C.UTF-8 @@ -63,6 +69,18 @@ echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} $ > /etc/apt/sources.list.d/pgdg-archive.list apt-get update -qq +# Resolve before installing our packages so their versions cannot influence the +# APT candidate. Archived package versions may differ between PostgreSQL majors. +declare -A old_versions +for v in ${PG_VERSIONS}; do + old="${OLD_TDIGEST:-$(apt-cache policy "postgresql-${v}-tdigest" | awk '/Candidate:/ {print $2}')}" + if [ -z "${old}" ] || [ "${old}" = '(none)' ]; then + echo "ERROR: no archived tdigest upgrade baseline for PG${v}" >&2; exit 1 + fi + old_versions[${v}]="${old}" + note "PG${v} upgrade baseline: ${old}" +done + psql_v() { su postgres -c "psql --cluster $1/main -qtAX -c \"$2\"" 2>&1; } ensure_cluster() { @@ -71,6 +89,19 @@ ensure_cluster() { pg_ctlcluster "$1" main start >/dev/null 2>&1 } +# apt --fix-broken may remove packages to resolve dependencies. Check that both +# runtime and symbols from the shipped set remain installed after each path. +check_packages() { + local v="$1" package expected actual + for package in "postgresql-${v}-tdigest" "postgresql-${v}-tdigest-dbgsym"; do + expected="$(dpkg-deb -f "${DEBS_DIR}/${package}"_*.deb Version)" + actual="$(dpkg-query -W -f='${Status} ${Version}' "${package}")" + if [ "${actual}" != "install ok installed ${expected}" ]; then + err "${package}: '${actual}' (expected installed ${expected})" + fi + done +} + # Median of 1..1000 is ~500. t-digest is approximate, so accept a small window. check_percentile() { local v="$1" p @@ -80,14 +111,15 @@ check_percentile() { note "[3/6] Scenario A: clean install" for v in ${PG_VERSIONS}; do - apt-get install -y -qq postgresql-${v} >/dev/null 2>&1 + apt-get install -y -qq "postgresql-${v}" >/dev/null 2>&1 done -dpkg -i --force-all "${DEBS_DIR}"/*.deb >/dev/null 2>&1 +dpkg -i "${debs[@]}" || apt-get --fix-broken install -y if ! apt-get --fix-broken install -y >/dev/null 2>&1; then err "apt --fix-broken install failed after clean install" fi for v in ${PG_VERSIONS}; do + check_packages "${v}" ensure_cluster "${v}" psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" @@ -99,26 +131,33 @@ for v in ${PG_VERSIONS}; do pg_ctlcluster "${v}" main stop >/dev/null 2>&1 done -note "[4/6] Scenario B: upgrade over PGDG tdigest ${OLD_TDIGEST}" +note "[4/6] Scenario B: upgrade over archived PGDG tdigest" for v in ${PG_VERSIONS}; do - apt-get remove -y -qq postgresql-${v}-tdigest >/dev/null 2>&1 + old="${old_versions[${v}]}" + apt-get remove -y -qq "postgresql-${v}-tdigest" >/dev/null 2>&1 pg_dropcluster "${v}" main >/dev/null 2>&1 pg_createcluster "${v}" main >/dev/null 2>&1 apt-get install -y -qq --allow-downgrades \ - postgresql-${v}-tdigest=${OLD_TDIGEST} >/dev/null 2>&1 + "postgresql-${v}-tdigest=${old}" \ + "postgresql-${v}-tdigest-dbgsym=${old}" >/dev/null ensure_cluster "${v}" psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 + ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" + [ "${ext}" = "${old%%-*}" ] || { + echo "ERROR: PG${v} upgrade baseline is '${ext}', expected ${old%%-*}" >&2; exit 1; + } pg_ctlcluster "${v}" main stop >/dev/null 2>&1 done -# Mirrors how consumers apply the set: force-all dpkg, then let apt settle. -dpkg -i --force-all "${DEBS_DIR}"/*.deb >/dev/null 2>&1 +# Install both runtime and symbols, then let apt settle dependencies. +dpkg -i "${debs[@]}" || apt-get --fix-broken install -y if ! apt-get --fix-broken install -y >/dev/null 2>&1; then - err "apt --fix-broken install failed after upgrade over ${OLD_TDIGEST}" + err "apt --fix-broken install failed after upgrade over archived packages" fi note "[5/6] Verifying the extension upgrades and works after the package upgrade" for v in ${PG_VERSIONS}; do + check_packages "${v}" ensure_cluster "${v}" out="$(psql_v "${v}" "ALTER EXTENSION tdigest UPDATE;")" ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" From fadea46d01ed91d40804b328040976235b76c380 Mon Sep 17 00:00:00 2001 From: Kemal Buyukkaya Date: Fri, 18 Sep 2026 14:12:20 +0000 Subject: [PATCH 5/5] focal tdigest: preserve patches and fix historical rebuilds Apply the full Debian patch series and enforce debhelper >= 13. Stage and validate artifacts before replacing previous tdigest output. Derive upgradeable revisions for same-upstream rebuilds and select compatible archived runtime/dbgsym baselines without attempting extension downgrades. --- dockerfiles/focal-tdigest-builder/Dockerfile | 3 +- scripts/build_tdigest_focal | 77 +++++++++++++------- scripts/smoke_test_focal_tdigest_debs | 60 +++++++++++---- 3 files changed, 101 insertions(+), 39 deletions(-) diff --git a/dockerfiles/focal-tdigest-builder/Dockerfile b/dockerfiles/focal-tdigest-builder/Dockerfile index 5f9daf5c..d1f0c266 100644 --- a/dockerfiles/focal-tdigest-builder/Dockerfile +++ b/dockerfiles/focal-tdigest-builder/Dockerfile @@ -59,7 +59,8 @@ RUN set -ex; \ # Fail the image build early if the archived focal-pgdg debhelper (>= 13) is not # what we picked up (debhelper-compat (= 13) is required by the packaging). -RUN dpkg-query -W -f='${Package} ${Version}\n' debhelper postgresql-common-dev dh-exec +RUN dpkg-query -W -f='${Package} ${Version}\n' debhelper postgresql-common-dev dh-exec \ + && dpkg --compare-versions "$(dpkg-query -W -f='${Version}' debhelper)" ge 13 COPY scripts/build_tdigest_focal /usr/local/bin/build_tdigest_focal COPY scripts/resolve_tdigest_focal /usr/local/bin/resolve_tdigest_focal diff --git a/scripts/build_tdigest_focal b/scripts/build_tdigest_focal index 54ea9fd7..4e3d7947 100755 --- a/scripts/build_tdigest_focal +++ b/scripts/build_tdigest_focal @@ -47,6 +47,7 @@ PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" # APT selects the newest source package in the focal archive unless overridden. PACKAGING_SRC_VERSION="${PACKAGING_SRC_VERSION:-}" +explicit_version="${TARGET_VERSION:-${DEB_REVISION:-}}" DEB_REVISION="${DEB_REVISION:-1.citus20.04+1}" TARGET_VERSION="${TARGET_VERSION:-${TDIGEST_VERSION}-${DEB_REVISION}}" @@ -81,6 +82,31 @@ PKG_DIR="$(find "${WORK}/packaging" -maxdepth 1 -type d -name 'tdigest-*' -exec PACKAGING_SRC_VERSION="$(dpkg-parsechangelog -l "${PKG_DIR}/debian/changelog" -S Version)" echo " selected packaging source: ${PACKAGING_SRC_VERSION}" +# Same-upstream rebuilds must sort above every matching archived binary, even +# when different PostgreSQL majors have different Debian revisions. Older +# upstream releases retain their historical ordering rather than gaining epochs. +baseline="" +for v in ${PG_VERSIONS}; do + versions="$(apt-cache madison "postgresql-${v}-tdigest" | awk '{print $3}')" + for version in ${versions}; do + upstream="${version#*:}" + upstream="${upstream%-*}" + if dpkg --compare-versions "${upstream}" eq "${TDIGEST_VERSION}"; then + if [ -z "${baseline}" ] || dpkg --compare-versions "${version}" gt "${baseline}"; then + baseline="${version}" + fi + fi + done +done +if [ -n "${baseline}" ] && ! dpkg --compare-versions "${TARGET_VERSION}" gt "${baseline}"; then + if [ -n "${explicit_version}" ]; then + echo "ERROR: explicit package version ${TARGET_VERSION} must be newer than ${baseline}" >&2 + exit 1 + fi + TARGET_VERSION="${baseline}+citus1" + echo " same-upstream rebuild version: ${TARGET_VERSION}" +fi + echo "==> [2/6] Fetch and verify upstream tdigest-${TDIGEST_VERSION}.${UPSTREAM_FORMAT}" cd "${WORK}" archive="tdigest-${TDIGEST_VERSION}.${UPSTREAM_FORMAT}" @@ -98,22 +124,10 @@ fi cp -a "${PKG_DIR}/debian" "${SRCDIR}/debian" cd "${SRCDIR}" -echo "==> [3/6] Triage quilt patches against the new upstream" -if [ -f debian/patches/series ]; then - KEPT="" - while read -r patch; do - [ -z "${patch}" ] && continue - case "${patch}" in \#*) continue ;; esac - if patch -p1 --dry-run --silent < "debian/patches/${patch}" >/dev/null 2>&1; then - echo " keep ${patch}"; KEPT="${KEPT}${patch}\n" - else - echo " drop ${patch} (does not apply to ${TDIGEST_VERSION})" - fi - done < debian/patches/series - printf "%b" "${KEPT}" > debian/patches/series -else - echo " no debian/patches/series -- nothing to triage" -fi +echo "==> [3/6] Apply the complete Debian patch series" +# Preserve series ordering, options, and dependencies between patches. A conflict +# must fail the build rather than silently remove a packaging or correctness fix. +dpkg-source --before-build . echo "==> [4/6] Target PostgreSQL majors: ${PG_VERSIONS}" # debian/control ships with every major PGDG built (10..17); regenerate it from @@ -152,9 +166,11 @@ mk-build-deps --install --remove \ echo "==> [6/6] Build binary packages (DEB_BUILD_OPTIONS='${BUILD_OPTIONS}')" dpkg-buildpackage -b -uc -us -echo "==> Collect artifacts into ${OUTPUT_DIR}" +echo "==> Stage artifacts for validation" +STAGING_DIR="${WORK}/artifacts" +mkdir "${STAGING_DIR}" for v in ${PG_VERSIONS}; do - cp -v "${WORK}/postgresql-${v}-tdigest_"*.deb "${OUTPUT_DIR}/" + cp -v "${WORK}/postgresql-${v}-tdigest_"*.deb "${STAGING_DIR}/" # Debug-symbol packages are emitted as .ddeb on Ubuntu. Ship them as .deb (the # on-disk format is identical) so they flow through the same signing, # verification and publishing as everything else. Consumers commonly install @@ -168,14 +184,14 @@ for v in ${PG_VERSIONS}; do echo "ERROR: expected exactly one PG${v} dbgsym package, found ${#debug_packages[@]}" >&2; exit 1; } dbg="${debug_packages[0]}" - cp -v "${dbg}" "${OUTPUT_DIR}/$(basename "${dbg%.*}").deb" + cp -v "${dbg}" "${STAGING_DIR}/$(basename "${dbg%.*}").deb" done -cp -v "${WORK}"/*.buildinfo "${WORK}"/*.changes "${OUTPUT_DIR}/" 2>/dev/null || true +cp -v "${WORK}"/*.buildinfo "${WORK}"/*.changes "${STAGING_DIR}/" echo "==> Verify runtime <-> dbgsym pairing" for v in ${PG_VERSIONS}; do - rt="$(dpkg-deb -f "${OUTPUT_DIR}/postgresql-${v}-tdigest_"*.deb Version)" - for dbg in "${OUTPUT_DIR}/postgresql-${v}-tdigest-dbgsym_"*.deb; do + rt="$(dpkg-deb -f "${STAGING_DIR}/postgresql-${v}-tdigest_"*.deb Version)" + for dbg in "${STAGING_DIR}/postgresql-${v}-tdigest-dbgsym_"*.deb; do dep="$(dpkg-deb -f "${dbg}" Depends)" if [ "${dep}" != "postgresql-${v}-tdigest (= ${rt})" ]; then echo "ERROR: PG${v} dbgsym wants '${dep}' but runtime is '${rt}'" >&2 @@ -184,7 +200,7 @@ for v in ${PG_VERSIONS}; do # Verify real DWARF symbols for this runtime's ELF build ID, not just an # empty package with the right dependency metadata. verify_dir="$(mktemp -d "${WORK}/verify.XXXXXX")" - dpkg-deb -x "${OUTPUT_DIR}/postgresql-${v}-tdigest_"*.deb "${verify_dir}" + dpkg-deb -x "${STAGING_DIR}/postgresql-${v}-tdigest_"*.deb "${verify_dir}" dpkg-deb -x "${dbg}" "${verify_dir}" build_id="$(readelf -n "${verify_dir}/usr/lib/postgresql/${v}/lib/tdigest.so" | awk '/Build ID:/ {print $3}')" [ -n "${build_id}" ] || { echo "ERROR: PG${v} runtime has no build ID" >&2; exit 1; } @@ -199,8 +215,19 @@ for v in ${PG_VERSIONS}; do done # Record exactly which source was built alongside the package artifacts. -printf '%s\n' "${resolved}" > "${OUTPUT_DIR}/tdigest-source.env" -printf '%s\n' "PACKAGING_SRC_VERSION=${PACKAGING_SRC_VERSION}" >> "${OUTPUT_DIR}/tdigest-source.env" +printf '%s\n' "${resolved}" > "${STAGING_DIR}/tdigest-source.env" +printf '%s\n' "PACKAGING_SRC_VERSION=${PACKAGING_SRC_VERSION}" \ + "TARGET_VERSION=${TARGET_VERSION}" >> "${STAGING_DIR}/tdigest-source.env" + +echo "==> Replace previous tdigest artifacts in ${OUTPUT_DIR}" +# Touch the persistent output only after the entire new set passes validation. +# Remove all previous majors/versions, but preserve unrelated files. +rm -f "${OUTPUT_DIR}"/postgresql-*-tdigest_*.deb \ + "${OUTPUT_DIR}"/postgresql-*-tdigest-dbgsym_*.deb \ + "${OUTPUT_DIR}"/postgresql-*-tdigest-dbgsym_*.ddeb \ + "${OUTPUT_DIR}"/tdigest_*.buildinfo "${OUTPUT_DIR}"/tdigest_*.changes \ + "${OUTPUT_DIR}/tdigest-source.env" +cp -v "${STAGING_DIR}"/* "${OUTPUT_DIR}/" echo "==> DONE. Packages:" ls -1 "${OUTPUT_DIR}"/*.deb diff --git a/scripts/smoke_test_focal_tdigest_debs b/scripts/smoke_test_focal_tdigest_debs index be565517..ddc5ed85 100755 --- a/scripts/smoke_test_focal_tdigest_debs +++ b/scripts/smoke_test_focal_tdigest_debs @@ -20,7 +20,7 @@ # DEBS_DIR directory holding the *.deb set (default /debs) # PG_VERSIONS majors to verify (default "11 12 13 14 15 16") # EXPECTED_TDIGEST expected extension version (required) -# OLD_TDIGEST PGDG version to upgrade from in test B (default archive candidate per major) +# OLD_TDIGEST PGDG baseline override (default newest compatible runtime/dbgsym pair) # PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) set -euo pipefail @@ -70,15 +70,45 @@ echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} $ apt-get update -qq # Resolve before installing our packages so their versions cannot influence the -# APT candidate. Archived package versions may differ between PostgreSQL majors. +# APT candidates. Prefer a genuine extension upgrade; fall back to same-version +# package replacement. Never attempt to downgrade an extension. declare -A old_versions +upgrade_majors=() for v in ${PG_VERSIONS}; do - old="${OLD_TDIGEST:-$(apt-cache policy "postgresql-${v}-tdigest" | awk '/Candidate:/ {print $2}')}" - if [ -z "${old}" ] || [ "${old}" = '(none)' ]; then - echo "ERROR: no archived tdigest upgrade baseline for PG${v}" >&2; exit 1 + versions="$(apt-cache madison "postgresql-${v}-tdigest" | awk '{print $3}')" + debug_versions="$(apt-cache madison "postgresql-${v}-tdigest-dbgsym" | awk '{print $3}')" + older="" same="" + for version in ${versions}; do + [ -z "${OLD_TDIGEST}" ] || [ "${version}" = "${OLD_TDIGEST}" ] || continue + grep -Fxq "${version}" <<< "${debug_versions}" || continue + upstream="${version#*:}" + upstream="${upstream%-*}" + if dpkg --compare-versions "${upstream}" lt "${EXPECTED_TDIGEST}"; then + if [ -z "${older}" ] || dpkg --compare-versions "${version}" gt "${older}"; then + older="${version}" + fi + elif dpkg --compare-versions "${upstream}" eq "${EXPECTED_TDIGEST}"; then + if [ -z "${same}" ] || dpkg --compare-versions "${version}" gt "${same}"; then + same="${version}" + fi + fi + done + old="${older:-${same}}" + if [ -z "${old}" ]; then + if [ -n "${OLD_TDIGEST}" ]; then + echo "ERROR: PG${v} OLD_TDIGEST=${OLD_TDIGEST} requires an available runtime/dbgsym pair no newer than ${EXPECTED_TDIGEST}" >&2 + exit 1 + fi + note "PG${v}: skipping upgrade scenario; no compatible archived runtime/dbgsym pair for ${EXPECTED_TDIGEST} (clean install still runs)" + continue fi old_versions[${v}]="${old}" - note "PG${v} upgrade baseline: ${old}" + upgrade_majors+=("${v}") + if [ -n "${older}" ]; then + note "PG${v} extension-upgrade baseline: ${old}" + else + note "PG${v} same-extension-version package-replacement baseline: ${old}" + fi done psql_v() { su postgres -c "psql --cluster $1/main -qtAX -c \"$2\"" 2>&1; } @@ -132,7 +162,7 @@ for v in ${PG_VERSIONS}; do done note "[4/6] Scenario B: upgrade over archived PGDG tdigest" -for v in ${PG_VERSIONS}; do +for v in "${upgrade_majors[@]}"; do old="${old_versions[${v}]}" apt-get remove -y -qq "postgresql-${v}-tdigest" >/dev/null 2>&1 pg_dropcluster "${v}" main >/dev/null 2>&1 @@ -143,20 +173,24 @@ for v in ${PG_VERSIONS}; do ensure_cluster "${v}" psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" - [ "${ext}" = "${old%%-*}" ] || { - echo "ERROR: PG${v} upgrade baseline is '${ext}', expected ${old%%-*}" >&2; exit 1; + upstream="${old#*:}" + upstream="${upstream%-*}" + [ "${ext}" = "${upstream}" ] || { + echo "ERROR: PG${v} upgrade baseline is '${ext}', expected ${upstream}" >&2; exit 1; } pg_ctlcluster "${v}" main stop >/dev/null 2>&1 done # Install both runtime and symbols, then let apt settle dependencies. -dpkg -i "${debs[@]}" || apt-get --fix-broken install -y -if ! apt-get --fix-broken install -y >/dev/null 2>&1; then - err "apt --fix-broken install failed after upgrade over archived packages" +if [ "${#upgrade_majors[@]}" -gt 0 ]; then + dpkg -i "${debs[@]}" || apt-get --fix-broken install -y + if ! apt-get --fix-broken install -y >/dev/null 2>&1; then + err "apt --fix-broken install failed after upgrade over archived packages" + fi fi note "[5/6] Verifying the extension upgrades and works after the package upgrade" -for v in ${PG_VERSIONS}; do +for v in "${upgrade_majors[@]}"; do check_packages "${v}" ensure_cluster "${v}" out="$(psql_v "${v}" "ALTER EXTENSION tdigest UPDATE;")"