diff --git a/.github/workflows/build-tdigest-focal.yml b/.github/workflows/build-tdigest-focal.yml new file mode 100644 index 00000000..efc0ca9a --- /dev/null +++ b/.github/workflows/build-tdigest-focal.yml @@ -0,0 +1,188 @@ +name: Build tdigest (focal) + +# Builds tdigest .deb packages for Ubuntu 20.04 (focal) from upstream source +# (PGDG removed focal entirely -- focal-pgdg 404s), then signs them with +# debsigs (--sign=maint) using the +# existing packaging key. +# +# There is no per-major matrix: the tdigest Debian packaging is multi-version by +# design (debian/pgversions + pg_buildext), so one source build emits +# postgresql--tdigest for every requested major in a single pass. That +# also means no assemble/de-duplicate job is needed -- each runtime package is +# self-contained and produced exactly once. + +on: + workflow_dispatch: + inputs: + tdigest_version: + description: "tdigest upstream version; blank or latest discovers the latest stable release" + required: false + default: "latest" + pg_versions: + description: "Space-separated PostgreSQL majors (supported: 11–16)" + required: false + default: "11 12 13 14 15 16" + tdigest_sha256: + description: "Optional SHA-256 for the GitHub tag tarball. Blank = use the release ZIP and its GitHub-published digest." + required: false + default: "" + run_tests: + description: "Run upstream regression suite (1=yes, much slower)" + required: false + default: "0" + push: + branches: + - tdigest-focal + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-and-sign: + name: Build & sign tdigest ${{ github.event.inputs.tdigest_version || 'latest' }} (focal) + runs-on: ubuntu-latest + outputs: + tdigest_version: ${{ steps.source.outputs.TDIGEST_VERSION }} + env: + PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }} + PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }} + TDIGEST_VERSION: ${{ github.event.inputs.tdigest_version || 'latest' }} + PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} + TDIGEST_SHA256: ${{ github.event.inputs.tdigest_sha256 || '' }} + RUN_TESTS: ${{ github.event.inputs.run_tests || '0' }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Validate majors and resolve tdigest source + id: source + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: bash scripts/resolve_tdigest_focal | tee -a "$GITHUB_ENV" "$GITHUB_OUTPUT" + + - name: Login to Docker Hub + uses: docker/login-action@v4 + with: + username: ${{ secrets.DOCKERHUB_USER_NAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build focal tdigest builder image + run: | + docker build -t focal-tdigest-builder \ + -f dockerfiles/focal-tdigest-builder/Dockerfile . + + - name: Build tdigest packages + run: | + mkdir -p packages + docker run --rm \ + -e TDIGEST_VERSION="${TDIGEST_VERSION}" \ + -e PG_VERSIONS="${PG_VERSIONS}" \ + -e TDIGEST_SHA256="${TDIGEST_SHA256}" \ + -e UPSTREAM_URL="${UPSTREAM_URL}" \ + -e UPSTREAM_FORMAT="${UPSTREAM_FORMAT}" \ + -e RUN_TESTS="${RUN_TESTS}" \ + -v "${PWD}/packages:/packages" \ + focal-tdigest-builder + echo "Built packages:" + ls -1 packages/focal/tdigest/*.deb + + - name: Sign packages (debsigs --sign=maint) + # Use the prebuilt, deployed debsigner image (the one all Citus signing + # uses). Its entrypoint signs exactly "/packages/*/*.deb" (one dir level + # deep), so mount the parent of the output dir: with + # "${PWD}/packages/focal:/packages" the debs land at + # /packages/tdigest/*.deb, which is what that glob expects. + run: | + if [ -z "${PACKAGING_SECRET_KEY}" ] || [ -z "${PACKAGING_PASSPHRASE}" ]; then + echo "::error::PACKAGING_SECRET_KEY / PACKAGING_PASSPHRASE secrets are not set" >&2 + exit 1 + fi + printf '%s' "${PACKAGING_PASSPHRASE}" | docker run --rm -i \ + -e PACKAGING_SECRET_KEY \ + -e PACKAGING_PASSPHRASE \ + -v "${PWD}/packages/focal:/packages" \ + citusdata/packaging:debsigner + + - name: Verify signatures are embedded + run: | + rc=0 + for deb in packages/focal/tdigest/*.deb; do + if ar t "$deb" | grep -q '^_gpgmaint$'; then + echo "signed: $deb" + else + echo "::error::missing _gpgmaint signature in $deb" >&2 + rc=1 + fi + done + exit $rc + + - name: Verify the set is self-contained + # Every runtime must ship its own extension control file and the full SQL + # tree, so a package can never be paired with another version's SQL and + # there is no separate -scripts package to depend on. tdigest is installed + # from the base script and reaches the built version through the shipped + # upgrade chain, whose terminal script names that version, so match + # either tdigest----.sql or a direct + # tdigest--.sql. + run: | + rc=0 + for v in ${PG_VERSIONS}; do + deb="$(ls "packages/focal/tdigest/postgresql-${v}-tdigest_"*.deb)" + contents="$(dpkg-deb -c "$deb")" + if ! grep -q 'extension/tdigest\.control' <<<"$contents"; then + echo "::error::PG${v} runtime ships no extension control file" >&2; rc=1 + fi + if ! grep -qE "extension/tdigest--([0-9.]+--)?${TDIGEST_VERSION}\.sql" <<<"$contents"; then + echo "::error::PG${v} runtime ships no SQL producing tdigest ${TDIGEST_VERSION}" >&2; rc=1 + fi + done + exit $rc + + - name: Upload signed packages + uses: actions/upload-artifact@v4 + with: + name: tdigest-focal-deb + path: | + packages/focal/tdigest/*.deb + packages/focal/tdigest/*.changes + packages/focal/tdigest/*.buildinfo + packages/focal/tdigest/tdigest-source.env + if-no-files-found: error + + install-smoke-test: + needs: build-and-sign + name: Install smoke test (focal) + runs-on: ubuntu-latest + env: + PG_VERSIONS: ${{ github.event.inputs.pg_versions || '11 12 13 14 15 16' }} + TDIGEST_VERSION: ${{ needs.build-and-sign.outputs.tdigest_version }} + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Download built packages + uses: actions/download-artifact@v4 + with: + name: tdigest-focal-deb + path: debs + + - name: Install and verify in a clean focal container + # The jobs above only prove the packages exist, are signed and are + # self-contained -- not that the extension can actually be created. This + # installs the set into a stock ubuntu:20.04 twice: clean, and over + # PGDG's archived tdigest (the in-place upgrade path), then runs + # CREATE EXTENSION / ALTER EXTENSION UPDATE on every major. + run: | + docker run --rm \ + -v "${PWD}/debs:/debs:ro" \ + -v "${PWD}/scripts/smoke_test_focal_tdigest_debs:/usr/local/bin/smoke_test_focal_tdigest_debs:ro" \ + -e DEBS_DIR=/debs \ + -e PG_VERSIONS="${PG_VERSIONS}" \ + -e EXPECTED_TDIGEST="${TDIGEST_VERSION}" \ + ubuntu:20.04 \ + /usr/local/bin/smoke_test_focal_tdigest_debs diff --git a/dockerfiles/focal-tdigest-builder/Dockerfile b/dockerfiles/focal-tdigest-builder/Dockerfile new file mode 100644 index 00000000..d1f0c266 --- /dev/null +++ b/dockerfiles/focal-tdigest-builder/Dockerfile @@ -0,0 +1,70 @@ +# vim:set ft=dockerfile: +# +# Builder image for tdigest packages targeting Ubuntu 20.04 (focal). One image +# builds every focal-buildable PostgreSQL major at once (PG 11..16 by default); +# the tdigest Debian packaging is multi-version by design (debian/pgversions + +# the pgxs_loop debhelper addon drive pg_buildext), so a single source build +# emits postgresql--tdigest for each major in one pass. +# +# Why this exists: +# apt.postgresql.org (PGDG) no longer ships focal binaries -- focal-pgdg 404s +# and the apt-archive mirror is frozen. New releases have to be rebuilt +# from upstream source. +# +# Strategy: +# - Upstream source: latest GitHub release asset and its published SHA-256, +# or a specific tag tarball with an explicitly supplied checksum. +# - Debian packaging: the frozen focal-era debian/ from PGDG's last focal +# tdigest source package, selected and fetched with `apt-get source` +# so it is authenticated by the archive's signed Release. +# - Build tooling restored from the PGDG *archive*, which keeps the removed +# focal-pgdg suite. +# +# The heavy lifting lives in scripts/build_tdigest_focal (the entrypoint). +FROM ubuntu:20.04 +ARG DEBIAN_FRONTEND=noninteractive + +# PGDG repository signing key fingerprint: +# B97B 0AFC AA1A 47F0 44F2 44A0 7FCC 7D46 ACCC 4CF8 +RUN set -ex; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl gnupg; \ + install -d /usr/share/keyrings; \ + curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg; \ + # 'main' carries the build tooling, the tdigest source package and the PG11 + # binaries; PG12..16 are each a separate component. + echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 11 12 13 14 15 16" \ + > /etc/apt/sources.list.d/pgdg-archive.list; \ + echo "deb-src [signed-by=/usr/share/keyrings/pgdg-archive.gpg] https://apt-archive.postgresql.org/pub/repos/apt focal-pgdg main 11 12 13 14 15 16" \ + >> /etc/apt/sources.list.d/pgdg-archive.list; \ + apt-get update; \ + # base build tooling; per-build Build-Depends are resolved at run time by + # scripts/build_tdigest_focal via mk-build-deps against debian/control. + apt-get install -y --no-install-recommends \ + build-essential \ + devscripts \ + equivs \ + fakeroot \ + quilt \ + dpkg-dev \ + debhelper \ + dh-exec \ + jq \ + unzip \ + postgresql-common-dev \ + postgresql-server-dev-all \ + xz-utils; \ + rm -rf /var/lib/apt/lists/* + +# Fail the image build early if the archived focal-pgdg debhelper (>= 13) is not +# what we picked up (debhelper-compat (= 13) is required by the packaging). +RUN dpkg-query -W -f='${Package} ${Version}\n' debhelper postgresql-common-dev dh-exec \ + && dpkg --compare-versions "$(dpkg-query -W -f='${Version}' debhelper)" ge 13 + +COPY scripts/build_tdigest_focal /usr/local/bin/build_tdigest_focal +COPY scripts/resolve_tdigest_focal /usr/local/bin/resolve_tdigest_focal +RUN chmod +x /usr/local/bin/build_tdigest_focal + +VOLUME /packages +ENTRYPOINT ["/usr/local/bin/build_tdigest_focal"] diff --git a/scripts/build_tdigest_focal b/scripts/build_tdigest_focal new file mode 100755 index 00000000..4e3d7947 --- /dev/null +++ b/scripts/build_tdigest_focal @@ -0,0 +1,233 @@ +#!/bin/bash +# +# build_tdigest_focal -- rebuild tdigest Debian packages for Ubuntu 20.04 (focal). +# +# PGDG removed focal entirely (focal-pgdg 404s), so current tdigest releases +# on focal have to be rebuilt from upstream source using archived packaging. +# +# Approach (see dockerfiles/focal-tdigest-builder/Dockerfile for the rationale): +# upstream release archive + the frozen focal-era debian/ packaging +# from PGDG's last focal tdigest source package +# -> dpkg-buildpackage inside a focal environment with the archived focal-pgdg +# build tooling available. +# +# One source build emits packages for *every* PostgreSQL major at once. tdigest +# is a plain PGXS extension whose Debian packaging is multi-version by design +# (debian/pgversions + the pgxs_loop debhelper addon drive pg_buildext), so a +# single source build emits postgresql--tdigest for each requested major +# in one pass. There is no per-major matrix here. +# +# Each runtime package ships its own extension control file and SQL; there is no +# separate -scripts package and no update-alternatives handling to worry about. +# +# Output: unsigned *.deb in ${OUTPUT_DIR} (default /packages/focal/tdigest). +# Signing is a separate step performed by the debsigner image (debsigs +# --sign=maint), so the build and the signing key never live in the same +# container. +# +# Examples: +# build_tdigest_focal # latest for PG11..16 +# TDIGEST_VERSION= build_tdigest_focal # specific release +# PG_VERSIONS="15 16" RUN_TESTS=1 build_tdigest_focal + +set -euo pipefail + +# ---------------------------------------------------------------------------- +# Inputs (override via env) +# ---------------------------------------------------------------------------- +# Validate inputs and resolve latest before doing any package installation. +resolved="$(bash "$(dirname "${BASH_SOURCE[0]}")/resolve_tdigest_focal")" +while IFS='=' read -r key value; do + export "${key}=${value}" +done <<< "${resolved}" + +# This builder supports PG11..16 only. +PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" + +# APT selects the newest source package in the focal archive unless overridden. +PACKAGING_SRC_VERSION="${PACKAGING_SRC_VERSION:-}" + +explicit_version="${TARGET_VERSION:-${DEB_REVISION:-}}" +DEB_REVISION="${DEB_REVISION:-1.citus20.04+1}" +TARGET_VERSION="${TARGET_VERSION:-${TDIGEST_VERSION}-${DEB_REVISION}}" + +# Set RUN_TESTS=1 to run the upstream regression suite (much slower, and pulls +# in postgresql-all for a full test matrix). +RUN_TESTS="${RUN_TESTS:-0}" + +OUTPUT_DIR="${OUTPUT_DIR:-/packages/focal/tdigest}" +WORK="${WORK_DIR:-/build}" + +export DEBEMAIL="${DEBEMAIL:-packaging@citusdata.com}" +export DEBFULLNAME="${DEBFULLNAME:-Citus Data}" +export DEBIAN_FRONTEND=noninteractive + +echo "==> Building tdigest ${TDIGEST_VERSION} as ${TARGET_VERSION} for PG${PG_VERSIONS// /,} (focal)" + +mkdir -p "${WORK}" "${OUTPUT_DIR}" +# Isolate each invocation, including the frozen packaging, from upstream source +# extraction and from stale artifacts left by earlier builds. +WORK="$(mktemp -d "${WORK}/tdigest.XXXXXX")" +# pg_virtualenv runs installcheck as postgres and loads staged libraries here. +chmod 755 "${WORK}" +mkdir "${WORK}/packaging" +cd "${WORK}/packaging" + +echo "==> [1/6] Fetch archived focal debian/ packaging" +apt-get update +# apt-get source validates against the archive's signed Release file. +apt-get source "tdigest${PACKAGING_SRC_VERSION:+=$PACKAGING_SRC_VERSION}" +PKG_DIR="$(find "${WORK}/packaging" -maxdepth 1 -type d -name 'tdigest-*' -exec test -d '{}/debian' \; -print | head -1)" +[ -n "${PKG_DIR}" ] || { echo "ERROR: could not locate unpacked packaging source" >&2; exit 1; } +PACKAGING_SRC_VERSION="$(dpkg-parsechangelog -l "${PKG_DIR}/debian/changelog" -S Version)" +echo " selected packaging source: ${PACKAGING_SRC_VERSION}" + +# Same-upstream rebuilds must sort above every matching archived binary, even +# when different PostgreSQL majors have different Debian revisions. Older +# upstream releases retain their historical ordering rather than gaining epochs. +baseline="" +for v in ${PG_VERSIONS}; do + versions="$(apt-cache madison "postgresql-${v}-tdigest" | awk '{print $3}')" + for version in ${versions}; do + upstream="${version#*:}" + upstream="${upstream%-*}" + if dpkg --compare-versions "${upstream}" eq "${TDIGEST_VERSION}"; then + if [ -z "${baseline}" ] || dpkg --compare-versions "${version}" gt "${baseline}"; then + baseline="${version}" + fi + fi + done +done +if [ -n "${baseline}" ] && ! dpkg --compare-versions "${TARGET_VERSION}" gt "${baseline}"; then + if [ -n "${explicit_version}" ]; then + echo "ERROR: explicit package version ${TARGET_VERSION} must be newer than ${baseline}" >&2 + exit 1 + fi + TARGET_VERSION="${baseline}+citus1" + echo " same-upstream rebuild version: ${TARGET_VERSION}" +fi + +echo "==> [2/6] Fetch and verify upstream tdigest-${TDIGEST_VERSION}.${UPSTREAM_FORMAT}" +cd "${WORK}" +archive="tdigest-${TDIGEST_VERSION}.${UPSTREAM_FORMAT}" +curl -4 -fsSL --retry 3 -o "${archive}" "${UPSTREAM_URL}" +echo "${TDIGEST_SHA256} ${archive}" | sha256sum -c - + +SRCDIR="${WORK}/tdigest-${TDIGEST_VERSION}" +if [ "${UPSTREAM_FORMAT}" = zip ]; then + unzip -q "${archive}" + mv "t-digest-${TDIGEST_VERSION}" "${SRCDIR}" +else + mkdir "${SRCDIR}" + tar xzf "${archive}" --strip-components=1 -C "${SRCDIR}" +fi +cp -a "${PKG_DIR}/debian" "${SRCDIR}/debian" +cd "${SRCDIR}" + +echo "==> [3/6] Apply the complete Debian patch series" +# Preserve series ordering, options, and dependencies between patches. A conflict +# must fail the build rather than silently remove a packaging or correctness fix. +dpkg-source --before-build . + +echo "==> [4/6] Target PostgreSQL majors: ${PG_VERSIONS}" +# debian/control ships with every major PGDG built (10..17); regenerate it from +# debian/control.in so only the requested majors are declared and built. +read -r -a majors <<< "${PG_VERSIONS}" +printf '%s\n' "${majors[@]}" > debian/pgversions +pg_buildext updatecontrol +echo " binary packages now declared:" +grep '^Package:' debian/control | sed 's/^/ /' + +echo "==> [5/6] Set version to ${TARGET_VERSION} and install Build-Depends" +# Explicit historical rebuilds may sort below the archived packaging version. +dch --force-bad-version --newversion "${TARGET_VERSION}" --distribution focal --force-distribution \ + "Rebuild of tdigest ${TDIGEST_VERSION} for focal (upstream PGDG focal-pgdg discontinued)." + +BUILD_OPTIONS="parallel=$(nproc)" +BUILD_PROFILES="" +if [ "${RUN_TESTS}" != "1" ]; then + # nocheck drops the postgresql-all Build-Depends (a full server + # matrix pulled in only for the upstream test suite) and skips dh_auto_test. + BUILD_OPTIONS="${BUILD_OPTIONS} nocheck" + BUILD_PROFILES="nocheck" +fi +export DEB_BUILD_OPTIONS="${BUILD_OPTIONS}" +export DEB_BUILD_PROFILES="${BUILD_PROFILES}" +# Keep debug information during compilation and let dh_strip split it into +# installable dbgsym packages. Never inherit noautodbgsym/nostrip settings. +export DEB_CFLAGS_MAINT_APPEND="${DEB_CFLAGS_MAINT_APPEND:-} -g" +export DH_STRIP_ENABLE=1 +export DH_BUILD_DDEBS=1 + +mk-build-deps --install --remove \ + --tool 'apt-get -o Debug::pkgProblemResolver=yes --yes --no-install-recommends' \ + debian/control + +echo "==> [6/6] Build binary packages (DEB_BUILD_OPTIONS='${BUILD_OPTIONS}')" +dpkg-buildpackage -b -uc -us + +echo "==> Stage artifacts for validation" +STAGING_DIR="${WORK}/artifacts" +mkdir "${STAGING_DIR}" +for v in ${PG_VERSIONS}; do + cp -v "${WORK}/postgresql-${v}-tdigest_"*.deb "${STAGING_DIR}/" + # Debug-symbol packages are emitted as .ddeb on Ubuntu. Ship them as .deb (the + # on-disk format is identical) so they flow through the same signing, + # verification and publishing as everything else. Consumers commonly install + # with an `*.deb` glob, which does not match `.ddeb`; because a dbgsym depends + # on its runtime with an exact `=` version, silently dropping it strands the + # previously installed dbgsym and breaks `apt --fix-broken install`. + shopt -s nullglob + debug_packages=("${WORK}/postgresql-${v}-tdigest-dbgsym_"*.ddeb "${WORK}/postgresql-${v}-tdigest-dbgsym_"*.deb) + shopt -u nullglob + [ "${#debug_packages[@]}" -eq 1 ] || { + echo "ERROR: expected exactly one PG${v} dbgsym package, found ${#debug_packages[@]}" >&2; exit 1; + } + dbg="${debug_packages[0]}" + cp -v "${dbg}" "${STAGING_DIR}/$(basename "${dbg%.*}").deb" +done +cp -v "${WORK}"/*.buildinfo "${WORK}"/*.changes "${STAGING_DIR}/" + +echo "==> Verify runtime <-> dbgsym pairing" +for v in ${PG_VERSIONS}; do + rt="$(dpkg-deb -f "${STAGING_DIR}/postgresql-${v}-tdigest_"*.deb Version)" + for dbg in "${STAGING_DIR}/postgresql-${v}-tdigest-dbgsym_"*.deb; do + dep="$(dpkg-deb -f "${dbg}" Depends)" + if [ "${dep}" != "postgresql-${v}-tdigest (= ${rt})" ]; then + echo "ERROR: PG${v} dbgsym wants '${dep}' but runtime is '${rt}'" >&2 + exit 1 + fi + # Verify real DWARF symbols for this runtime's ELF build ID, not just an + # empty package with the right dependency metadata. + verify_dir="$(mktemp -d "${WORK}/verify.XXXXXX")" + dpkg-deb -x "${STAGING_DIR}/postgresql-${v}-tdigest_"*.deb "${verify_dir}" + dpkg-deb -x "${dbg}" "${verify_dir}" + build_id="$(readelf -n "${verify_dir}/usr/lib/postgresql/${v}/lib/tdigest.so" | awk '/Build ID:/ {print $3}')" + [ -n "${build_id}" ] || { echo "ERROR: PG${v} runtime has no build ID" >&2; exit 1; } + symbols="${verify_dir}/usr/lib/debug/.build-id/${build_id:0:2}/${build_id:2}.debug" + sections="$(readelf -S "${symbols}")" + if ! grep -q '\.debug_info' <<< "${sections}"; then + echo "ERROR: PG${v} dbgsym contains no DWARF debug information" >&2; exit 1 + fi + rm -rf "${verify_dir}" + done + echo " PG${v} OK (${rt})" +done + +# Record exactly which source was built alongside the package artifacts. +printf '%s\n' "${resolved}" > "${STAGING_DIR}/tdigest-source.env" +printf '%s\n' "PACKAGING_SRC_VERSION=${PACKAGING_SRC_VERSION}" \ + "TARGET_VERSION=${TARGET_VERSION}" >> "${STAGING_DIR}/tdigest-source.env" + +echo "==> Replace previous tdigest artifacts in ${OUTPUT_DIR}" +# Touch the persistent output only after the entire new set passes validation. +# Remove all previous majors/versions, but preserve unrelated files. +rm -f "${OUTPUT_DIR}"/postgresql-*-tdigest_*.deb \ + "${OUTPUT_DIR}"/postgresql-*-tdigest-dbgsym_*.deb \ + "${OUTPUT_DIR}"/postgresql-*-tdigest-dbgsym_*.ddeb \ + "${OUTPUT_DIR}"/tdigest_*.buildinfo "${OUTPUT_DIR}"/tdigest_*.changes \ + "${OUTPUT_DIR}/tdigest-source.env" +cp -v "${STAGING_DIR}"/* "${OUTPUT_DIR}/" + +echo "==> DONE. Packages:" +ls -1 "${OUTPUT_DIR}"/*.deb diff --git a/scripts/resolve_tdigest_focal b/scripts/resolve_tdigest_focal new file mode 100644 index 00000000..70f66848 --- /dev/null +++ b/scripts/resolve_tdigest_focal @@ -0,0 +1,71 @@ +#!/bin/bash +# Resolve once per build; stdout is also suitable for GITHUB_ENV/GITHUB_OUTPUT. +# Automatic builds use the release asset's GitHub-published SHA-256 digest. +# Explicit checksums retain support for historical GitHub tag tarballs. +set -euo pipefail + +PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" +read -r -a majors <<< "${PG_VERSIONS}" +[[ "${PG_VERSIONS}" != *$'\n'* && ${#majors[@]} -gt 0 ]] || { + echo "ERROR: PG_VERSIONS must be a space-separated list of majors 11–16" >&2; exit 1; +} +for v in "${majors[@]}"; do + case "${v}" in + 11|12|13|14|15|16) ;; + *) echo "ERROR: unsupported PostgreSQL major '${v}'; supported range is 11–16" >&2; exit 1 ;; + esac +done + +TDIGEST_VERSION="${TDIGEST_VERSION:-latest}" +TDIGEST_SHA256="${TDIGEST_SHA256:-}" +UPSTREAM_URL="${UPSTREAM_URL:-}" +UPSTREAM_FORMAT="${UPSTREAM_FORMAT:-tar.gz}" +api=https://api.github.com/repos/tvondra/tdigest/releases +headers=(-H 'Accept: application/vnd.github+json') +if [ -n "${GITHUB_TOKEN:-}" ]; then + headers+=(-H "Authorization: Bearer ${GITHUB_TOKEN}") +fi + +if [ "${TDIGEST_VERSION}" = latest ]; then + release="$(curl -4 -fsSL --retry 3 "${headers[@]}" "${api}/latest")" + TDIGEST_VERSION="$(jq -er '.tag_name | sub("^v"; "")' <<< "${release}")" +fi +[[ "${TDIGEST_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || { + echo "ERROR: expected a stable tdigest version (major.minor.patch) or latest" >&2; exit 1; +} + +if [ -z "${TDIGEST_SHA256}" ]; then + [ -z "${UPSTREAM_URL}" ] || { + echo "ERROR: UPSTREAM_URL requires an explicit TDIGEST_SHA256" >&2; exit 1; + } + if [ -z "${release:-}" ]; then + release="$(curl -4 -fsSL --retry 3 "${headers[@]}" "${api}/tags/v${TDIGEST_VERSION}")" + fi + asset="$(jq -er --arg name "t-digest-${TDIGEST_VERSION}.zip" ' + .assets[] | select(.name == $name and .state == "uploaded") + ' <<< "${release}")" || { + echo "ERROR: release has no source asset; provide TDIGEST_SHA256 for the tag tarball" >&2; exit 1; + } + TDIGEST_SHA256="$(jq -er '.digest | select(startswith("sha256:")) | ltrimstr("sha256:")' <<< "${asset}")" || { + echo "ERROR: release asset has no SHA-256 digest; provide TDIGEST_SHA256 for the tag tarball" >&2; exit 1; + } + UPSTREAM_URL="$(jq -er '.browser_download_url' <<< "${asset}")" + UPSTREAM_FORMAT=zip +else + UPSTREAM_URL="${UPSTREAM_URL:-https://github.com/tvondra/tdigest/archive/refs/tags/v${TDIGEST_VERSION}.tar.gz}" +fi + +[[ "${TDIGEST_SHA256}" =~ ^[[:xdigit:]]{64}$ ]] || { + echo "ERROR: TDIGEST_SHA256 must contain exactly 64 hexadecimal digits" >&2; exit 1; +} +[[ "${UPSTREAM_URL}" == https://* && "${UPSTREAM_URL}" != *[[:space:]]* ]] || { + echo "ERROR: UPSTREAM_URL must be an HTTPS URL without whitespace" >&2; exit 1; +} +case "${UPSTREAM_FORMAT}" in + tar.gz|zip) ;; + *) echo "ERROR: unsupported UPSTREAM_FORMAT '${UPSTREAM_FORMAT}'" >&2; exit 1 ;; +esac + +echo "Resolved tdigest ${TDIGEST_VERSION} (${UPSTREAM_FORMAT}, SHA-256 ${TDIGEST_SHA256})" >&2 +printf '%s\n' "TDIGEST_VERSION=${TDIGEST_VERSION}" "TDIGEST_SHA256=${TDIGEST_SHA256}" \ + "UPSTREAM_URL=${UPSTREAM_URL}" "UPSTREAM_FORMAT=${UPSTREAM_FORMAT}" diff --git a/scripts/smoke_test_focal_tdigest_debs b/scripts/smoke_test_focal_tdigest_debs new file mode 100755 index 00000000..ddc5ed85 --- /dev/null +++ b/scripts/smoke_test_focal_tdigest_debs @@ -0,0 +1,211 @@ +#!/bin/bash +# +# smoke_test_focal_tdigest_debs -- install the built tdigest .deb set inside a +# clean Ubuntu 20.04 (focal) container and prove it actually works. +# +# The build pipeline already checks the packages exist, are signed, and pair +# correctly with their dbgsym. None of that proves the extension is usable, so +# this installs the set into a stock ubuntu:20.04 and runs CREATE EXTENSION plus +# a real aggregate query on every major. +# +# It tests both paths that matter: +# A. clean install on a stock focal + PGDG PostgreSQL +# B. install *over* PGDG's archived tdigest and ALTER EXTENSION ... UPDATE, which +# is what an in-place upgrade on an existing host actually does +# +# Usage (inside a stock ubuntu:20.04 container, as root): +# DEBS_DIR=/debs EXPECTED_TDIGEST= smoke_test_focal_tdigest_debs +# +# Overridable via environment: +# DEBS_DIR directory holding the *.deb set (default /debs) +# PG_VERSIONS majors to verify (default "11 12 13 14 15 16") +# EXPECTED_TDIGEST expected extension version (required) +# OLD_TDIGEST PGDG baseline override (default newest compatible runtime/dbgsym pair) +# PGDG_ARCHIVE_SUITE archive suite to enable (default focal-pgdg) + +set -euo pipefail + +DEBS_DIR="${DEBS_DIR:-/debs}" +PG_VERSIONS="${PG_VERSIONS:-11 12 13 14 15 16}" +EXPECTED_TDIGEST="${EXPECTED_TDIGEST:?Set EXPECTED_TDIGEST to the resolved build version}" +OLD_TDIGEST="${OLD_TDIGEST:-}" +PGDG_ARCHIVE_SUITE="${PGDG_ARCHIVE_SUITE:-focal-pgdg}" +PGDG_ARCHIVE_URL="${PGDG_ARCHIVE_URL:-https://apt-archive.postgresql.org/pub/repos/apt}" +for v in ${PG_VERSIONS}; do + case "${v}" in + 11|12|13|14|15|16) ;; + *) echo "ERROR: unsupported PostgreSQL major '${v}'; supported range is 11–16" >&2; exit 1 ;; + esac +done + +export DEBIAN_FRONTEND=noninteractive +export LANG=C.UTF-8 + +fail=0 +note() { echo "==> $*"; } +err() { echo "::error::$*" >&2; fail=1; } + +shopt -s nullglob +debs=("${DEBS_DIR}"/*.deb) +shopt -u nullglob +[ ${#debs[@]} -gt 0 ] || { echo "ERROR: no .deb files found in ${DEBS_DIR}" >&2; exit 1; } + +note "[1/6] Preparing container environment (${#debs[@]} packages)" +# postgresql-NN.postinst ends in `invoke-rc.d postgresql start $VERSION` under +# `set -e`, and there is no systemd in a container. A policy-rc.d denying the +# action makes invoke-rc.d return 0 so the postinst still creates the cluster; +# clusters are started explicitly below. +printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d +chmod +x /usr/sbin/policy-rc.d + +apt-get update -qq +apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg >/dev/null + +note "[2/6] Enabling the PGDG archive" +install -d /usr/share/keyrings +curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ + | gpg --dearmor -o /usr/share/keyrings/pgdg-archive.gpg +echo "deb [signed-by=/usr/share/keyrings/pgdg-archive.gpg] ${PGDG_ARCHIVE_URL} ${PGDG_ARCHIVE_SUITE} main 11 12 13 14 15 16" \ + > /etc/apt/sources.list.d/pgdg-archive.list +apt-get update -qq + +# Resolve before installing our packages so their versions cannot influence the +# APT candidates. Prefer a genuine extension upgrade; fall back to same-version +# package replacement. Never attempt to downgrade an extension. +declare -A old_versions +upgrade_majors=() +for v in ${PG_VERSIONS}; do + versions="$(apt-cache madison "postgresql-${v}-tdigest" | awk '{print $3}')" + debug_versions="$(apt-cache madison "postgresql-${v}-tdigest-dbgsym" | awk '{print $3}')" + older="" same="" + for version in ${versions}; do + [ -z "${OLD_TDIGEST}" ] || [ "${version}" = "${OLD_TDIGEST}" ] || continue + grep -Fxq "${version}" <<< "${debug_versions}" || continue + upstream="${version#*:}" + upstream="${upstream%-*}" + if dpkg --compare-versions "${upstream}" lt "${EXPECTED_TDIGEST}"; then + if [ -z "${older}" ] || dpkg --compare-versions "${version}" gt "${older}"; then + older="${version}" + fi + elif dpkg --compare-versions "${upstream}" eq "${EXPECTED_TDIGEST}"; then + if [ -z "${same}" ] || dpkg --compare-versions "${version}" gt "${same}"; then + same="${version}" + fi + fi + done + old="${older:-${same}}" + if [ -z "${old}" ]; then + if [ -n "${OLD_TDIGEST}" ]; then + echo "ERROR: PG${v} OLD_TDIGEST=${OLD_TDIGEST} requires an available runtime/dbgsym pair no newer than ${EXPECTED_TDIGEST}" >&2 + exit 1 + fi + note "PG${v}: skipping upgrade scenario; no compatible archived runtime/dbgsym pair for ${EXPECTED_TDIGEST} (clean install still runs)" + continue + fi + old_versions[${v}]="${old}" + upgrade_majors+=("${v}") + if [ -n "${older}" ]; then + note "PG${v} extension-upgrade baseline: ${old}" + else + note "PG${v} same-extension-version package-replacement baseline: ${old}" + fi +done + +psql_v() { su postgres -c "psql --cluster $1/main -qtAX -c \"$2\"" 2>&1; } + +ensure_cluster() { + pg_lsclusters -h | awk '{print $1"/"$2}' | grep -qx "$1/main" \ + || pg_createcluster "$1" main >/dev/null 2>&1 + pg_ctlcluster "$1" main start >/dev/null 2>&1 +} + +# apt --fix-broken may remove packages to resolve dependencies. Check that both +# runtime and symbols from the shipped set remain installed after each path. +check_packages() { + local v="$1" package expected actual + for package in "postgresql-${v}-tdigest" "postgresql-${v}-tdigest-dbgsym"; do + expected="$(dpkg-deb -f "${DEBS_DIR}/${package}"_*.deb Version)" + actual="$(dpkg-query -W -f='${Status} ${Version}' "${package}")" + if [ "${actual}" != "install ok installed ${expected}" ]; then + err "${package}: '${actual}' (expected installed ${expected})" + fi + done +} + +# Median of 1..1000 is ~500. t-digest is approximate, so accept a small window. +check_percentile() { + local v="$1" p + p="$(psql_v "${v}" "SELECT round(tdigest_percentile(i, 100, 0.5)) FROM generate_series(1,1000) i;")" + awk -v p="${p}" 'BEGIN{exit !(p+0>=480 && p+0<=520)}' +} + +note "[3/6] Scenario A: clean install" +for v in ${PG_VERSIONS}; do + apt-get install -y -qq "postgresql-${v}" >/dev/null 2>&1 +done +dpkg -i "${debs[@]}" || apt-get --fix-broken install -y +if ! apt-get --fix-broken install -y >/dev/null 2>&1; then + err "apt --fix-broken install failed after clean install" +fi + +for v in ${PG_VERSIONS}; do + check_packages "${v}" + ensure_cluster "${v}" + psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 + ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" + if [ "${ext}" = "${EXPECTED_TDIGEST}" ] && check_percentile "${v}"; then + echo " PG${v} OK ext=${ext}" + else + err "PG${v} clean install: ext='${ext}' (expected ${EXPECTED_TDIGEST}) or percentile check failed" + fi + pg_ctlcluster "${v}" main stop >/dev/null 2>&1 +done + +note "[4/6] Scenario B: upgrade over archived PGDG tdigest" +for v in "${upgrade_majors[@]}"; do + old="${old_versions[${v}]}" + apt-get remove -y -qq "postgresql-${v}-tdigest" >/dev/null 2>&1 + pg_dropcluster "${v}" main >/dev/null 2>&1 + pg_createcluster "${v}" main >/dev/null 2>&1 + apt-get install -y -qq --allow-downgrades \ + "postgresql-${v}-tdigest=${old}" \ + "postgresql-${v}-tdigest-dbgsym=${old}" >/dev/null + ensure_cluster "${v}" + psql_v "${v}" "CREATE EXTENSION tdigest;" >/dev/null 2>&1 + ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" + upstream="${old#*:}" + upstream="${upstream%-*}" + [ "${ext}" = "${upstream}" ] || { + echo "ERROR: PG${v} upgrade baseline is '${ext}', expected ${upstream}" >&2; exit 1; + } + pg_ctlcluster "${v}" main stop >/dev/null 2>&1 +done + +# Install both runtime and symbols, then let apt settle dependencies. +if [ "${#upgrade_majors[@]}" -gt 0 ]; then + dpkg -i "${debs[@]}" || apt-get --fix-broken install -y + if ! apt-get --fix-broken install -y >/dev/null 2>&1; then + err "apt --fix-broken install failed after upgrade over archived packages" + fi +fi + +note "[5/6] Verifying the extension upgrades and works after the package upgrade" +for v in "${upgrade_majors[@]}"; do + check_packages "${v}" + ensure_cluster "${v}" + out="$(psql_v "${v}" "ALTER EXTENSION tdigest UPDATE;")" + ext="$(psql_v "${v}" "SELECT extversion FROM pg_extension WHERE extname='tdigest';")" + if [ "${ext}" = "${EXPECTED_TDIGEST}" ] && check_percentile "${v}"; then + echo " PG${v} OK ext=${ext}" + else + err "PG${v} after upgrade: ext='${ext}' (expected ${EXPECTED_TDIGEST}) ${out}" + fi + pg_ctlcluster "${v}" main stop >/dev/null 2>&1 +done + +note "[6/6] Result" +if [ "${fail}" -ne 0 ]; then + echo "SMOKE TEST FAILED" >&2 + exit 1 +fi +echo "SMOKE TEST PASSED"