diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 576628a9..2ecb544c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: curl -L https://dl.dagger.io/dagger/install.sh | BIN_DIR=$HOME/.local/bin sh - name: Run CI task run: | - task ci + task all:ci publish: runs-on: ${{ vars.CI_RUNNERS || 'ubuntu-latest' }} @@ -76,7 +76,7 @@ jobs: curl -L https://dl.dagger.io/dagger/install.sh | BIN_DIR=$HOME/.local/bin sh - name: Write manifest run: | - task manifest + task core:manifest - name: Publish images if: | github.event_name == 'workflow_dispatch' || @@ -85,9 +85,65 @@ jobs: REGISTRY_USER: ${{ github.actor }} REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} run: | - task publish + task core:publish - name: Attach manifest to workflow run uses: actions/upload-artifact@v7 with: name: manifest.yaml path: ./manifest.yaml + + integration-tests: + name: Integration Tests + runs-on: ${{ vars.CI_RUNNERS || 'ubuntu-latest' }} + permissions: + contents: read + steps: + - name: Cleanup Disk + if: vars.CI_RUNNERS == '' || vars.CI_RUNNERS == 'ubuntu-latest' + uses: jlumbroso/free-disk-space@v1.3.1 + with: + android: true + dotnet: true + haskell: true + tool-cache: true + large-packages: false + swap-storage: false + - name: Cleanup docker cache + if: vars.CI_RUNNERS == '' || vars.CI_RUNNERS == 'ubuntu-latest' + run: | + docker system prune -a -f + - name: Checkout plugin-barman-cloud + uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + ref: ${{ github.event.pull_request.head.sha }} + - name: Checkout CloudNativePG + uses: actions/checkout@v7.0.1 + with: + repository: cloudnative-pg/cloudnative-pg + path: cloudnativepg + fetch-depth: 0 + - name: Install Task + uses: arduino/setup-task@v3.0.0 + - name: Install Dagger + env: + # renovate: datasource=github-tags depName=dagger/dagger versioning=semver + DAGGER_VERSION: 0.21.8 + run: | + curl -L https://dl.dagger.io/dagger/install.sh | BIN_DIR=$HOME/.local/bin sh + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: cloudnativepg/go.mod + - name: Start Kind cluster + id: kind-cluster + working-directory: cloudnativepg + run: | + hack/setup-cluster.sh create load deploy + CLUSTERS=$(kind get clusters) + echo "clusters=$CLUSTERS" >> "$GITHUB_OUTPUT" + - name: Run integration tests + env: + KIND_CLUSTER_NAME: ${{ steps.kind-cluster.outputs.clusters }} + run: | + task integration:e2e diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 24f62d34..1de77b67 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -39,8 +39,8 @@ jobs: REGISTRY_USER: ${{ github.actor }} REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} run: | - task publish - task manifest + task core:publish + task core:manifest - name: Attach manifest to workflow run uses: actions/upload-artifact@v7 with: diff --git a/.github/workflows/release-publish.yml b/.github/workflows/release-publish.yml index ddae8570..93660893 100644 --- a/.github/workflows/release-publish.yml +++ b/.github/workflows/release-publish.yml @@ -29,10 +29,10 @@ jobs: REGISTRY_USER: ${{ github.actor }} REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} run: | - task publish - task manifest + task core:publish + task core:manifest - name: Attach manifest to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - task upload-manifest-to-release + task core:upload-manifest-to-release diff --git a/Makefile b/Makefile deleted file mode 100644 index a351bd34..00000000 --- a/Makefile +++ /dev/null @@ -1,200 +0,0 @@ -# Image URL to use all building/pushing image targets -IMG ?= plugin-barman-cloud:latest -# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. -ENVTEST_K8S_VERSION = 1.31.0 - -# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) -ifeq (,$(shell go env GOBIN)) -GOBIN=$(shell go env GOPATH)/bin -else -GOBIN=$(shell go env GOBIN) -endif - -# CONTAINER_TOOL defines the container tool to be used for building images. -# Be aware that the target commands are only tested with Docker which is -# scaffolded by default. However, you might want to replace it to use other -# tools. (i.e. podman) -CONTAINER_TOOL ?= docker - -# Setting SHELL to bash allows bash commands to be executed by recipes. -# Options are set to exit when a recipe line exits non-zero or a piped command fails. -SHELL = /usr/bin/env bash -o pipefail -.SHELLFLAGS = -ec - -.PHONY: all -all: build - -##@ General - -# The help target prints out all targets with their descriptions organized -# beneath their categories. The categories are represented by '##@' and the -# target descriptions by '##'. The awk command is responsible for reading the -# entire set of makefiles included in this invocation, looking for lines of the -# file as xyz: ## something, and then pretty-format the target and help. Then, -# if there's a line with ##@ something, that gets pretty-printed as a category. -# More info on the usage of ANSI control characters for terminal formatting: -# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters -# More info on the awk command: -# http://linuxcommand.org/lc3_adv_awk.php - -.PHONY: help -help: ## Display this help. - @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) - -##@ Development - -.PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. - $(CONTROLLER_GEN) rbac:roleName=plugin-barman-cloud crd webhook paths="./api/..." paths="./internal/controller/..." output:crd:artifacts:config=config/crd/bases - -.PHONY: generate -generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. - $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./api/..." - -.PHONY: fmt -fmt: ## Run go fmt against code. - go fmt ./... - -.PHONY: vet -vet: ## Run go vet against code. - go vet ./... - -.PHONY: test -test: manifests generate fmt vet envtest ## Run tests. - KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out - -# Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors. -.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up. -test-e2e: - go test ./test/e2e/ -v -ginkgo.v - -.PHONY: lint -lint: golangci-lint ## Run golangci-lint linter - $(GOLANGCI_LINT) run - -.PHONY: lint-fix -lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes - $(GOLANGCI_LINT) run --fix - -##@ Build - -.PHONY: build -build: manifests generate fmt vet ## Build manager binary. - go build -o bin/manager cmd/main.go - -.PHONY: run -run: manifests generate fmt vet ## Run a controller from your host. - go run ./cmd/main.go - -# If you wish to build the manager image targeting other platforms you can use the --platform flag. -# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. -# More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -.PHONY: docker-build -docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . - -.PHONY: docker-push -docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} - -# PLATFORMS defines the target platforms for the manager image be built to provide support to multiple -# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: -# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/ -# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/ -# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=> then the export will fail) -# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option. -PLATFORMS ?= linux/arm64,linux/amd64,linux/s390x,linux/ppc64le -.PHONY: docker-buildx -docker-buildx: ## Build and push docker image for the manager for cross-platform support - # copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile - sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - - $(CONTAINER_TOOL) buildx create --name plugin-barman-cloud-builder - $(CONTAINER_TOOL) buildx use plugin-barman-cloud-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . - - $(CONTAINER_TOOL) buildx rm plugin-barman-cloud-builder - rm Dockerfile.cross - -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - -##@ Deployment - -ifndef ignore-not-found - ignore-not-found = false -endif - -.PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - - -.PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -.PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - - -.PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - - -##@ Dependencies - -## Location to install dependencies to -LOCALBIN ?= $(shell pwd)/bin -$(LOCALBIN): - mkdir -p $(LOCALBIN) - -## Tool Binaries -KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize -CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen -ENVTEST ?= $(LOCALBIN)/setup-envtest -GOLANGCI_LINT = $(LOCALBIN)/golangci-lint - -## Tool Versions -KUSTOMIZE_VERSION ?= v5.4.3 -CONTROLLER_TOOLS_VERSION ?= v0.16.1 -ENVTEST_VERSION ?= release-0.19 -GOLANGCI_LINT_VERSION ?= v1.64.8 - -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) - -.PHONY: controller-gen -controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. -$(CONTROLLER_GEN): $(LOCALBIN) - $(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION)) - -.PHONY: envtest -envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. -$(ENVTEST): $(LOCALBIN) - $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) - -.PHONY: golangci-lint -golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. -$(GOLANGCI_LINT): $(LOCALBIN) - $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) - -# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist -# $1 - target path with name of binary -# $2 - package url which can be installed -# $3 - specific version of package -define go-install-tool -@[ -f "$(1)-$(3)" ] || { \ -set -e; \ -package=$(2)@$(3) ;\ -echo "Downloading $${package}" ;\ -rm -f $(1) || true ;\ -GOBIN=$(LOCALBIN) go install $${package} ;\ -mv $(1) $(1)-$(3) ;\ -} ;\ -ln -sf $(1)-$(3) $(1) -endef diff --git a/Taskfile.yml b/Taskfile.yml index b80daf29..90103b47 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -12,41 +12,172 @@ vars: E2E_KUBERNETES_VERSION: v1.36.1 E2E_CLUSTER_NAME: barman-cloud-plugin-e2e-{{.E2E_KUBERNETES_VERSION}} REGISTRY_NETWORK: barman-cloud-plugin - REGISTRY_NAME: registry.barman-cloud-plugin - REGISTRY_PORT: 5000 + REGISTRY_NAME: '{{ .REGISTRY_NAME | default "registry.dev" }}' + REGISTRY_PORT: '{{ .REGISTRY_PORT | default 5000 }}' + GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY | default "cloudnative-pg/plugin-barman-cloud"}}' DAGGER_ENGINE_CONTAINER_NAME: e2e-dagger-engine + tasks: - lint: + + #################################### + ## ALL + #################################### + + all:ci: + desc: Run the CI pipeline + deps: + - core:ci + + #################################### + ## CORE + #################################### + + core:ci: + desc: Run the CI core pipeline + deps: + - internal:spellcheck + - internal:commitlint + - internal:uncommitted + - internal:lint + - internal:go-test + - documentation:docusaurus-build-test + - documentation:docusaurus-version-is-aligned + + # run the go linters + core:lint: + desc: Run the linters + deps: + - internal:lint + + # run the go linters + core:uncommitted: + desc: Check for uncommitted changes + deps: + - internal:uncommitted + + # run the spellcheck + core:spellcheck: + desc: Run the spellcheck + deps: + - internal:spellcheck + + # Generate the manifest from the current branch + core:manifest: + desc: Generate the manifest to be uploaded to the release or as a workflow artifact + deps: + - internal:controller-gen + vars: + GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY | default "cloudnative-pg/plugin-barman-cloud"}}' + CURRENT_BRANCH: + sh: git rev-parse --abbrev-ref HEAD + GITHUB_REF: '{{.GITHUB_REF | default (printf "refs/heads/%s" .CURRENT_BRANCH)}}' + GITHUB_REF_NAME: '{{.GITHUB_REF_NAME | default .CURRENT_BRANCH}}' + cmds: + - task: internal:manifest + vars: + GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY}}' + GITHUB_REF: '{{.GITHUB_REF}}' + GITHUB_REF_NAME: '{{.GITHUB_REF_NAME}}' + + core:go-test: + desc: Run the Go tests + deps: + - internal:go-test + + core:publish: + desc: Publish the container images + requires: + # We expect this to run in a GitHub workflow, so we put a few GitHub-specific vars here + # to prevent running this task locally by accident. + vars: + - CI + - REGISTRY_USER + - REGISTRY_PASSWORD + vars: + GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY | default "cloudnative-pg/plugin-barman-cloud"}}' + CURRENT_BRANCH: + sh: git rev-parse --abbrev-ref HEAD + GITHUB_REF: '{{.GITHUB_REF | default (printf "refs/heads/%s" .CURRENT_BRANCH)}}' + GITHUB_REF_NAME: '{{.GITHUB_REF_NAME | default .CURRENT_BRANCH}}' + cmds: + - task: internal:publish + vars: + GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY}}' + GITHUB_REF: '{{.GITHUB_REF}}' + GITHUB_REF_NAME: '{{.GITHUB_REF_NAME}}' + REGISTRY_USER: '{{.REGISTRY_USER}}' + REGISTRY_PASSWORD: '{{.REGISTRY_PASSWORD}}' + + core:upload-manifest-to-release: + desc: Upload the manifest to the release + requires: + # We expect this to run in a GitHub workflow, so we put a few GitHub-specific vars here + # to prevent running this task locally by accident. + vars: + - CI + - GITHUB_TOKEN + env: + # renovate: datasource=git-refs depName=gh lookupName=https://github.com/sagikazarmark/daggerverse + DAGGER_GH_SHA: ff27cd50f6b4eed2e3753c520632cd6099e1ce52 + preconditions: + - sh: "[[ {{.GITHUB_REF}} =~ 'refs/tags/v.*' ]]" + msg: not a tag, failing + vars: + GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY | default "cloudnative-pg/plugin-barman-cloud"}}' + CURRENT_BRANCH: + sh: git rev-parse --abbrev-ref HEAD + GITHUB_REF: '{{.GITHUB_REF | default (printf "refs/heads/%s" .CURRENT_BRANCH)}}' + GITHUB_REF_NAME: '{{.GITHUB_REF_NAME | default .CURRENT_BRANCH}}' + cmds: + - > + GITHUB_REF= dagger -s call -m github.com/sagikazarmark/daggerverse/gh@${DAGGER_GH_SHA} + with-source --source . + run --repo {{.GITHUB_REPOSITORY}} --token env:GITHUB_TOKEN + --cmd "release upload {{.GITHUB_REF_NAME}} manifest.yaml" + + core:build-images: + desc: Build the container images for the plugin + deps: + - internal:build-plugin-image + - internal:build-sidecar-image + + #################################### + ## INTERNAL + #################################### + + internal:lint: desc: Run golangci-lint env: # renovate: datasource=git-refs depName=golangci-lint lookupName=https://github.com/sagikazarmark/daggerverse currentValue=main DAGGER_GOLANGCI_LINT_SHA: ff27cd50f6b4eed2e3753c520632cd6099e1ce52 - # renovate: datasource=docker depName=golangci/golangci-lint versioning=semver - GOLANGCI_LINT_VERSION: v2.12.2 + # renovate: datasource=docker depName=golangci/golangci-lint versioning=docker + GOLANGCI_LINT_VERSION: v2.12.2@sha256:5cceeef04e53efe1470638d4b4b4f5ceefd574955ab3941b2d9a68a8c9ad5240 + internal: true cmds: - > - GITHUB_REF= dagger -sc "github.com/sagikazarmark/daggerverse/golangci-lint@${DAGGER_GOLANGCI_LINT_SHA} + echo $(pwd); + GITHUB_REF= dagger -c "github.com/sagikazarmark/daggerverse/golangci-lint@${DAGGER_GOLANGCI_LINT_SHA} --version ${GOLANGCI_LINT_VERSION} | run . --config .golangci.yml | stdout" - sources: - - ./**/*.go - wordlist-ordered: + internal:wordlist-ordered: desc: Order the word list file + internal: true cmds: - LANG=C LC_ALL=C sort .wordlist.txt > .wordlist.txt.new - mv -f .wordlist.txt.new .wordlist.txt sources: - .wordlist.txt - spellcheck: + internal:spellcheck: desc: Run spellcheck deps: - - wordlist-ordered + - internal:wordlist-ordered env: # renovate: datasource=git-refs depName=spellcheck lookupName=https://github.com/cloudnative-pg/daggerverse currentValue=main DAGGER_SPELLCHECK_SHA: 26cd57fb6ad66e88c036b675b0478d38df93962d + internal: true cmds: - > GITHUB_REF= dagger -s call -m github.com/cloudnative-pg/daggerverse/spellcheck@${DAGGER_SPELLCHECK_SHA} @@ -56,84 +187,44 @@ tasks: - .spellcheck.yaml - .wordlist.txt - commitlint: + internal:commitlint: desc: Check for conventional commits env: # renovate: datasource=git-refs depName=commitlint lookupName=https://github.com/cloudnative-pg/daggerverse currentValue=main DAGGER_COMMITLINT_SHA: 26cd57fb6ad66e88c036b675b0478d38df93962d + internal: true cmds: - > GITHUB_REF= dagger -s call -m github.com/cloudnative-pg/daggerverse/commitlint@${DAGGER_COMMITLINT_SHA} lint --source . --args "--from=origin/main" stdout - uncommitted: + internal:uncommitted: desc: Check for uncommitted changes deps: - - manifest-main - - apidoc - - wordlist-ordered + - task: internal:manifest + vars: + GITHUB_REPOSITORY: cloudnative-pg/plugin-barman-cloud + GITHUB_REF: main + GITHUB_REF_NAME: main + - documentation:apidoc + - internal:wordlist-ordered env: # renovate: datasource=git-refs depName=uncommitted lookupName=https://github.com/cloudnative-pg/daggerverse currentValue=main DAGGER_UNCOMMITTED_SHA: 26cd57fb6ad66e88c036b675b0478d38df93962d + internal: true cmds: - GITHUB_REF= dagger -s call -m github.com/cloudnative-pg/daggerverse/uncommitted@${DAGGER_UNCOMMITTED_SHA} check-uncommitted --source . stdout sources: - ./** - apidoc: - desc: Update the API Reference section of the documentation - deps: - - controller-gen - env: - # renovate: datasource=git-refs depName=crd-gen-refs lookupName=https://github.com/cloudnative-pg/daggerverse currentValue=main - DAGGER_CRDGENREF_SHA: 26cd57fb6ad66e88c036b675b0478d38df93962d - # renovate: datasource=go depName=github.com/elastic/crd-ref-docs - CRDREFDOCS_VERSION: v0.3.0 - cmds: - - > - GITHUB_REF= dagger -s call -m github.com/cloudnative-pg/daggerverse/crd-ref-docs@${DAGGER_CRDGENREF_SHA} - --version ${CRDREFDOCS_VERSION} - generate - --src . - --source-path api/v1 - --config-file hack/crd-gen-refs/config.yaml - --renderer markdown - --templates-dir hack/crd-gen-refs/markdown - file --path out.md - export --path web/docs/plugin-barman-cloud.v1.md - sources: - - ./api/**/*.go - - ./hack/docs/crd-gen-refs/**/* - - ./Taskfile.yml - - docusaurus-build-test: - desc: Check that the documentation builds - deps: - - apidoc - env: - # renovate: datasource=git-refs depName=docusaurus lookupName=https://github.com/levlaz/daggerverse currentValue=main - DAGGER_DOCUSAURUS_SHA: 93f57fccbd5afd22fa917107ab7697f3e7db0e5a - cmds: - - GITHUB_REF= dagger -s call -m github.com/levlaz/daggerverse/docusaurus@${DAGGER_DOCUSAURUS_SHA} - --src web --yarn build - src: - - ./web/**/* - - docusaurus-version-is-aligned: - desc: Verify that a documentation version exists for the current version of the plugin - cmds: - - $(GITHUB_REF= dagger -s call -m dagger/check-doc-version has-version-documentation --src .) - src: - - .release-please-manifest.json - - ./web/versions.json - - go-test: + internal:go-test: desc: Run go test env: - # renovate: datasource=git-refs depname=kubernetes packageName=https://github.com/kubernetes/kubernetes versioning=semver + # renovate: datasource=git-refs depName=kubernetes packageName=https://github.com/kubernetes/kubernetes versioning=semver K8S_VERSION: 1.31.0 # renovate: datasource=git-refs depName=controller-runtime packageName=https://github.com/kubernetes-sigs/controller-runtime versioning=semver SETUP_ENVTEST_VERSION: 0.24.1 + internal: true cmds: - > GITHUB_REF= dagger -s call -m ./dagger/gotest @@ -145,11 +236,12 @@ tasks: sources: - ./**/*.go - generate-certs: + internal:generate-certs: desc: Generate certificates for the local registry + internal: true run: once cmds: - - > + - > mkdir -p certs && docker volume create certs && docker run -v certs:/certs -w /certs --name certs --entrypoint=/bin/sh alpine/openssl -c ' @@ -168,20 +260,22 @@ tasks: - docker volume inspect certs - test -f certs/ca.pem - start-build-network: + internal:start-build-network: desc: Create a docker network for image building used by the dagger engine and the registry run: once + internal: true cmds: - docker network create {{ .REGISTRY_NETWORK}} status: - docker network inspect {{ .REGISTRY_NETWORK }} - start-registry: + internal:start-registry: desc: Start a container registry run: once + internal: true deps: - - generate-certs - - start-build-network + - internal:generate-certs + - internal:start-build-network env: # renovate: datasource=docker depName=registry versioning=semver REGISTRY_VERSION: 2 @@ -196,160 +290,53 @@ tasks: status: - \[ "$(docker inspect -f {{`'{{.State.Running}}'`}} "{{ .REGISTRY_NAME }}" 2> /dev/null )" == 'true' \] - - # Start a dagger engine that mounts the CA certificate for the local registry. - start-dagger-engine-for-local-builds: - desc: Start a dagger engine mounting the CA - run: once + internal:registry-network-connect: + desc: Make sure the registry is connected to the registry network + summary: If we started the registry on our own, it should already be connected. However, if the registry was started + somewhere else, it may not be on the expected network. + internal: true deps: - - generate-certs - - start-build-network - vars: - # renovate: datasource=github-tags depName=dagger/dagger versioning=semver - DAGGER_VERSION: 0.21.8 - DAGGER_ENGINE_IMAGE: registry.dagger.io/engine:v{{ .DAGGER_VERSION }} + - internal:start-registry + run: once cmds: - - > - docker run -d -v /var/lib/dagger --name "{{ .DAGGER_ENGINE_CONTAINER_NAME }}" - --network={{ .REGISTRY_NETWORK }} - -v certs:/usr/local/share/ca-certificates/ - --privileged {{ .DAGGER_ENGINE_IMAGE }} + - docker network connect {{ .REGISTRY_NETWORK }} {{ .REGISTRY_NAME }} || true status: - - \[ "$(docker inspect -f {{`'{{.State.Running}}'`}} "{{ .DAGGER_ENGINE_CONTAINER_NAME }}" 2> /dev/null )" == 'true' \] + - docker network inspect {{ .REGISTRY_NETWORK }} -f '{{ range .Containers }}{{ .Name }} {{ end }}' | grep "{{ .REGISTRY_NAME }}" # We build an image and push it to a local registry. # The name is always `plugin-barman-cloud:testing`. - build-plugin-image: - desc: Build the operator container image for the plugin + internal:build-plugin-image: + desc: Build the operator container image for the plugin using the local Docker engine + internal: true deps: - - start-registry - - start-dagger-engine-for-local-builds + - internal:registry-network-connect env: # renovate: datasource=git-refs depName=docker lookupName=https://github.com/purpleclay/daggerverse currentValue=main DAGGER_DOCKER_SHA: ee12c1a4a2630e194ec20c5a9959183e3a78c192 - _EXPERIMENTAL_DAGGER_RUNNER_HOST: docker-container://{{ .DAGGER_ENGINE_CONTAINER_NAME }} cmds: - > - GITHUB_REF= dagger -s call -m github.com/purpleclay/daggerverse/docker@${DAGGER_DOCKER_SHA} - build --dir . --file containers/Dockerfile.plugin --args GO_VERSION={{.GO_VERSION}} --platform linux/amd64 - publish --ref {{ .REGISTRY_NAME }}:{{ .REGISTRY_PORT }}/plugin-barman-cloud --tags testing + dagger run docker buildx build -f containers/Dockerfile.plugin --build-arg GO_VERSION={{.GO_VERSION}} + --tag localhost:{{.REGISTRY_PORT}}/plugin-barman-cloud:testing --push . # We build an image and push it to a local registry. # The name is always `sidecar-barman-cloud:testing`. - build-sidecar-image: - desc: Build the sidecar container image for the plugin + internal:build-sidecar-image: + desc: Build the sidecar container image for the plugin using the local Docker engine + internal: true deps: - - start-registry - - start-dagger-engine-for-local-builds + - internal:registry-network-connect env: # renovate: datasource=git-refs depName=docker lookupName=https://github.com/purpleclay/daggerverse currentValue=main DAGGER_DOCKER_SHA: ee12c1a4a2630e194ec20c5a9959183e3a78c192 - _EXPERIMENTAL_DAGGER_RUNNER_HOST: docker-container://{{ .DAGGER_ENGINE_CONTAINER_NAME }} cmds: - > - GITHUB_REF= dagger -s call -m github.com/purpleclay/daggerverse/docker@${DAGGER_DOCKER_SHA} - build --dir . --file containers/Dockerfile.sidecar --args GO_VERSION={{.GO_VERSION}} --platform linux/amd64 - publish --ref {{ .REGISTRY_NAME }}:{{ .REGISTRY_PORT }}/sidecar-barman-cloud --tags testing - - build-images: - desc: Build the container images for the plugin - deps: - - build-plugin-image - - build-sidecar-image - - # Install kind if not at the expected version. - install-kind: - desc: Install kind - run: once - vars: - # renovate: datasource=git-refs depName=kind lookupName=https://github.com/kubernetes-sigs/kind versioning=semver - KIND_VERSION: v0.32.0 - cmds: - - go install sigs.k8s.io/kind@{{.KIND_VERSION}} - - kind version | grep -q {{.KIND_VERSION}} - status: - - kind version | grep -q {{.KIND_VERSION}} + dagger run docker buildx build -f containers/Dockerfile.sidecar --build-arg GO_VERSION={{.GO_VERSION}} + --tag localhost:{{.REGISTRY_PORT}}/sidecar-barman-cloud:testing --push . - start-kind-cluster: - desc: Start a kind cluster - deps: - - install-kind - - start-build-network - - generate-certs - run: once - cmds: - - > - kind create cluster --name {{ .E2E_CLUSTER_NAME }} - --image kindest/node:{{ .E2E_KUBERNETES_VERSION }} - --config hack/kind-config.yaml - --wait 5m - - > - for node in $(kind get nodes --name {{ .E2E_CLUSTER_NAME }} ); do - docker network connect {{ .REGISTRY_NETWORK }} $node; - docker exec $node sh -c "update-ca-certificates"; - done - status: - - kind get clusters | grep -q {{ .E2E_CLUSTER_NAME }} - - # TODO: see if it is possible to daggerize this. It will have to manage docker to make kind work. - # TODO: add a task to clean up the kind cluster for new test runs. - # Run the e2e tests. This task will start a kind cluster, deploy the plugin, and run the tests. - # Running the e2e tests requires: - # * The registry to have a valid TLS certificate. - # * The registry to be in the same network of the dagger-engine. - # * The dagger-engine to mount the CA. - # * The kind cluster to mount the CA. - e2e-external-kind: - desc: Run e2e tests in a local kind cluster - deps: - - build-images - - start-kind-cluster - vars: - KUBECONFIG_PATH: - sh: mktemp -t kubeconfig-XXXXX - env: - _EXPERIMENTAL_DAGGER_RUNNER_HOST: docker-container://{{ .DAGGER_ENGINE_CONTAINER_NAME }} - cmds: - - kind get kubeconfig --internal --name {{ .E2E_CLUSTER_NAME }} > {{ .KUBECONFIG_PATH }} - - > - GITHUB_REF= dagger call -m dagger/e2e/ run - --source . - --kubeconfig {{.KUBECONFIG_PATH}} - --go-version {{.GO_VERSION}} - - e2e-ephemeral: - desc: Run e2e tests in an ephemeral k3s cluster - deps: - - build-images - env: - _EXPERIMENTAL_DAGGER_RUNNER_HOST: docker-container://{{ .DAGGER_ENGINE_CONTAINER_NAME }} - cmds: - - > - GITHUB_REF= dagger call -m dagger/e2e/ run-ephemeral - --source . - --ca certs/ca.pem - --registry {{.REGISTRY_NAME}}:{{.REGISTRY_PORT}} - --go-version {{.GO_VERSION}} - - ci: - desc: Run the CI pipeline - deps: - - spellcheck - - commitlint - - uncommitted - - lint - - go-test - - e2e-ephemeral - - docusaurus-build-test - - docusaurus-version-is-aligned - - publish: + internal:publish: desc: Build and publish a container image for the plugin requires: - # We expect this to run in a GitHub workflow, so we put a few GitHub-specific vars here - # to prevent running this task locally by accident. vars: - - CI - GITHUB_REPOSITORY - GITHUB_REF - GITHUB_REF_NAME @@ -366,6 +353,7 @@ tasks: env: # renovate: datasource=git-refs depName=docker lookupName=https://github.com/purpleclay/daggerverse currentValue=main DAGGER_DOCKER_SHA: ee12c1a4a2630e194ec20c5a9959183e3a78c192 + internal: true cmds: - > GITHUB_REF= dagger call -m github.com/purpleclay/daggerverse/docker@${DAGGER_DOCKER_SHA} @@ -378,12 +366,13 @@ tasks: build --dir . --file containers/Dockerfile.sidecar --args GO_VERSION={{.GO_VERSION}} --platform linux/amd64 --platform linux/arm64 publish --ref {{.SIDECAR_IMAGE_NAME}} --tags {{.IMAGE_VERSION}} - controller-gen: + internal:controller-gen: desc: Run controller-gen run: once env: # renovate: datasource=git-refs depName=controller-gen lookupName=https://github.com/cloudnative-pg/daggerverse currentValue=main DAGGER_CONTROLLER_GEN_SHA: 26cd57fb6ad66e88c036b675b0478d38df93962d + internal: true cmds: - > GITHUB_REF= dagger -s call -m github.com/cloudnative-pg/daggerverse/controller-gen@${DAGGER_CONTROLLER_GEN_SHA} @@ -404,41 +393,7 @@ tasks: - ./config/rbac/role.yaml - ./config/crd/bases/*.yaml - manifest-main: - deps: - - controller-gen - desc: Generate the manifest for the main branch - vars: - GITHUB_REPOSITORY: cloudnative-pg/plugin-barman-cloud - GITHUB_REF: main - GITHUB_REF_NAME: main - cmds: - - task: manifest-internal - vars: - GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY}}' - GITHUB_REF: '{{.GITHUB_REF}}' - GITHUB_REF_NAME: '{{.GITHUB_REF_NAME}}' - - manifest: - desc: Generate the manifest to be uploaded to the release or as a workflow artifact - deps: - - controller-gen - requires: - # We expect this to run in a GitHub workflow, so we put a few GitHub-specific vars here - # to prevent running this task locally by accident. - vars: - - CI - - GITHUB_REPOSITORY - - GITHUB_REF - - GITHUB_REF_NAME - cmds: - - task: manifest-internal - vars: - GITHUB_REPOSITORY: '{{.GITHUB_REPOSITORY}}' - GITHUB_REF: '{{.GITHUB_REF}}' - GITHUB_REF_NAME: '{{.GITHUB_REF_NAME}}' - - manifest-internal: + internal:manifest: desc: Update the image in the Kustomization label: manifest-internal-{{.GITHUB_REF_NAME}} internal: true @@ -474,31 +429,142 @@ tasks: generates: - ./manifest.yaml - upload-manifest-to-release: - desc: Upload the manifest to the release + internal:clean: + desc: Remove autogenerated artifacts + internal: true + cmds: + - rm -rf .task/ + + #################################### + ## DOCUMENTATION + #################################### + + documentation:apidoc: + desc: Update the API Reference section of the documentation + deps: + - internal:controller-gen + env: + # renovate: datasource=git-refs depName=crd-gen-refs lookupName=https://github.com/cloudnative-pg/daggerverse currentValue=main + DAGGER_CRDGENREF_SHA: 26cd57fb6ad66e88c036b675b0478d38df93962d + # renovate: datasource=go depName=github.com/elastic/crd-ref-docs + CRDREFDOCS_VERSION: v0.3.0 + cmds: + - > + GITHUB_REF= dagger -s call -m github.com/cloudnative-pg/daggerverse/crd-ref-docs@${DAGGER_CRDGENREF_SHA} + --version ${CRDREFDOCS_VERSION} + generate + --src . + --source-path api/v1 + --config-file hack/crd-gen-refs/config.yaml + --renderer markdown + --templates-dir hack/crd-gen-refs/markdown + file --path out.md + export --path web/docs/plugin-barman-cloud.v1.md + sources: + - ./api/**/*.go + - ./hack/docs/crd-gen-refs/**/* + - ./Taskfile.yml + + documentation:docusaurus-build-test: + desc: Check that the documentation builds + deps: + - documentation:apidoc + env: + # renovate: datasource=git-refs depName=docusaurus lookupName=https://github.com/levlaz/daggerverse currentValue=main + DAGGER_DOCUSAURUS_SHA: 93f57fccbd5afd22fa917107ab7697f3e7db0e5a + cmds: + - GITHUB_REF= dagger -s call -m github.com/levlaz/daggerverse/docusaurus@${DAGGER_DOCUSAURUS_SHA} + --src web --yarn build + sources: + - ./web/**/* + + documentation:docusaurus-version-is-aligned: + desc: Verify that a documentation version exists for the current version of the plugin + cmds: + - $(GITHUB_REF= dagger -s call -m dagger/check-doc-version has-version-documentation --src .) + sources: + - .release-please-manifest.json + - ./web/versions.json + + + #################################### + ## integration + #################################### + + # Deploy the plugin-barman-cloud to kind cluster + # To use KIND_CLUSTER_NAME=pg-operator-e2e-v1-33-1 task integration:deploy-to-kind + integration:deploy-to-kind: + desc: Deploy the operator on a kind cluster generated by the CNPG hack setup script + summary: | + Deploys the plugin-barman-cloud operator to a Kind cluster for integration testing. This task assumes a + Kind cluster has been previously created by the CloudNativePG hack/setup-cluster.sh script. It loads + the appropriate kubeconfig and applies the operator manifest via kustomize. + deps: + - internal:build-plugin-image + - internal:build-sidecar-image + run: once requires: - # We expect this to run in a GitHub workflow, so we put a few GitHub-specific vars here - # to prevent running this task locally by accident. vars: - - CI - - GITHUB_REPOSITORY - - GITHUB_REF - - GITHUB_REF_NAME - - GITHUB_TOKEN - env: - # renovate: datasource=git-refs depName=gh lookupName=https://github.com/sagikazarmark/daggerverse - DAGGER_GH_SHA: ff27cd50f6b4eed2e3753c520632cd6099e1ce52 - preconditions: - - sh: "[[ {{.GITHUB_REF}} =~ 'refs/tags/v.*' ]]" - msg: not a tag, failing + - KIND_CLUSTER_NAME + vars: + KUBECONFIG_PATH: + sh: mktemp -t kubeconfig-XXXXX + KIND_HOSTPORT: + # This command extracts the host port for the control plane of the Kind cluster. + # Redundant curly braces are used to escape the template syntax in Taskfile. + sh: docker inspect --format '{{"{{"}} (index (index .HostConfig.PortBindings "6443/tcp") 0).HostPort {{"}}"}}' {{ .KIND_CLUSTER_NAME }}-control-plane + PLUGIN_IMAGE_REPOSITORY: '{{ .PLUGIN_IMAGE_REPOSITORY | default (printf "%s:%s/plugin-barman-cloud" .REGISTRY_NAME .REGISTRY_PORT) }}' + PLUGIN_IMAGE_TAG: '{{ .PLUGIN_IMAGE_TAG | default "testing" }}' + SIDECAR_IMAGE_REPOSITORY: '{{ .SIDECAR_IMAGE_REPOSITORY | default (printf "%s:%s/sidecar-barman-cloud" .REGISTRY_NAME .REGISTRY_PORT) }}' + SIDECAR_IMAGE_TAG: '{{ .SIDECAR_IMAGE_TAG | default "testing" }}' + # renovate: datasource=docker depName=alpine/k8s versioning=docker + ALPINE_K8S_VERSION: 1.36.2@sha256:44ef4942e171939b9c665a4a84beb80e2dcdb9a24330d4651cfdfd2e9deecc47 + # renovate: datasource=github-releases depName=cert-manager/cert-manager versioning=semver + CERT_MANAGER_VERSION: v1.21.1 + # Cache busting timestamp to prevent Dagger from caching the deployment steps + CACHE_BUST: + sh: date +%s cmds: - - > - GITHUB_REF= dagger -s call -m github.com/sagikazarmark/daggerverse/gh@${DAGGER_GH_SHA} - with-source --source . - run --repo {{.GITHUB_REPOSITORY}} --token env:GITHUB_TOKEN - --cmd "release upload {{.GITHUB_REF_NAME}} manifest.yaml" + - kind get kubeconfig --name {{ .KIND_CLUSTER_NAME }} > {{ .KUBECONFIG_PATH }} + - | + dagger shell << "." + container | from alpine/k8s:{{ .ALPINE_K8S_VERSION }} | with-mounted-file /kube/.config $(host | file {{ .KUBECONFIG_PATH }}) | + with-directory /src . | + with-env-variable KUBECONFIG /kube/.config | + with-service-binding localhost tcp://127.0.0.1:{{ .KIND_HOSTPORT }} | + with-exec -- sh -c "kubectl config set-cluster kind-{{ .KIND_CLUSTER_NAME }} --server=https://localhost:{{ .KIND_HOSTPORT }}" | + with-exec -- sh -c "kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/{{ .CERT_MANAGER_VERSION }}/cert-manager.yaml" | + with-exec -- sh -c "kubectl wait --for=condition=Available --timeout=120s -n cert-manager deployment --all" | + with-env-variable CACHE_BUST {{ .CACHE_BUST }} | + with-workdir /src/kubernetes | + with-exec -- sh -c "kustomize edit set image plugin-barman-cloud={{ .PLUGIN_IMAGE_REPOSITORY }}:{{ .PLUGIN_IMAGE_TAG }}" | + with-exec -- sh -c "kustomize edit set secret plugin-barman-cloud --from-literal=SIDECAR_IMAGE={{ .SIDECAR_IMAGE_REPOSITORY }}:{{ .SIDECAR_IMAGE_TAG }}" | + with-exec -- sh -c "kustomize build . | kubectl apply -f -" + . + - rm -f {{ .KUBECONFIG_PATH }} - clean: - desc: Remove autogenerated artifacts + # Run e2e tests on the kind cluster specified by KIND_CLUSTER_NAME + integration:e2e: + desc: Run the e2e tests + summary: | + Run e2e against the Kind cluster specified by KIND_CLUSTER_NAME + deps: + - integration:deploy-to-kind + run: once + requires: + vars: + - KIND_CLUSTER_NAME + vars: + KUBECONFIG_PATH: + sh: mktemp -t kubeconfig-XXXXX + KIND_HOSTPORT: + # This command extracts the host port for the control plane of the Kind cluster. + # Redundant curly braces are used to escape the template syntax in Taskfile. + sh: docker inspect --format '{{"{{"}} (index (index .HostConfig.PortBindings "6443/tcp") 0).HostPort {{"}}"}}' {{ .KIND_CLUSTER_NAME }}-control-plane cmds: - - rm -rf .task/ + - kind get kubeconfig --name {{ .KIND_CLUSTER_NAME }} > {{ .KUBECONFIG_PATH }} + - defer: rm -f {{ .KUBECONFIG_PATH }} + - > + kubectl --kubeconfig={{.KUBECONFIG_PATH}} config set-cluster kind-{{.KIND_CLUSTER_NAME}} + --server=https://127.0.0.1:{{.KIND_HOSTPORT}} --insecure-skip-tls-verify=true + - KUBECONFIG={{.KUBECONFIG_PATH}} go test ./test/e2e/... -v -ginkgo.v -timeout 45m diff --git a/dagger/e2e/main.go b/dagger/e2e/main.go index 5046ed47..d4113ef5 100644 --- a/dagger/e2e/main.go +++ b/dagger/e2e/main.go @@ -72,7 +72,7 @@ func (m *E2E) RunEphemeral( source *dagger.Directory, // registry is a private registry // +optional - // +default="registry.barman-cloud-plugin:5000" + // +default="registry.dev:5000" registry string, // ca is the certificate authority for the registry // +optional diff --git a/scripts/cleanup.sh b/hack/cleanup.sh similarity index 100% rename from scripts/cleanup.sh rename to hack/cleanup.sh diff --git a/scripts/minio-delete.sh b/hack/minio-delete.sh similarity index 100% rename from scripts/minio-delete.sh rename to hack/minio-delete.sh diff --git a/scripts/run.sh b/hack/run.sh similarity index 100% rename from scripts/run.sh rename to hack/run.sh diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 0f2bfdf7..4dc26af2 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -64,7 +64,7 @@ var _ = SynchronizedBeforeSuite(func(ctx SpecContext) []byte { Images: []kustomizeTypes.Image{ { Name: "ghcr.io/cloudnative-pg/plugin-barman-cloud-testing", - NewName: "registry.barman-cloud-plugin:5000/plugin-barman-cloud", + NewName: "registry.dev:5000/plugin-barman-cloud", NewTag: "testing", }, }, @@ -74,7 +74,7 @@ var _ = SynchronizedBeforeSuite(func(ctx SpecContext) []byte { Name: "plugin-barman-cloud", Behavior: "replace", KvPairSources: kustomizeTypes.KvPairSources{ - LiteralSources: []string{"SIDECAR_IMAGE=registry.barman-cloud-plugin:5000/sidecar-barman-cloud:testing"}, + LiteralSources: []string{"SIDECAR_IMAGE=registry.dev:5000/sidecar-barman-cloud:testing"}, }, }, }, diff --git a/test/e2e/internal/e2etestenv/main.go b/test/e2e/internal/e2etestenv/main.go index 5400d364..9a7b9b59 100644 --- a/test/e2e/internal/e2etestenv/main.go +++ b/test/e2e/internal/e2etestenv/main.go @@ -98,7 +98,7 @@ func WithIgnoreExistingResources(ignore bool) SetupOption { func defaultSetupOptions() SetupOptions { return SetupOptions{ - CertManagerVersion: certmanager.DefaultVersion, + // CertManagerVersion: certmanager.DefaultVersion, } }