pipeline {
	agent none

	options {
		// Every build needs a lot of executors. If there is something newer to
		// build, there is no point in finishing what has been built before.
		disableConcurrentBuilds(abortPrevious: true)
	}

	stages {
		/*
			Run the build and test suite on various distributions...
		*/
		stage("Run Tests on Multiple Distributions") {
			matrix {
				axes {
					axis {
						name "DISTRO"
						values \
							"archlinux:base-devel", \
							"debian:sid", \
							"debian:trixie", \
							"debian:bookworm", \
							"fedora:rawhide", \
							"fedora:45", \
							"fedora:44", \
							"ubuntu:26.10", \
							"ubuntu:26.04"
					}

					axis {
						name "COMPILER"
						values "gcc", "clang"
					}
				}

				agent {
					docker {
						image "${DISTRO}"

						// Run as root inside the containers to install dependencies
						args "--init --privileged -u root"

						customWorkspace "${JOB_NAME}/${BUILD_ID}/${env.DISTRO.replace(":", "-")}/${env.COMPILER}"
					}
				}

				stages {
					stage("Install Dependencies") {
						steps {
							script {
								// Arch Linux
								if (env.DISTRO.contains("archlinux")) {
									installBuildDepsArchLinux(env.DISTRO, env.COMPILER)

								// Fedora, etc.
								} else if (env.DISTRO.contains("fedora")) {
									installBuildDepsRedHat(env.DISTRO, env.COMPILER)

								// Debian & Ubuntu
								} else if (env.DISTRO.contains("debian") || env.DISTRO.contains("ubuntu")) {
									installBuildDepsDebian(env.DISTRO, env.COMPILER)
								}
							}
						}
					}

					stage("Configure") {
						steps {
							// Run autogen
							sh "./autogen.sh"

							// Run ./configure...
							sh "CC=${env.COMPILER} ./configure --prefix=/usr --enable-debug"
						}

						post {
							failure {
								archiveArtifacts artifacts: "config.log",
									allowEmptyArchive: true

								echo "config.log has been archived"
							}
						}
					}

					stage("Build") {
						steps {
							sh "make -j\$(nproc)"
						}
					}

					stage("Check") {
						steps {
							// Fail the build if any of the tests fail
							// Don't let a test that hangs block the agent forever
							timeout(time: 15, unit: "MINUTES") {
								sh "make -j\$(nproc) check VERBOSE=1"
							}
						}

						post {
							always {
								script {
									archiveTestLogs("logs/${env.DISTRO.replace(":", "-")}/${env.COMPILER}")
								}
							}
						}
					}

					// Build again without liburing to check that it is optional
					stage("Build without io_uring") {
						steps {
							sh "make distclean"
							sh "CC=${env.COMPILER} ./configure --prefix=/usr --enable-debug --without-uring"
							sh "make -j\$(nproc)"
						}
					}
				}

				// Cleanup the workspace afterwards
				post {
					always {
						script {
							cleanupWorkspace()
						}
					}
				}
			}
		}

		stage("Coverage Tests") {
			parallel {
				/*
					Run the tests with the address and undefined behaviour sanitizers...
				*/
				stage("Address Sanitizer") {
					agent {
						docker {
							image "debian:trixie"

							// Run as root inside the containers to install dependencies
							args "--init --privileged -u root"

							customWorkspace "${JOB_NAME}/${BUILD_ID}/asan"
						}
					}

					stages {
						stage("Install Dependencies") {
							steps {
								script {
									installBuildDepsDebian("trixie", "gcc")
								}
							}
						}

						stage("Configure") {
							steps {
								sh "./autogen.sh"
								sh """
									./configure \
										--prefix=/usr \
										--enable-asan \
										--enable-ubsan \
										--enable-debug
								"""
							}
						}

						stage("Build") {
							steps {
								sh "make -j\$(nproc)"
							}
						}

						stage("Check") {
							steps {
								// Fail the build if the sanitizers find anything
								// Don't let a test that hangs block the agent forever
								timeout(time: 15, unit: "MINUTES") {
									sh "make -j\$(nproc) check VERBOSE=1"
								}
							}

							post {
								failure {
									script {
										archiveTestLogs("logs/asan")
									}
								}
							}
						}
					}

					// Cleanup the workspace afterwards
					post {
						always {
							script {
								cleanupWorkspace()
							}
						}
					}
				}

				/*
					Every worker is a thread of its own, so we want to
					know if they are getting into each other's way...
				*/
				stage("Thread Sanitizer") {
					agent {
						docker {
							image "debian:trixie"

							// Run as root inside the containers to install dependencies
							args "--init --privileged -u root"

							customWorkspace "${JOB_NAME}/${BUILD_ID}/tsan"
						}
					}

					stages {
						stage("Install Dependencies") {
							steps {
								script {
									installBuildDepsDebian("trixie", "gcc")
								}
							}
						}

						stage("Configure") {
							steps {
								sh "./autogen.sh"
								sh """
									./configure \
										--prefix=/usr \
										--enable-tsan \
										--enable-debug
								"""
							}
						}

						stage("Build") {
							steps {
								sh "make -j\$(nproc)"
							}
						}

						stage("Check") {
							steps {
								// Fail the build if the sanitizer finds anything
								// Don't let a test that hangs block the agent forever
								timeout(time: 15, unit: "MINUTES") {
									sh "make -j\$(nproc) check VERBOSE=1"
								}
							}

							post {
								failure {
									script {
										archiveTestLogs("logs/tsan")
									}
								}
							}
						}
					}

					// Cleanup the workspace afterwards
					post {
						always {
							script {
								cleanupWorkspace()
							}
						}
					}
				}

				/*
					Run fireperf through Clang's Static Analyzer...
				*/
				stage("Clang Static Analyzer") {
					agent {
						docker {
							image "debian:trixie"

							// Run as root inside the containers to install dependencies
							args "--init -u root"

							customWorkspace "${JOB_NAME}/${BUILD_ID}/clang-static-analyzer"
						}
					}

					stages {
						stage("Install Dependencies") {
							steps {
								script {
									installBuildDepsDebian("trixie", "clang")

									// Install Clang Tools
									sh "apt-get install -y clang-tools"
								}
							}
						}

						stage("Configure") {
							steps {
								sh "./autogen.sh"
								sh "scan-build ./configure --prefix=/usr --enable-debug"
							}
						}

						stage("Build") {
							steps {
								sh "scan-build -o scan-build-output make -j\$(nproc)"
							}
						}

						stage("Publish Report") {
							steps {
								// There is nothing to archive if the analyzer has not found
								// anything, as it then removes its (empty) report
								archiveArtifacts artifacts: "scan-build-output/**/*", allowEmptyArchive: true

								// What the analyzer finds is something that we should fix, but
								// not something that we must fix before anything else. So the
								// build does not fail, but it shows that there is something.
								script {
									def reports = sh(
										returnStdout: true,
										script: "find scan-build-output -name 'report-*.html' 2>/dev/null | wc -l"
									).trim()

									if (reports != "0") {
										unstable("The static analyzer has found ${reports} problem(s)")
									}
								}
							}
						}
					}

					// Cleanup the workspace afterwards
					post {
						always {
							script {
								cleanupWorkspace()
							}
						}
					}
				}

				/*
					Check that the tarball has everything it needs and that
					we can build outside of the source tree...
				*/
				stage("Distribution Check") {
					agent {
						docker {
							image "debian:trixie"

							// Run as root inside the containers to install dependencies
							args "--init --privileged -u root"

							customWorkspace "${JOB_NAME}/${BUILD_ID}/distcheck"
						}
					}

					stages {
						stage("Install Dependencies") {
							steps {
								script {
									installBuildDepsDebian("trixie", "gcc")
								}
							}
						}

						stage("Configure") {
							steps {
								sh "./autogen.sh"
								sh "./configure --prefix=/usr --enable-debug"
							}
						}

						stage("Check") {
							steps {
								// Don't let a test that hangs block the agent forever
								timeout(time: 15, unit: "MINUTES") {
									sh "make -j\$(nproc) distcheck"
								}
							}
						}

						// Keep the tarball that has just been checked. If this is
						// a release, it is being published later.
						stage("Source Tarball") {
							steps {
								// We ask Git whether this is a release
								sh "apt-get install -y --no-install-recommends git"

								script {
									archiveSourceTarball()
								}
							}
						}

						// Build the man pages, which are being published
						// once everything else has passed
						stage("Man Pages") {
							steps {
								sh "make -j\$(nproc) man"

								// The pages as HTML, and as what man reads
								stash includes: "man/*.html, man/*.7, man/*.8", name: "man-pages"
							}
						}
					}

					// Cleanup the workspace afterwards
					post {
						always {
							script {
								cleanupWorkspace()
							}
						}
					}
				}

				stage("LCOV") {
					agent {
						docker {
							image "debian:trixie"

							// Run as root inside the containers to install dependencies
							args "--init --privileged -u root"

							customWorkspace "${JOB_NAME}/${BUILD_ID}/lcov"
						}
					}

					stages {
						stage("Install Dependencies") {
							steps {
								script {
									installBuildDepsDebian("trixie", "gcc")

									// Install lcov
									sh "apt-get install -y lcov"
								}
							}
						}

						stage("Configure") {
							steps {
								sh "./autogen.sh"
								sh """
									./configure \
										--prefix=/usr \
										--enable-coverage \
										--enable-debug
								"""
							}
						}

						stage("Build") {
							steps {
								sh "make -j\$(nproc)"
							}
						}

						stage("Check") {
							steps {
								// Don't let a test that hangs block the agent forever
								timeout(time: 15, unit: "MINUTES") {
									sh "make -j\$(nproc) check VERBOSE=1 || true"
								}
							}
						}

						stage("Publish Report") {
							steps {
								// Capture coverage data
								sh """
									lcov \
										--capture \
										--directory . \
										--ignore-errors inconsistent,inconsistent \
										--output-file coverage.info
								"""

								// Generate HTML report
								sh """
									genhtml \
										coverage.info \
										--ignore-errors inconsistent,corrupt \
										--output-directory lcov
								"""

								// Upload the report
								archiveArtifacts artifacts: "lcov/**/*"
							}
						}
					}

					// Cleanup the workspace afterwards
					post {
						always {
							script {
								cleanupWorkspace()
							}
						}
					}
				}
			}
		}

		stage("Debian Packages") {
			stages {
				// The packages are being built for all branches, but they are only
				// being published for the master branch and for releases (see below).
				stage("Build Debian Packages") {
					steps {
						script {
							debianBuildAllPackages()
						}
					}
				}

				/*
					Signs the packages and publishes them on
					https://repo.ipfire.org/debian/fireperf/

					There are two repositories: "master" gets every build of the
					master branch, and "stable" only gets what has been built from
					a tag, which is a release. To use one of them, import the key
					and add this to /etc/apt/sources.list.d/fireperf.list:

					  deb [signed-by=/usr/share/keyrings/fireperf.asc] https://repo.ipfire.org/debian/fireperf/stable trixie main
				*/
				stage("Master Debian Repository") {
					agent any

					// We don't need to check out the source for this stage
					options {
						skipDefaultCheckout()
					}

					environment {
						GNUPGHOME  = "${WORKSPACE}/.gnupg"
						KRB5CCNAME = "${WORKSPACE}/.krb5cc"

						// Our signing key
						GPG_KEY_ID = "CB4E92A2819141F2635ED4B14F61249A8485F921"
					}

					// Only what is in the master branch and releases are being
					// published. Other branches are there to find out whether
					// something builds. (There is no name of a branch if this
					// is not a multibranch job, which then only builds master.)
					when {
						beforeAgent true

						anyOf {
							branch "master"
							buildingTag()
							expression {
								return env.BRANCH_NAME == null
							}
						}
					}

					steps {
						// Cleanup the workspace
						cleanWs()

						script {
							debianPublishPackages()
						}
					}

					// Don't leave any keys or tickets behind if anything has failed
					post {
						always {
							sh "rm -rf $GNUPGHOME"
							sh "kdestroy || :"
						}
					}
				}
			}
		}

		stage("Arch Linux Packages") {
			stages {
				// The packages are being built for all branches, but they are only
				// being published for the master branch and for releases (see below).
				stage("Build Arch Linux Packages") {
					steps {
						script {
							archlinuxBuildPackages()
						}
					}
				}

				/*
					Signs the packages and publishes them on
					https://repo.ipfire.org/archlinux/fireperf/master/ (and on
					.../stable/ if this is a release)

					To use the repository, import the key and add this to
					/etc/pacman.conf:

					  [fireperf]
					  Server = https://repo.ipfire.org/archlinux/fireperf/stable/$arch
				*/
				stage("Master Arch Linux Repository") {
					agent any

					// We don't need to check out the source for this stage
					options {
						skipDefaultCheckout()
					}

					environment {
						GNUPGHOME  = "${WORKSPACE}/.gnupg"
						KRB5CCNAME = "${WORKSPACE}/.krb5cc"

						// The key that signs the packages for Arch Linux
						ARCHLINUX_GPG_KEY_ID = "274205B524A108FD0416482AA49D90BBC489A8E7"
					}

					// See "Master Debian Repository"
					when {
						beforeAgent true

						anyOf {
							branch "master"
							buildingTag()
							expression {
								return env.BRANCH_NAME == null
							}
						}
					}

					steps {
						// Cleanup the workspace
						cleanWs()

						script {
							archlinuxPublishPackages()
						}
					}

					// Don't leave any keys or tickets behind if anything has failed
					post {
						always {
							sh "rm -rf $GNUPGHOME"
							sh "kdestroy || :"
						}
					}
				}
			}
		}

		stage("RPM Packages") {
			stages {
				// The packages are being built for all branches, but they are only
				// being published for the master branch and for releases (see below).
				stage("Build RPM Packages") {
					steps {
						script {
							rpmBuildAllPackages()
						}
					}
				}

				/*
					Signs the packages and publishes them on
					https://repo.ipfire.org/<distro>/fireperf/master/<release>/ (and
					on .../stable/... if this is a release)

					RHEL (and its derivatives) and openSUSE have a key each, which
					Jenkins expects as "Secret file" credentials (without a passphrase)
					called "fireperf-rpm-el-key" and "fireperf-rpm-opensuse-key". The
					public key is being published as RPM-GPG-KEY-fireperf in each
					repository. To use one of them on RHEL, add this to
					/etc/yum.repos.d/fireperf.repo:

					  [fireperf]
					  name=fireperf
					  baseurl=https://repo.ipfire.org/el/fireperf/stable/$releasever/
					  gpgcheck=1
					  repo_gpgcheck=1
					  gpgkey=https://repo.ipfire.org/el/fireperf/stable/$releasever/RPM-GPG-KEY-fireperf
				*/
				stage("Master RPM Repository") {
					agent any

					// We don't need to check out the source for this stage
					options {
						skipDefaultCheckout()
					}

					environment {
						GNUPGHOME  = "${WORKSPACE}/.gnupg"
						KRB5CCNAME = "${WORKSPACE}/.krb5cc"
					}

					// See "Master Debian Repository"
					when {
						beforeAgent true

						anyOf {
							branch "master"
							buildingTag()
							expression {
								return env.BRANCH_NAME == null
							}
						}
					}

					steps {
						// Cleanup the workspace
						cleanWs()

						script {
							rpmPublishPackages()
						}
					}

					// Don't leave any keys or tickets behind if anything has failed
					post {
						always {
							sh "rm -rf $GNUPGHOME"
							sh "kdestroy || :"
						}
					}
				}
			}
		}

		/*
			Publish the man pages on https://man-pages.ipfire.org/fireperf/

			This is the last stage, so that the pages are only being
			updated if everything else has passed.
		*/
		stage("Publish Man Pages") {
			agent any

			// We don't need to check out the source for this stage
			options {
				skipDefaultCheckout()
			}

			environment {
				KRB5CCNAME = "${WORKSPACE}/.krb5cc"
			}

			// The pages that are online are the ones of the master branch
			when {
				beforeAgent true

				anyOf {
					branch "master"
					expression {
						return env.BRANCH_NAME == null
					}
				}
			}

			steps {
				// Cleanup the workspace
				cleanWs()

				// Fetch the pages that have been built before
				unstash "man-pages"

				// Keep them with the build, too
				archiveArtifacts artifacts: "man/*.html, man/*.7, man/*.8"

				// Fetch a Kerberos ticket
				withCredentials([file(credentialsId: "jenkins.keytab", variable: "KEYTAB")]) {
					sh 'kinit -kV -t $KEYTAB jenkins@IPFIRE.ORG'
				}

				// Publish files
				sh """
					rsync \
						--verbose \
						--recursive \
						--delete \
						--delay-updates \
						man/ \
						pakfire@fs01.haj.ipfire.org:/pub/man-pages/fireperf/
				"""

				// Destroy the Kerberos credentials
				sh "kdestroy"
			}
		}

		/*
			Sign the source tarball of a release and publish it on
			https://source.ipfire.org/releases/fireperf/

			This is the last stage, so that a release is only being
			published if everything else has passed.
		*/
		stage("Publish Source Tarball") {
			agent any

			// We don't need to check out the source for this stage
			options {
				skipDefaultCheckout()
			}

			environment {
				GNUPGHOME  = "${WORKSPACE}/.gnupg"
				KRB5CCNAME = "${WORKSPACE}/.krb5cc"

				// The key that signs our releases, which is not one
				// of the keys that sign the package repositories:
				// fireperf Release Signing Key <info@ipfire.org>
				GPG_KEY_ID = "276051E908658F1CD54AB53190BE00ED9965EC34"
			}

			// Releases are being built from the master branch, or from a tag
			when {
				beforeAgent true

				anyOf {
					branch "master"
					buildingTag()
				}
			}

			steps {
				// Cleanup the workspace
				cleanWs()

				script {
					publishSourceTarball()
				}
			}
		}
	}
}

// Copies the logs of all tests into a directory and keeps them with the build.
// This is a function because whatever is in the pipeline above is being compiled
// into one single method which must not become larger than 64 kB.
def archiveTestLogs(directory) {
	sh """
		mkdir -pv ${directory}
		find . -name "*.log" -not -name "config.log" -not -path "./logs/*" | xargs --no-run-if-empty \
			cp --verbose --parents --target-directory=${directory}/
	"""

	archiveArtifacts artifacts: "${directory}/**/*", allowEmptyArchive: true

	echo "The test logs have been archived"
}

/*
	Removes the workspace of a stage that has been running in a container.

	Whatever has been created in there belongs to root, but the agent runs as
	an unprivileged user and cannot remove it. cleanWs() does not tell, and
	leaves everything behind on the agent. That is why we have to remove it
	from the inside where we are root.
*/
def cleanupWorkspace() {
	sh "find . -xdev -mindepth 1 -delete || :"

	cleanWs()
}

// Keeps the tarball that "make distcheck" has created with the build, and
// remembers whether this is a release, which is when the commit has a tag
// that is the version of the tarball
def archiveSourceTarball() {
	sh """
		mkdir -pv source

		cp -v fireperf-*.tar.xz source/

		cd source

		if tag="\$(git -c safe.directory='*' describe --exact-match --tags HEAD 2>/dev/null)"; then
			if [ ! -e "fireperf-\$tag.tar.xz" ]; then
				echo "The tag (\$tag) does not match the version of the tarball:" >&2
				ls >&2
				exit 1
			fi

			touch RELEASE
		fi
	"""

	// Keep the tarball with the build
	archiveArtifacts artifacts: "source/fireperf-*"

	// Stash it for when it is being published
	stash includes: "source/**/*", name: "source"
}

// Signs the tarball and publishes it if this is a release
def publishSourceTarball() {
	unstash "source"

	if (!fileExists("source/RELEASE")) {
		echo "This is not a release, not publishing the source tarball"
		return
	}

	sh "rm -f source/RELEASE"

	// Create the GPG stash directory
	sh """
		mkdir -p \$GNUPGHOME
		chmod 700 \$GNUPGHOME
	"""

	// Import the GPG key
	withCredentials([file(credentialsId: "${env.GPG_KEY_ID}", variable: "GPG_KEY_FILE")]) {
		// Jenkins prefers to have single quotes here so that $GPG_KEY_FILE won't be expanded
		sh 'gpg --import --batch < $GPG_KEY_FILE'
	}

	// Sign the tarball
	sh """
		for tarball in source/fireperf-*.tar.xz; do
			gpg --batch \
				--armor \
				--detach-sign \
				--local-user ${env.GPG_KEY_ID} \
				--output \$tarball.asc \
				\$tarball
		done
	"""

	// Export the public key, so that the signatures can be verified
	sh "gpg --batch --armor --export ${env.GPG_KEY_ID} > source/fireperf-release-key.asc"

	// Remove the GPG key material as soon as possible
	sh "rm -rf \$GNUPGHOME"

	// Keep the signature with the build
	archiveArtifacts artifacts: "source/fireperf-*"

	// Fetch a Kerberos ticket
	withCredentials([file(credentialsId: "jenkins.keytab", variable: "KEYTAB")]) {
		sh 'kinit -kV -t $KEYTAB jenkins@IPFIRE.ORG'
	}

	// Publish files, but never replace a release that has been published before
	sh """
		rsync \
			--verbose \
			--recursive \
			--mkpath \
			--ignore-existing \
			source/ \
			pakfire@fs01.haj.ipfire.org:/pub/sources/releases/fireperf
	"""

	// Destroy the Kerberos credentials
	sh "kdestroy"
}

// Installs everything we need on RHEL/Fedora/etc.
def installBuildDepsRedHat(distro, compiler) {
	// Install basic development tools
	sh "dnf group install -y development-tools"

	// Install our own build and runtime dependencies
	sh """
		dnf install -y \
			asciidoc \
			autoconf \
			automake \
			docbook-style-xsl \
			gawk \
			gettext \
			gettext-devel \
			intltool \
			libtool \
			libxslt \
			pkg-config \
			python3 \
			${compiler} \
			\
			liburing-devel \
			ncurses-devel
	"""
}

// Installs everything we need on Arch Linux
def installBuildDepsArchLinux(distro, compiler) {
	// Ensure that the keyring is always up to date
	sh "pacman -Sy --needed --noconfirm archlinux-keyring"

	// Install all updates, because pacman isn't very smart
	sh "pacman -Syu --needed --noconfirm"

	// Install packages
	sh """
		pacman -Sy \
			--needed \
			--noconfirm \
			asciidoc \
			autoconf \
			automake \
			docbook-xsl \
			gettext \
			intltool \
			libtool \
			libxslt \
			pkg-config \
			python \
			${compiler} \
			\
			liburing \
			ncurses
	"""
}

// Installs everything we need on Debian & Ubuntu
def installBuildDepsDebian(distro, compiler) {
	sh "apt-get update"
	// iperf and iperf3 are there for the tests to talk to. The package
	// of iperf3 asks a question when it is being installed.
	sh """
		DEBIAN_FRONTEND=noninteractive apt-get install -y \
			--no-install-recommends \
			asciidoc \
			autoconf \
			automake \
			autopoint \
			build-essential \
			docbook-xml \
			docbook-xsl \
			gettext \
			intltool \
			iperf \
			iperf3 \
			libtool \
			pkg-config \
			python3 \
			xsltproc \
			${compiler} \
			\
			libncurses-dev \
			liburing-dev
	"""
}

/*
	Packages

	Everything that runs once per distribution or architecture is down here, as
	the "pipeline" block above is being compiled into one method that cannot be
	larger than 64 kB, and a matrix is being copied for each of its cells.
*/

/*
	Runs something in a container of its own, with the source checked out.

	Everything gets a workspace of its own which nothing else has ever used,
	in a directory of its own: the packages for Debian are being written to
	the directory above the source and must not find anything there that
	another build has left behind or is writing at the same time.

	The tests need to be able to do things that a container normally may not,
	like dropping capabilities and changing limits, which is why it is privileged.
*/
def inContainer(image, name, Closure body) {
	node {
		ws("${env.JOB_NAME}/${env.BUILD_ID}/${name}/fireperf") {
			checkout scm

			// Run as root inside the containers to install dependencies
			docker.image(image).inside("--init --privileged -u root") {
				try {
					body()

				// Cleanup the workspace afterwards
				} finally {
					cleanupWorkspace()
				}
			}
		}
	}
}

// Publishes packages in a channel ("master" for every build of the master branch,
// and "stable" for releases) on https://repo.ipfire.org/<distro>/fireperf/<channel>
def publishPackages(directory, distro, release) {
	// Fetch a Kerberos ticket
	withCredentials([file(credentialsId: "jenkins.keytab", variable: "KEYTAB")]) {
		sh 'kinit -kV -t $KEYTAB jenkins@IPFIRE.ORG'
	}

	sh """
		channels="master"

		# This has been built from a tag of its own
		if [ -n "\${TAG_NAME:-}" ]; then
			channels="stable"

		# This is the master branch, and what is in it has a tag
		elif [ -e ${release} ]; then
			channels="master stable"
		fi

		rm -f ${release}

		for channel in \$channels; do
			rsync \
				--verbose \
				--recursive \
				--mkpath \
				--delete \
				--delete-excluded \
				--delay-updates \
				${directory}/ \
				pakfire@fs01.haj.ipfire.org:/pub/repo/${distro}/fireperf/\$channel
		done
	"""

	// Destroy the Kerberos credentials
	sh "kdestroy"
}

// Imports a key for signing from the credentials of Jenkins
def importSigningKey(credentials) {
	sh """
		mkdir -p \$GNUPGHOME
		chmod 700 \$GNUPGHOME
	"""

	withCredentials([file(credentialsId: credentials, variable: "GPG_KEY_FILE")]) {
		// Jenkins prefers to have single quotes here so that $GPG_KEY_FILE won't be expanded
		sh 'gpg --import --batch < $GPG_KEY_FILE'
	}
}

/*
	The releases of Debian that we build for, and the architectures of each that
	we can build (cross-compiling everything but amd64, and running the tests in
	QEMU). See "Architectures" in https://deb.debian.org/debian/dists/
*/
def debianReleases() {
	return [
		"sid"      : ["amd64", "arm64", "armhf", "i386", "loong64", "ppc64el", "riscv64", "s390x"],
		"forky"    : ["amd64", "arm64", "armhf", "i386", "loong64", "ppc64el", "riscv64", "s390x"],
		"trixie"   : ["amd64", "arm64", "armel", "armhf", "i386", "ppc64el", "riscv64", "s390x"],
		"bookworm" : ["amd64", "arm64", "armhf", "i386", "ppc64el"],
	]
}

// The suite that a release of Debian is in
def debianSuite(release) {
	def suites = [
		"sid"      : "unstable",
		"forky"    : "testing",
		"trixie"   : "stable",
		"bookworm" : "oldstable",
	]

	return suites[release]
}

// Builds the packages for all releases of Debian and all of their architectures
def debianBuildAllPackages() {
	def releases = debianReleases()

	def branches = [:]

	for (r in releases.keySet()) {
		for (a in releases[r]) {
			// These must not change while we are waiting to be run
			def release = r
			def arch = a

			branches["${release} (${arch})"] = {
				inContainer("debian:${release}", "debian-${release}/${arch}") {
					debianBuildPackages(release, arch)
				}
			}
		}
	}

	parallel branches
}

// Builds the packages for one architecture of one release of Debian
def debianBuildPackages(release, arch) {
	stage("Setup Build Environment") {
		// Add the architecture
		sh "dpkg --add-architecture ${arch}"
		sh "apt-get update"

		// Install required packages
		sh """
			apt-get install -y \
				apt-utils \
				build-essential \
				devscripts \
				git \
				lintian \
				qemu-user-static
		"""

		// Install the compilers. There is no crossbuild-essential
		// for all architectures, but the compilers that it would
		// have installed are there.
		sh """
			if [ "${arch}" != "\$(dpkg --print-architecture)" ]; then
				if ! apt-get install -y crossbuild-essential-${arch}; then
					triplet="\$(dpkg-architecture -a${arch} -qDEB_HOST_GNU_TYPE)"

					apt-get install -y gcc-\$triplet dpkg-cross
				fi
			fi
		"""
	}

	stage("Install Build Dependencies") {
		// In testing and unstable, a library is not always being rebuilt for
		// all architectures at the same time, and its documentation then
		// differs between them. dpkg refuses to install those next to each
		// other, which is what we have to do. This does not matter to what
		// we build.
		sh """
			options=""

			case "${release}" in
				sid|forky)
					options="-o Dpkg::Options::=--force-overwrite"
					;;
			esac

			# Tell apt that we are cross-compiling
			profiles=""

			if [ "${arch}" != "\$(dpkg --print-architecture)" ]; then
				profiles="-Pcross"
			fi

			apt-get build-dep -y \$options \$profiles -a${arch} .

			# The tests run what we have built in QEMU, which needs the libraries
			# that it has been linked against, but the cross compiler has only
			# brought them along for itself. libatomic is needed on some
			# architectures.
			if [ "${arch}" != "\$(dpkg --print-architecture)" ]; then
				apt-get install -y \$options libatomic1:${arch}
			fi
		"""
	}

	stage("Tag") {
		// A release has the version that the changelog says. Everything
		// else has the number of commits appended to that, so that it is
		// newer than what has been built before.
		sh """
			if git -c safe.directory='*' describe --exact-match --tags HEAD >/dev/null 2>&1; then
				echo "This is a release"
			else
				commits="\$(git -c safe.directory='*' rev-list --count HEAD)"

				dch -m "Jenkins Build ${env.BUILD_ID} (\$commits commits)" -l .build-\$commits.
			fi
		"""
	}

	try {
		stage("Build") {
			// Create the source tarball from the checked out source
			sh """
				version="\$(dpkg-parsechangelog --show-field Version | sed "s/-[^-]*\$//")";
				tar \
					--create \
					--verbose \
					--xz \
					--file="../fireperf_\$version.orig.tar.xz" \
					--transform="s|^|fireperf-\$version/|" \
					*
			"""

			// Build the packages. The test suite is being run as part of the
			// build (in QEMU if we are cross-compiling), and fails the build.
			timeout(time: 90, unit: "MINUTES") {
				sh """
					options=""

					if [ "${arch}" != "\$(dpkg --print-architecture)" ]; then
						options="-Pcross"

						export DEB_BUILD_OPTIONS="crossbuildcanrunhostbinaries"
					fi

					dpkg-buildpackage \
						--host-arch ${arch} \
						--build=full \
						--unsigned-source \
						--unsigned-changes \
						\$options
				"""
			}
		}

	// Keep the logs of the tests, whatever has happened
	} finally {
		archiveTestLogs("logs/debian-${release}/${arch}")
	}

	stage("Lint") {
		// Show what lintian has to say, but don't fail the build because of it
		sh "lintian --allow-root ../*.changes || true"
	}

	stage("Create Repository") {
		// Create a repository and generate Packages
		sh "mkdir -pv \
			packages/debian/dists/${release}/main/binary-${arch} \
			packages/debian/dists/${release}/main/source \
			packages/debian/pool/${release}/main/${arch}"

		// Copy all packages
		sh "cp -v ../*.deb packages/debian/pool/${release}/main/${arch}"

		// Generate and compress Packages, Sources and Contents
		sh """
			cd packages/debian

			apt-ftparchive packages pool/${release}/main/${arch} \
				> dists/${release}/main/binary-${arch}/Packages
			xz -v9 < dists/${release}/main/binary-${arch}/Packages \
				> dists/${release}/main/binary-${arch}/Packages.xz

			apt-ftparchive sources pool/${release}/main/${arch} \
				> dists/${release}/main/source/Sources
			xz -v9 < dists/${release}/main/source/Sources \
				> dists/${release}/main/source/Sources.xz

			apt-ftparchive contents pool/${release}/main/${arch} \
				> dists/${release}/main/Contents-${arch}
			xz -v9 < dists/${release}/main/Contents-${arch} \
				> dists/${release}/main/Contents-${arch}.xz
		"""

		// Keep the packages with the build
		archiveArtifacts artifacts: "packages/debian/**/*"

		// Tell whoever publishes this if it is a release
		sh """
			if git -c safe.directory='*' describe --exact-match --tags HEAD >/dev/null 2>&1; then
				touch packages/debian/RELEASE
			fi
		"""

		// Stash the packages
		stash includes: "packages/debian/**/*", name: "${release}-${arch}"

		// The packages and the tarball are outside of the workspace
		sh "rm -vf ../*.deb ../*.dsc ../*.changes ../*.buildinfo ../*.tar.* || :"
	}
}

// Signs the repositories for all releases of Debian, and publishes them
def debianPublishPackages() {
	importSigningKey(env.GPG_KEY_ID)

	def releases = debianReleases()

	for (distro in releases.keySet()) {
		def arches = releases[distro]
		def suite = debianSuite(distro)

		for (arch in arches) {
			unstash "${distro}-${arch}"
		}

		// Create the Release file, and sign it
		sh """
			(
				echo "Origin: fireperf Repository"
				echo "Label: fireperf Repository"
				echo "Suite: ${suite}"
				echo "Codename: ${distro}"
				echo "Version: 1.0"
				echo "Architectures: ${arches.join(" ")}"
				echo "Components: main"
				echo "Description: fireperf Jenkins Repository"

				# Do the rest automatically
				cd packages/debian && apt-ftparchive release dists/${distro}
			) >> packages/debian/dists/${distro}/Release

			gpg --batch \
				--clearsign \
				--local-user ${env.GPG_KEY_ID} \
				--output packages/debian/dists/${distro}/InRelease \
				packages/debian/dists/${distro}/Release

			gpg --batch \
				--armor \
				--detach-sign \
				--local-user ${env.GPG_KEY_ID} \
				--output packages/debian/dists/${distro}/Release.gpg \
				packages/debian/dists/${distro}/Release
		"""
	}

	// Export the public key
	sh "gpg --batch --export --armor ${env.GPG_KEY_ID} > packages/debian/fireperf.asc"

	// Remove the GPG key material as soon as possible
	sh "rm -rf \$GNUPGHOME"

	// Keep the signed repository with the build
	archiveArtifacts artifacts: "packages/debian/**/*"

	publishPackages("packages/debian", "debian", "packages/debian/RELEASE")
}

// Builds the packages for Arch Linux, which we only do on x86_64
def archlinuxBuildPackages() {
	inContainer("archlinux:base-devel", "archlinux/x86_64") {
		stage("Setup Build Environment") {
			// Ensure that the keyring is always up to date
			sh "pacman -Sy --needed --noconfirm archlinux-keyring"

			// Install all updates
			sh "pacman -Syu --needed --noconfirm"

			// makepkg refuses to run as root, so we need an unprivileged
			// user which is allowed to install the build dependencies
			sh """
				useradd --create-home build
				echo "build ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/build
				chown -R build .
			"""
		}

		try {
			stage("Build") {
				sh """
					su build -c "cd contrib/archlinux && \
						MAKEFLAGS=-j\$(nproc) makepkg --syncdeps --noconfirm"
				"""
			}

		// Keep the logs of the tests, whatever has happened
		} finally {
			archiveTestLogs("logs/archlinux/x86_64")
		}

		stage("Create Repository") {
			sh "mkdir -pv packages/archlinux/x86_64"

			// Copy all packages
			sh "cp -v contrib/archlinux/*.pkg.tar.zst packages/archlinux/x86_64"

			// Create the repository database
			sh """
				repo-add \
					packages/archlinux/x86_64/fireperf.db.tar.zst \
					packages/archlinux/x86_64/*.pkg.tar.zst
			"""

			// pacman asks for fireperf.db and fireperf.files, which are
			// symbolic links. Those do not survive being stashed and
			// being copied to the server, so we make them files.
			sh """
				cd packages/archlinux/x86_64

				for name in fireperf.db fireperf.files; do
					rm -f \$name \$name.tar.zst.old
					cp -v \$name.tar.zst \$name
				done
			"""

			// Keep the packages with the build
			archiveArtifacts artifacts: "packages/archlinux/**/*"

			// Tell whoever publishes this if it is a release
			sh """
				if git -c safe.directory='*' describe --exact-match --tags HEAD >/dev/null 2>&1; then
					touch packages/archlinux/RELEASE
				fi
			"""

			// Stash the packages
			stash includes: "packages/archlinux/**/*", name: "archlinux-x86_64"
		}
	}
}

// Signs the packages for Arch Linux and publishes them
def archlinuxPublishPackages() {
	importSigningKey(env.ARCHLINUX_GPG_KEY_ID)

	unstash "archlinux-x86_64"

	// pacman wants a detached signature next to every package
	// and next to the databases, and it does not read ASCII armor
	sh """
		for file in packages/archlinux/*/*.pkg.tar.zst \
				packages/archlinux/*/fireperf.db \
				packages/archlinux/*/fireperf.db.tar.zst \
				packages/archlinux/*/fireperf.files \
				packages/archlinux/*/fireperf.files.tar.zst; do
			gpg --batch \
				--detach-sign \
				--no-armor \
				--local-user ${env.ARCHLINUX_GPG_KEY_ID} \
				--output \$file.sig \
				\$file
		done
	"""

	// Export the public key
	sh "gpg --batch --export --armor ${env.ARCHLINUX_GPG_KEY_ID} > packages/archlinux/fireperf.asc"

	// Remove the GPG key material as soon as possible
	sh "rm -rf \$GNUPGHOME"

	// Keep the signatures with the build
	archiveArtifacts artifacts: "packages/archlinux/**/*"

	publishPackages("packages/archlinux", "archlinux", "packages/archlinux/RELEASE")
}

/*
	The distributions that we build RPM packages for, and the repository that
	they go into, which is https://repo.ipfire.org/<distro>/fireperf/<channel>/<release>/

	The release is what dnf and zypper call $releasever, so that one
	configuration works for all releases of a distribution, except for
	Tumbleweed, whose $releasever is the date of the snapshot.
*/
def rpmReleases() {
	return [
		[ image : "almalinux:10",        distro : "el",       release : "10" ],
		[ image : "almalinux:9",         distro : "el",       release : "9" ],
		[ image : "opensuse/tumbleweed", distro : "opensuse", release : "tumbleweed" ],
	]
}

// The distributions that we build RPM packages for
def rpmDistros() {
	def distros = []

	for (release in rpmReleases()) {
		if (!distros.contains(release.distro))
			distros.add(release.distro)
	}

	return distros
}

// Builds the RPM packages for all distributions, which we only do on x86_64
def rpmBuildAllPackages() {
	def branches = [:]

	for (r in rpmReleases()) {
		// This must not change while we are waiting to be run
		def release = r

		branches["${release.distro}-${release.release}"] = {
			inContainer(release.image, "rpm/${release.distro}-${release.release}") {
				rpmBuildPackages(release)
			}
		}
	}

	parallel branches
}

// Builds the RPM packages for one release of a distribution
def rpmBuildPackages(release) {
	def repo = "packages/${release.distro}/${release.release}"

	stage("Install Build Dependencies") {
		if (release.distro == "opensuse") {
			sh "zypper --non-interactive install rpm-build git"
			sh "rpmspec -q --buildrequires contrib/rpm/fireperf.spec \
				| xargs -d '\\n' zypper --non-interactive install"

		} else {
			// RHEL and its derivatives need EPEL and CodeReady Builder
			if (release.distro == "el") {
				sh "dnf install -y epel-release dnf-plugins-core"
				sh "dnf config-manager --set-enabled crb"
			}

			sh "dnf install -y rpm-build git"
			sh "rpmspec -q --buildrequires contrib/rpm/fireperf.spec \
				| xargs -d '\\n' dnf install -y"
		}
	}

	stage("Build") {
		timeout(time: 60, unit: "MINUTES") {
			sh "contrib/rpm/build.sh ${repo}"
		}
	}

	stage("Create Repository") {
		// Keep the packages with the build
		archiveArtifacts artifacts: "${repo}/*"

		// Tell whoever publishes this if it is a release
		sh """
			if git -c safe.directory='*' describe --exact-match --tags HEAD >/dev/null 2>&1; then
				touch packages/RELEASE
			fi
		"""

		// Stash the packages
		stash includes: "packages/**/*", name: "rpm-${release.distro}-${release.release}"
	}
}

/*
	Signs all packages of one release, creates the repository, signs that,
	too, and exports the public key into it
*/
def rpmSignRepository(release) {
	def repo = "packages/${release.distro}/${release.release}"

	// Every distribution has a key of its own
	importSigningKey("fireperf-rpm-${release.distro}-key")

	// The key that we have just imported is the only one there is
	sh """
		key="\$(gpg --batch --list-secret-keys --with-colons | awk -F: '/^fpr/ { print \$10; exit }')"

		rpmsign --define "_gpg_name \$key" --addsign ${repo}/*.rpm

		createrepo_c ${repo}

		gpg --batch \
			--armor \
			--detach-sign \
			--local-user \$key \
			--output ${repo}/repodata/repomd.xml.asc \
			${repo}/repodata/repomd.xml

		# Export the public key
		gpg --batch --export --armor \$key > ${repo}/RPM-GPG-KEY-fireperf
	"""

	// Remove the GPG key material as soon as possible
	sh "rm -rf \$GNUPGHOME"
}

/*
	Signs the RPM packages, creates a repository for each distribution, signs
	that, and publishes everything
*/
def rpmPublishPackages() {
	for (release in rpmReleases()) {
		unstash "rpm-${release.distro}-${release.release}"
	}

	// The tools for this are not on the host
	docker.image("fedora:latest").inside("-u root") {
		sh "dnf install -y createrepo_c gnupg2 rpm-sign"

		for (release in rpmReleases()) {
			rpmSignRepository(release)
		}

		// Give everything back to the user that Jenkins runs as
		sh "chown -R \$(stat -c %u:%g .) packages"
	}

	// Keep the signed packages with the build
	archiveArtifacts artifacts: "packages/**/*"

	// The RELEASE marker is shared by all distributions
	for (distro in rpmDistros()) {
		sh "if [ -e packages/RELEASE ]; then touch packages/${distro}/RELEASE; fi"
	}

	sh "rm -f packages/RELEASE"

	for (distro in rpmDistros()) {
		publishPackages("packages/${distro}", distro, "packages/${distro}/RELEASE")
	}
}
