From c69c9c4f4eaaea302e41e3ffd5f7b6d4bcc6a37a Mon Sep 17 00:00:00 2001 From: Eric Engestrom Date: Wed, 25 Sep 2019 22:54:59 +0100 Subject: [PATCH] gitlab-ci: pre-build containers to improve CI run time and reliability MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Shaves about 50% off the build time on both debian and arch builds. (yeah, I know, it's very small anyway compared to mesa, but we might want to add more things in the future) This also makes the build no longer dependent on external websites: once the image is build, only fdo-internal services are used. Signed-off-by: Eric Engestrom Reviewed-by: Michel Dänzer --- .gitlab-ci.yml | 178 +++++++++++++++++++++-------------- .gitlab-ci/arch-install.sh | 14 +++ .gitlab-ci/debian-install.sh | 43 +++++++++ 3 files changed, 162 insertions(+), 73 deletions(-) create mode 100644 .gitlab-ci/arch-install.sh create mode 100644 .gitlab-ci/debian-install.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 47fe3cd2..05aa7438 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,77 +1,109 @@ -.artifacts-meson: &artifacts-meson - when: always - paths: - - _build/meson-logs +# This is the tag of the docker image used for the build jobs. If the +# image doesn't exist yet, the containers stage generates it. +# +# In order to generate a new image, one should generally change the tag. +# While removing the image from the registry would also work, that's not +# recommended except for ephemeral images during development: Replacing +# an image after a significant amount of time might pull in newer +# versions of gcc/clang or other packages, which might break the build +# with older commits using the same tag. +# +# After merging a change resulting in generating a new image to the +# main repository, it's recommended to remove the image from the source +# repository's container registry, so that the image from the main +# repository's registry will be used there as well. +variables: + UPSTREAM_REPO: mesa/drm + DEBIAN_TAG: "2019-10-20" + DEBIAN_VERSION: buster-slim + DEBIAN_IMAGE: "$CI_REGISTRY_IMAGE/debian/$DEBIAN_VERSION:$DEBIAN_TAG" + ARCH_TAG: "2019-10-20" + ARCH_VERSION: rolling + ARCH_IMAGE: "$CI_REGISTRY_IMAGE/archlinux/$ARCH_VERSION:$ARCH_TAG" -.meson-build: &meson-build - - meson _build - -D amdgpu=true - -D cairo-tests=true - -D etnaviv=true - -D exynos=true - -D freedreno=true - -D freedreno-kgsl=true - -D intel=true - -D libkms=true - -D man-pages=true - -D nouveau=true - -D omap=true - -D radeon=true - -D tegra=true - -D udev=true - -D valgrind=true - -D vc4=true - -D vmwgfx=true - - ninja -C _build - - ninja -C _build test +include: + - project: 'wayland/ci-templates' + ref: 1f7f57c64ff4ebbf7292e3b7a13600518b8cb24c + file: '/templates/debian.yml' + - project: 'wayland/ci-templates' + ref: 1f7f57c64ff4ebbf7292e3b7a13600518b8cb24c + file: '/templates/arch.yml' -latest-meson: +stages: + - containers + - build + + +# When & how to run the CI +.ci-run-policy: + retry: + max: 2 + when: + - runner_system_failure + +# CONTAINERS + +debian: + stage: containers + extends: + - .ci-run-policy + - .debian@container-ifnot-exists + variables: + GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image + DEBIAN_EXEC: 'bash .gitlab-ci/debian-install.sh' + +arch: + stage: containers + extends: + - .ci-run-policy + - .arch@container-ifnot-exists + variables: + GIT_STRATEGY: none # no need to pull the whole tree for rebuilding the image + ARCH_EXEC: 'bash .gitlab-ci/arch-install.sh' + + +# BUILD + +.meson-build: stage: build - image: archlinux/base:latest - before_script: - - pacman -Syu --noconfirm --needed - base-devel - meson - libpciaccess - libxslt docbook-xsl - valgrind - libatomic_ops - cairo cunit - script: *meson-build + extends: .ci-run-policy + variables: + GIT_DEPTH: 10 + script: + - meson _build + -D amdgpu=true + -D cairo-tests=true + -D etnaviv=true + -D exynos=true + -D freedreno=true + -D freedreno-kgsl=true + -D intel=true + -D libkms=true + -D man-pages=true + -D nouveau=true + -D omap=true + -D radeon=true + -D tegra=true + -D udev=true + -D valgrind=true + -D vc4=true + -D vmwgfx=true + - ninja -C _build + - ninja -C _build test + - DESTDIR=$PWD/install ninja -C _build install + artifacts: + when: on_failure + paths: + - _build/meson-logs/* -oldest-meson: - stage: build - image: debian:stable - artifacts: *artifacts-meson - before_script: - - printf > /etc/dpkg/dpkg.cfg.d/99-exclude-cruft "%s\n" - 'path-exclude=/usr/share/doc/*' - 'path-exclude=/usr/share/man/*' - - printf > /usr/sbin/policy-rc.d "%s\n" - '#!/bin/sh' - 'exit 101' - - chmod +x /usr/sbin/policy-rc.d - - apt-get update - - apt-get -y --no-install-recommends install - build-essential - pkg-config - xsltproc - libxslt1-dev docbook-xsl - valgrind - libatomic-ops-dev - libcairo2-dev libcunit1-dev - ninja-build - python3 python3-pip - wget - # We need `--no-check-certificate` here because Debian's CA list is - # too old to know about LetsEncrypt's CA, so it refuses to connect - # to FreeDesktop.org - - LIBPCIACCESS_VERSION=libpciaccess-0.10 && - wget --no-check-certificate https://xorg.freedesktop.org/releases/individual/lib/$LIBPCIACCESS_VERSION.tar.bz2 && - tar -jxvf $LIBPCIACCESS_VERSION.tar.bz2 && - (cd $LIBPCIACCESS_VERSION && ./configure --prefix=$HOME/prefix && make install) - - pip3 install wheel setuptools - - pip3 install meson==0.43 - - export PKG_CONFIG_PATH=$HOME/prefix/lib/pkgconfig:$HOME/prefix/share/pkgconfig - - export LD_LIBRARY_PATH="$HOME/prefix/lib:$LD_LIBRARY_PATH" - script: *meson-build +meson-debian: + extends: .meson-build + image: $DEBIAN_IMAGE + needs: + - debian + +meson-arch: + extends: .meson-build + image: $ARCH_IMAGE + needs: + - arch diff --git a/.gitlab-ci/arch-install.sh b/.gitlab-ci/arch-install.sh new file mode 100644 index 00000000..70605464 --- /dev/null +++ b/.gitlab-ci/arch-install.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -o errexit +set -o xtrace + +pacman -Syu --noconfirm --needed \ + base-devel \ + cairo \ + cunit \ + docbook-xsl \ + libatomic_ops \ + libpciaccess \ + libxslt \ + meson \ + valgrind diff --git a/.gitlab-ci/debian-install.sh b/.gitlab-ci/debian-install.sh new file mode 100644 index 00000000..05b63a15 --- /dev/null +++ b/.gitlab-ci/debian-install.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -o errexit +set -o xtrace + +export DEBIAN_FRONTEND=noninteractive + +apt-get install -y \ + ca-certificates + +sed -i -e 's/http:\/\/deb/https:\/\/deb/g' /etc/apt/sources.list +echo 'deb https://deb.debian.org/debian buster-backports main' >/etc/apt/sources.list.d/backports.list + +apt-get update + +# Use newer packages from backports by default +cat >/etc/apt/preferences <