From b35d030dcbface162f2f5e0a1ddbd80531a47e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Malte=20J=C3=BCrgens?= Date: Sat, 23 Jul 2022 21:18:42 +0200 Subject: [PATCH] Fix build errors by implmenting dind on gitlab ci --- .gitlab-ci.yml | 34 ++++++---------------------------- Makefile | 20 +++++++++++++++++--- assets/dind.Dockerfile | 9 +++++++++ assets/linux.Dockerfile | 3 ++- assets/linux.mk | 1 + assets/macos.Dockerfile | 2 ++ assets/macos.mozconfig | 4 ---- source_release | 2 +- 8 files changed, 38 insertions(+), 37 deletions(-) create mode 100644 assets/dind.Dockerfile diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3c63454..5309b63 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,15 +9,17 @@ Build Docker Images: when: manual only: - master - image: ubuntu + image: registry.gitlab.com/librewolf-community/browser/bsys5/dind + # Needed because we aren't using our custom runners here services: - docker:dind + variables: + DOCKER_HOST: tcp://docker:2375 parallel: matrix: - DISTRO: - debian11 - ubuntu20 - - ubuntu21 - ubuntu22 - mint20 - fedora35 @@ -25,17 +27,7 @@ Build Docker Images: - tumbleweed - macos-x86_64 - macos-aarch64 - variables: - DOCKER_HOST: tcp://docker:2375 - # Install docker and make - before_script: - - apt-get update - - apt-get install -y ca-certificates curl gnupg lsb-release - - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - - echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null - - apt-get update - - apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin make - # Build container + - dind script: - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD registry.gitlab.com - make docker-$DISTRO @@ -50,7 +42,6 @@ Build: - DISTRO: - debian11 - ubuntu20 - - ubuntu21 - ubuntu22 - mint20 - fedora35 @@ -60,8 +51,7 @@ Build: - macos-aarch64 variables: SOURCE_URL: $SOURCE_URL - use_docker: "false" - image: registry.gitlab.com/librewolf-community/browser/bsys5/$DISTRO + image: registry.gitlab.com/librewolf-community/browser/bsys5/dind tags: - autoscale except: @@ -112,8 +102,6 @@ Release: upload_package librewolf-$FILEVER.en-US.debian11.x86_64.deb.sha256sum upload_package librewolf-$FILEVER.en-US.ubuntu20.x86_64.deb upload_package librewolf-$FILEVER.en-US.ubuntu20.x86_64.deb.sha256sum - upload_package librewolf-$FILEVER.en-US.ubuntu21.x86_64.deb - upload_package librewolf-$FILEVER.en-US.ubuntu21.x86_64.deb.sha256sum upload_package librewolf-$FILEVER.en-US.ubuntu22.x86_64.deb upload_package librewolf-$FILEVER.en-US.ubuntu22.x86_64.deb.sha256sum upload_package librewolf-$FILEVER.en-US.mint20.x86_64.deb @@ -145,12 +133,6 @@ Release: - name: Ubuntu 20 (sha256sum) link_type: other url: $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu20.x86_64.deb.sha256sum - - name: Ubuntu 21 (deb) - link_type: package - url: $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu21.x86_64.deb - - name: Ubuntu 21 (sha256sum) - link_type: other - url: $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu21.x86_64.deb.sha256sum - name: Ubuntu 22 (deb) link_type: package url: $CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu22.x86_64.deb @@ -222,10 +204,6 @@ Update Repositories: "file": "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu20.x86_64.deb", "distros": ["focal"] }, - { - "file": "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu21.x86_64.deb", - "distros": ["impish"] - }, { "file": "$CI_API_V4_URL/projects/$CI_PROJECT_ID/packages/generic/librewolf/$FILEVER/librewolf-$FILEVER.en-US.ubuntu22.x86_64.deb", "distros": ["jammy"] diff --git a/Makefile b/Makefile index 95721ca..60179e2 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY : help clean veryclean prune docker push rmi build update work docker-debian11 debian11 docker-mint20 mint20 docker-ubuntu20 ubuntu20 docker-ubuntu21 ubuntu21 docker-ubuntu22 ubuntu22 docker-fedora34 fedora34 docker-fedora35 fedora35 docker-fedora36 fedora36 docker-macos-x86_64 macos-x86_64 docker-macos-aarch64 macos-aarch64 docker-tumbleweed tumbleweed tarball +.PHONY : help clean veryclean prune docker push rmi build update work docker-debian11 debian11 docker-mint20 mint20 docker-ubuntu20 ubuntu20 docker-ubuntu21 ubuntu21 docker-ubuntu22 ubuntu22 docker-fedora34 fedora34 docker-fedora35 fedora35 docker-fedora36 fedora36 docker-macos-x86_64 macos-x86_64 docker-macos-aarch64 macos-aarch64 docker-tumbleweed tumbleweed tarball docker-dind version:=$(shell cat version) release:=$(shell cat release) @@ -24,6 +24,7 @@ help : @echo " [docker-macos-x86_64]" @echo " [docker-macos-aarch64]" @echo " [docker-tumbleweed]" + @echo " [docker-dind]" @echo "" @echo "build targets:" @echo " [debian11]" @@ -53,7 +54,7 @@ veryclean : clean prune : docker system prune --all --force -docker : docker-debian11 docker-mint20 docker-ubuntu20 docker-ubuntu21 docker-ubuntu22 docker-fedora34 docker-fedora35 docker-fedora36 docker-macos-x86_64 docker-macos-aarch64 +docker : docker-debian11 docker-mint20 docker-ubuntu20 docker-ubuntu21 docker-ubuntu22 docker-fedora34 docker-fedora35 docker-fedora36 docker-tumbleweed docker-macos-x86_64 docker-macos-aarch64 docker-dind build : ${MAKE} clean @@ -73,6 +74,8 @@ build : ${MAKE} clean ${MAKE} fedora36 ${MAKE} clean + ${MAKE} tumbleweed + ${MAKE} clean ${MAKE} macos-x86_64 ${MAKE} clean ${MAKE} macos-aarch64 @@ -87,8 +90,10 @@ push : docker push registry.gitlab.com/librewolf-community/browser/bsys5/fedora34 docker push registry.gitlab.com/librewolf-community/browser/bsys5/fedora35 docker push registry.gitlab.com/librewolf-community/browser/bsys5/fedora36 + docker push registry.gitlab.com/librewolf-community/browser/bsys5/tumbleweed docker push registry.gitlab.com/librewolf-community/browser/bsys5/macos-x86_64 docker push registry.gitlab.com/librewolf-community/browser/bsys5/macos-aarch64 + docker push registry.gitlab.com/librewolf-community/browser/bsys5/dind rmi : docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/debian11 @@ -99,8 +104,10 @@ rmi : docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/fedora34 docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/fedora35 docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/fedora36 + docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/tumbleweed docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/macos-x86_64 docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/macos-aarch64 + docker rmi registry.gitlab.com/librewolf-community/browser/bsys5/dind update : @wget -q -O version "https://gitlab.com/librewolf-community/browser/source/-/raw/main/version" @@ -121,7 +128,6 @@ $(tarball) : work : $(tarball) mkdir work (cd work && tar xf ../$(tarball)) - echo ac_add_options --enable-bootstrap >> work/librewolf-$(version)-$(source_release)/mozconfig @@ -207,3 +213,11 @@ docker-macos-aarch64 : ${MAKE} -f assets/macos.mk arch=aarch64 docker macos-aarch64 : ${MAKE} -f assets/macos.mk arch=aarch64 build + + +# +# Docker in Docker (for GitLab CI) +# + +docker-dind : + docker build -f assets/dind.Dockerfile -t registry.gitlab.com/librewolf-community/browser/bsys5/dind:latest . diff --git a/assets/dind.Dockerfile b/assets/dind.Dockerfile new file mode 100644 index 0000000..37c2a5a --- /dev/null +++ b/assets/dind.Dockerfile @@ -0,0 +1,9 @@ +FROM ubuntu:jammy + +RUN apt-get update &&\ + apt-get install -y ca-certificates curl gnupg lsb-release &&\ + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg &&\ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null &&\ + apt-get update &&\ + apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin make wget lbzip2 &&\ + apt-get clean diff --git a/assets/linux.Dockerfile b/assets/linux.Dockerfile index 8b6432e..fd8de85 100644 --- a/assets/linux.Dockerfile +++ b/assets/linux.Dockerfile @@ -16,7 +16,7 @@ ENV TZ=Europe/Amsterdam # dependencies needed to run ./mach bootstrap RUN ( apt-get -y update && apt-get -y upgrade && apt-get -y install python3 python3-dev python3-pip wget dpkg-sig ; true) RUN ( dnf -y upgrade && dnf -y install python3 python3-devel wget rpm-build rpm-sign ; true) -RUN ( zypper -n in mercurial python3 python3-pip python3-devel wget rpm-build lld ; true) +RUN ( zypper -n in mercurial python3 python3-pip python3-devel wget rpm-build ; true) # run the bootstrap RUN cd /tmp &&\ @@ -24,6 +24,7 @@ RUN cd /tmp &&\ tar xf librewolf-$version-$source_release.source.tar.gz &&\ cd librewolf-$version-$source_release &&\ MOZBUILD_STATE_PATH=$HOME/.mozbuild ./mach --no-interactive bootstrap --application-choice=browser &&\ + (cd /root/.mozbuild && /tmp/librewolf-$version-$source_release/mach artifact toolchain --from-build sysroot-wasm32-wasi) &&\ /root/.cargo/bin/cargo install cbindgen &&\ cd .. &&\ rm -rf librewolf-$version-$source_release librewolf-$version-$source_release.source.tar.gz diff --git a/assets/linux.mk b/assets/linux.mk index 94315e9..5393f4d 100644 --- a/assets/linux.mk +++ b/assets/linux.mk @@ -18,6 +18,7 @@ build : $(outfile) $(outfile).sha256sum $(outfile) : ${MAKE} work + if [ "$(distro)" != "tumbleweed" ]; then ln -sfv ./lw/mozconfig.new.without-bootstrap work/librewolf-$(version)-$(source_release)/mozconfig; fi ifeq ($(use_docker),false) (cd work/librewolf-$(version)-$(source_release) && ./mach build && echo 'Packaging... (output hidden)' && cat browser/locales/shipped-locales | xargs ./mach package-multi-locale --locales >/dev/null) else diff --git a/assets/macos.Dockerfile b/assets/macos.Dockerfile index a463aed..7bcfc5d 100644 --- a/assets/macos.Dockerfile +++ b/assets/macos.Dockerfile @@ -35,11 +35,13 @@ RUN ./mach artifact toolchain --from-build linux64-libdmg RUN ./mach artifact toolchain --from-build linux64-cctools-port RUN ./mach artifact toolchain --from-build linux64-hfsplus RUN ./mach artifact toolchain --from-build linux64-binutils +RUN ./mach artifact toolchain --from-build sysroot-wasm32-wasi ## RUN ./mach artifact toolchain --from-build linux64-clang-macosx-cross RUN /root/.cargo/bin/cargo install cbindgen RUN /root/.cargo/bin/rustup target add $arch-apple-darwin RUN cp -r binutils /root/.mozbuild RUN cp -r cctools /root/.mozbuild +RUN cp -r sysroot-wasm32-wasi /root/.mozbuild ## RUN cp -r clang /root/.mozbuild RUN cp -r dmg /root/.mozbuild RUN cp -r hfsplus-tools /root/.mozbuild diff --git a/assets/macos.mozconfig b/assets/macos.mozconfig index e236eaa..180e0d9 100644 --- a/assets/macos.mozconfig +++ b/assets/macos.mozconfig @@ -55,7 +55,3 @@ mk_add_options MOZ_CRASHREPORTER=0 mk_add_options MOZ_DATA_REPORTING=0 mk_add_options MOZ_SERVICES_HEALTHREPORT=0 mk_add_options MOZ_TELEMETRY_REPORTING=0 - -# wasi: we ask maintainers to all use this as wasi sysroot.. -# There is a script 'setup-wasi-linux.sh' that attempts to download the wasi sdk and place it into the location pointed to below. -ac_add_options --with-wasi-sysroot=/root/.mozbuild/wrlb/wasi-sysroot diff --git a/source_release b/source_release index 00750ed..b8626c4 100644 --- a/source_release +++ b/source_release @@ -1 +1 @@ -3 +4