From 084ac001de74ee0a816076b2c7dd171891656e98 Mon Sep 17 00:00:00 2001 From: teidesu Date: Thu, 16 Jan 2025 04:03:14 +0300 Subject: [PATCH] chore(koi): dind -> buildkitd in actions-runner --- .../koi/services/actions-runner/buildkitd.nix | 32 ++++++++++++++++ hosts/koi/services/actions-runner/default.nix | 36 +++++++++--------- .../{image-dind => image-buildkit}/Dockerfile | 11 +++--- .../image-buildkit/registry-login.sh | 12 ++++++ .../image-dind/start-dockerd.sh | 21 ---------- secrets/forgejo-runners-token.age | Bin 258 -> 0 bytes 6 files changed, 66 insertions(+), 46 deletions(-) create mode 100644 hosts/koi/services/actions-runner/buildkitd.nix rename hosts/koi/services/actions-runner/{image-dind => image-buildkit}/Dockerfile (69%) create mode 100755 hosts/koi/services/actions-runner/image-buildkit/registry-login.sh delete mode 100755 hosts/koi/services/actions-runner/image-dind/start-dockerd.sh delete mode 100644 secrets/forgejo-runners-token.age diff --git a/hosts/koi/services/actions-runner/buildkitd.nix b/hosts/koi/services/actions-runner/buildkitd.nix new file mode 100644 index 0000000..06dd96c --- /dev/null +++ b/hosts/koi/services/actions-runner/buildkitd.nix @@ -0,0 +1,32 @@ +{ pkgs, ... }: + +{ + virtualisation.oci-containers.containers.act-runner-buildkitd = { + image = "moby/buildkit:v0.19.0-rc2-rootless"; + cmd = [ + "--oci-worker-no-process-sandbox" + "--addr=unix:///var/run/act-runner-buildkit/buildkitd.sock" + ]; + user = "1000:1000"; + extraOptions = [ + "--security-opt=seccomp=unconfined" + "--security-opt=apparmor=unconfined" + "--mount=type=bind,source=/var/lib/act-runner-buildkit,target=/home/user/.local/share/buildkit" + "--mount=type=bind,source=/var/run/act-runner-buildkit,target=/var/run/act-runner-buildkit" + ]; + }; + + systemd.services.act-runner-buildkit-clear-cache = { + serviceConfig = { + Type = "oneshot"; + User = "1000"; + ExecStart = "${pkgs.buildkit}/bin/buildctl --addr=unix:///var/run/act-runner-buildkit/buildkitd.sock prune"; + }; + startAt = "Mon 03:00"; + }; + + systemd.tmpfiles.rules = [ + "d /var/lib/act-runner-buildkit 0700 1000 1000 -" + "d /var/run/act-runner-buildkit 0700 1000 1000 -" + ]; +} \ No newline at end of file diff --git a/hosts/koi/services/actions-runner/default.nix b/hosts/koi/services/actions-runner/default.nix index d405682..4ebba67 100644 --- a/hosts/koi/services/actions-runner/default.nix +++ b/hosts/koi/services/actions-runner/default.nix @@ -1,28 +1,23 @@ { config, pkgs, ... }: -let - UID = 1126; -in { - desu.secrets.forgejo-runners-token = {}; +{ + imports = [ ./buildkitd.nix ]; desu.secrets.forgejo-runners-token-sf = {}; - users.users.actions-runner = { - isNormalUser = true; - uid = 1126; - }; - - systemd.services.actions-runner-build-dind = { - description = "dind image builder for actions runner"; + systemd.services.actions-runner-build-buildkit = { + description = "buildkit image builder for actions runner"; after = [ "docker.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; - ExecStart = "${pkgs.docker}/bin/docker build -t local/actions-runner-dind ${pkgs.copyPathToStore ./image-dind}"; + ExecStart = "${pkgs.docker}/bin/docker build -t local/actions-runner-buildkit ${pkgs.copyPathToStore ./image-buildkit}"; }; }; - systemd.services.gitea-runner-koi.requires = [ "actions-runner-build-dind.service" ]; - systemd.services.gitea-runner-koi-stupid-fish.requires = [ "actions-runner-build-dind.service" ]; + systemd.services.gitea-runner-koi-buildkit.requires = [ + "actions-runner-build-buildkit.service" + "docker-act-runner-buildkitd.service" + ]; services.gitea-actions-runner = { package = pkgs.forgejo-runner; @@ -43,17 +38,20 @@ in { }; }; - # a separate runner for dind because it requires privileged mode and act-runner doesnt support setting --privileged for certain images - instances.koi-dind = { - name = "koi-dind"; + instances.koi-buildkit = { + name = "koi-buildkit"; enable = true; url = "https://git.stupid.fish"; tokenFile = config.desu.secrets.forgejo-runners-token-sf.path; labels = [ - "docker-dind:docker://local/actions-runner-dind" + "buildkit:docker://local/actions-runner-buildkit" ]; settings = { - container.privileged = true; + runner.capacity = 4; + container = { + valid_volumes = [ "/var/run/act-runner-buildkit" ]; + options = "--user=1000:1000 --mount=type=bind,source=/var/run/act-runner-buildkit,target=/var/run/buildkit"; + }; }; }; }; diff --git a/hosts/koi/services/actions-runner/image-dind/Dockerfile b/hosts/koi/services/actions-runner/image-buildkit/Dockerfile similarity index 69% rename from hosts/koi/services/actions-runner/image-dind/Dockerfile rename to hosts/koi/services/actions-runner/image-buildkit/Dockerfile index f20e5f7..c71ea83 100644 --- a/hosts/koi/services/actions-runner/image-dind/Dockerfile +++ b/hosts/koi/services/actions-runner/image-buildkit/Dockerfile @@ -1,20 +1,19 @@ FROM node:23.4.0-alpine AS node -FROM docker:27-dind-rootless +FROM moby/buildkit:master-rootless USER root COPY --from=node /usr/local/bin/node /usr/local/bin/node COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules COPY --from=node /usr/local/include/node /usr/local/include/node -COPY ./start-dockerd.sh /opt/start-dockerd.sh +COPY ./registry-login.sh /opt/registry-login.sh RUN apk add libstdc++ bash && \ ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx && \ - ln -s /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack && \ - ln -s /run/user/1000/docker.sock /var/run/docker.sock + ln -s /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack -ENV DOCKER_HOST=unix:///run/user/1000/docker.sock +ENV BUILDKIT_HOST=unix:///var/run/buildkit/buildkitd.sock -USER rootless \ No newline at end of file +USER user \ No newline at end of file diff --git a/hosts/koi/services/actions-runner/image-buildkit/registry-login.sh b/hosts/koi/services/actions-runner/image-buildkit/registry-login.sh new file mode 100755 index 0000000..e8bbb82 --- /dev/null +++ b/hosts/koi/services/actions-runner/image-buildkit/registry-login.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ -z "$1" -o -z "$2" -o -z "$3" ]; then + echo "Usage: $0 " + exit 1 +fi + +BASE64_AUTH=$(echo -n "$2:$3" | base64) +mkdir -p /home/user/.docker +echo "{\"auths\": {\"$1\": {\"auth\": \"$BASE64_AUTH\"}}}" > /home/user/.docker/config.json \ No newline at end of file diff --git a/hosts/koi/services/actions-runner/image-dind/start-dockerd.sh b/hosts/koi/services/actions-runner/image-dind/start-dockerd.sh deleted file mode 100755 index 44b8f8f..0000000 --- a/hosts/koi/services/actions-runner/image-dind/start-dockerd.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail - -if docker info &> /dev/null; then - exit 0 -fi - -nohup /usr/local/bin/dockerd-entrypoint.sh > /home/rootless/dockerd.log 2>&1 & -export DOCKER_HOST=unix:///run/user/1000/docker.sock - -# wait for docker to start -retry=0 -while ! docker info &> /dev/null; do - sleep 1 - retry=$((retry + 1)) - if [ $retry -gt 15 ]; then - echo "Failed to start dockerd after 15 seconds" - exit 1 - fi -done diff --git a/secrets/forgejo-runners-token.age b/secrets/forgejo-runners-token.age deleted file mode 100644 index de18fd70e21ffc8a29c2408ffff390c3ab34d5ec..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 258 zcmV+d0sa1AXJsvAZewzJaCB*JZZ2- zQfe<*F?e}LOlDJKOlf9waAq-XS9eBsL{>&lZ)9|Fc5PX9OfPpfM>GmfRBvupL`61bH86KabWU_fVQ4XUL0WNfV=+-NV`B