From 488f089a3a52c3a335ba50032831e27508ca4977 Mon Sep 17 00:00:00 2001 From: teidesu Date: Sat, 4 Jan 2025 22:52:20 +0300 Subject: [PATCH] chore(koi/forgejo): improved dind image and added actions cleanup script --- .../containers/forgejo/clear-actions-logs.mjs | 48 +++++++++++++++++++ hosts/koi/containers/forgejo/default.nix | 15 ++++++ hosts/koi/services/actions-runner/default.nix | 43 ++++++++++------- .../Dockerfile} | 12 ++++- .../image-dind/start-dockerd.sh | 21 ++++++++ 5 files changed, 120 insertions(+), 19 deletions(-) create mode 100644 hosts/koi/containers/forgejo/clear-actions-logs.mjs rename hosts/koi/services/actions-runner/{Dockerfile.dind => image-dind/Dockerfile} (67%) create mode 100755 hosts/koi/services/actions-runner/image-dind/start-dockerd.sh diff --git a/hosts/koi/containers/forgejo/clear-actions-logs.mjs b/hosts/koi/containers/forgejo/clear-actions-logs.mjs new file mode 100644 index 0000000..7a432d5 --- /dev/null +++ b/hosts/koi/containers/forgejo/clear-actions-logs.mjs @@ -0,0 +1,48 @@ +// forgejo doesn't have a built-in way to clear old actions logs, so we have to do it manually +// https://github.com/go-gitea/gitea/issues/24256, didnt find a separate issue for forgejo + +import * as fs from 'fs' +import * as path from 'path' + +const LOGS_DIR = '/srv/forgejo/data/data/actions_log' +const RETENTION = 30 * 24 * 60 * 60 * 1000 // 30 days + +function handleLogsDir(dir) { + // the actions_log dir structure is something like this: + // actions_log/owner/repo/job-id-hex/job-id.log.zst + // (todo: verify this is the case) + let found = false + const subdirs = fs.readdirSync(dir, { withFileTypes: true }) + for (const subdir of subdirs) { + if (!subdir.isDirectory()) continue + + const fullPath = path.join(dir, subdir.name) + + // validate that the structure is how we expect it + const jobId = parseInt(subdir.name, 16) + if (isNaN(jobId)) { + console.error(`invalid job id at: ${fullPath}`) + process.exit(1) + } + + const children = fs.readdirSync(fullPath, { withFileTypes: true }) + if (children.length !== 1 || !children[0].isFile() || children[0].name !== `${jobId}.log.zst`) { + console.error(`Invalid actions_log dir: ${dir}`) + process.exit(1) + } + + const stat = fs.statSync(fullPath) + if (stat.mtimeMs < Date.now() - RETENTION) { + console.log(`deleting old (${new Date(stat.mtimeMs).toISOString()}) actions log: ${fullPath}`) + fs.rmSync(fullPath, { recursive: true }) + found = true + } + } + +} + +for (const owner of fs.readdirSync(LOGS_DIR)) { + for (const repo of fs.readdirSync(path.join(LOGS_DIR, owner))) { + handleLogsDir(path.join(LOGS_DIR, owner, repo)) + } +} \ No newline at end of file diff --git a/hosts/koi/containers/forgejo/default.nix b/hosts/koi/containers/forgejo/default.nix index ab809e6..75ad3e0 100644 --- a/hosts/koi/containers/forgejo/default.nix +++ b/hosts/koi/containers/forgejo/default.nix @@ -47,6 +47,15 @@ in { systemd.services.docker-forgejo.after = [ "postgresql.service" "gocryptfs.service" ]; + systemd.services.forgejo-clear-actions-logs = { + serviceConfig = { + Type = "oneshot"; + User = "forgejo"; + ExecStart = "${pkgs.nodejs_22}/bin/nodejs ${./clear-actions-logs.mjs}"; + }; + startAt = "03:00"; + }; + systemd.tmpfiles.rules = [ "d /srv/forgejo/repos 0700 ${builtins.toString UID} ${builtins.toString UID} -" ]; @@ -58,6 +67,12 @@ in { locations."/" = { proxyPass = "http://forgejo.docker:3000$request_uri"; proxyWebsockets = true; + + extraConfig = '' + client_max_body_size 1g; + proxy_read_timeout 120s; + proxy_send_timeout 120s; + ''; }; }; diff --git a/hosts/koi/services/actions-runner/default.nix b/hosts/koi/services/actions-runner/default.nix index ef6e35a..230011b 100644 --- a/hosts/koi/services/actions-runner/default.nix +++ b/hosts/koi/services/actions-runner/default.nix @@ -1,39 +1,32 @@ { config, pkgs, ... }: -{ +let + UID = 1126; +in { desu.secrets.forgejo-runners-token = {}; desu.secrets.forgejo-runners-token-sf = {}; + users.users.actions-runner = { + isNormalUser = true; + uid = 1126; + }; + systemd.services.actions-runner-build-dind = { description = "dind image builder for actions runner"; after = [ "docker.service" ]; wantedBy = [ "multi-user.target" ]; serviceConfig = { Type = "oneshot"; - ExecStart = "${pkgs.docker}/bin/docker build -t local/actions-runner-dind -f ${./Dockerfile.dind} ."; + ExecStart = "${pkgs.docker}/bin/docker build -t local/actions-runner-dind ${pkgs.copyPathToStore ./image-dind}"; }; }; systemd.services.gitea-runner-koi.requires = [ "actions-runner-build-dind.service" ]; + systemd.services.gitea-runner-koi-stupid-fish.requires = [ "actions-runner-build-dind.service" ]; services.gitea-actions-runner = { package = pkgs.forgejo-runner; instances.koi = { - name = "koi"; - enable = true; - url = "https://codeberg.org"; - tokenFile = config.desu.secrets.forgejo-runners-token.path; - labels = [ - "node18:docker://node:18-bullseye" - "node20:docker://node:20-bullseye" - "node22:docker://node:22-bullseye" - "docker:docker://local/actions-runner-dind" - ]; - settings = { - runner.capacity = 8; - }; - }; - instances.koi-stupid-fish = { name = "koi"; enable = true; url = "https://git.stupid.fish"; @@ -42,10 +35,26 @@ "node18:docker://node:18-bullseye" "node20:docker://node:20-bullseye" "node22:docker://node:22-bullseye" + # fun fact: the actual image doesnt matter! it's only used to determine the runner + "docker:docker://node:22-bullseye" ]; settings = { runner.capacity = 8; }; }; + + # a separate runner for dind because it requires privileged mode and act-runner doesnt support setting --privileged for certain images + instances.koi-dind = { + name = "koi-dind"; + enable = true; + url = "https://git.stupid.fish"; + tokenFile = config.desu.secrets.forgejo-runners-token-sf.path; + labels = [ + "docker-dind:docker://local/actions-runner-dind" + ]; + settings = { + container.privileged = true; + }; + }; }; } \ No newline at end of file diff --git a/hosts/koi/services/actions-runner/Dockerfile.dind b/hosts/koi/services/actions-runner/image-dind/Dockerfile similarity index 67% rename from hosts/koi/services/actions-runner/Dockerfile.dind rename to hosts/koi/services/actions-runner/image-dind/Dockerfile index 3185be5..f20e5f7 100644 --- a/hosts/koi/services/actions-runner/Dockerfile.dind +++ b/hosts/koi/services/actions-runner/image-dind/Dockerfile @@ -1,12 +1,20 @@ FROM node:23.4.0-alpine AS node -FROM docker:27-dind +FROM docker:27-dind-rootless + +USER root COPY --from=node /usr/local/bin/node /usr/local/bin/node COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules COPY --from=node /usr/local/include/node /usr/local/include/node +COPY ./start-dockerd.sh /opt/start-dockerd.sh RUN apk add libstdc++ bash && \ ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \ ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx && \ - ln -s /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack \ No newline at end of file + ln -s /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack && \ + ln -s /run/user/1000/docker.sock /var/run/docker.sock + +ENV DOCKER_HOST=unix:///run/user/1000/docker.sock + +USER rootless \ No newline at end of file diff --git a/hosts/koi/services/actions-runner/image-dind/start-dockerd.sh b/hosts/koi/services/actions-runner/image-dind/start-dockerd.sh new file mode 100755 index 0000000..44b8f8f --- /dev/null +++ b/hosts/koi/services/actions-runner/image-dind/start-dockerd.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if docker info &> /dev/null; then + exit 0 +fi + +nohup /usr/local/bin/dockerd-entrypoint.sh > /home/rootless/dockerd.log 2>&1 & +export DOCKER_HOST=unix:///run/user/1000/docker.sock + +# wait for docker to start +retry=0 +while ! docker info &> /dev/null; do + sleep 1 + retry=$((retry + 1)) + if [ $retry -gt 15 ]; then + echo "Failed to start dockerd after 15 seconds" + exit 1 + fi +done