chore(koi/forgejo): improved dind image and added actions cleanup script

hosts/koi/containers/forgejo/clear-actions-logs.mjs

@@ -0,0 +1,48 @@
+// forgejo doesn't have a built-in way to clear old actions logs, so we have to do it manually
+// https://github.com/go-gitea/gitea/issues/24256, didnt find a separate issue for forgejo
+
+import * as fs from 'fs'
+import * as path from 'path'
+
+const LOGS_DIR = '/srv/forgejo/data/data/actions_log'
+const RETENTION = 30 * 24 * 60 * 60 * 1000 // 30 days
+
+function handleLogsDir(dir) {
+    // the actions_log dir structure is something like this:
+    // actions_log/owner/repo/job-id-hex/job-id.log.zst
+    // (todo: verify this is the case)
+    let found = false
+    const subdirs = fs.readdirSync(dir, { withFileTypes: true })
+    for (const subdir of subdirs) {
+        if (!subdir.isDirectory()) continue
+
+        const fullPath = path.join(dir, subdir.name)
+
+        // validate that the structure is how we expect it
+        const jobId = parseInt(subdir.name, 16)
+        if (isNaN(jobId)) {
+            console.error(`invalid job id at: ${fullPath}`)
+            process.exit(1)
+        }
+
+        const children = fs.readdirSync(fullPath, { withFileTypes: true })
+        if (children.length !== 1 || !children[0].isFile() || children[0].name !== `${jobId}.log.zst`) {
+            console.error(`Invalid actions_log dir: ${dir}`)
+            process.exit(1)
+        }
+
+        const stat = fs.statSync(fullPath)
+        if (stat.mtimeMs < Date.now() - RETENTION) {
+            console.log(`deleting old (${new Date(stat.mtimeMs).toISOString()}) actions log: ${fullPath}`)
+            fs.rmSync(fullPath, { recursive: true })
+            found = true
+        }
+    }
+
+}
+
+for (const owner of fs.readdirSync(LOGS_DIR)) {
+    for (const repo of fs.readdirSync(path.join(LOGS_DIR, owner))) {
+        handleLogsDir(path.join(LOGS_DIR, owner, repo))
+    }
+}

hosts/koi/containers/forgejo/default.nix

@@ -47,6 +47,15 @@ in {
 
   systemd.services.docker-forgejo.after = [ "postgresql.service" "gocryptfs.service" ];
 
+  systemd.services.forgejo-clear-actions-logs = {
+    serviceConfig = {
+      Type = "oneshot";
+      User = "forgejo";
+      ExecStart = "${pkgs.nodejs_22}/bin/nodejs ${./clear-actions-logs.mjs}";
+    };
+    startAt = "03:00";
+  };
+
   systemd.tmpfiles.rules = [
     "d /srv/forgejo/repos 0700 ${builtins.toString UID} ${builtins.toString UID} -"
   ];
@@ -58,6 +67,12 @@ in {
     locations."/" = {
       proxyPass = "http://forgejo.docker:3000$request_uri";
       proxyWebsockets = true;
+
+      extraConfig = ''
+        client_max_body_size 1g;
+        proxy_read_timeout 120s;
+        proxy_send_timeout 120s;
+      '';
     };
   };
 

hosts/koi/services/actions-runner/default.nix

@@ -1,39 +1,32 @@
 { config, pkgs, ... }: 
 
-{
+let 
+  UID = 1126;
+in {
   desu.secrets.forgejo-runners-token = {};
   desu.secrets.forgejo-runners-token-sf = {};
 
+  users.users.actions-runner = {
+    isNormalUser = true;
+    uid = 1126;
+  };
+
   systemd.services.actions-runner-build-dind = {
     description = "dind image builder for actions runner";
     after = [ "docker.service" ];
     wantedBy = [ "multi-user.target" ];
     serviceConfig = {
       Type = "oneshot";
-      ExecStart = "${pkgs.docker}/bin/docker build -t local/actions-runner-dind -f ${./Dockerfile.dind} .";
+      ExecStart = "${pkgs.docker}/bin/docker build -t local/actions-runner-dind ${pkgs.copyPathToStore ./image-dind}";
     };
   };
 
   systemd.services.gitea-runner-koi.requires = [ "actions-runner-build-dind.service" ];
+  systemd.services.gitea-runner-koi-stupid-fish.requires = [ "actions-runner-build-dind.service" ];
 
   services.gitea-actions-runner = {
     package = pkgs.forgejo-runner;
     instances.koi = {
-      name = "koi";
-      enable = true;
-      url = "https://codeberg.org";
-      tokenFile = config.desu.secrets.forgejo-runners-token.path;
-      labels = [
-        "node18:docker://node:18-bullseye"
-        "node20:docker://node:20-bullseye"
-        "node22:docker://node:22-bullseye"
-        "docker:docker://local/actions-runner-dind"
-      ];
-      settings = {
-        runner.capacity = 8;
-      };
-    };
-    instances.koi-stupid-fish = {
       name = "koi";
       enable = true;
       url = "https://git.stupid.fish";
@@ -42,10 +35,26 @@
         "node18:docker://node:18-bullseye"
         "node20:docker://node:20-bullseye"
         "node22:docker://node:22-bullseye"
+        # fun fact: the actual image doesnt matter! it's only used to determine the runner
+        "docker:docker://node:22-bullseye"
       ];
       settings = {
         runner.capacity = 8;
       };
     };
+
+    # a separate runner for dind because it requires privileged mode and act-runner doesnt support setting --privileged for certain images
+    instances.koi-dind = {
+      name = "koi-dind";
+      enable = true;
+      url = "https://git.stupid.fish";
+      tokenFile = config.desu.secrets.forgejo-runners-token-sf.path;
+      labels = [
+        "docker-dind:docker://local/actions-runner-dind"
+      ];
+      settings = {
+        container.privileged = true;
+      };
+    };
   };
 }

hosts/koi/services/actions-runner/image-dind/Dockerfile

@@ -1,12 +1,20 @@
 FROM node:23.4.0-alpine AS node
 
-FROM docker:27-dind
+FROM docker:27-dind-rootless
+
+USER root 
 
 COPY --from=node /usr/local/bin/node /usr/local/bin/node
 COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules
 COPY --from=node /usr/local/include/node /usr/local/include/node
+COPY ./start-dockerd.sh /opt/start-dockerd.sh
 
 RUN apk add libstdc++ bash && \
     ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm && \
     ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx && \
-    ln -s /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack
+    ln -s /usr/local/lib/node_modules/corepack/dist/corepack.js /usr/local/bin/corepack && \
+    ln -s /run/user/1000/docker.sock /var/run/docker.sock
+
+ENV DOCKER_HOST=unix:///run/user/1000/docker.sock
+
+USER rootless

hosts/koi/services/actions-runner/image-dind/start-dockerd.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+if docker info &> /dev/null; then
+  exit 0
+fi
+
+nohup /usr/local/bin/dockerd-entrypoint.sh > /home/rootless/dockerd.log 2>&1 &
+export DOCKER_HOST=unix:///run/user/1000/docker.sock
+
+# wait for docker to start
+retry=0
+while ! docker info &> /dev/null; do
+  sleep 1
+  retry=$((retry + 1))
+  if [ $retry -gt 15 ]; then
+    echo "Failed to start dockerd after 15 seconds"
+    exit 1
+  fi
+done