feat(koi): backups

hosts/koi/configuration.nix

@@ -23,6 +23,7 @@
     ./services/landing
     ./services/geesefs.nix
     ./services/actions-runner
+    ./services/autorestic
 
     ./containers/torrent.nix
     ./containers/soulseek

hosts/koi/containers/conduwuit/config.toml

@@ -9,7 +9,9 @@ sentry_traces_sample_rate = 0.01
 sentry_attach_stacktrace = false
 
 
-database_path = "/data"
+database_path = "/data/db"
+database_backup_path = "/data/backup"
+database_backups_to_keep = 1
 database_backend = "rocksdb"
 
 port = 6167

hosts/koi/services/autorestic/config.yaml

@@ -0,0 +1,87 @@
+version: 2
+
+global:
+  forget:
+    keep-daily: 30
+    keep-weekly: 52
+
+backends:
+  rclone:
+    type: rclone
+    path: backups:backups/koi
+    requireKey: true
+
+# todo: move this to `options.desu.autorestic.locations`
+locations:
+  conduwuit:
+    # while conduwuit does support online backups, it's not really useful
+    # since it needs to be invoked manually from the management room via "!admin server backup-database"
+    from: /srv/conduwuit/db
+    to: rclone
+    hooks:
+      prevalidate:
+        - systemctl stop docker-conduwuit
+      after:
+        - systemctl start docker-conduwuit
+    cron: '0 6 * * *'
+    options:
+      backup:
+        exclude:
+          - /srv/conduwuit/db/media
+  # media is safe to backup online, so we do this as a separate location to avoid too much downtime
+  conduwuit-media:
+    from: /srv/conduwuit/db/media
+    to: rclone
+    cron: '0 6 * * *'
+  mautrix-telegram:
+    from: /srv/mautrix-telegram/
+    to: rclone
+    cron: '0 6 * * *'
+  postgres:
+    from: /tmp/pg-backup/
+    to: rclone
+    cron: '0 6,18 * * *'
+    hooks:
+      prevalidate:
+        - mkdir -p /tmp/pg-backup
+        - chown postgres:postgres /tmp/pg-backup
+        - sudo -u postgres pg_dumpall -f /tmp/pg-backup/backup.sql
+      after:
+        - rm -rf /tmp/pg-backup
+  navidrome-db:
+    from: /srv/navidrome/navidrome.db.bak
+    to: rclone
+    hooks:
+      prevalidate:
+        - sqlite3 /srv/navidrome/navidrome.db ".backup /srv/navidrome/navidrome.db.bak"
+      after:
+        - rm /srv/navidrome/navidrome.db.bak
+    cron: '0 6 * * *'
+  bluesky-pds:
+    from: /srv/bluesky-pds/data/
+    to: rclone
+    cron: '0 6 * * *'
+  sftpgo:
+    from: /srv/sftpgo/
+    to: rclone
+    cron: '0 6 * * *'
+  siyuan-teidesu:
+    from: /srv/siyuan-teidesu
+    to: rclone
+    cron: '0 6 * * *'
+  slskd:
+    from: /srv/slskd/
+    to: rclone
+    cron: '0 6 * * *'
+  verdaccio:
+    from: /srv/verdaccio/
+    to: rclone
+    cron: '0 6 * * *'
+  teisu:
+    from: /srv/teisu/
+    to: rclone
+    cron: '0 6 * * *'
+  vaultwarden:
+    from: /srv/vaultwarden/
+    to: rclone
+    cron: '0 6 * * *'

hosts/koi/services/autorestic/default.nix

@@ -0,0 +1,33 @@
+{ pkgs, config, lib, ... }:
+
+
+let 
+  autoresticWrapped = pkgs.writeShellScriptBin "autorestic" ''
+    set -euo pipefail
+
+    set -o allexport
+    source ${config.desu.secrets.autorestic-env.path}
+    set +o allexport
+    export PATH="${lib.makeBinPath [ pkgs.restic pkgs.rclone pkgs.sqlite ]}:$PATH"
+
+    # crutch: autorestic requires a lockfile to be beside the config file
+    mkdir -p /etc/autorestic
+    chmod -R 700 /etc/autorestic
+    cp ${./config.yaml} /etc/autorestic/config.yaml
+
+    exec ${pkgs.autorestic}/bin/autorestic -c /etc/autorestic/config.yaml $@
+  '';
+in {
+  desu.secrets.autorestic-env.owner = "root";
+
+  systemd.services.autorestic = {
+    serviceConfig = {
+      Type = "oneshot";
+      User = "root";
+      ExecStart = "${autoresticWrapped}/bin/autorestic --ci cron";
+    };
+    startAt = "*:0/5";
+  };
+
+  environment.systemPackages = [ autoresticWrapped ];
+}

hosts/koi/services/geesefs.nix

@@ -3,6 +3,7 @@
 {
   imports = [
     (abs "services/geesefs.nix")
+    (abs "services/rclone-mount.nix")
     (abs "services/gocryptfs.nix")
   ];
 
@@ -35,11 +36,11 @@
       "--max-parallel-parts" "32"
       "--part-sizes" "25"
       "--large-read-cutoff" "40960"
-      "--enable-patch"
     ];
     bucket = "desu-priv";
     mountPoint = "/mnt/s3-desu-priv";
   };
+
   systemd.services.geesefs.after = [ "coredns.service" ];
 
   services.gocryptfs = {

readme.md

@@ -19,6 +19,7 @@ note to self on what needs to be installed on the host manually:
 - `/etc/vms/bnuuy.img` - qcow2 image of an ubuntu cloud image (e.g. `https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img`)
 - `/etc/secureboot/keys` - secure boot keys, generated with `sudo nix-shell -p sbctl --run "sbctl create-keys"`
 - to enroll fde onto tpm: `sudo systemd-cryptenroll /dev/nvme0n1p2 --tpm2-device=auto --tpm2-pcrs=0+2+7`
+- rclone config (for backups): `sudo nix-shell -p rclone --run "rclone config"` – you need to set up a remote called `backups`, which will be used as the destination for all backups
 
 ### teidesu-osx
 `cp /var/run/current-system/Library/Fonts/* /Library/Fonts` - copy nix-managed fonts to system fonts (waiting for [this PR](https://github.com/LnL7/nix-darwin/pull/754))

services/geesefs.nix

@@ -36,7 +36,7 @@
       cfg.bucket
       cfg.mountPoint
     ];
-  in {
+  in lib.mkIf cfg.enable {
     systemd.services.${cfg.serviceName} = {
       description = "${cfg.serviceName} Daemon";
       after = [ "network.target" ];
@@ -48,7 +48,7 @@
         Type = "forking";
         GuessMainPID = true;
         ExecStart = "${cfg.package}/bin/geesefs ${builtins.concatStringsSep " " (map lib.escapeShellArg allArgs)}";
-        ExecStop = "fusermount -uz ${lib.escapeShellArg cfg.mountPoint}";
+        ExecStop = "${pkgs.fuse}/bin/fusermount -uz ${lib.escapeShellArg cfg.mountPoint}";
         Restart = "on-failure";
       };
     };

services/gocryptfs.nix

@@ -46,7 +46,7 @@
 
   config = let 
     cfg = config.services.gocryptfs;
-  in {
+  in lib.mkIf cfg.enable {
     systemd.services.${cfg.serviceName} = {
       description = "${cfg.serviceName} daemon";
       wantedBy = [ "multi-user.target" ];

services/rclone-mount.nix

@@ -0,0 +1,64 @@
+{ config, lib, pkgs, ... }:
+
+{
+  options.services.rclone-mount = with lib; {
+    enable = mkEnableOption "rclone";
+    package = mkOption {
+      type = types.package;
+      default = pkgs.rclone;
+      defaultText = "pkgs.rclone";
+      description = "rclone package";
+    };
+    config = mkOption {
+      type = types.attrs;
+      description = "rclone config";
+    };
+    remote = mkOption {
+      type = types.str;
+      description = "rclone remote path to mount";
+    };
+    mountPoint = mkOption {
+      type = types.str;
+      description = "rclone mount point";
+    };
+    extraArgs = mkOption {
+      type = types.listOf types.str;
+      default = [ ];
+      description = "extra rclone arguments";
+    };
+    serviceName = mkOption {
+      type = types.str;
+      default = "rclone-mount";
+      description = "rclone service name";
+    };
+  };
+
+  config = let 
+    cfg = config.services.rclone-mount;
+
+    allArgs = cfg.extraArgs ++ [
+      cfg.remote
+      cfg.mountPoint
+    ];
+
+    rcloneConfig = (pkgs.formats.ini {}).generate "rclone.conf" cfg.config;
+  in lib.mkIf cfg.enable {
+    systemd.services.${cfg.serviceName} = {
+      description = "rclone-mount Daemon";
+      after = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      path = [ pkgs.fuse ];
+      serviceConfig = {
+        User = "root";
+        Group = "root";
+        ExecStart = "${cfg.package}/bin/rclone --config ${rcloneConfig} mount ${builtins.concatStringsSep " " (map lib.escapeShellArg allArgs)}";
+        ExecStop = "${pkgs.fuse}/bin/fusermount -uz ${lib.escapeShellArg cfg.mountPoint}";
+        Restart = "on-failure";
+      };
+    };
+
+    systemd.tmpfiles.rules = [
+      "d ${cfg.mountPoint} 0777 root root -"
+    ];
+  };
+}