From 72dae4c786794596e550ce6a9666bb46efa18f5c Mon Sep 17 00:00:00 2001 From: teidesu Date: Mon, 5 Aug 2024 10:26:36 +0300 Subject: [PATCH] chore(koi): initial migration to shared postgres - postgres on the host - improved docker networking - sharkey: moved from docker-compose to nix-managed containers --- hosts/koi/configuration.nix | 5 +- .../containers/sharkey/.config/default.yml | 8 +-- hosts/koi/containers/sharkey/Dockerfile | 4 +- hosts/koi/containers/sharkey/default.nix | 66 +++++++++++++++++-- .../containers/sharkey/docker-compose.yaml | 65 ------------------ hosts/koi/partials/docker.nix | 14 ++++ hosts/koi/services/postgresql.nix | 39 +++++++++++ lib/containers.nix | 17 ++++- 8 files changed, 136 insertions(+), 82 deletions(-) delete mode 100644 hosts/koi/containers/sharkey/docker-compose.yaml create mode 100644 hosts/koi/partials/docker.nix create mode 100644 hosts/koi/services/postgresql.nix diff --git a/hosts/koi/configuration.nix b/hosts/koi/configuration.nix index 4cfa4eb..3106000 100755 --- a/hosts/koi/configuration.nix +++ b/hosts/koi/configuration.nix @@ -11,11 +11,13 @@ ./hardware-configuration.nix ./partials/fde.nix + ./partials/docker.nix ./services/coredns.nix ./services/sing-box.nix ./services/nginx.nix ./services/phpfront.nix + ./services/postgresql.nix ./services/landing ./containers/torrent.nix @@ -76,9 +78,6 @@ allowedBridges = [ "br0" ]; }; - virtualisation.docker.enable = true; - virtualisation.oci-containers.backend = "docker"; - boot.extraModprobeConfig = '' options kvm_amd avic=1 nested=0 options kvm ignore_msrs=N report_ignored_msrs=Y diff --git a/hosts/koi/containers/sharkey/.config/default.yml b/hosts/koi/containers/sharkey/.config/default.yml index 911051a..07db093 100644 --- a/hosts/koi/containers/sharkey/.config/default.yml +++ b/hosts/koi/containers/sharkey/.config/default.yml @@ -2,7 +2,7 @@ url: https://very.stupid.fish port: 80 db: - host: db + host: 172.17.0.1 port: 5432 db: misskey user: misskey @@ -11,14 +11,14 @@ db: dbReplications: false redis: - host: redis + host: sharkey-redis.docker port: 6379 # ┌───────────────────────────┐ #───┘ MeiliSearch configuration └───────────────────────────── meilisearch: - host: meilisearch + host: sharkey-meili.docker port: 7700 apiKey: misskeymeilisearch index: '' @@ -31,7 +31,7 @@ clusterLimit: 2 maxNoteLength: 30000 -proxy: 'http://10.42.0.2:7890' +proxy: 'http://172.17.0.1:7890' proxyBypassHosts: - api.deepl.com - api-free.deepl.com diff --git a/hosts/koi/containers/sharkey/Dockerfile b/hosts/koi/containers/sharkey/Dockerfile index d7b22b7..1192fb4 100644 --- a/hosts/koi/containers/sharkey/Dockerfile +++ b/hosts/koi/containers/sharkey/Dockerfile @@ -58,8 +58,8 @@ RUN node /patches/patch-locale.js FROM node:${NODE_VERSION} -ARG UID="991" -ARG GID="991" +ARG UID="1104" +ARG GID="1104" RUN apk add ffmpeg tini jemalloc \ && corepack enable \ diff --git a/hosts/koi/containers/sharkey/default.nix b/hosts/koi/containers/sharkey/default.nix index b037835..e9f660c 100644 --- a/hosts/koi/containers/sharkey/default.nix +++ b/hosts/koi/containers/sharkey/default.nix @@ -1,17 +1,69 @@ { abs, pkgs, ... }@inputs: -{ - imports = [ - ((import (abs "lib/containers.nix") inputs).mkDockerComposeContainer { - directory = ./.; - }) - ]; +let + UID = 1104; + trivial = import (abs "lib/trivial.nix") inputs; + + context = trivial.storeDirectory ./.; +in { + users.users.misskey = { + isNormalUser = true; + uid = UID; + }; systemd.tmpfiles.rules = [ "d /mnt/puffer/Sharkey 0777 root root -" "d /srv/Sharkey 0777 root root -" ]; + services.postgresql.ensureUsers = [ + { name = "misskey"; ensureDBOwnership = true; } + ]; + services.postgresql.ensureDatabases = [ "misskey" ]; + desu.postgresql.ensurePasswords.misskey = "misskey"; + + virtualisation.oci-containers.containers.sharkey-redis = { + image = "docker.io/redis:7.0-alpine"; + volumes = [ + "/srv/Sharkey/redis:/data" + ]; + user = builtins.toString UID; + }; + + virtualisation.oci-containers.containers.sharkey-meili = { + image = "getmeili/meilisearch:v1.3.4"; + volumes = [ + "/srv/Sharkey/meili_data:/meili_data" + ]; + environment = { + MEILI_NO_ANALYTICS = "true"; + MEILI_ENV = "production"; + MEILI_MASTER_KEY = "misskeymeilisearch"; + }; + user = builtins.toString UID; + }; + + # not really reproducible but fuck it i figured it's the best way lol. + # im **not** rewriting that 100 lines dockerfile + systemd.services.docker-sharkey.serviceConfig.ExecStartPre = [ + (pkgs.writeShellScript "build-sharkey" '' + docker build -t local/sharkey ${context} + '') + ]; + systemd.services.docker-sharkey.after = [ "postgresql.service" ]; + virtualisation.oci-containers.containers.sharkey = { + dependsOn = [ "sharkey-redis" "sharkey-meili" ]; + image = "local/sharkey"; + volumes = [ + "/mnt/puffer/Sharkey/files:/sharkey/files" + "${context}/.config:/sharkey/.config:ro" + ]; + environment = { + NODE_ENV = "production"; + }; + user = builtins.toString UID; + }; + services.nginx.virtualHosts."very.stupid.fish" = { forceSSL = true; useACMEHost = "stupid.fish"; @@ -22,7 +74,7 @@ ''; locations."/" = { - proxyPass = "http://web.sharkey.docker$request_uri"; + proxyPass = "http://sharkey.docker$request_uri"; proxyWebsockets = true; }; }; diff --git a/hosts/koi/containers/sharkey/docker-compose.yaml b/hosts/koi/containers/sharkey/docker-compose.yaml deleted file mode 100644 index e207e02..0000000 --- a/hosts/koi/containers/sharkey/docker-compose.yaml +++ /dev/null @@ -1,65 +0,0 @@ -version: "3" - -services: - web: - build: - context: . - restart: unless-stopped - depends_on: - db: - condition: service_healthy - redis: - condition: service_healthy - meilisearch: - condition: service_started - networks: - - calcnet - environment: - NODE_ENV: production - volumes: - - /mnt/puffer/Sharkey/files:/sharkey/files - - .config:/sharkey/.config:ro - - redis: - restart: unless-stopped - image: docker.io/redis:7.0-alpine - networks: - - calcnet - volumes: - - /srv/Sharkey/redis:/data - healthcheck: - test: "redis-cli ping" - interval: 5s - retries: 20 - - db: - restart: unless-stopped - image: docker.io/postgres:15-alpine - networks: - - calcnet - environment: - POSTGRES_PASSWORD: misskey - POSTGRES_USER: misskey - POSTGRES_DB: misskey - volumes: - - /srv/Sharkey/db:/var/lib/postgresql/data - healthcheck: - test: "pg_isready -U $$POSTGRES_USER -d $$POSTGRES_DB" - interval: 5s - retries: 20 - - meilisearch: - restart: always - image: getmeili/meilisearch:v1.3.4 - environment: - - MEILI_NO_ANALYTICS=true - - MEILI_ENV=production - - MEILI_MASTER_KEY=misskeymeilisearch - networks: - - calcnet - volumes: - - /srv/Sharkey/meili_data:/meili_data - -networks: - calcnet: {} - diff --git a/hosts/koi/partials/docker.nix b/hosts/koi/partials/docker.nix new file mode 100644 index 0000000..e932321 --- /dev/null +++ b/hosts/koi/partials/docker.nix @@ -0,0 +1,14 @@ +{ ... }: + +{ + virtualisation.docker.enable = true; + virtualisation.docker.daemon.settings = { + # docker for whatever reason decides not to use system resolver if we have 127.0.0.1 in resolv.conf + # and fallbacks to google dns (src: https://github.com/moby/moby/issues/6388#issuecomment-46343580) + # but we want it to use it. so pin the cidr used by docker and force the gateway as the default dns :D + fixed-cidr = "172.17.0.1/16"; + default-gateway = "172.17.0.1"; + dns = ["172.17.0.1"]; + }; + virtualisation.oci-containers.backend = "docker"; +} \ No newline at end of file diff --git a/hosts/koi/services/postgresql.nix b/hosts/koi/services/postgresql.nix new file mode 100644 index 0000000..7e913b9 --- /dev/null +++ b/hosts/koi/services/postgresql.nix @@ -0,0 +1,39 @@ +{ pkgs, config, lib, ... }: + +let + cfg = config.desu.postgresql; +in { + options.desu.postgresql = with lib; { + ensurePasswords = mkOption { + type = types.attrsOf (types.str); + default = {}; + }; + }; + + config = { + services.postgresql = { + enable = true; + enableJIT = true; + enableTCPIP = true; + package = pkgs.postgresql_15; + dataDir = "/srv/postgres"; + + authentication = '' + host all all 172.17.0.1/16 md5 + ''; + }; + + # expose postgres to docker containers + networking.firewall.extraCommands = '' + iptables -A nixos-fw -p tcp --dport 5432 -j nixos-fw-accept -i docker0 + ''; + + systemd.services.postgresql.postStart = + builtins.concatStringsSep "\n" ( + lib.attrsets.mapAttrsToList ( + # who cares about injections LOL. also i hate bash + user: password: ''$PSQL -tAc 'ALTER user "${user}" with password '"'"'${password}'"'"';' '' + ) cfg.ensurePasswords + ); + }; +} \ No newline at end of file diff --git a/lib/containers.nix b/lib/containers.nix index 775e1c1..ebdcea0 100644 --- a/lib/containers.nix +++ b/lib/containers.nix @@ -62,6 +62,7 @@ in , env ? { } , envFiles ? [ ] , extraFlags ? [ ] + , after ? [ ] }: let # referencing the file directly would make the service dependant @@ -92,7 +93,7 @@ in { systemd.services."docker-compose-${name}" = { wantedBy = if autoStart then [ "multi-user.target" ] else [ ]; - after = [ "docker.service" "docker.socket" ]; + after = [ "docker.service" "docker.socket" ] ++ after; serviceConfig = { WorkingDirectory = storeDir; ExecStart = "${pkgs.docker}/bin/docker compose ${cmdlineBeforeUp} up ${cmdline}"; @@ -100,4 +101,18 @@ in } // (extraConfig.serviceConfig or { }); } // (builtins.removeAttrs extraConfig [ "serviceConfig" ]); }; + + # buildDockerfile = { name, context }: builtins.derivation { + # name = "${name}-image"; + # # __noChroot = true; + # src = context; + # builder = pkgs.writeShellScript "builder.sh" (let + # docker = "${pkgs.docker}/bin/docker"; + # in '' + # ${docker} build -t ${name} $src + # ${docker} save -o $out ${name} + # ${docker} image rm ${name} + # ''); + # system = pkgs.system; + # }; }