Move stuff to various S3 stores instead of local minio (#5)

Move most things to Cloudflare R2, whilst gotosocial goes to local.

Reviewed-on: #5
Co-authored-by: Natsu Kagami <nki@nkagami.me>
Co-committed-by: Natsu Kagami <nki@nkagami.me>

modules/cloud/authentik/default.nix

@@ -48,7 +48,14 @@ in
   };
 
   config = mkIf cfg.enable {
-    systemd.services.arion-authentik.serviceConfig.EnvironmentFile = cfg.envFile;
+    systemd.services.arion-authentik = {
+      serviceConfig.EnvironmentFile = cfg.envFile;
+      serviceConfig.Type = "notify";
+      serviceConfig.NotifyAccess = "all";
+      script = lib.mkBefore ''
+        ${lib.getExe pkgs.wait4x} http http://127.0.0.1:${toString cfg.port} --expect-status-code 200 -t 0 -q -- systemd-notify --ready &
+      '';
+    };
     virtualisation.arion.projects.authentik.settings = {
       services.postgresql.service = {
         image = images.postgresql;

modules/cloud/conduit/heisenbridge.nix

@@ -33,7 +33,7 @@ with lib; {
     {
       systemd.services.heisenbridge = {
         description = "Matrix<->IRC bridge";
-        requires = [ "matrix-synapse.service" ]; # So the registration file can be used by Synapse
+        requires = [ "matrix-conduit-nkagami.service" "matrix-synapse.service" ]; # So the registration file can be used by Synapse
         wantedBy = [ "multi-user.target" ];
 
         serviceConfig = rec {

modules/cloud/gotosocial/default.nix

@@ -4,6 +4,7 @@ let
   cfg = config.cloud.gotosocial;
 
   dbUser = "gotosocial";
+  storageLocation = "/mnt/data/gotosocial";
 in
 {
   options.cloud.gotosocial = {
@@ -74,6 +75,9 @@ in
         # Media
         media-emoji-remote-max-size = 256 * 1024 /* bytes */;
         media-emoji-local-max-size = 256 * 1024 /* bytes */;
+        media-remote-cache-days = 7;
+        media-cleanup-from = "00:00";
+        media-cleanup-every = "24h";
         # OIDC
         oidc-enabled = true;
         oidc-idp-name = "DTTH";
@@ -82,10 +86,22 @@ in
         http-client.block-ips = [ "11.0.0.0/24" ];
         # Advanced
         advanced-rate-limit-requests = 0;
+        # Storage
+        storage-backend = "local";
+        storage-local-base-path = "${storageLocation}/storage";
         # instance-inject-mastodon-version = true;
       };
     };
-    systemd.services.gotosocial.requires = mkAfter [ "minio.service" "postgresql.service" ];
+    systemd.services.gotosocial.requires = mkAfter [ "postgresql.service" "arion-authentik.service" ];
-    systemd.services.gotosocial.after = mkAfter [ "minio.service" "postgresql.service" ];
+    systemd.services.gotosocial.after = mkAfter [ "postgresql.service" "arion-authentik.service" ];
+    systemd.services.gotosocial.unitConfig = {
+      RequiresMountsFor = [ storageLocation ];
+      ReadWritePaths = [ storageLocation ];
+    };
+    systemd.tmpfiles.settings."10-gotosocial".${storageLocation}.d = {
+      user = dbUser;
+      group = dbUser;
+      mode = "0700";
+    };
   };
 }

modules/cloud/outline/r2.patch

@@ -0,0 +1,183 @@
+commit 8c7f8c28fabc174a71499a4737579b24b5c4b244
+Author: Natsu Kagami <nki@nkagami.me>
+Date:   Mon Oct 21 02:17:36 2024 +0200
+
+    Support R2
+
+diff --git a/.env.sample b/.env.sample
+index eb57ad85c..94ffcee07 100644
+--- a/.env.sample
++++ b/.env.sample
+@@ -66,6 +66,8 @@ AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569
+ AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here
+ AWS_S3_FORCE_PATH_STYLE=true
+ AWS_S3_ACL=private
++AWS_S3_R2=true
++AWS_S3_R2_PUBLIC_URL=http://s3:4569
+ 
+ # –––––––––––––– AUTHENTICATION ––––––––––––––
+ 
+diff --git a/app/utils/files.ts b/app/utils/files.ts
+index 6607a6b12..5138f68ad 100644
+--- a/app/utils/files.ts
++++ b/app/utils/files.ts
+@@ -63,8 +63,13 @@ export const uploadFile = async (
+     xhr.addEventListener("loadend", () => {
+       resolve(xhr.readyState === 4 && xhr.status >= 200 && xhr.status < 400);
+     });
+-    xhr.open("POST", data.uploadUrl, true);
+-    xhr.send(formData);
++    xhr.open(data.method, data.uploadUrl, true);
++    xhr.setRequestHeader("Content-Type", file.type);
++    if (data.method === "POST") {
++      xhr.send(formData);
++    } else {
++      xhr.send(file);
++    }
+   });
+ 
+   if (!success) {
+diff --git a/server/env.ts b/server/env.ts
+index 5b420f2e1..4ea1e8d3c 100644
+--- a/server/env.ts
++++ b/server/env.ts
+@@ -519,6 +519,14 @@ export class Environment {
+     environment.AWS_S3_UPLOAD_BUCKET_NAME
+   );
+ 
++  @IsOptional()
++  public AWS_S3_R2 = this.toBoolean(environment.AWS_S3_R2 ?? "false");
++
++  @IsOptional()
++  public AWS_S3_R2_PUBLIC_URL = this.toOptionalString(
++    environment.AWS_S3_R2_PUBLIC_URL
++  );
++
+   /**
+    * Whether to force path style URLs for S3 objects, this is required for some
+    * S3-compatible storage providers.
+diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts
+index 5e6c27594..b7620f440 100644
+--- a/server/routes/api/attachments/attachments.ts
++++ b/server/routes/api/attachments/attachments.ts
+@@ -3,6 +3,7 @@ import { v4 as uuidv4 } from "uuid";
+ import { AttachmentPreset } from "@shared/types";
+ import { bytesToHumanReadable } from "@shared/utils/files";
+ import { AttachmentValidation } from "@shared/validations";
++import env from "@server/env";
+ import { AuthorizationError, ValidationError } from "@server/errors";
+ import auth from "@server/middlewares/authentication";
+ import { rateLimiter } from "@server/middlewares/rateLimiter";
+@@ -90,16 +91,30 @@ router.post(
+       { transaction }
+     );
+ 
+-    const presignedPost = await FileStorage.getPresignedPost(
+-      key,
+-      acl,
+-      maxUploadSize,
+-      contentType
+-    );
++    let uploadUrl;
++    let method;
++    let presignedPost = {
++      fields: {},
++    };
++    if (env.AWS_S3_R2) {
++      uploadUrl = await FileStorage.getPresignedPut(key);
++      method = "PUT";
++    } else {
++      uploadUrl = FileStorage.getUploadUrl();
++      method = "POST";
++
++      presignedPost = await FileStorage.getPresignedPost(
++        key,
++        acl,
++        maxUploadSize,
++        contentType
++      );
++    }
+ 
+     ctx.body = {
+       data: {
+-        uploadUrl: FileStorage.getUploadUrl(),
++        uploadUrl,
++        method,
+         form: {
+           "Cache-Control": "max-age=31557600",
+           "Content-Type": contentType,
+diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts
+index ce0287ebc..a1931c83d 100644
+--- a/server/storage/files/BaseStorage.ts
++++ b/server/storage/files/BaseStorage.ts
+@@ -26,6 +26,8 @@ export default abstract class BaseStorage {
+     contentType: string
+   ): Promise<Partial<PresignedPost>>;
+ 
++  public abstract getPresignedPut(key: string): Promise<string>;
++
+   /**
+    * Returns a promise that resolves with a stream for reading a file from the storage provider.
+    *
+diff --git a/server/storage/files/LocalStorage.ts b/server/storage/files/LocalStorage.ts
+index 83cf98c50..324e60dd9 100644
+--- a/server/storage/files/LocalStorage.ts
++++ b/server/storage/files/LocalStorage.ts
+@@ -30,6 +30,10 @@ export default class LocalStorage extends BaseStorage {
+     });
+   }
+ 
++  public async getPresignedPut(key: string) {
++    return this.getUrlForKey(key);
++  }
++
+   public getUploadUrl() {
+     return "/api/files.create";
+   }
+diff --git a/server/storage/files/S3Storage.ts b/server/storage/files/S3Storage.ts
+index a42442e0c..d55ef5472 100644
+--- a/server/storage/files/S3Storage.ts
++++ b/server/storage/files/S3Storage.ts
+@@ -4,6 +4,7 @@ import {
+   S3Client,
+   DeleteObjectCommand,
+   GetObjectCommand,
++  PutObjectCommand,
+   ObjectCannedACL,
+ } from "@aws-sdk/client-s3";
+ import { Upload } from "@aws-sdk/lib-storage";
+@@ -58,6 +59,16 @@ export default class S3Storage extends BaseStorage {
+     return createPresignedPost(this.client, params);
+   }
+ 
++  public async getPresignedPut(key: string) {
++    const params = {
++      Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
++      Key: key,
++    };
++
++    const command = new PutObjectCommand(params);
++    return await getSignedUrl(this.client, command, { expiresIn: 3600 });
++  }
++
+   private getPublicEndpoint(isServerUpload?: boolean) {
+     if (env.AWS_S3_ACCELERATE_URL) {
+       return env.AWS_S3_ACCELERATE_URL;
+@@ -137,10 +148,17 @@ export default class S3Storage extends BaseStorage {
+     );
+   }
+ 
++  public getR2ObjectUrl = async (key: string) =>
++    env.AWS_S3_R2_PUBLIC_URL + "/" + key;
++
+   public getSignedUrl = async (
+     key: string,
+     expiresIn = S3Storage.defaultSignedUrlExpires
+   ) => {
++    if (env.AWS_S3_R2) {
++      return this.getR2ObjectUrl(key);
++    }
++
+     const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
+     const params = {
+       Bucket: this.getBucket(),

nki-personal-do/configuration.nix

@@ -24,8 +24,11 @@
     ./invidious.nix
     ./owncast.nix
     ./peertube.nix
+    ./outline.nix
   ];
 
+  system.stateVersion = "21.11";
+
   common.linux.enable = false; # Don't enable the "common linux" module, this is a special machine.
 
   # Personal user
@@ -189,74 +192,13 @@
     protocol = "udp";
   };
 
-
-  # Outline
-  sops.secrets.minio-secret-key = { owner = "root"; mode = "0444"; };
-  sops.secrets.authentik-oidc-client-secret = { owner = "outline"; };
-  sops.secrets."outline/smtp-password" = { owner = "outline"; };
-  services.outline = {
-    enable = true;
-    package = pkgs.outline.overrideAttrs (attrs: {
-      patches = if builtins.hasAttr "patches" attrs then attrs.patches else [ ] ++ [ ../modules/cloud/outline/dtth-wiki.patch ];
-    });
-    databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable";
-    redisUrl = "local";
-    publicUrl = "https://wiki.dtth.ch";
-    port = 18729;
-    storage = {
-      accessKey = "minio";
-      secretKeyFile = config.sops.secrets.minio-secret-key.path;
-      region = config.services.minio.region;
-      uploadBucketUrl = "https://s3.dtth.ch";
-      uploadBucketName = "dtth-outline";
-      uploadMaxSize = 50 * 1024 * 1000;
-    };
-    maximumImportSize = 50 * 1024 * 1000;
-
-    oidcAuthentication = {
-      clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7";
-      clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path;
-      authUrl = "https://auth.dtth.ch/application/o/authorize/";
-      tokenUrl = "https://auth.dtth.ch/application/o/token/";
-      userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/";
-      displayName = "DTTH Account";
-    };
-
-    smtp = {
-      fromEmail = "DTTH Wiki <dtth.wiki@nkagami.me>";
-      replyEmail = "";
-      host = "mx1.nkagami.me";
-      username = "dtth.wiki@nkagami.me";
-      passwordFile = config.sops.secrets."outline/smtp-password".path;
-      port = 465;
-      secure = true;
-    };
-
-    forceHttps = false;
-  };
-  cloud.postgresql.databases = [ "outline" ];
-  systemd.services.outline.requires = [ "postgresql.service" ];
-  cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; };
-
   # GoToSocial
-  sops.secrets.gts-env = { };
+  sops.secrets.gts-env = { restartUnits = [ "gotosocial.service" ]; };
   cloud.gotosocial = {
     enable = true;
     envFile = config.sops.secrets.gts-env.path;
   };
 
-  # Minio
-  sops.secrets.minio-credentials = { };
-  services.minio = {
-    enable = true;
-    listenAddress = ":61929";
-    consoleAddress = ":62929";
-    rootCredentialsFile = config.sops.secrets.minio-credentials.path;
-    dataDir = lib.mkForce [ "/mnt/data/minio" ];
-  };
-  cloud.traefik.hosts.minio = { host = "s3.dtth.ch"; port = 61929; };
-  system.stateVersion = "21.11";
-
   # ntfy
   cloud.traefik.hosts.ntfy-sh = { host = "ntfy.nkagami.me"; port = 11161; noCloudflare = true; };
   services.ntfy-sh = {

nki-personal-do/gitea.nix

@@ -98,6 +98,7 @@ in
   };
   users.groups.${user} = { };
   sops.secrets."gitea/signing-key".owner = user;
+  sops.secrets."gitea/minio-secret-key".owner = user;
   sops.secrets."gitea/mailer-password".owner = user;
   # database
   cloud.postgresql.databases = [ user ];
@@ -174,6 +175,17 @@ in
         PATH = "${pkgs.git}/bin/git";
       };
 
+      storage = {
+        STORAGE_TYPE = "minio";
+        MINIO_USE_SSL = "true";
+        MINIO_ENDPOINT = "60c0807121eb35ef52cdcd4a33735fa6.r2.cloudflarestorage.com";
+        MINIO_ACCESS_KEY_ID = "704c29ade7a8b438b77ab520da2799ca";
+        MINIO_SECRET_ACCESS_KEY = "#miniosecretkey#";
+        MINIO_BUCKET = "dtth-gitea";
+        MINIO_LOCATION = "auto";
+        MINIO_CHECKSUM_ALGORITHM = "md5"; # R2 moment
+      };
+
       federation.ENABLED = true;
       DEFAULT.APP_NAME = "DTTHGit";
     };
@@ -203,17 +215,25 @@ in
     environment.GNUPGHOME = "${config.services.gitea.stateDir}/.gnupg";
     # https://github.com/NixOS/nixpkgs/commit/93c1d370db28ad4573fb9890c90164ba55391ce7
     serviceConfig.SystemCallFilter = mkForce "~@clock @cpu-emulation @debug @keyring @module @mount @obsolete @raw-io @reboot @setuid @swap";
-    preStart = ''
+    preStart =
-      # Import the signing subkey
+      let
-      if cat ${config.services.gitea.stateDir}/.gnupg/gpg.conf | grep -q ${signingKey}; then
+        configFile = "${config.services.forgejo.customDir}/conf/app.ini";
-        echo "Keys already imported"
+      in
-        # imported
+      ''
-      else
+        # Update minio secret key
-        echo "Import your keys!"
+        chmod u+w ${configFile} && \
-        ${pkgs.gnupg}/bin/gpg --quiet --import ${secrets."gitea/signing-key".path}
+        ${lib.getExe pkgs.replace-secret} '#miniosecretkey#' '${config.sops.secrets."gitea/minio-secret-key".path}' '${configFile}' && \
-        echo "trusted-key ${signingKey}" >> ${config.services.gitea.stateDir}/.gnupg/gpg.conf
+        chmod u-w ${configFile}
-        exit 1
+        # Import the signing subkey
-      fi
+        if cat ${config.services.forgejo.stateDir}/.gnupg/gpg.conf | grep -q ${signingKey}; then
-    '';
+          echo "Keys already imported"
+          # imported
+        else
+          echo "Import your keys!"
+          ${pkgs.gnupg}/bin/gpg --quiet --import ${secrets."gitea/signing-key".path}
+          echo "trusted-key ${signingKey}" >> ${config.services.forgejo.stateDir}/.gnupg/gpg.conf
+          exit 1
+        fi
+      '';
   };
 }

nki-personal-do/hardware-configuration.nix

@@ -9,9 +9,11 @@
   swapDevices = [{ device = "/var/swapfile"; size = 4 * 1024; priority = 1024; }];
   zramSwap.enable = true;
   # volumes
+  services.btrfs.autoScrub.enable = true;
   fileSystems.data = {
-    device = "/dev/disk/by-id/scsi-0HC_Volume_31812942";
+    device = "/dev/disk/by-id/scsi-0HC_Volume_101470796";
-    fsType = "ext4";
+    fsType = "btrfs";
     mountPoint = "/mnt/data";
+    options = [ "compress=zstd" ];
   };
 }

nki-personal-do/nextcloud.nix

@@ -1,67 +0,0 @@
-{ lib, pkgs, config, ... }:
-with lib;
-let
-  user = "nextcloud";
-  host = "cloud.dtth.ch";
-  port = 61155;
-
-  secrets = config.sops.secrets;
-in
-{
-  sops.secrets."nextcloud/admin-password" = { owner = user; };
-  sops.secrets."nextcloud/minio-secret-key" = { owner = user; key = "minio-secret-key"; };
-  # database
-  cloud.postgresql.databases = [ user ];
-  # traefik
-  cloud.traefik.hosts.nextcloud = {
-    inherit port host;
-  };
-  systemd.services.nextcloud.requires = [ "postgresql.service" ];
-  services.nextcloud = {
-    enable = true;
-    hostName = host;
-    package = pkgs.nextcloud26;
-    enableBrokenCiphersForSSE = false;
-
-    home = "/mnt/data/nextcloud";
-    https = true;
-    database.createLocally = false;
-
-    extraApps = with pkgs.nextcloud26Packages.apps; {
-      inherit calendar contacts deck forms groupfolders news tasks;
-      sociallogin = pkgs.fetchNextcloudApp rec {
-        url = "https://github.com/zorn-v/nextcloud-social-login/releases/download/v5.4.3/release.tar.gz";
-        sha256 = "sha256-ZKwtF9j9WFIk3MZgng9DmN00A73S2Rb4qbehL9adaZo=";
-      };
-    };
-
-    config = {
-      # Database
-      dbtype = "pgsql";
-      dbname = user;
-      dbuser = user;
-      dbhost = "/run/postgresql";
-      # User
-      adminuser = "nki";
-      adminpassFile = secrets."nextcloud/admin-password".path;
-      # General
-      overwriteProtocol = "https";
-      defaultPhoneRegion = "VN";
-
-      objectstore.s3 = {
-        enable = true;
-        bucket = "nextcloud-dtth";
-        autocreate = true;
-        key = "minio";
-        secretFile = config.sops.secrets."nextcloud/minio-secret-key".path;
-        hostname = "s3.dtth.ch";
-        port = 443;
-        useSsl = true;
-        usePathStyle = true;
-        region = "us-east-1";
-      };
-    };
-  };
-  services.nginx.virtualHosts.${host}.listen = [{ inherit port; addr = "127.0.0.1"; }];
-}
-

nki-personal-do/outline.nix

@@ -0,0 +1,56 @@
+{ config, pkgs, ... }: {
+  sops.secrets.authentik-oidc-client-secret = { owner = "outline"; };
+  sops.secrets."outline/smtp-password" = { owner = "outline"; };
+  sops.secrets."outline/s3-secret-key" = { owner = "outline"; };
+
+  services.outline = {
+    enable = true;
+    package = pkgs.outline.overrideAttrs (attrs: {
+      patches = attrs.patches or [ ] ++ [
+        ../modules/cloud/outline/dtth-wiki.patch
+        ../modules/cloud/outline/r2.patch
+      ];
+    });
+    databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable";
+    redisUrl = "local";
+    publicUrl = "https://wiki.dtth.ch";
+    port = 18729;
+    storage = {
+      accessKey = "6ef730e13f172d2ed6ed77f0b5b9bad9";
+      secretKeyFile = config.sops.secrets."outline/s3-secret-key".path;
+      region = "auto";
+      uploadBucketUrl = "https://60c0807121eb35ef52cdcd4a33735fa6.r2.cloudflarestorage.com";
+      uploadBucketName = "dtth-outline";
+      uploadMaxSize = 50 * 1024 * 1000;
+    };
+    maximumImportSize = 50 * 1024 * 1000;
+
+    oidcAuthentication = {
+      clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7";
+      clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path;
+      authUrl = "https://auth.dtth.ch/application/o/authorize/";
+      tokenUrl = "https://auth.dtth.ch/application/o/token/";
+      userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/";
+      displayName = "DTTH Account";
+    };
+
+    smtp = {
+      fromEmail = "DTTH Wiki <dtth.wiki@nkagami.me>";
+      replyEmail = "";
+      host = "mx1.nkagami.me";
+      username = "dtth.wiki@nkagami.me";
+      passwordFile = config.sops.secrets."outline/smtp-password".path;
+      port = 465;
+      secure = true;
+    };
+
+    forceHttps = false;
+  };
+  cloud.postgresql.databases = [ "outline" ];
+  systemd.services.outline.requires = [ "postgresql.service" ];
+  systemd.services.outline.environment = {
+    AWS_S3_R2 = "true";
+    AWS_S3_R2_PUBLIC_URL = "https://s3.wiki.dtth.ch";
+  };
+  cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; };
+}

overlay.nix

@@ -25,7 +25,7 @@ let
 
   overlay-versioning = final: prev: {
     gotosocial = prev.gotosocial.overrideAttrs (attrs: rec {
-      version = "0.17.0";
+      version = "0.17.1";
       ldflags = [
         "-s"
         "-w"
@@ -35,13 +35,13 @@ let
 
       web-assets = final.fetchurl {
         url = "https://github.com/superseriousbusiness/gotosocial/releases/download/v${version}/gotosocial_${version}_web-assets.tar.gz";
-        hash = "sha256-ASqPIf98qdnkh3j72ifQN3mWnzNCTRcUegmrStvQ08Q=";
+        hash = "sha256-rGntLlIbgfCtdqpD7tnvAY8qwF+BpYbQWfAGMhdOTgY=";
       };
       src = final.fetchFromGitHub {
         owner = "superseriousbusiness";
         repo = "gotosocial";
         rev = "v${version}";
-        hash = "sha256-uyqP3zhjcXKejGFAwZoTn2kY8IpX0QAAXNzb1VG6ve8=";
+        hash = "sha256-oWWsCs9jgd244yzWhgLkuHp7kY0BQ8+Ay6KpuBVG+U8=";
       };
       postInstall = ''
         tar xf ${web-assets}