Move outline to R2

modules/cloud/outline/r2.patch

@@ -0,0 +1,183 @@
+commit 8c7f8c28fabc174a71499a4737579b24b5c4b244
+Author: Natsu Kagami <nki@nkagami.me>
+Date:   Mon Oct 21 02:17:36 2024 +0200
+
+    Support R2
+
+diff --git a/.env.sample b/.env.sample
+index eb57ad85c..94ffcee07 100644
+--- a/.env.sample
++++ b/.env.sample
+@@ -66,6 +66,8 @@ AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569
+ AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here
+ AWS_S3_FORCE_PATH_STYLE=true
+ AWS_S3_ACL=private
++AWS_S3_R2=true
++AWS_S3_R2_PUBLIC_URL=http://s3:4569
+ 
+ # –––––––––––––– AUTHENTICATION ––––––––––––––
+ 
+diff --git a/app/utils/files.ts b/app/utils/files.ts
+index 6607a6b12..5138f68ad 100644
+--- a/app/utils/files.ts
++++ b/app/utils/files.ts
+@@ -63,8 +63,13 @@ export const uploadFile = async (
+     xhr.addEventListener("loadend", () => {
+       resolve(xhr.readyState === 4 && xhr.status >= 200 && xhr.status < 400);
+     });
+-    xhr.open("POST", data.uploadUrl, true);
+-    xhr.send(formData);
++    xhr.open(data.method, data.uploadUrl, true);
++    xhr.setRequestHeader("Content-Type", file.type);
++    if (data.method === "POST") {
++      xhr.send(formData);
++    } else {
++      xhr.send(file);
++    }
+   });
+ 
+   if (!success) {
+diff --git a/server/env.ts b/server/env.ts
+index 5b420f2e1..4ea1e8d3c 100644
+--- a/server/env.ts
++++ b/server/env.ts
+@@ -519,6 +519,14 @@ export class Environment {
+     environment.AWS_S3_UPLOAD_BUCKET_NAME
+   );
+ 
++  @IsOptional()
++  public AWS_S3_R2 = this.toBoolean(environment.AWS_S3_R2 ?? "false");
++
++  @IsOptional()
++  public AWS_S3_R2_PUBLIC_URL = this.toOptionalString(
++    environment.AWS_S3_R2_PUBLIC_URL
++  );
++
+   /**
+    * Whether to force path style URLs for S3 objects, this is required for some
+    * S3-compatible storage providers.
+diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts
+index 5e6c27594..b7620f440 100644
+--- a/server/routes/api/attachments/attachments.ts
++++ b/server/routes/api/attachments/attachments.ts
+@@ -3,6 +3,7 @@ import { v4 as uuidv4 } from "uuid";
+ import { AttachmentPreset } from "@shared/types";
+ import { bytesToHumanReadable } from "@shared/utils/files";
+ import { AttachmentValidation } from "@shared/validations";
++import env from "@server/env";
+ import { AuthorizationError, ValidationError } from "@server/errors";
+ import auth from "@server/middlewares/authentication";
+ import { rateLimiter } from "@server/middlewares/rateLimiter";
+@@ -90,16 +91,30 @@ router.post(
+       { transaction }
+     );
+ 
+-    const presignedPost = await FileStorage.getPresignedPost(
+-      key,
+-      acl,
+-      maxUploadSize,
+-      contentType
+-    );
++    let uploadUrl;
++    let method;
++    let presignedPost = {
++      fields: {},
++    };
++    if (env.AWS_S3_R2) {
++      uploadUrl = await FileStorage.getPresignedPut(key);
++      method = "PUT";
++    } else {
++      uploadUrl = FileStorage.getUploadUrl();
++      method = "POST";
++
++      presignedPost = await FileStorage.getPresignedPost(
++        key,
++        acl,
++        maxUploadSize,
++        contentType
++      );
++    }
+ 
+     ctx.body = {
+       data: {
+-        uploadUrl: FileStorage.getUploadUrl(),
++        uploadUrl,
++        method,
+         form: {
+           "Cache-Control": "max-age=31557600",
+           "Content-Type": contentType,
+diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts
+index ce0287ebc..a1931c83d 100644
+--- a/server/storage/files/BaseStorage.ts
++++ b/server/storage/files/BaseStorage.ts
+@@ -26,6 +26,8 @@ export default abstract class BaseStorage {
+     contentType: string
+   ): Promise<Partial<PresignedPost>>;
+ 
++  public abstract getPresignedPut(key: string): Promise<string>;
++
+   /**
+    * Returns a promise that resolves with a stream for reading a file from the storage provider.
+    *
+diff --git a/server/storage/files/LocalStorage.ts b/server/storage/files/LocalStorage.ts
+index 83cf98c50..324e60dd9 100644
+--- a/server/storage/files/LocalStorage.ts
++++ b/server/storage/files/LocalStorage.ts
+@@ -30,6 +30,10 @@ export default class LocalStorage extends BaseStorage {
+     });
+   }
+ 
++  public async getPresignedPut(key: string) {
++    return this.getUrlForKey(key);
++  }
++
+   public getUploadUrl() {
+     return "/api/files.create";
+   }
+diff --git a/server/storage/files/S3Storage.ts b/server/storage/files/S3Storage.ts
+index a42442e0c..d55ef5472 100644
+--- a/server/storage/files/S3Storage.ts
++++ b/server/storage/files/S3Storage.ts
+@@ -4,6 +4,7 @@ import {
+   S3Client,
+   DeleteObjectCommand,
+   GetObjectCommand,
++  PutObjectCommand,
+   ObjectCannedACL,
+ } from "@aws-sdk/client-s3";
+ import { Upload } from "@aws-sdk/lib-storage";
+@@ -58,6 +59,16 @@ export default class S3Storage extends BaseStorage {
+     return createPresignedPost(this.client, params);
+   }
+ 
++  public async getPresignedPut(key: string) {
++    const params = {
++      Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
++      Key: key,
++    };
++
++    const command = new PutObjectCommand(params);
++    return await getSignedUrl(this.client, command, { expiresIn: 3600 });
++  }
++
+   private getPublicEndpoint(isServerUpload?: boolean) {
+     if (env.AWS_S3_ACCELERATE_URL) {
+       return env.AWS_S3_ACCELERATE_URL;
+@@ -137,10 +148,17 @@ export default class S3Storage extends BaseStorage {
+     );
+   }
+ 
++  public getR2ObjectUrl = async (key: string) =>
++    env.AWS_S3_R2_PUBLIC_URL + "/" + key;
++
+   public getSignedUrl = async (
+     key: string,
+     expiresIn = S3Storage.defaultSignedUrlExpires
+   ) => {
++    if (env.AWS_S3_R2) {
++      return this.getR2ObjectUrl(key);
++    }
++
+     const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
+     const params = {
+       Bucket: this.getBucket(),

nki-personal-do/configuration.nix

@@ -24,6 +24,7 @@
     ./invidious.nix
     ./owncast.nix
     ./peertube.nix
+    ./outline.nix
   ];
 
   common.linux.enable = false; # Don't enable the "common linux" module, this is a special machine.
@@ -189,55 +190,6 @@
     protocol = "udp";
   };
 
-
-  # Outline
-  sops.secrets.minio-secret-key = { owner = "root"; mode = "0444"; };
-  sops.secrets.authentik-oidc-client-secret = { owner = "outline"; };
-  sops.secrets."outline/smtp-password" = { owner = "outline"; };
-  services.outline = {
-    enable = true;
-    package = pkgs.outline.overrideAttrs (attrs: {
-      patches = if builtins.hasAttr "patches" attrs then attrs.patches else [ ] ++ [ ../modules/cloud/outline/dtth-wiki.patch ];
-    });
-    databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable";
-    redisUrl = "local";
-    publicUrl = "https://wiki.dtth.ch";
-    port = 18729;
-    storage = {
-      accessKey = "minio";
-      secretKeyFile = config.sops.secrets.minio-secret-key.path;
-      region = config.services.minio.region;
-      uploadBucketUrl = "https://s3.dtth.ch";
-      uploadBucketName = "dtth-outline";
-      uploadMaxSize = 50 * 1024 * 1000;
-    };
-    maximumImportSize = 50 * 1024 * 1000;
-
-    oidcAuthentication = {
-      clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7";
-      clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path;
-      authUrl = "https://auth.dtth.ch/application/o/authorize/";
-      tokenUrl = "https://auth.dtth.ch/application/o/token/";
-      userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/";
-      displayName = "DTTH Account";
-    };
-
-    smtp = {
-      fromEmail = "DTTH Wiki <dtth.wiki@nkagami.me>";
-      replyEmail = "";
-      host = "mx1.nkagami.me";
-      username = "dtth.wiki@nkagami.me";
-      passwordFile = config.sops.secrets."outline/smtp-password".path;
-      port = 465;
-      secure = true;
-    };
-
-    forceHttps = false;
-  };
-  cloud.postgresql.databases = [ "outline" ];
-  systemd.services.outline.requires = [ "postgresql.service" ];
-  cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; };
-
   # GoToSocial
   sops.secrets.gts-env = { };
   cloud.gotosocial = {

nki-personal-do/outline.nix

@@ -0,0 +1,56 @@
+{ config, pkgs, ... }: {
+  sops.secrets.authentik-oidc-client-secret = { owner = "outline"; };
+  sops.secrets."outline/smtp-password" = { owner = "outline"; };
+  sops.secrets."outline/s3-secret-key" = { owner = "outline"; };
+
+  services.outline = {
+    enable = true;
+    package = pkgs.outline.overrideAttrs (attrs: {
+      patches = attrs.patches or [ ] ++ [
+        ../modules/cloud/outline/dtth-wiki.patch
+        ../modules/cloud/outline/r2.patch
+      ];
+    });
+    databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable";
+    redisUrl = "local";
+    publicUrl = "https://wiki.dtth.ch";
+    port = 18729;
+    storage = {
+      accessKey = "6ef730e13f172d2ed6ed77f0b5b9bad9";
+      secretKeyFile = config.sops.secrets."outline/s3-secret-key".path;
+      region = "auto";
+      uploadBucketUrl = "https://60c0807121eb35ef52cdcd4a33735fa6.r2.cloudflarestorage.com";
+      uploadBucketName = "dtth-outline";
+      uploadMaxSize = 50 * 1024 * 1000;
+    };
+    maximumImportSize = 50 * 1024 * 1000;
+
+    oidcAuthentication = {
+      clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7";
+      clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path;
+      authUrl = "https://auth.dtth.ch/application/o/authorize/";
+      tokenUrl = "https://auth.dtth.ch/application/o/token/";
+      userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/";
+      displayName = "DTTH Account";
+    };
+
+    smtp = {
+      fromEmail = "DTTH Wiki <dtth.wiki@nkagami.me>";
+      replyEmail = "";
+      host = "mx1.nkagami.me";
+      username = "dtth.wiki@nkagami.me";
+      passwordFile = config.sops.secrets."outline/smtp-password".path;
+      port = 465;
+      secure = true;
+    };
+
+    forceHttps = false;
+  };
+  cloud.postgresql.databases = [ "outline" ];
+  systemd.services.outline.requires = [ "postgresql.service" ];
+  systemd.services.outline.environment = {
+    AWS_S3_R2 = "true";
+    AWS_S3_R2_PUBLIC_URL = "https://s3.wiki.dtth.ch";
+  };
+  cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; };
+}

nki-personal-do/secrets/secrets.yaml

@@ -11,6 +11,7 @@ mail-users: ENC[AES256_GCM,data:qKLi42k8LT6ojxbPXQgbi6FlI2I6ge6qJn0aNj/Lp9iRjjnn
 youmubot-env: ENC[AES256_GCM,data:EQ9e6lmCrjofHiHyN5Qe4b2oplP9/3JKl0vuFp54Hw9aYIS7j3nqzWLCvV54ZK7j1PcQ+CQorjeCVMV0TUy1f1Pf3qjrLkdOdV7ICq540gdfXOeXuhAx2EILpGkwIYOdKmTMSO3l2QkOlM02RNOn1lq/DogAydkEq7gJ7qSWnUEr45oNCa1+LamH8vcbDmIyzUWWXyA5EQ==,iv:fnNGZ6OaZ4D71SvWPRynsMpO1IsvxjQ3XtrswNSY+Wo=,tag:cN/ZnKrjSfD6AbU9pYNl+Q==,type:str]
 outline:
     smtp-password: ENC[AES256_GCM,data:zpIi6jVB2Y7ksBOR8SGFgjOD1x3aS6dKa6taLKB8v2l9p92iWDti75qgB1puglmmq8mCzz8KXLrM0Bv7W8GWRg==,iv:6tKINzQcApmNuIbNn0kSzFJtwn3rky/uFG2Ff3lazUk=,tag:kjB6qB87tRQVpy32Pt3D5A==,type:str]
+    s3-secret-key: ENC[AES256_GCM,data:dH1Uh3G3RNqITOvsecOW0my3xM3H6xhKYONcwORNPBZmlvSWYvhZUxkOghlH9sYHLIU4yb31QO7npi01Sn3kww==,iv:cV4xqzS5/3HseODY3hS/ycjI6HccsrSGz5Dh9exqNIA=,tag:FMGR9NiTn5S2fTxNSQYBDw==,type:str]
 heisenbridge: ENC[AES256_GCM,data:rJY7gpcOY8nODR3KlYW1rEs54mKxr+AjNBeg1/2vTG0Gzpuvjgbnn5UVJS+P8uej/P4HfeFtlQSFZCEy8cXcwvwq97ppVliCGL4GMLRWaFmop35feC8t2ovh79cy/vKC7drASeGvWYNUmGRjboPuKA8W5LARa0HVDPGDLIEMVgJfYry/YKR3gsGmLzU7Mx1yLO6M/EFOJQJc84bSuu+CPSZcyUVF4SSNBiaDU5/NazlqaA9KWL6Xzu1MD2LEYdEFkRfitNgYj2m2gLd9voyGV4cfaCqJvYjJPwuZeZUoqCpDnom2JoV29q/Yq/gmyumPgOvriGxLsYBqV14MaCcE6KXE2uLicD+I/5or1AxepVDVjG9NoSgho1HpLvpRhMSCeXLk9+U+ykH3QA+0M+VVu9pswMMVQifnTtXZRM6pWxOnRVAzGf2tGDo4jy36S7pHaRn7SJcrljjWLfwHuNiu7E2uZhMrkcCjnjcBA9Xrb3drDQYVHya7XcoD4wOBHBDvVZwhYkNdkS3oYkom8A==,iv:fO1onfon3EdSNC/LjN1aWxpHBYq5aa0F/h0V6gl88ac=,tag:NL9p2nhIlEqgOdvUDM19Dg==,type:str]
 matrix-discord-bridge: ENC[AES256_GCM,data:/rlSjD6inKfak7HKKghH5ays5RjKmb9czGsoIOYHyTZC4A5EMucCbfn8DL1gkYXgvRHJ+QglGX/BGo5ebaxSj6nF60+aW87UG31KggOt5kkMuWsPsjvrufoc5IlNfWnXIWmqf8cdC01hmHEp7biUpI8CcfEZiD9OkOxbZcRfYqW+ttnzplFniRBjGPVZfL5g4DBbuJen5MuOrrMDo5CT+78n,iv:r9VBbDCAAElisCaDehrB6PhJHsaaHjdrk3103lmBT7o=,tag:WoNMMfyMifsL56yWq3MUOg==,type:str]
 authentik-env: ENC[AES256_GCM,data:CjxTaqIcpBX7ea9L3tgJDELr8HBPJdxXsrOfhsiH4cXwCEzktsNKHjF7l95ZFgI5O08q4Vlbln5Dg4xPEx33nwUesEbQrT5d+n+2YaAxmm/WInrYzF+jB7HYTXASb3rY9PWgd2C3v+YPBkJetHlTUc/k19Q7lOQRNw==,iv:cG8Bi2eCsS+v94tSJBsqp+bjVLzXZvvwX1QVVSYExL8=,tag:VmbfcxCcfi3IpKjg3f8QPw==,type:str]
@@ -75,8 +76,8 @@ sops:
             by9kZFlTRVdCZFkxYTVVb0RIRk8zUlkKCqMw9oL9RaYBV5Hhy3o8Nm5xmGrPH8Sd
             hv36sxRFFNZT/DCKaHaSRbT3mfpBZSTXJt1dgl4nZe6whH54t/1KmA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-10-20T23:19:07Z"
+    lastmodified: "2024-10-21T00:39:40Z"
-    mac: ENC[AES256_GCM,data:7k0W6cV3HXVmcKjhDBcw+skzTukIay4vpa2cDEWUyLlvEUw3sR0yoKwgYACh4J63UEjcXfnLqQlR2jUkOQ3iigX/gvqSkjKcmfCPvqAnqe9CB/DOVgUufXOOcoNnJXu4G99St3Jgqazaq0xOxG1mXMkbejwPWMsDuqzGuw5v2gE=,iv:HkORvujIH+OePQDzTNqI541y9SEwkdIvxo4gh4RhOt0=,tag:a1p9LkQf6oazfri/SNcbqw==,type:str]
+    mac: ENC[AES256_GCM,data:LtQXhFPm8SFuq7GZIRJyYmzUBcQFRP1UkfkZ2K6eGv0BE72cAN7n1XlxU5Ujj9G1rTjumaquCWmD7h0cmh4ufJnAjAatSn2XOwVAK8+2STd52YQE2sidlHJBlrNrvo4TICusIl+m5Z9E97G420SH6E846Wv+tPQBF9t5HQQgo24=,iv:/7vfawv3rzn2l28MrJcEYRNdMV/QDHThbP2gA1b+jZk=,tag:pdpItbrshuzVtrKWQS949g==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.9.1