Move outline to R2

2024-10-21 03:21:33 +02:00 · 2024-10-21 03:21:33 +02:00 · 54700e75cd
parent ff70514a3d
commit 54700e75cd
4 changed files with 243 additions and 51 deletions
--- a/modules/cloud/outline/r2.patch
+++ b/modules/cloud/outline/r2.patch
@ -0,0 +1,183 @@
+commit 8c7f8c28fabc174a71499a4737579b24b5c4b244
+Author: Natsu Kagami <nki@nkagami.me>
+Date:   Mon Oct 21 02:17:36 2024 +0200
+
+    Support R2
+
+diff --git a/.env.sample b/.env.sample
+index eb57ad85c..94ffcee07 100644
+--- a/.env.sample
+++ b/.env.sample
+@@ -66,6 +66,8 @@ AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569
+ AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here
+ AWS_S3_FORCE_PATH_STYLE=true
+ AWS_S3_ACL=private
+AWS_S3_R2=true
+AWS_S3_R2_PUBLIC_URL=http://s3:4569
+ 
+ # –––––––––––––– AUTHENTICATION ––––––––––––––
+ 
+diff --git a/app/utils/files.ts b/app/utils/files.ts
+index 6607a6b12..5138f68ad 100644
+--- a/app/utils/files.ts
+++ b/app/utils/files.ts
+@@ -63,8 +63,13 @@ export const uploadFile = async (
+     xhr.addEventListener("loadend", () => {
+       resolve(xhr.readyState === 4 && xhr.status >= 200 && xhr.status < 400);
+     });
+-    xhr.open("POST", data.uploadUrl, true);
+-    xhr.send(formData);
+    xhr.open(data.method, data.uploadUrl, true);
+    xhr.setRequestHeader("Content-Type", file.type);
+    if (data.method === "POST") {
+      xhr.send(formData);
+    } else {
+      xhr.send(file);
+    }
+   });
+ 
+   if (!success) {
+diff --git a/server/env.ts b/server/env.ts
+index 5b420f2e1..4ea1e8d3c 100644
+--- a/server/env.ts
+++ b/server/env.ts
+@@ -519,6 +519,14 @@ export class Environment {
+     environment.AWS_S3_UPLOAD_BUCKET_NAME
+   );
+ 
+  @IsOptional()
+  public AWS_S3_R2 = this.toBoolean(environment.AWS_S3_R2 ?? "false");
+
+  @IsOptional()
+  public AWS_S3_R2_PUBLIC_URL = this.toOptionalString(
+    environment.AWS_S3_R2_PUBLIC_URL
+  );
+
+   /**
+    * Whether to force path style URLs for S3 objects, this is required for some
+    * S3-compatible storage providers.
+diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts
+index 5e6c27594..b7620f440 100644
+--- a/server/routes/api/attachments/attachments.ts
+++ b/server/routes/api/attachments/attachments.ts
+@@ -3,6 +3,7 @@ import { v4 as uuidv4 } from "uuid";
+ import { AttachmentPreset } from "@shared/types";
+ import { bytesToHumanReadable } from "@shared/utils/files";
+ import { AttachmentValidation } from "@shared/validations";
+import env from "@server/env";
+ import { AuthorizationError, ValidationError } from "@server/errors";
+ import auth from "@server/middlewares/authentication";
+ import { rateLimiter } from "@server/middlewares/rateLimiter";
+@@ -90,16 +91,30 @@ router.post(
+       { transaction }
+     );
+ 
+-    const presignedPost = await FileStorage.getPresignedPost(
+-      key,
+-      acl,
+-      maxUploadSize,
+-      contentType
+-    );
+    let uploadUrl;
+    let method;
+    let presignedPost = {
+      fields: {},
+    };
+    if (env.AWS_S3_R2) {
+      uploadUrl = await FileStorage.getPresignedPut(key);
+      method = "PUT";
+    } else {
+      uploadUrl = FileStorage.getUploadUrl();
+      method = "POST";
+
+      presignedPost = await FileStorage.getPresignedPost(
+        key,
+        acl,
+        maxUploadSize,
+        contentType
+      );
+    }
+ 
+     ctx.body = {
+       data: {
+-        uploadUrl: FileStorage.getUploadUrl(),
+        uploadUrl,
+        method,
+         form: {
+           "Cache-Control": "max-age=31557600",
+           "Content-Type": contentType,
+diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts
+index ce0287ebc..a1931c83d 100644
+--- a/server/storage/files/BaseStorage.ts
+++ b/server/storage/files/BaseStorage.ts
+@@ -26,6 +26,8 @@ export default abstract class BaseStorage {
+     contentType: string
+   ): Promise<Partial<PresignedPost>>;
+ 
+  public abstract getPresignedPut(key: string): Promise<string>;
+
+   /**
+    * Returns a promise that resolves with a stream for reading a file from the storage provider.
+    *
+diff --git a/server/storage/files/LocalStorage.ts b/server/storage/files/LocalStorage.ts
+index 83cf98c50..324e60dd9 100644
+--- a/server/storage/files/LocalStorage.ts
+++ b/server/storage/files/LocalStorage.ts
+@@ -30,6 +30,10 @@ export default class LocalStorage extends BaseStorage {
+     });
+   }
+ 
+  public async getPresignedPut(key: string) {
+    return this.getUrlForKey(key);
+  }
+
+   public getUploadUrl() {
+     return "/api/files.create";
+   }
+diff --git a/server/storage/files/S3Storage.ts b/server/storage/files/S3Storage.ts
+index a42442e0c..d55ef5472 100644
+--- a/server/storage/files/S3Storage.ts
+++ b/server/storage/files/S3Storage.ts
+@@ -4,6 +4,7 @@ import {
+   S3Client,
+   DeleteObjectCommand,
+   GetObjectCommand,
+  PutObjectCommand,
+   ObjectCannedACL,
+ } from "@aws-sdk/client-s3";
+ import { Upload } from "@aws-sdk/lib-storage";
+@@ -58,6 +59,16 @@ export default class S3Storage extends BaseStorage {
+     return createPresignedPost(this.client, params);
+   }
+ 
+  public async getPresignedPut(key: string) {
+    const params = {
+      Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME,
+      Key: key,
+    };
+
+    const command = new PutObjectCommand(params);
+    return await getSignedUrl(this.client, command, { expiresIn: 3600 });
+  }
+
+   private getPublicEndpoint(isServerUpload?: boolean) {
+     if (env.AWS_S3_ACCELERATE_URL) {
+       return env.AWS_S3_ACCELERATE_URL;
+@@ -137,10 +148,17 @@ export default class S3Storage extends BaseStorage {
+     );
+   }
+ 
+  public getR2ObjectUrl = async (key: string) =>
+    env.AWS_S3_R2_PUBLIC_URL + "/" + key;
+
+   public getSignedUrl = async (
+     key: string,
+     expiresIn = S3Storage.defaultSignedUrlExpires
+   ) => {
+    if (env.AWS_S3_R2) {
+      return this.getR2ObjectUrl(key);
+    }
+
+     const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/);
+     const params = {
+       Bucket: this.getBucket(),
--- a/nki-personal-do/configuration.nix
+++ b/nki-personal-do/configuration.nix
@ -24,6 +24,7 @@
    ./invidious.nix
    ./owncast.nix
    ./peertube.nix
+    ./outline.nix
  ];

  common.linux.enable = false; # Don't enable the "common linux" module, this is a special machine.
@ -189,55 +190,6 @@
    protocol = "udp";
  };

-
-  # Outline
-  sops.secrets.minio-secret-key = { owner = "root"; mode = "0444"; };
-  sops.secrets.authentik-oidc-client-secret = { owner = "outline"; };
-  sops.secrets."outline/smtp-password" = { owner = "outline"; };
-  services.outline = {
-    enable = true;
-    package = pkgs.outline.overrideAttrs (attrs: {
-      patches = if builtins.hasAttr "patches" attrs then attrs.patches else [ ] ++ [ ../modules/cloud/outline/dtth-wiki.patch ];
-    });
-    databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable";
-    redisUrl = "local";
-    publicUrl = "https://wiki.dtth.ch";
-    port = 18729;
-    storage = {
-      accessKey = "minio";
-      secretKeyFile = config.sops.secrets.minio-secret-key.path;
-      region = config.services.minio.region;
-      uploadBucketUrl = "https://s3.dtth.ch";
-      uploadBucketName = "dtth-outline";
-      uploadMaxSize = 50 * 1024 * 1000;
-    };
-    maximumImportSize = 50 * 1024 * 1000;
-
-    oidcAuthentication = {
-      clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7";
-      clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path;
-      authUrl = "https://auth.dtth.ch/application/o/authorize/";
-      tokenUrl = "https://auth.dtth.ch/application/o/token/";
-      userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/";
-      displayName = "DTTH Account";
-    };
-
-    smtp = {
-      fromEmail = "DTTH Wiki <dtth.wiki@nkagami.me>";
-      replyEmail = "";
-      host = "mx1.nkagami.me";
-      username = "dtth.wiki@nkagami.me";
-      passwordFile = config.sops.secrets."outline/smtp-password".path;
-      port = 465;
-      secure = true;
-    };
-
-    forceHttps = false;
-  };
-  cloud.postgresql.databases = [ "outline" ];
-  systemd.services.outline.requires = [ "postgresql.service" ];
-  cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; };
-
  # GoToSocial
  sops.secrets.gts-env = { };
  cloud.gotosocial = {
--- a/nki-personal-do/outline.nix
+++ b/nki-personal-do/outline.nix
@ -0,0 +1,56 @@
+{ config, pkgs, ... }: {
+  sops.secrets.authentik-oidc-client-secret = { owner = "outline"; };
+  sops.secrets."outline/smtp-password" = { owner = "outline"; };
+  sops.secrets."outline/s3-secret-key" = { owner = "outline"; };
+
+  services.outline = {
+    enable = true;
+    package = pkgs.outline.overrideAttrs (attrs: {
+      patches = attrs.patches or [ ] ++ [
+        ../modules/cloud/outline/dtth-wiki.patch
+        ../modules/cloud/outline/r2.patch
+      ];
+    });
+    databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable";
+    redisUrl = "local";
+    publicUrl = "https://wiki.dtth.ch";
+    port = 18729;
+    storage = {
+      accessKey = "6ef730e13f172d2ed6ed77f0b5b9bad9";
+      secretKeyFile = config.sops.secrets."outline/s3-secret-key".path;
+      region = "auto";
+      uploadBucketUrl = "https://60c0807121eb35ef52cdcd4a33735fa6.r2.cloudflarestorage.com";
+      uploadBucketName = "dtth-outline";
+      uploadMaxSize = 50 * 1024 * 1000;
+    };
+    maximumImportSize = 50 * 1024 * 1000;
+
+    oidcAuthentication = {
+      clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7";
+      clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path;
+      authUrl = "https://auth.dtth.ch/application/o/authorize/";
+      tokenUrl = "https://auth.dtth.ch/application/o/token/";
+      userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/";
+      displayName = "DTTH Account";
+    };
+
+    smtp = {
+      fromEmail = "DTTH Wiki <dtth.wiki@nkagami.me>";
+      replyEmail = "";
+      host = "mx1.nkagami.me";
+      username = "dtth.wiki@nkagami.me";
+      passwordFile = config.sops.secrets."outline/smtp-password".path;
+      port = 465;
+      secure = true;
+    };
+
+    forceHttps = false;
+  };
+  cloud.postgresql.databases = [ "outline" ];
+  systemd.services.outline.requires = [ "postgresql.service" ];
+  systemd.services.outline.environment = {
+    AWS_S3_R2 = "true";
+    AWS_S3_R2_PUBLIC_URL = "https://s3.wiki.dtth.ch";
+  };
+  cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; };
+}
--- a/nki-personal-do/secrets/secrets.yaml
+++ b/nki-personal-do/secrets/secrets.yaml
@ -11,6 +11,7 @@ mail-users: ENC[AES256_GCM,data:qKLi42k8LT6ojxbPXQgbi6FlI2I6ge6qJn0aNj/Lp9iRjjnn
 youmubot-env: ENC[AES256_GCM,data:EQ9e6lmCrjofHiHyN5Qe4b2oplP9/3JKl0vuFp54Hw9aYIS7j3nqzWLCvV54ZK7j1PcQ+CQorjeCVMV0TUy1f1Pf3qjrLkdOdV7ICq540gdfXOeXuhAx2EILpGkwIYOdKmTMSO3l2QkOlM02RNOn1lq/DogAydkEq7gJ7qSWnUEr45oNCa1+LamH8vcbDmIyzUWWXyA5EQ==,iv:fnNGZ6OaZ4D71SvWPRynsMpO1IsvxjQ3XtrswNSY+Wo=,tag:cN/ZnKrjSfD6AbU9pYNl+Q==,type:str]
 outline:
    smtp-password: ENC[AES256_GCM,data:zpIi6jVB2Y7ksBOR8SGFgjOD1x3aS6dKa6taLKB8v2l9p92iWDti75qgB1puglmmq8mCzz8KXLrM0Bv7W8GWRg==,iv:6tKINzQcApmNuIbNn0kSzFJtwn3rky/uFG2Ff3lazUk=,tag:kjB6qB87tRQVpy32Pt3D5A==,type:str]
+    s3-secret-key: ENC[AES256_GCM,data:dH1Uh3G3RNqITOvsecOW0my3xM3H6xhKYONcwORNPBZmlvSWYvhZUxkOghlH9sYHLIU4yb31QO7npi01Sn3kww==,iv:cV4xqzS5/3HseODY3hS/ycjI6HccsrSGz5Dh9exqNIA=,tag:FMGR9NiTn5S2fTxNSQYBDw==,type:str]
 heisenbridge: ENC[AES256_GCM,data:rJY7gpcOY8nODR3KlYW1rEs54mKxr+AjNBeg1/2vTG0Gzpuvjgbnn5UVJS+P8uej/P4HfeFtlQSFZCEy8cXcwvwq97ppVliCGL4GMLRWaFmop35feC8t2ovh79cy/vKC7drASeGvWYNUmGRjboPuKA8W5LARa0HVDPGDLIEMVgJfYry/YKR3gsGmLzU7Mx1yLO6M/EFOJQJc84bSuu+CPSZcyUVF4SSNBiaDU5/NazlqaA9KWL6Xzu1MD2LEYdEFkRfitNgYj2m2gLd9voyGV4cfaCqJvYjJPwuZeZUoqCpDnom2JoV29q/Yq/gmyumPgOvriGxLsYBqV14MaCcE6KXE2uLicD+I/5or1AxepVDVjG9NoSgho1HpLvpRhMSCeXLk9+U+ykH3QA+0M+VVu9pswMMVQifnTtXZRM6pWxOnRVAzGf2tGDo4jy36S7pHaRn7SJcrljjWLfwHuNiu7E2uZhMrkcCjnjcBA9Xrb3drDQYVHya7XcoD4wOBHBDvVZwhYkNdkS3oYkom8A==,iv:fO1onfon3EdSNC/LjN1aWxpHBYq5aa0F/h0V6gl88ac=,tag:NL9p2nhIlEqgOdvUDM19Dg==,type:str]
 matrix-discord-bridge: ENC[AES256_GCM,data:/rlSjD6inKfak7HKKghH5ays5RjKmb9czGsoIOYHyTZC4A5EMucCbfn8DL1gkYXgvRHJ+QglGX/BGo5ebaxSj6nF60+aW87UG31KggOt5kkMuWsPsjvrufoc5IlNfWnXIWmqf8cdC01hmHEp7biUpI8CcfEZiD9OkOxbZcRfYqW+ttnzplFniRBjGPVZfL5g4DBbuJen5MuOrrMDo5CT+78n,iv:r9VBbDCAAElisCaDehrB6PhJHsaaHjdrk3103lmBT7o=,tag:WoNMMfyMifsL56yWq3MUOg==,type:str]
 authentik-env: ENC[AES256_GCM,data:CjxTaqIcpBX7ea9L3tgJDELr8HBPJdxXsrOfhsiH4cXwCEzktsNKHjF7l95ZFgI5O08q4Vlbln5Dg4xPEx33nwUesEbQrT5d+n+2YaAxmm/WInrYzF+jB7HYTXASb3rY9PWgd2C3v+YPBkJetHlTUc/k19Q7lOQRNw==,iv:cG8Bi2eCsS+v94tSJBsqp+bjVLzXZvvwX1QVVSYExL8=,tag:VmbfcxCcfi3IpKjg3f8QPw==,type:str]
@ -75,8 +76,8 @@ sops:
            by9kZFlTRVdCZFkxYTVVb0RIRk8zUlkKCqMw9oL9RaYBV5Hhy3o8Nm5xmGrPH8Sd
            hv36sxRFFNZT/DCKaHaSRbT3mfpBZSTXJt1dgl4nZe6whH54t/1KmA==
            -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-10-20T23:19:07Z"
-    mac: ENC[AES256_GCM,data:7k0W6cV3HXVmcKjhDBcw+skzTukIay4vpa2cDEWUyLlvEUw3sR0yoKwgYACh4J63UEjcXfnLqQlR2jUkOQ3iigX/gvqSkjKcmfCPvqAnqe9CB/DOVgUufXOOcoNnJXu4G99St3Jgqazaq0xOxG1mXMkbejwPWMsDuqzGuw5v2gE=,iv:HkORvujIH+OePQDzTNqI541y9SEwkdIvxo4gh4RhOt0=,tag:a1p9LkQf6oazfri/SNcbqw==,type:str]
+    lastmodified: "2024-10-21T00:39:40Z"
+    mac: ENC[AES256_GCM,data:LtQXhFPm8SFuq7GZIRJyYmzUBcQFRP1UkfkZ2K6eGv0BE72cAN7n1XlxU5Ujj9G1rTjumaquCWmD7h0cmh4ufJnAjAatSn2XOwVAK8+2STd52YQE2sidlHJBlrNrvo4TICusIl+m5Z9E97G420SH6E846Wv+tPQBF9t5HQQgo24=,iv:/7vfawv3rzn2l28MrJcEYRNdMV/QDHThbP2gA1b+jZk=,tag:pdpItbrshuzVtrKWQS949g==,type:str]
    pgp: []
    unencrypted_suffix: _unencrypted
    version: 3.9.1