From 54700e75cd857c5efe039c085f411739d8a19107 Mon Sep 17 00:00:00 2001 From: Natsu Kagami Date: Mon, 21 Oct 2024 03:21:33 +0200 Subject: [PATCH] Move outline to R2 --- modules/cloud/outline/r2.patch | 183 +++++++++++++++++++++++++++ nki-personal-do/configuration.nix | 50 +------- nki-personal-do/outline.nix | 56 ++++++++ nki-personal-do/secrets/secrets.yaml | 5 +- 4 files changed, 243 insertions(+), 51 deletions(-) create mode 100644 modules/cloud/outline/r2.patch create mode 100644 nki-personal-do/outline.nix diff --git a/modules/cloud/outline/r2.patch b/modules/cloud/outline/r2.patch new file mode 100644 index 0000000..e62abb5 --- /dev/null +++ b/modules/cloud/outline/r2.patch @@ -0,0 +1,183 @@ +commit 8c7f8c28fabc174a71499a4737579b24b5c4b244 +Author: Natsu Kagami +Date: Mon Oct 21 02:17:36 2024 +0200 + + Support R2 + +diff --git a/.env.sample b/.env.sample +index eb57ad85c..94ffcee07 100644 +--- a/.env.sample ++++ b/.env.sample +@@ -66,6 +66,8 @@ AWS_S3_UPLOAD_BUCKET_URL=http://s3:4569 + AWS_S3_UPLOAD_BUCKET_NAME=bucket_name_here + AWS_S3_FORCE_PATH_STYLE=true + AWS_S3_ACL=private ++AWS_S3_R2=true ++AWS_S3_R2_PUBLIC_URL=http://s3:4569 + + # –––––––––––––– AUTHENTICATION –––––––––––––– + +diff --git a/app/utils/files.ts b/app/utils/files.ts +index 6607a6b12..5138f68ad 100644 +--- a/app/utils/files.ts ++++ b/app/utils/files.ts +@@ -63,8 +63,13 @@ export const uploadFile = async ( + xhr.addEventListener("loadend", () => { + resolve(xhr.readyState === 4 && xhr.status >= 200 && xhr.status < 400); + }); +- xhr.open("POST", data.uploadUrl, true); +- xhr.send(formData); ++ xhr.open(data.method, data.uploadUrl, true); ++ xhr.setRequestHeader("Content-Type", file.type); ++ if (data.method === "POST") { ++ xhr.send(formData); ++ } else { ++ xhr.send(file); ++ } + }); + + if (!success) { +diff --git a/server/env.ts b/server/env.ts +index 5b420f2e1..4ea1e8d3c 100644 +--- a/server/env.ts ++++ b/server/env.ts +@@ -519,6 +519,14 @@ export class Environment { + environment.AWS_S3_UPLOAD_BUCKET_NAME + ); + ++ @IsOptional() ++ public AWS_S3_R2 = this.toBoolean(environment.AWS_S3_R2 ?? "false"); ++ ++ @IsOptional() ++ public AWS_S3_R2_PUBLIC_URL = this.toOptionalString( ++ environment.AWS_S3_R2_PUBLIC_URL ++ ); ++ + /** + * Whether to force path style URLs for S3 objects, this is required for some + * S3-compatible storage providers. +diff --git a/server/routes/api/attachments/attachments.ts b/server/routes/api/attachments/attachments.ts +index 5e6c27594..b7620f440 100644 +--- a/server/routes/api/attachments/attachments.ts ++++ b/server/routes/api/attachments/attachments.ts +@@ -3,6 +3,7 @@ import { v4 as uuidv4 } from "uuid"; + import { AttachmentPreset } from "@shared/types"; + import { bytesToHumanReadable } from "@shared/utils/files"; + import { AttachmentValidation } from "@shared/validations"; ++import env from "@server/env"; + import { AuthorizationError, ValidationError } from "@server/errors"; + import auth from "@server/middlewares/authentication"; + import { rateLimiter } from "@server/middlewares/rateLimiter"; +@@ -90,16 +91,30 @@ router.post( + { transaction } + ); + +- const presignedPost = await FileStorage.getPresignedPost( +- key, +- acl, +- maxUploadSize, +- contentType +- ); ++ let uploadUrl; ++ let method; ++ let presignedPost = { ++ fields: {}, ++ }; ++ if (env.AWS_S3_R2) { ++ uploadUrl = await FileStorage.getPresignedPut(key); ++ method = "PUT"; ++ } else { ++ uploadUrl = FileStorage.getUploadUrl(); ++ method = "POST"; ++ ++ presignedPost = await FileStorage.getPresignedPost( ++ key, ++ acl, ++ maxUploadSize, ++ contentType ++ ); ++ } + + ctx.body = { + data: { +- uploadUrl: FileStorage.getUploadUrl(), ++ uploadUrl, ++ method, + form: { + "Cache-Control": "max-age=31557600", + "Content-Type": contentType, +diff --git a/server/storage/files/BaseStorage.ts b/server/storage/files/BaseStorage.ts +index ce0287ebc..a1931c83d 100644 +--- a/server/storage/files/BaseStorage.ts ++++ b/server/storage/files/BaseStorage.ts +@@ -26,6 +26,8 @@ export default abstract class BaseStorage { + contentType: string + ): Promise>; + ++ public abstract getPresignedPut(key: string): Promise; ++ + /** + * Returns a promise that resolves with a stream for reading a file from the storage provider. + * +diff --git a/server/storage/files/LocalStorage.ts b/server/storage/files/LocalStorage.ts +index 83cf98c50..324e60dd9 100644 +--- a/server/storage/files/LocalStorage.ts ++++ b/server/storage/files/LocalStorage.ts +@@ -30,6 +30,10 @@ export default class LocalStorage extends BaseStorage { + }); + } + ++ public async getPresignedPut(key: string) { ++ return this.getUrlForKey(key); ++ } ++ + public getUploadUrl() { + return "/api/files.create"; + } +diff --git a/server/storage/files/S3Storage.ts b/server/storage/files/S3Storage.ts +index a42442e0c..d55ef5472 100644 +--- a/server/storage/files/S3Storage.ts ++++ b/server/storage/files/S3Storage.ts +@@ -4,6 +4,7 @@ import { + S3Client, + DeleteObjectCommand, + GetObjectCommand, ++ PutObjectCommand, + ObjectCannedACL, + } from "@aws-sdk/client-s3"; + import { Upload } from "@aws-sdk/lib-storage"; +@@ -58,6 +59,16 @@ export default class S3Storage extends BaseStorage { + return createPresignedPost(this.client, params); + } + ++ public async getPresignedPut(key: string) { ++ const params = { ++ Bucket: env.AWS_S3_UPLOAD_BUCKET_NAME, ++ Key: key, ++ }; ++ ++ const command = new PutObjectCommand(params); ++ return await getSignedUrl(this.client, command, { expiresIn: 3600 }); ++ } ++ + private getPublicEndpoint(isServerUpload?: boolean) { + if (env.AWS_S3_ACCELERATE_URL) { + return env.AWS_S3_ACCELERATE_URL; +@@ -137,10 +148,17 @@ export default class S3Storage extends BaseStorage { + ); + } + ++ public getR2ObjectUrl = async (key: string) => ++ env.AWS_S3_R2_PUBLIC_URL + "/" + key; ++ + public getSignedUrl = async ( + key: string, + expiresIn = S3Storage.defaultSignedUrlExpires + ) => { ++ if (env.AWS_S3_R2) { ++ return this.getR2ObjectUrl(key); ++ } ++ + const isDocker = env.AWS_S3_UPLOAD_BUCKET_URL.match(/http:\/\/s3:/); + const params = { + Bucket: this.getBucket(), diff --git a/nki-personal-do/configuration.nix b/nki-personal-do/configuration.nix index 8d65ae4..88d2bf0 100644 --- a/nki-personal-do/configuration.nix +++ b/nki-personal-do/configuration.nix @@ -24,6 +24,7 @@ ./invidious.nix ./owncast.nix ./peertube.nix + ./outline.nix ]; common.linux.enable = false; # Don't enable the "common linux" module, this is a special machine. @@ -189,55 +190,6 @@ protocol = "udp"; }; - - # Outline - sops.secrets.minio-secret-key = { owner = "root"; mode = "0444"; }; - sops.secrets.authentik-oidc-client-secret = { owner = "outline"; }; - sops.secrets."outline/smtp-password" = { owner = "outline"; }; - services.outline = { - enable = true; - package = pkgs.outline.overrideAttrs (attrs: { - patches = if builtins.hasAttr "patches" attrs then attrs.patches else [ ] ++ [ ../modules/cloud/outline/dtth-wiki.patch ]; - }); - databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable"; - redisUrl = "local"; - publicUrl = "https://wiki.dtth.ch"; - port = 18729; - storage = { - accessKey = "minio"; - secretKeyFile = config.sops.secrets.minio-secret-key.path; - region = config.services.minio.region; - uploadBucketUrl = "https://s3.dtth.ch"; - uploadBucketName = "dtth-outline"; - uploadMaxSize = 50 * 1024 * 1000; - }; - maximumImportSize = 50 * 1024 * 1000; - - oidcAuthentication = { - clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7"; - clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path; - authUrl = "https://auth.dtth.ch/application/o/authorize/"; - tokenUrl = "https://auth.dtth.ch/application/o/token/"; - userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/"; - displayName = "DTTH Account"; - }; - - smtp = { - fromEmail = "DTTH Wiki "; - replyEmail = ""; - host = "mx1.nkagami.me"; - username = "dtth.wiki@nkagami.me"; - passwordFile = config.sops.secrets."outline/smtp-password".path; - port = 465; - secure = true; - }; - - forceHttps = false; - }; - cloud.postgresql.databases = [ "outline" ]; - systemd.services.outline.requires = [ "postgresql.service" ]; - cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; }; - # GoToSocial sops.secrets.gts-env = { }; cloud.gotosocial = { diff --git a/nki-personal-do/outline.nix b/nki-personal-do/outline.nix new file mode 100644 index 0000000..de5e64c --- /dev/null +++ b/nki-personal-do/outline.nix @@ -0,0 +1,56 @@ +{ config, pkgs, ... }: { + sops.secrets.authentik-oidc-client-secret = { owner = "outline"; }; + sops.secrets."outline/smtp-password" = { owner = "outline"; }; + sops.secrets."outline/s3-secret-key" = { owner = "outline"; }; + + services.outline = { + enable = true; + package = pkgs.outline.overrideAttrs (attrs: { + patches = attrs.patches or [ ] ++ [ + ../modules/cloud/outline/dtth-wiki.patch + ../modules/cloud/outline/r2.patch + ]; + }); + databaseUrl = "postgres://outline:outline@localhost/outline?sslmode=disable"; + redisUrl = "local"; + publicUrl = "https://wiki.dtth.ch"; + port = 18729; + storage = { + accessKey = "6ef730e13f172d2ed6ed77f0b5b9bad9"; + secretKeyFile = config.sops.secrets."outline/s3-secret-key".path; + region = "auto"; + uploadBucketUrl = "https://60c0807121eb35ef52cdcd4a33735fa6.r2.cloudflarestorage.com"; + uploadBucketName = "dtth-outline"; + uploadMaxSize = 50 * 1024 * 1000; + }; + maximumImportSize = 50 * 1024 * 1000; + + oidcAuthentication = { + clientId = "3a0c10e00cdcb4a1194315577fa208a747c1a5f7"; + clientSecretFile = config.sops.secrets.authentik-oidc-client-secret.path; + authUrl = "https://auth.dtth.ch/application/o/authorize/"; + tokenUrl = "https://auth.dtth.ch/application/o/token/"; + userinfoUrl = "https://auth.dtth.ch/application/o/userinfo/"; + displayName = "DTTH Account"; + }; + + smtp = { + fromEmail = "DTTH Wiki "; + replyEmail = ""; + host = "mx1.nkagami.me"; + username = "dtth.wiki@nkagami.me"; + passwordFile = config.sops.secrets."outline/smtp-password".path; + port = 465; + secure = true; + }; + + forceHttps = false; + }; + cloud.postgresql.databases = [ "outline" ]; + systemd.services.outline.requires = [ "postgresql.service" ]; + systemd.services.outline.environment = { + AWS_S3_R2 = "true"; + AWS_S3_R2_PUBLIC_URL = "https://s3.wiki.dtth.ch"; + }; + cloud.traefik.hosts.outline = { host = "wiki.dtth.ch"; port = 18729; }; +} diff --git a/nki-personal-do/secrets/secrets.yaml b/nki-personal-do/secrets/secrets.yaml index 95e2f61..b3b3e2c 100644 --- a/nki-personal-do/secrets/secrets.yaml +++ b/nki-personal-do/secrets/secrets.yaml @@ -11,6 +11,7 @@ mail-users: ENC[AES256_GCM,data:qKLi42k8LT6ojxbPXQgbi6FlI2I6ge6qJn0aNj/Lp9iRjjnn youmubot-env: ENC[AES256_GCM,data:EQ9e6lmCrjofHiHyN5Qe4b2oplP9/3JKl0vuFp54Hw9aYIS7j3nqzWLCvV54ZK7j1PcQ+CQorjeCVMV0TUy1f1Pf3qjrLkdOdV7ICq540gdfXOeXuhAx2EILpGkwIYOdKmTMSO3l2QkOlM02RNOn1lq/DogAydkEq7gJ7qSWnUEr45oNCa1+LamH8vcbDmIyzUWWXyA5EQ==,iv:fnNGZ6OaZ4D71SvWPRynsMpO1IsvxjQ3XtrswNSY+Wo=,tag:cN/ZnKrjSfD6AbU9pYNl+Q==,type:str] outline: smtp-password: ENC[AES256_GCM,data:zpIi6jVB2Y7ksBOR8SGFgjOD1x3aS6dKa6taLKB8v2l9p92iWDti75qgB1puglmmq8mCzz8KXLrM0Bv7W8GWRg==,iv:6tKINzQcApmNuIbNn0kSzFJtwn3rky/uFG2Ff3lazUk=,tag:kjB6qB87tRQVpy32Pt3D5A==,type:str] + s3-secret-key: ENC[AES256_GCM,data:dH1Uh3G3RNqITOvsecOW0my3xM3H6xhKYONcwORNPBZmlvSWYvhZUxkOghlH9sYHLIU4yb31QO7npi01Sn3kww==,iv:cV4xqzS5/3HseODY3hS/ycjI6HccsrSGz5Dh9exqNIA=,tag:FMGR9NiTn5S2fTxNSQYBDw==,type:str] heisenbridge: ENC[AES256_GCM,data:rJY7gpcOY8nODR3KlYW1rEs54mKxr+AjNBeg1/2vTG0Gzpuvjgbnn5UVJS+P8uej/P4HfeFtlQSFZCEy8cXcwvwq97ppVliCGL4GMLRWaFmop35feC8t2ovh79cy/vKC7drASeGvWYNUmGRjboPuKA8W5LARa0HVDPGDLIEMVgJfYry/YKR3gsGmLzU7Mx1yLO6M/EFOJQJc84bSuu+CPSZcyUVF4SSNBiaDU5/NazlqaA9KWL6Xzu1MD2LEYdEFkRfitNgYj2m2gLd9voyGV4cfaCqJvYjJPwuZeZUoqCpDnom2JoV29q/Yq/gmyumPgOvriGxLsYBqV14MaCcE6KXE2uLicD+I/5or1AxepVDVjG9NoSgho1HpLvpRhMSCeXLk9+U+ykH3QA+0M+VVu9pswMMVQifnTtXZRM6pWxOnRVAzGf2tGDo4jy36S7pHaRn7SJcrljjWLfwHuNiu7E2uZhMrkcCjnjcBA9Xrb3drDQYVHya7XcoD4wOBHBDvVZwhYkNdkS3oYkom8A==,iv:fO1onfon3EdSNC/LjN1aWxpHBYq5aa0F/h0V6gl88ac=,tag:NL9p2nhIlEqgOdvUDM19Dg==,type:str] matrix-discord-bridge: ENC[AES256_GCM,data:/rlSjD6inKfak7HKKghH5ays5RjKmb9czGsoIOYHyTZC4A5EMucCbfn8DL1gkYXgvRHJ+QglGX/BGo5ebaxSj6nF60+aW87UG31KggOt5kkMuWsPsjvrufoc5IlNfWnXIWmqf8cdC01hmHEp7biUpI8CcfEZiD9OkOxbZcRfYqW+ttnzplFniRBjGPVZfL5g4DBbuJen5MuOrrMDo5CT+78n,iv:r9VBbDCAAElisCaDehrB6PhJHsaaHjdrk3103lmBT7o=,tag:WoNMMfyMifsL56yWq3MUOg==,type:str] authentik-env: ENC[AES256_GCM,data:CjxTaqIcpBX7ea9L3tgJDELr8HBPJdxXsrOfhsiH4cXwCEzktsNKHjF7l95ZFgI5O08q4Vlbln5Dg4xPEx33nwUesEbQrT5d+n+2YaAxmm/WInrYzF+jB7HYTXASb3rY9PWgd2C3v+YPBkJetHlTUc/k19Q7lOQRNw==,iv:cG8Bi2eCsS+v94tSJBsqp+bjVLzXZvvwX1QVVSYExL8=,tag:VmbfcxCcfi3IpKjg3f8QPw==,type:str] @@ -75,8 +76,8 @@ sops: by9kZFlTRVdCZFkxYTVVb0RIRk8zUlkKCqMw9oL9RaYBV5Hhy3o8Nm5xmGrPH8Sd hv36sxRFFNZT/DCKaHaSRbT3mfpBZSTXJt1dgl4nZe6whH54t/1KmA== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-10-20T23:19:07Z" - mac: ENC[AES256_GCM,data:7k0W6cV3HXVmcKjhDBcw+skzTukIay4vpa2cDEWUyLlvEUw3sR0yoKwgYACh4J63UEjcXfnLqQlR2jUkOQ3iigX/gvqSkjKcmfCPvqAnqe9CB/DOVgUufXOOcoNnJXu4G99St3Jgqazaq0xOxG1mXMkbejwPWMsDuqzGuw5v2gE=,iv:HkORvujIH+OePQDzTNqI541y9SEwkdIvxo4gh4RhOt0=,tag:a1p9LkQf6oazfri/SNcbqw==,type:str] + lastmodified: "2024-10-21T00:39:40Z" + mac: ENC[AES256_GCM,data:LtQXhFPm8SFuq7GZIRJyYmzUBcQFRP1UkfkZ2K6eGv0BE72cAN7n1XlxU5Ujj9G1rTjumaquCWmD7h0cmh4ufJnAjAatSn2XOwVAK8+2STd52YQE2sidlHJBlrNrvo4TICusIl+m5Z9E97G420SH6E846Wv+tPQBF9t5HQQgo24=,iv:/7vfawv3rzn2l28MrJcEYRNdMV/QDHThbP2gA1b+jZk=,tag:pdpItbrshuzVtrKWQS949g==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.9.1