digital ocean spaces/aws S3 integratoin

lib/api/archive.ts

@@ -3,15 +3,13 @@ import { prisma } from "@/lib/api/db";
 import puppeteer from "puppeteer-extra";
 import AdblockerPlugin from "puppeteer-extra-plugin-adblocker";
 import StealthPlugin from "puppeteer-extra-plugin-stealth";
-import fs from "fs";
+import createFile from "@/lib/api/storage/createFile";
 
 export default async function archive(
   url: string,
   collectionId: number,
   linkId: number
 ) {
-  const archivePath = `data/archives/${collectionId}/${linkId}`;
-
   const browser = await puppeteer.launch();
 
   try {
@@ -42,11 +40,14 @@ export default async function archive(
         fullPage: true,
       });
 
-      fs.writeFile(archivePath + ".pdf", pdf, function (err) {
-        console.log(err);
+      createFile({
+        data: screenshot,
+        filePath: `archives/${collectionId}/${linkId}.png`,
       });
-      fs.writeFile(archivePath + ".png", screenshot, function (err) {
-        console.log(err);
+
+      createFile({
+        data: pdf,
+        filePath: `archives/${collectionId}/${linkId}.pdf`,
       });
     }
 

lib/api/controllers/collections/deleteCollection.ts

@@ -1,7 +1,7 @@
 import { prisma } from "@/lib/api/db";
 import getPermission from "@/lib/api/getPermission";
 import { Collection, UsersAndCollections } from "@prisma/client";
-import fs from "fs";
+import removeFolder from "@/lib/api/storage/removeFolder";
 
 export default async function deleteCollection(
   collection: { id: number },
@@ -56,13 +56,7 @@ export default async function deleteCollection(
       },
     });
 
-    try {
-      fs.rmdirSync(`data/archives/${collectionId}`, { recursive: true });
-    } catch (error) {
-      console.log(
-        "Collection's archive directory wasn't deleted most likely because it didn't exist..."
-      );
-    }
+    removeFolder({ filePath: `archives/${collectionId}` });
 
     return await prisma.collection.delete({
       where: {

lib/api/controllers/collections/postCollection.ts

@@ -1,6 +1,6 @@
 import { prisma } from "@/lib/api/db";
 import { CollectionIncludingMembersAndLinkCount } from "@/types/global";
-import { existsSync, mkdirSync } from "fs";
+import createFolder from "@/lib/api/storage/createFolder";
 
 export default async function postCollection(
   collection: CollectionIncludingMembersAndLinkCount,
@@ -66,9 +66,7 @@ export default async function postCollection(
     },
   });
 
-  const collectionPath = `data/archives/${newCollection.id}`;
-  if (!existsSync(collectionPath))
-    mkdirSync(collectionPath, { recursive: true });
+  createFolder({ filePath: `archives/${newCollection.id}` });
 
   return { response: newCollection, status: 200 };
 }

lib/api/controllers/links/deleteLink.ts

@@ -1,8 +1,8 @@
 import { prisma } from "@/lib/api/db";
 import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
-import fs from "fs";
 import { Collection, Link, UsersAndCollections } from "@prisma/client";
 import getPermission from "@/lib/api/getPermission";
+import removeFile from "@/lib/api/storage/removeFile";
 
 export default async function deleteLink(
   link: LinkIncludingShortenedCollectionAndTags,
@@ -33,13 +33,8 @@ export default async function deleteLink(
     },
   });
 
-  fs.unlink(`data/archives/${link.collectionId}/${link.id}.pdf`, (err) => {
-    if (err) console.log(err);
-  });
-
-  fs.unlink(`data/archives/${link.collectionId}/${link.id}.png`, (err) => {
-    if (err) console.log(err);
-  });
+  removeFile({ filePath: `archives/${link.collectionId}/${link.id}.pdf` });
+  removeFile({ filePath: `archives/${link.collectionId}/${link.id}.png` });
 
   return { response: deleteLink, status: 200 };
 }

lib/api/controllers/links/postLink.ts

@@ -1,10 +1,9 @@
 import { prisma } from "@/lib/api/db";
 import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
-import getTitle from "../../getTitle";
-import archive from "../../archive";
+import getTitle from "@/lib/api/getTitle";
+import archive from "@/lib/api/archive";
 import { Collection, Link, UsersAndCollections } from "@prisma/client";
 import getPermission from "@/lib/api/getPermission";
-import { existsSync, mkdirSync } from "fs";
 
 export default async function postLink(
   link: LinkIncludingShortenedCollectionAndTags,
@@ -84,10 +83,6 @@ export default async function postLink(
     include: { tags: true, collection: true },
   });
 
-  const collectionPath = `data/archives/${newLink.collectionId}`;
-  if (!existsSync(collectionPath))
-    mkdirSync(collectionPath, { recursive: true });
-
   archive(newLink.url, newLink.collectionId, newLink.id);
 
   return { response: newLink, status: 200 };

lib/api/controllers/users/updateUser.ts

@@ -1,8 +1,8 @@
 import { prisma } from "@/lib/api/db";
 import { AccountSettings } from "@/types/global";
-import fs from "fs";
-import path from "path";
 import bcrypt from "bcrypt";
+import removeFile from "@/lib/api/storage/removeFile";
+import createFile from "@/lib/api/storage/createFile";
 
 export default async function updateUser(
   user: AccountSettings,
@@ -43,15 +43,12 @@ export default async function updateUser(
   if (profilePic.startsWith("data:image/jpeg;base64")) {
     if (user.profilePic.length < 1572864) {
       try {
-        const filePath = path.join(
-          process.cwd(),
-          `data/uploads/avatar/${userId}.jpg`
-        );
-
         const base64Data = profilePic.replace(/^data:image\/jpeg;base64,/, "");
 
-        fs.writeFile(filePath, base64Data, "base64", function (err) {
-          console.log(err);
+        await createFile({
+          filePath: `uploads/avatar/${userId}.jpg`,
+          data: base64Data,
+          isBase64: true,
         });
       } catch (err) {
         console.log("Error saving image:", err);
@@ -64,9 +61,7 @@ export default async function updateUser(
       };
     }
   } else if (profilePic == "") {
-    fs.unlink(`data/uploads/avatar/${userId}.jpg`, (err) => {
-      if (err) console.log(err);
-    });
+    removeFile({ filePath: `uploads/avatar/${userId}.jpg` });
   }
 
   // Other settings

lib/api/storage/createFile.ts

@@ -0,0 +1,40 @@
+import { PutObjectCommand, PutObjectCommandInput } from "@aws-sdk/client-s3";
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+
+export default async function createFile({
+  filePath,
+  data,
+  isBase64,
+}: {
+  filePath: string;
+  data: Buffer | string;
+  isBase64?: boolean;
+}) {
+  if (s3Client) {
+    const bucketParams: PutObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filePath,
+      Body: isBase64 ? Buffer.from(data as string, "base64") : data,
+    };
+
+    try {
+      await s3Client.send(new PutObjectCommand(bucketParams));
+
+      return true;
+    } catch (err) {
+      console.log("Error", err);
+      return false;
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    fs.writeFile(creationPath, data, isBase64 ? "base64" : {}, function (err) {
+      if (err) console.log(err);
+    });
+
+    return true;
+  }
+}

lib/api/storage/createFolder.ts

@@ -0,0 +1,14 @@
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+
+export default function createFolder({ filePath }: { filePath: string }) {
+  if (s3Client) {
+    // Do nothing, S3 builds files recursively
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    fs.mkdirSync(creationPath, { recursive: true });
+  }
+}

lib/api/storage/readFile.ts

@@ -0,0 +1,74 @@
+import { GetObjectCommand, GetObjectCommandInput } from "@aws-sdk/client-s3";
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+
+export default async function readFile({ filePath }: { filePath: string }) {
+  let contentType:
+    | "text/plain"
+    | "image/jpeg"
+    | "image/png"
+    | "application/pdf";
+
+  if (s3Client) {
+    const bucketParams: GetObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filePath,
+    };
+
+    try {
+      const response = await s3Client.send(new GetObjectCommand(bucketParams));
+      const data = await streamToBuffer(response.Body);
+
+      if (filePath.endsWith(".pdf")) {
+        contentType = "application/pdf";
+      } else if (filePath.endsWith(".png")) {
+        contentType = "image/png";
+      } else {
+        // if (filePath.endsWith(".jpg"))
+        contentType = "image/jpeg";
+      }
+
+      return { file: data, contentType };
+    } catch (err) {
+      console.log("Error", err);
+
+      contentType = "text/plain";
+
+      return {
+        file: "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.",
+        contentType,
+      };
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    const file = fs.existsSync(creationPath)
+      ? fs.readFileSync(creationPath)
+      : "File not found, it's possible that the file you're looking for either doesn't exist or hasn't been created yet.";
+
+    if (file.toString().startsWith("File not found")) {
+      contentType = "text/plain";
+    } else if (filePath.endsWith(".pdf")) {
+      contentType = "application/pdf";
+    } else if (filePath.endsWith(".png")) {
+      contentType = "image/png";
+    } else {
+      // if (filePath.endsWith(".jpg"))
+      contentType = "image/jpeg";
+    }
+
+    return { file, contentType };
+  }
+}
+
+// Turn the file's body into buffer
+const streamToBuffer = (stream: any) => {
+  const chunks: any = [];
+  return new Promise((resolve, reject) => {
+    stream.on("data", (chunk: any) => chunks.push(Buffer.from(chunk)));
+    stream.on("error", (err: any) => reject(err));
+    stream.on("end", () => resolve(Buffer.concat(chunks)));
+  });
+};

lib/api/storage/removeFile.ts

@@ -0,0 +1,26 @@
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+import { PutObjectCommandInput, DeleteObjectCommand } from "@aws-sdk/client-s3";
+
+export default async function removeFile({ filePath }: { filePath: string }) {
+  if (s3Client) {
+    const bucketParams: PutObjectCommandInput = {
+      Bucket: process.env.BUCKET_NAME,
+      Key: filePath,
+    };
+
+    try {
+      await s3Client.send(new DeleteObjectCommand(bucketParams));
+    } catch (err) {
+      console.log("Error", err);
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    fs.unlink(creationPath, (err) => {
+      if (err) console.log(err);
+    });
+  }
+}

lib/api/storage/removeFolder.ts

@@ -0,0 +1,59 @@
+import fs from "fs";
+import path from "path";
+import s3Client from "./s3Client";
+import {
+  DeleteObjectsCommand,
+  DeleteObjectsCommandInput,
+  ListObjectsCommand,
+} from "@aws-sdk/client-s3";
+
+async function emptyS3Directory(bucket: string, dir: string) {
+  if (s3Client) {
+    const listParams = {
+      Bucket: bucket,
+      Prefix: dir,
+    };
+
+    const deleteParams: DeleteObjectsCommandInput = {
+      Bucket: bucket,
+      Delete: { Objects: [] },
+    };
+
+    const listedObjects = await s3Client.send(
+      new ListObjectsCommand(listParams)
+    );
+
+    if (listedObjects.Contents?.length === 0) return;
+
+    listedObjects.Contents?.forEach(({ Key }) => {
+      deleteParams.Delete?.Objects?.push({ Key });
+    });
+
+    console.log(listedObjects);
+
+    await s3Client.send(new DeleteObjectsCommand(deleteParams));
+
+    if (listedObjects.IsTruncated) await emptyS3Directory(bucket, dir);
+  }
+}
+
+export default async function removeFolder({ filePath }: { filePath: string }) {
+  if (s3Client) {
+    try {
+      await emptyS3Directory(process.env.BUCKET_NAME as string, filePath);
+    } catch (err) {
+      console.log("Error", err);
+    }
+  } else {
+    const storagePath = process.env.STORAGE_FOLDER;
+    const creationPath = path.join(process.cwd(), storagePath + "/" + filePath);
+
+    try {
+      fs.rmdirSync(creationPath, { recursive: true });
+    } catch (error) {
+      console.log(
+        "Collection's archive directory wasn't deleted most likely because it didn't exist..."
+      );
+    }
+  }
+}

lib/api/storage/s3Client.ts

@@ -0,0 +1,19 @@
+import { S3 } from "@aws-sdk/client-s3";
+
+const s3Client: S3 | undefined =
+  process.env.SPACES_ENDPOINT &&
+  process.env.SPACES_REGION &&
+  process.env.SPACES_KEY &&
+  process.env.SPACES_SECRET
+    ? new S3({
+        forcePathStyle: false,
+        endpoint: process.env.SPACES_ENDPOINT,
+        region: process.env.SPACES_REGION,
+        credentials: {
+          accessKeyId: process.env.SPACES_KEY,
+          secretAccessKey: process.env.SPACES_SECRET,
+        },
+      })
+    : undefined;
+
+export default s3Client;