improved archive handler

lib/api/archiveHandler.ts

@@ -2,7 +2,7 @@ import { LaunchOptions, chromium, devices } from "playwright";
 import { prisma } from "./db";
 import sendToWayback from "./preservationScheme/sendToWayback";
 import { Collection, Link, User } from "@prisma/client";
-import validateUrlSize from "./validateUrlSize";
+import fetchHeaders from "./fetchHeaders";
 import createFolder from "./storage/createFolder";
 import { removeFiles } from "./manageLinkFiles";
 import handleMonolith from "./preservationScheme/handleMonolith";
@@ -65,17 +65,9 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
       (async () => {
         const user = link.collection?.owner;
 
-        const validatedUrl = link.url
-          ? await validateUrlSize(link.url)
-          : undefined;
+        const header = link.url ? await fetchHeaders(link.url) : undefined;
 
-        if (
-          validatedUrl === null &&
-          process.env.IGNORE_URL_SIZE_LIMIT !== "true"
-        )
-          throw "Something went wrong while retrieving the file size.";
-
-        const contentType = validatedUrl?.get("content-type");
+        const contentType = header?.get("content-type");
         let linkType = "url";
         let imageExtension = "png";
 

lib/api/controllers/links/postLink.ts

@@ -1,8 +1,7 @@
 import { prisma } from "@/lib/api/db";
 import { LinkIncludingShortenedCollectionAndTags } from "@/types/global";
-import getTitle from "@/lib/shared/getTitle";
+import fetchTitleAndHeaders from "@/lib/shared/fetchTitleAndHeaders";
 import createFolder from "@/lib/api/storage/createFolder";
-import validateUrlSize from "../../validateUrlSize";
 import setLinkCollection from "../../setLinkCollection";
 
 const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000;
@@ -70,17 +69,12 @@ export default async function postLink(
       status: 400,
     };
 
-  const title =
-    !(link.name && link.name !== "") && link.url
-      ? await getTitle(link.url)
-      : "";
+  const { title, headers } = await fetchTitleAndHeaders(link.url || "");
 
   const name =
     link.name && link.name !== "" ? link.name : link.url ? title : "";
 
-  const validatedUrl = link.url ? await validateUrlSize(link.url) : undefined;
-
-  const contentType = validatedUrl?.get("content-type");
+  const contentType = headers?.get("content-type");
   let linkType = "url";
   let imageExtension = "png";
 

lib/api/fetchHeaders.ts

@@ -2,7 +2,7 @@ import fetch from "node-fetch";
 import https from "https";
 import { SocksProxyAgent } from "socks-proxy-agent";
 
-export default async function validateUrlSize(url: string) {
+export default async function fetchHeaders(url: string) {
   if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
 
   try {
@@ -29,13 +29,17 @@ export default async function validateUrlSize(url: string) {
       };
     }
 
-    const response = await fetch(url, fetchOpts);
+    const responsePromise = fetch(url, fetchOpts);
 
-    const totalSizeMB =
-      Number(response.headers.get("content-length")) / Math.pow(1024, 2);
-    if (totalSizeMB > Number(process.env.NEXT_PUBLIC_MAX_FILE_BUFFER || 10))
-      return null;
-    else return response.headers;
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error("Fetch header timeout"));
+      }, 10 * 1000); // Stop after 10 seconds
+    });
+
+    const response = await Promise.race([responsePromise, timeoutPromise]);
+
+    return (response as Response)?.headers || null;
   } catch (err) {
     console.log(err);
     return null;

lib/api/preservationScheme/imageHandler.ts

@@ -1,6 +1,7 @@
 import { Link } from "@prisma/client";
 import { prisma } from "../db";
 import createFile from "../storage/createFile";
+import generatePreview from "../generatePreview";
 
 const imageHandler = async ({ url, id }: Link, extension: string) => {
   const image = await fetch(url as string).then((res) => res.blob());
@@ -18,6 +19,8 @@ const imageHandler = async ({ url, id }: Link, extension: string) => {
   });
 
   if (linkExists) {
+    await generatePreview(buffer, linkExists.collectionId, id);
+
     await createFile({
       data: buffer,
       filePath: `archives/${linkExists.collectionId}/${id}.${extension}`,