Merge pull request #478 from go-compile/main

feat: Socks5 proxy support

lib/api/archiveHandler.ts

@@ -1,4 +1,4 @@
-import { chromium, devices } from "playwright";
+import { LaunchOptions, chromium, devices } from "playwright";
 import { prisma } from "./db";
 import createFile from "./storage/createFile";
 import sendToWayback from "./sendToWayback";
@@ -20,11 +20,23 @@ type LinksAndCollectionAndOwner = Link & {
 const BROWSER_TIMEOUT = Number(process.env.BROWSER_TIMEOUT) || 5;
 
 export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
-  const browser = await chromium.launch();
+  // allow user to configure a proxy
+  let browserOptions: LaunchOptions = {};
+  if (process.env.PROXY) {
+    browserOptions.proxy = {
+      server: process.env.PROXY,
+      bypass: process.env.PROXY_BYPASS,
+      username: process.env.PROXY_USERNAME,
+      password: process.env.PROXY_PASSWORD,
+    };
+  }
+
+  const browser = await chromium.launch(browserOptions);
   const context = await browser.newContext({
     ...devices["Desktop Chrome"],
     ignoreHTTPSErrors: process.env.IGNORE_HTTPS_ERRORS === "true",
   });
+
   const page = await context.newPage();
 
   const timeoutPromise = new Promise((_, reject) => {
@@ -241,6 +253,13 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
                 })
               );
             }
+
+            // apply administrator's defined pdf margins or default to 15px
+            const margins = {
+              top: process.env.PDF_MARGIN_TOP || "15px",
+              bottom: process.env.PDF_MARGIN_BOTTOM || "15px",
+            };
+
             if (user.archiveAsPDF && !link.pdf?.startsWith("archive")) {
               processingPromises.push(
                 page
@@ -248,7 +267,7 @@ export default async function archiveHandler(link: LinksAndCollectionAndOwner) {
                     width: "1366px",
                     height: "1931px",
                     printBackground: true,
-                    margin: { top: "15px", bottom: "15px" },
+                    margin: margins,
                   })
                   .then((pdf) => {
                     return createFile({

lib/shared/getTitle.ts

@@ -1,5 +1,7 @@
 import fetch from "node-fetch";
 import https from "https";
+import { SocksProxyAgent } from "socks-proxy-agent";
+
 export default async function getTitle(url: string) {
   try {
     const httpsAgent = new https.Agent({
@@ -7,9 +9,26 @@ export default async function getTitle(url: string) {
         process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
     });
 
-    const response = await fetch(url, {
+    // fetchOpts allows a proxy to be defined
+    let fetchOpts = {
       agent: httpsAgent,
-    });
+    };
+
+    if (process.env.PROXY) {
+      // parse proxy url
+      let proxy = new URL(process.env.PROXY);
+      // if authentication set, apply to proxy URL
+      if (process.env.PROXY_USERNAME) {
+        proxy.username = process.env.PROXY_USERNAME;
+        proxy.password = process.env.PROXY_PASSWORD || "";
+      }
+
+      // add socks5 proxy to fetchOpts
+      fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
+    }
+
+    const response = await fetch(url, fetchOpts);
+
     const text = await response.text();
 
     // regular expression to find the <title> tag