diff --git a/.env.sample b/.env.sample index bafdc9fd..0e692b0d 100644 --- a/.env.sample +++ b/.env.sample @@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT= BROWSER_TIMEOUT= IGNORE_UNAUTHORIZED_CA= IGNORE_HTTPS_ERRORS= +IGNORE_URL_SIZE_LIMIT= # AWS S3 Settings SPACES_KEY= diff --git a/components/SettingsSidebar.tsx b/components/SettingsSidebar.tsx index 9e169dfd..ca6f6cfb 100644 --- a/components/SettingsSidebar.tsx +++ b/components/SettingsSidebar.tsx @@ -4,7 +4,7 @@ import { useRouter } from "next/router"; import React, { useEffect, useState } from "react"; export default function SettingsSidebar({ className }: { className?: string }) { - const LINKWARDEN_VERSION = "v2.5.1"; + const LINKWARDEN_VERSION = "v2.5.2"; const { collections } = useCollectionStore(); diff --git a/lib/api/validateUrlSize.ts b/lib/api/validateUrlSize.ts index eae0b432..d5826b87 100644 --- a/lib/api/validateUrlSize.ts +++ b/lib/api/validateUrlSize.ts @@ -1,17 +1,35 @@ import fetch from "node-fetch"; import https from "https"; +import { SocksProxyAgent } from "socks-proxy-agent"; export default async function validateUrlSize(url: string) { + if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null; + try { const httpsAgent = new https.Agent({ rejectUnauthorized: process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true, }); - const response = await fetch(url, { + let fetchOpts = { method: "HEAD", agent: httpsAgent, - }); + }; + + if (process.env.PROXY) { + let proxy = new URL(process.env.PROXY); + if (process.env.PROXY_USERNAME) { + proxy.username = process.env.PROXY_USERNAME; + proxy.password = process.env.PROXY_PASSWORD || ""; + } + + fetchOpts = { + method: "HEAD", + agent: new SocksProxyAgent(proxy.toString()), + }; + } + + const response = await fetch(url, fetchOpts); const totalSizeMB = Number(response.headers.get("content-length")) / Math.pow(1024, 2); diff --git a/lib/shared/getTitle.ts b/lib/shared/getTitle.ts index 82fee373..01488fdd 100644 --- a/lib/shared/getTitle.ts +++ b/lib/shared/getTitle.ts @@ -27,14 +27,25 @@ export default async function getTitle(url: string) { fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies } - const response = await fetch(url, fetchOpts); + const responsePromise = fetch(url, fetchOpts); + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error("Fetch title timeout")); + }, 10 * 1000); // Stop after 10 seconds + }); - const text = await response.text(); + const response = await Promise.race([responsePromise, timeoutPromise]); - // regular expression to find the tag - let match = text.match(/<title.*>([^<]*)<\/title>/); - if (match) return match[1]; - else return ""; + if ((response as any)?.status) { + const text = await (response as any).text(); + + // regular expression to find the <title> tag + let match = text.match(/<title.*>([^<]*)<\/title>/); + if (match) return match[1]; + else return ""; + } else { + return ""; + } } catch (err) { console.log(err); } diff --git a/package.json b/package.json index 6b03cf3a..39069be9 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "linkwarden", - "version": "2.5.1", + "version": "0.0.0", "main": "index.js", "repository": "https://github.com/linkwarden/linkwarden.git", "author": "Daniel31X13 <daniel31x13@gmail.com>", diff --git a/types/enviornment.d.ts b/types/enviornment.d.ts index 899f7a4b..e8c5a7c6 100644 --- a/types/enviornment.d.ts +++ b/types/enviornment.d.ts @@ -13,6 +13,7 @@ declare global { MAX_LINKS_PER_USER?: string; ARCHIVE_TAKE_COUNT?: string; IGNORE_UNAUTHORIZED_CA?: string; + IGNORE_URL_SIZE_LIMIT?: string; SPACES_KEY?: string; SPACES_SECRET?: string;