Merge pull request #563 from linkwarden/hotfix/title-fetching

update version number

.env.sample

@@ -21,6 +21,7 @@ ARCHIVE_TAKE_COUNT=
 BROWSER_TIMEOUT=
 IGNORE_UNAUTHORIZED_CA=
 IGNORE_HTTPS_ERRORS=
+IGNORE_URL_SIZE_LIMIT=
 
 # AWS S3 Settings
 SPACES_KEY=

ARCHITECTURE.md

@@ -0,0 +1,45 @@
+# Architecture
+
+This is a summary of the architecture of Linkwarden. It's intended as a primer for collaborators to get a high-level understanding of the project.
+
+When you start Linkwarden, there are mainly two components that run:
+
+- The NextJS app, This is the main app and it's responsible for serving the frontend and handling the API routes.
+- [The Background Worker](https://github.com/linkwarden/linkwarden/blob/main/scripts/worker.ts), This is a separate `ts-node` process that runs in the background and is responsible for archiving links.
+
+## Main Tech Stack
+
+- [NextJS](https://github.com/vercel/next.js)
+- [TypeScript](https://github.com/microsoft/TypeScript)
+- [Tailwind](https://github.com/tailwindlabs/tailwindcss)
+- [DaisyUI](https://github.com/saadeghi/daisyui)
+- [Prisma](https://github.com/prisma/prisma)
+- [Playwright](https://github.com/microsoft/playwright)
+- [Zustand](https://github.com/pmndrs/zustand)
+
+## Folder Structure
+
+Here's a summary of the main files and folders in the project:
+
+```
+linkwarden
+├── components         # React components
+├── hooks              # React reusable hooks
+├── layouts            # Layouts for pages
+├── lib
+│   ├── api            # Server-side functions (controllers, etc.)
+│   ├── client         # Client-side functions
+│   └── shared         # Shared functions between client and server
+├── pages              # Pages and API routes
+├── prisma             # Prisma schema and migrations
+├── scripts
+│   ├── migration      # Scripts for breaking changes
+│   └── worker.ts      # Background worker for archiving links
+├── store              # Zustand stores
+├── styles             # Styles
+└── types              # TypeScript types
+```
+
+## Versioning
+
+We use semantic versioning for the project. You can track the changes from the [Releases](https://github.com/linkwarden/linkwarden/releases).

components/LinkViews/LinkComponents/LinkDate.tsx

@@ -6,14 +6,13 @@ export default function LinkDate({
 }: {
   link: LinkIncludingShortenedCollectionAndTags;
 }) {
-  const formattedDate = new Date(link.createdAt as string).toLocaleString(
-    "en-US",
-    {
-      year: "numeric",
-      month: "short",
-      day: "numeric",
-    }
-  );
+  const formattedDate = new Date(
+    (link.importDate || link.createdAt) as string
+  ).toLocaleString("en-US", {
+    year: "numeric",
+    month: "short",
+    day: "numeric",
+  });
 
   return (
     <div className="flex items-center gap-1 text-neutral">

components/ReadableView.tsx

@@ -34,6 +34,8 @@ export default function ReadableView({ link }: Props) {
   const [imageError, setImageError] = useState<boolean>(false);
   const [colorPalette, setColorPalette] = useState<RGBColor[]>();
 
+  const [date, setDate] = useState<Date | string>();
+
   const colorThief = new ColorThief();
 
   const router = useRouter();
@@ -54,6 +56,8 @@ export default function ReadableView({ link }: Props) {
     };
 
     fetchLinkContent();
+
+    setDate(link.importDate || link.createdAt);
   }, [link]);
 
   useEffect(() => {
@@ -211,8 +215,8 @@ export default function ReadableView({ link }: Props) {
           </div>
 
           <p className="min-w-fit text-sm text-neutral">
-            {link?.createdAt
-              ? new Date(link?.createdAt).toLocaleString("en-US", {
+            {date
+              ? new Date(date).toLocaleString("en-US", {
                   year: "numeric",
                   month: "long",
                   day: "numeric",

components/SettingsSidebar.tsx

@@ -4,7 +4,7 @@ import { useRouter } from "next/router";
 import React, { useEffect, useState } from "react";
 
 export default function SettingsSidebar({ className }: { className?: string }) {
-  const LINKWARDEN_VERSION = "v2.5.1";
+  const LINKWARDEN_VERSION = "v2.5.2";
 
   const { collections } = useCollectionStore();
 

lib/api/controllers/links/postLink.ts

@@ -48,6 +48,7 @@ export default async function postLink(
         return { response: "Collection is not accessible.", status: 401 };
 
       link.collection.id = findCollection.id;
+      link.collection.ownerId = findCollection.ownerId;
     } else {
       const collection = await prisma.collection.create({
         data: {

lib/api/controllers/migration/importFromHTMLFile.ts

@@ -2,6 +2,7 @@ import { prisma } from "@/lib/api/db";
 import createFolder from "@/lib/api/storage/createFolder";
 import { JSDOM } from "jsdom";
 import { parse, Node, Element, TextNode } from "himalaya";
+import { writeFileSync } from "fs";
 
 const MAX_LINKS_PER_USER = Number(process.env.MAX_LINKS_PER_USER) || 30000;
 
@@ -36,7 +37,9 @@ export default async function importFromHTMLFile(
 
   const jsonData = parse(document.documentElement.outerHTML);
 
-  for (const item of jsonData) {
+  const processedArray = processNodes(jsonData);
+
+  for (const item of processedArray) {
     console.log(item);
     await processBookmarks(userId, item as Element);
   }
@@ -74,7 +77,9 @@ async function processBookmarks(
       } else if (item.type === "element" && item.tagName === "a") {
         // process link
 
-        const linkUrl = item?.attributes.find((e) => e.key === "href")?.value;
+        const linkUrl = item?.attributes.find(
+          (e) => e.key.toLowerCase() === "href"
+        )?.value;
         const linkName = (
           item?.children.find((e) => e.type === "text") as TextNode
         )?.content;
@@ -82,14 +87,33 @@ async function processBookmarks(
           .find((e) => e.key === "tags")
           ?.value.split(",");
 
+        // set date if available
+        const linkDateValue = item?.attributes.find(
+          (e) => e.key.toLowerCase() === "add_date"
+        )?.value;
+
+        const linkDate = linkDateValue
+          ? new Date(Number(linkDateValue) * 1000)
+          : undefined;
+
+        let linkDesc =
+          (
+            (
+              item?.children?.find(
+                (e) => e.type === "element" && e.tagName === "dd"
+              ) as Element
+            )?.children[0] as TextNode
+          )?.content || "";
+
         if (linkUrl && parentCollectionId) {
           await createLink(
             userId,
             linkUrl,
             parentCollectionId,
             linkName,
-            "",
-            linkTags
+            linkDesc,
+            linkTags,
+            linkDate
           );
         } else if (linkUrl) {
           // create a collection named "Imported Bookmarks" and add the link to it
@@ -100,8 +124,9 @@ async function processBookmarks(
             linkUrl,
             collectionId,
             linkName,
-            "",
-            linkTags
+            linkDesc,
+            linkTags,
+            linkDate
           );
         }
 
@@ -160,7 +185,8 @@ const createLink = async (
   collectionId: number,
   name?: string,
   description?: string,
-  tags?: string[]
+  tags?: string[],
+  importDate?: Date
 ) => {
   await prisma.link.create({
     data: {
@@ -193,6 +219,48 @@ const createLink = async (
               }),
             }
           : undefined,
+      importDate: importDate || undefined,
     },
   });
 };
+
+function processNodes(nodes: Node[]) {
+  const findAndProcessDL = (node: Node) => {
+    if (node.type === "element" && node.tagName === "dl") {
+      processDLChildren(node);
+    } else if (
+      node.type === "element" &&
+      node.children &&
+      node.children.length
+    ) {
+      node.children.forEach((child) => findAndProcessDL(child));
+    }
+  };
+
+  const processDLChildren = (dlNode: Element) => {
+    dlNode.children.forEach((child, i) => {
+      if (child.type === "element" && child.tagName === "dt") {
+        const nextSibling = dlNode.children[i + 1];
+        if (
+          nextSibling &&
+          nextSibling.type === "element" &&
+          nextSibling.tagName === "dd"
+        ) {
+          const aElement = child.children.find(
+            (el) => el.type === "element" && el.tagName === "a"
+          );
+          if (aElement && aElement.type === "element") {
+            // Add the 'dd' element as a child of the 'a' element
+            aElement.children.push(nextSibling);
+            // Remove the 'dd' from the parent 'dl' to avoid duplicate processing
+            dlNode.children.splice(i + 1, 1);
+            // Adjust the loop counter due to the removal
+          }
+        }
+      }
+    });
+  };
+
+  nodes.forEach(findAndProcessDL);
+  return nodes;
+}

lib/api/validateUrlSize.ts

@@ -1,17 +1,35 @@
 import fetch from "node-fetch";
 import https from "https";
+import { SocksProxyAgent } from "socks-proxy-agent";
 
 export default async function validateUrlSize(url: string) {
+  if (process.env.IGNORE_URL_SIZE_LIMIT === "true") return null;
+
   try {
     const httpsAgent = new https.Agent({
       rejectUnauthorized:
         process.env.IGNORE_UNAUTHORIZED_CA === "true" ? false : true,
     });
 
-    const response = await fetch(url, {
+    let fetchOpts = {
       method: "HEAD",
       agent: httpsAgent,
-    });
+    };
+
+    if (process.env.PROXY) {
+      let proxy = new URL(process.env.PROXY);
+      if (process.env.PROXY_USERNAME) {
+        proxy.username = process.env.PROXY_USERNAME;
+        proxy.password = process.env.PROXY_PASSWORD || "";
+      }
+
+      fetchOpts = {
+        method: "HEAD",
+        agent: new SocksProxyAgent(proxy.toString()),
+      };
+    }
+
+    const response = await fetch(url, fetchOpts);
 
     const totalSizeMB =
       Number(response.headers.get("content-length")) / Math.pow(1024, 2);

lib/shared/getTitle.ts

@@ -27,14 +27,25 @@ export default async function getTitle(url: string) {
       fetchOpts = { agent: new SocksProxyAgent(proxy.toString()) }; //TODO: add support for http/https proxies
     }
 
-    const response = await fetch(url, fetchOpts);
+    const responsePromise = fetch(url, fetchOpts);
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error("Fetch title timeout"));
+      }, 10 * 1000); // Stop after 10 seconds
+    });
 
-    const text = await response.text();
+    const response = await Promise.race([responsePromise, timeoutPromise]);
 
-    // regular expression to find the <title> tag
-    let match = text.match(/<title.*>([^<]*)<\/title>/);
-    if (match) return match[1];
-    else return "";
+    if ((response as any)?.status) {
+      const text = await (response as any).text();
+
+      // regular expression to find the <title> tag
+      let match = text.match(/<title.*>([^<]*)<\/title>/);
+      if (match) return match[1];
+      else return "";
+    } else {
+      return "";
+    }
   } catch (err) {
     console.log(err);
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "linkwarden",
-  "version": "2.5.1",
+  "version": "0.0.0",
   "main": "index.js",
   "repository": "https://github.com/linkwarden/linkwarden.git",
   "author": "Daniel31X13 <daniel31x13@gmail.com>",
@@ -13,7 +13,7 @@
     "dev": "concurrently -k -P \"next dev {@}\" \"yarn worker:dev\" --",
     "worker:dev": "nodemon --skip-project scripts/worker.ts",
     "worker:prod": "ts-node --transpile-only --skip-project scripts/worker.ts",
-    "start": "concurrently -k -P \"next start {@}\" \"yarn worker:prod\" --",
+    "start": "concurrently -P \"next start {@}\" \"yarn worker:prod\" --",
     "build": "next build",
     "lint": "next lint",
     "format": "prettier --write \"**/*.{ts,tsx,js,json,md}\""

prisma/migrations/20240327070238_add_import_date_field_for_links/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "Link" ADD COLUMN     "importDate" TIMESTAMP(3);

prisma/schema.prisma

@@ -128,6 +128,7 @@ model Link {
   pdf           String?
   readable      String?
   lastPreserved DateTime?
+  importDate    DateTime?
   createdAt     DateTime   @default(now())
   updatedAt     DateTime   @default(now()) @updatedAt
 }

types/enviornment.d.ts

@@ -13,6 +13,7 @@ declare global {
       MAX_LINKS_PER_USER?: string;
       ARCHIVE_TAKE_COUNT?: string;
       IGNORE_UNAUTHORIZED_CA?: string;
+      IGNORE_URL_SIZE_LIMIT?: string;
 
       SPACES_KEY?: string;
       SPACES_SECRET?: string;

types/global.ts

@@ -7,10 +7,16 @@ type OptionalExcluding<T, TRequired extends keyof T> = Partial<T> &
 export interface LinkIncludingShortenedCollectionAndTags
   extends Omit<
     Link,
-    "id" | "createdAt" | "collectionId" | "updatedAt" | "lastPreserved"
+    | "id"
+    | "createdAt"
+    | "collectionId"
+    | "updatedAt"
+    | "lastPreserved"
+    | "importDate"
   > {
   id?: number;
   createdAt?: string;
+  importDate?: string;
   collectionId?: number;
   tags: Tag[];
   pinnedBy?: {