improvements

2023-12-13 06:59:36 -05:00
parent ca3eb29c48
commit a001f70b9d
11 changed files with 138 additions and 69 deletions
@@ -15,8 +15,9 @@ type LinksAndCollectionAndOwner = Link & {
 };

 async function processBatch() {
-  const links = await prisma.link.findMany({
+  const linksOldToNew = await prisma.link.findMany({
    where: {
+      url: { not: null },
      OR: [
        {
          collection: {
@@ -71,6 +72,63 @@ async function processBatch() {
    },
  });

+  const linksNewToOld = await prisma.link.findMany({
+    where: {
+      url: { not: null },
+      OR: [
+        {
+          collection: {
+            owner: {
+              archiveAsScreenshot: true,
+            },
+          },
+          screenshotPath: null,
+        },
+        {
+          collection: {
+            owner: {
+              archiveAsScreenshot: true,
+            },
+          },
+          screenshotPath: "pending",
+        },
+        ///////////////////////
+        {
+          collection: {
+            owner: {
+              archiveAsPDF: true,
+            },
+          },
+          pdfPath: null,
+        },
+        {
+          collection: {
+            owner: {
+              archiveAsPDF: true,
+            },
+          },
+          pdfPath: "pending",
+        },
+        ///////////////////////
+        {
+          readabilityPath: null,
+        },
+        {
+          readabilityPath: "pending",
+        },
+      ],
+    },
+    take: archiveTakeCount,
+    orderBy: { createdAt: "desc" },
+    include: {
+      collection: {
+        include: {
+          owner: true,
+        },
+      },
+    },
+  });
+
  const archiveLink = async (link: LinksAndCollectionAndOwner) => {
    try {
      console.log(
@@ -94,7 +152,9 @@ async function processBatch() {
  };

  // Process each link in the batch concurrently
-  const processingPromises = links.map((e) => archiveLink(e));
+  const processingPromises = [...linksOldToNew, ...linksNewToOld].map((e) =>
+    archiveLink(e)
+  );

  await Promise.allSettled(processingPromises);
 }