Added archive support! (Beta) + UI change

api/modules/getData.js

@@ -0,0 +1,35 @@
+const puppeteer = require('puppeteer');
+const { PuppeteerBlocker } = require('@cliqz/adblocker-puppeteer');
+const fetch = require('cross-fetch');
+const config = require('../../src/config.json');
+const fs = require('fs');
+
+const screenshotDirectory = config.api.storage_location + '/Webmarker/screenshot\'s/';
+const pdfDirectory = config.api.storage_location + '/Webmarker/pdf\'s/';
+
+if (!fs.existsSync(screenshotDirectory)){
+  fs.mkdirSync(screenshotDirectory, { recursive: true });
+}
+
+if (!fs.existsSync(pdfDirectory)){
+  fs.mkdirSync(pdfDirectory, { recursive: true });
+}
+
+module.exports = async (link, id) => {
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+
+  await PuppeteerBlocker.fromPrebuiltAdsAndTracking(fetch).then((blocker) => {
+    blocker.enableBlockingInPage(page);
+  });
+  
+  await page.goto(link, { waitUntil: 'load', timeout: 0 });
+
+  const title = await page.title();
+  await page.screenshot({ path: screenshotDirectory + id + '.png', fullPage: true});
+  await page.pdf({ path: pdfDirectory + id + '.pdf', format: 'a4' });
+
+  await browser.close();
+
+  return title;
+}

api/package.json

@@ -9,12 +9,13 @@
   "author": "",
   "license": "MIT",
   "dependencies": {
-    "cheerio": "^1.0.0-rc.10",
+    "@cliqz/adblocker-puppeteer": "^1.23.8",
     "cors": "^2.8.5",
+    "cross-fetch": "^3.1.5",
     "express": "^4.17.3",
     "mongodb": "^4.5.0",
-    "phantom": "^6.3.0",
-    "url-parse": "^1.5.10"
+    "puppeteer": "^14.1.1",
+    "uuid": "^8.3.2"
   },
   "devDependencies": {
     "nodemon": "^2.0.15"

api/server.js

@@ -1,58 +1,48 @@
 const express = require('express');
 const app = express();
-const { MongoClient, ObjectId } = require('mongodb');
-const request = require('request');
-const cheerio = require('cheerio');
-const URL = require('url-parse');
+const { MongoClient } = require('mongodb');
 const cors = require('cors');
-const config = require('../src/config.json')
+const config = require('../src/config.json');
+const getData = require('./modules/getData.js')
 
-const port = config.server.port;
+const port = config.api.port;
 
-const uri = config.server.mongodb_full_address;
-const database = config.server.database_name;
-const collection = config.server.collection_name;
+const URI = config.api.mongodb_URI;
+const database = config.api.database_name;
+const collection = config.api.collection_name;
 
-const client = new MongoClient(uri);
+const client = new MongoClient(URI);
 
 app.use(cors());
 
 app.use(express.json());
 
-app.get('/get', async (req, res) => {
+app.get('/api', async (req, res) => {
   const data = await getDoc();
   res.send(data);
 });
 
-app.post('/post', (req, res) => {
-  let title;
+app.post('/api', async (req, res) => {
   const pageToVisit = req.body.link;
-  request(pageToVisit, (error, response, body) => {
-    try {
-      if(response.statusCode === 200) {
-        // Parse the document body
-        const $ = cheerio.load(body);
 
-        req.body.title = $('title').text();
-        insertDoc(req.body);
-      } else {
-          req.body.title = null;
-          insertDoc(req.body);
-        } 
-      } catch (error) {
-        console.log(error);
-        req.body.title = null;
-        insertDoc(req.body);
-      }
-  });
+  try {
+    const dataResult = await getData(pageToVisit, req.body._id);
+    req.body.title = dataResult;
+    insertDoc(req.body);
+  } catch (err) {
+    console.log(err);
+    insertDoc(req.body);
+  } finally {
+    res.send('DONE!');
+  }
 });
 
-app.delete('/delete', async (req, res) => {
+app.delete('/api', async (req, res) => {
   const id = req.body.id.toString();
 
   await deleteDoc(id);
 
-  res.send(`Bookmark with ObjectId "${id}" deleted.`);
+  res.send(`Bookmark with _id:${id} deleted.`);
 });
 
 async function insertDoc(doc) {
@@ -85,7 +75,7 @@ async function deleteDoc(doc) {
   try {
     const db = client.db(database);
     const list = db.collection(collection);
-    const result = await list.deleteOne({"_id": ObjectId(doc)});
+    const result = await list.deleteOne({"_id": doc});
 
     return result;
   }