add -a switch to pup to cache fetches

Author: brewingcode

lib/exec/pup.coffee

@@ -1,6 +1,8 @@
+require "#{process.env.DOTFILES}/lib/node-globals"
 puppeteer = require 'puppeteer'
-pr = require 'bluebird'
-fs = require 'fs'
+crypto = require 'crypto'
+mkdirp = require 'mkdirp'
+slugify = require 'slugify'
 
 usage = """
 usage: pup [options] URL [more urls...]
@@ -16,6 +18,8 @@ OPTIONS
             to set regex flags like "i" for case-insensitive
 -i          Ignore navigation failures, log anyway
 -v          Verbose
+-a NUM      Return cached content, unless it is older than NUM seconds. If
+            NUM is zero or less, always return cached content
 
 API
 
@@ -28,6 +32,8 @@ API
 
 debug = false
 
+cache = data_root + '/pup-cache'
+
 log = (x...) -> console.warn(new Date(), x...) if debug
 
 _page = (argv) ->
@@ -84,17 +90,49 @@ _page = (argv) ->
 
   return [ browser, page ]
 
+cachepath = (url) ->
+  h = crypto.createHash('sha1')
+  h.update(url)
+  sha1 = h.digest('hex')
+  dir = "#{cache}/" + sha1.match(/\w{5}/g).join('/')
+  mkdirp.sync(dir)
+  "#{dir}/#{slugify(url)}"
+
 get = (argv) ->
+  content = argv._.map -> null
+
+  if argv.a?
+    argv._.forEach (url, i) ->
+      file = cachepath(url)
+      console.warn "cachepath: #{file}" if debug
+      if fs.existsSync(file)
+        mtime = moment(fs.statSync(file).mtime)
+        dur = mtime.diff(moment())
+        if argv.a <= 0 or dur/1000 < argv.a
+          content[i] = fs.readFileSync(file)
+          age = moment.duration(dur).humanize(true)
+          console.warn "cache hit: #{content[i].length} bytes from #{age}"
+      if not content[i] and debug
+        console.warn "cache miss"
+
+  if content.filter(Boolean).length is content.length
+    return content
+
   try
     [browser, page] = await _page(argv)
-    content = []
-    await pr.each argv._, (url) ->
+    await pr.each argv._, (url, i) ->
+      return if content[i]
+      console.warn ".goto(url)" if debug
       try
         await page.goto url, waitUntil:'networkidle0'
       catch e
         if not argv['ignore-nav-fail']
           throw e
-      content.push await page.content()
+      content[i] = await page.content()
+
+      if argv.a?
+        fs.writeFileSync(cachepath(url), content[i])
+
     await browser.close()
     return content
   catch e

lib/package.json

@@ -13,6 +13,7 @@
     "jsdom": "^16.6.0",
     "lodash": "^4.17.21",
     "minimist": "^1.2.5",
+    "mkdirp": "^1.0.4",
     "moment": "^2.29.1",
     "moment-timezone": "^0.5.33",
     "parse-domain": "2.3.4",
@@ -23,6 +24,7 @@
     "request-debug": "^0.2.0",
     "request-promise": "^4.2.5",
     "rss-generator": "^0.0.3",
+    "slugify": "^1.6.5",
     "sqlite-async": "^1.1.3",
     "sqlite3": "^5.0.2"
   },

lib/yarn.lock

@@ -5520,6 +5520,11 @@ slash@^2.0.0:
   resolved "https://registry.yarnpkg.com/slash/-/slash-2.0.0.tgz#de552851a1759df3a8f206535442f5ec4ddeab44"
   integrity sha512-ZYKh3Wh2z1PpEXWr0MpSBZ0V6mZHAQfYevttO11c51CaWjGTaadiKZ+wVt1PbMlDV5qhMFslpZCemhwOK7C89A==
 
+slugify@^1.6.5:
+  version "1.6.5"
+  resolved "https://registry.yarnpkg.com/slugify/-/slugify-1.6.5.tgz#c8f5c072bf2135b80703589b39a3d41451fbe8c8"
+  integrity sha512-8mo9bslnBO3tr5PEVFzMPIWwWnipGS0xVbYf65zxDqfNwmzYn1LpiKNrR6DlClusuvo+hDHd1zKpmfAe83NQSQ==
+
 snapdragon-node@^2.0.1:
   version "2.1.1"
   resolved "https://registry.yarnpkg.com/snapdragon-node/-/snapdragon-node-2.1.1.tgz#6c175f86ff14bdb0724563e8f3c1b021a286853b"