feat: don't pack into asar node module that contains executable/binary files

Close #602

Author: develar

package.json

@@ -69,9 +69,10 @@
     "electron-osx-sign": "^0.4.0-beta4",
     "electron-winstaller-fixed": "~3.1.0",
     "extract-zip": "^1.5.0",
-    "fs-extra-p": "^1.0.5",
+    "fs-extra-p": "^1.0.6",
     "hosted-git-info": "^2.1.5",
     "image-size": "^0.5.0",
+    "isbinaryfile": "^3.0.0",
     "lodash.template": "^4.3.0",
     "mime": "^1.3.4",
     "minimatch": "^3.0.2",
@@ -120,7 +121,7 @@
     "pre-git": "^3.10.0",
     "should": "^10.0.0",
     "ts-babel": "^1.0.3",
-    "tslint": "3.14.0-dev.1",
+    "tslint": "3.14.0",
     "typescript": "^2.1.0-dev.20160726",
     "whitespace": "^2.0.0"
   },

src/asarUtil.ts

@@ -1,17 +1,21 @@
-import { AsarFileInfo, listPackage, statFile, AsarOptions, AsarFileMetadata, createPackageFromFiles } from "asar-electron-builder"
+import { AsarFileInfo, listPackage, statFile, AsarOptions } from "asar-electron-builder"
 import { statOrNull } from "./util/util"
-import { lstat, readdir } from "fs-extra-p"
+import { lstat, readdir, readFile, mkdirp, Stats } from "fs-extra-p"
 import { Promise as BluebirdPromise } from "bluebird"
 import * as path from "path"
-import { Stats } from "fs"
 import pathSorter = require("path-sort")
+import { log } from "./util/log"
+import { Minimatch } from "minimatch"
+
+const isBinaryFile: any = BluebirdPromise.promisify(require("isbinaryfile"))
 
 //noinspection JSUnusedLocalSymbols
 const __awaiter = require("./util/awaiter")
 
 const concurrency = {concurrency: 50}
+const NODE_MODULES_PATTERN = path.sep + "node_modules" + path.sep
 
-function walk(dirPath: string, consumer: (file: string, stat: Stats) => void, filter: (file: string) => boolean): BluebirdPromise<any> {
+function walk(dirPath: string, consumer: (file: string, stat: Stats) => void, filter: (file: string) => boolean, addRootToResult?: boolean): BluebirdPromise<Array<string>> {
   return readdir(dirPath)
     .then(names => {
       return BluebirdPromise.map(names, name => {
@@ -21,31 +25,184 @@ function walk(dirPath: string, consumer: (file: string, stat: Stats) => void, fi
         }
 
         return lstat(filePath)
-          .then(stat => {
+          .then((stat): any => {
             consumer(filePath, stat)
             if (stat.isDirectory()) {
-              return walk(filePath, consumer, filter)
+              return walk(filePath, consumer, filter, true)
             }
-            return null
+            return filePath
           })
       }, concurrency)
     })
+    .then(list => {
+      list.sort((a, b) => {
+        // files before directories
+        if (Array.isArray(a) && Array.isArray(b)) {
+          return 0
+        }
+        else if (a == null || Array.isArray(a)) {
+          return 1
+        }
+        else if (b == null || Array.isArray(b)) {
+          return -1
+        }
+        else {
+          return a.localeCompare(b)
+        }
+      })
+
+      const result: Array<string> = addRootToResult ? [dirPath] : []
+      for (let item of list) {
+        if (item != null) {
+          if (Array.isArray(item)) {
+            result.push.apply(result, item)
+          }
+          else {
+            result.push(item)
+          }
+        }
+      }
+      return result
+    })
 }
 
 export async function createAsarArchive(src: string, resourcesPath: string, options: AsarOptions, filter: (file: string) => boolean): Promise<any> {
-  const metadata: { [key: string]: AsarFileMetadata; } = {}
-  const files: Array<string> = []
-  await walk(src, (it, stat) => {
-      files.push(it)
-      metadata[it] = {
-        type: stat.isFile() ? "file" : (stat.isDirectory() ? "directory" : "link"),
-        stat: stat,
-      }
-    },
-    filter)
+  const metadata = new Map<string, Stats>()
+  const files = await walk(src, (it, stat) => {
+    metadata.set(it, stat)
+  }, filter)
 
   // sort files to minimize file change (i.e. asar file is not changed dramatically on small change)
-  await BluebirdPromise.promisify(createPackageFromFiles)(src, path.join(resourcesPath, "app.asar"), pathSorter(files), metadata, options)
+  await createPackageFromFiles(src, path.join(resourcesPath, "app.asar"), options.ordering == null ? files : await order(src, files, options), metadata, options)
+}
+
+const Filesystem = require("asar-electron-builder/lib/filesystem")
+const writeFilesystem: any = BluebirdPromise.promisify(require("asar-electron-builder/lib/disk").writeFilesystem)
+
+function isUnpackDir(path: string, pattern: Minimatch, rawPattern: string): boolean {
+  return path.indexOf(rawPattern) === 0 || pattern.match(path)
+}
+
+async function order(src: string, filenames: Array<string>, options: any) {
+  const orderingFiles = (await readFile(options.ordering, "utf8")).split("\n").map(function (line) {
+    if (line.indexOf(':') !== -1) {
+      line = line.split(':').pop()!
+    }
+    line = line.trim()
+    if (line[0] === '/') {
+      line = line.slice(1)
+    }
+    return line
+  })
+
+  const ordering: Array<string> = []
+  for (let file of orderingFiles) {
+    let pathComponents = file.split(path.sep)
+    let str = src
+    for (let pathComponent of pathComponents) {
+      str = path.join(str, pathComponent)
+      ordering.push(str)
+    }
+  }
+
+  const filenamesSorted: Array<string> = []
+  let missing = 0
+  const total = filenames.length
+  for (let file of ordering) {
+    if (!filenamesSorted.includes(file) && filenames.includes(file)) {
+      filenamesSorted.push(file)
+    }
+  }
+  for (let file of filenames) {
+    if (!filenamesSorted.includes(file)) {
+      filenamesSorted.push(file)
+      missing += 1;
+    }
+  }
+  log(`Ordering file has ${((total - missing) / total * 100)}% coverage.`)
+  return filenamesSorted
+}
+
+async function createPackageFromFiles(src: string, dest: string, files: Array<string>, metadata: Map<string, Stats>, options: any) {
+  // search auto unpacked dir
+  const autoUnpackDirs = new Set<string>()
+  if (options.smartUnpack !== false) {
+    for (let file of files) {
+      const index = file.lastIndexOf(NODE_MODULES_PATTERN)
+      if (index < 0) {
+        continue
+      }
+
+      const nextSlashIndex = file.indexOf(path.sep, index + NODE_MODULES_PATTERN.length + 1)
+      if (nextSlashIndex < 0) {
+        continue
+      }
+
+      const nodeModuleDir = file.substring(0, nextSlashIndex)
+      if (autoUnpackDirs.has(nodeModuleDir) || !metadata.get(file)!.isFile()) {
+        continue
+      }
+
+      const ext = path.extname(file)
+      let shouldUnpack = false
+      if (ext === ".dll" || ext === ".exe") {
+        shouldUnpack = true
+      }
+      else if (ext === "") {
+        shouldUnpack = await isBinaryFile(file)
+      }
+
+      if (!shouldUnpack) {
+        continue
+      }
+
+      log(`${path.relative(src, nodeModuleDir)} is not packed into asar archive - contains executable code`)
+      autoUnpackDirs.add(nodeModuleDir)
+    }
+  }
+
+  const unpackDir = options.unpackDir == null ? null : new Minimatch(options.unpackDir)
+  const unpack = options.unpack == null ? null : new Minimatch(options.unpack, {
+    matchBase: true
+  })
+
+  const regularFiles: Array<any> = []
+  const filesystem = new Filesystem(src)
+  for (let file of files) {
+    const stat = metadata.get(file)!
+    if (stat.isFile()) {
+      let shouldUnpack = unpack != null && unpack.match(file)
+      if (!shouldUnpack) {
+        const dir = path.dirname(file)
+        shouldUnpack = autoUnpackDirs.has(dir) || (unpackDir != null && isUnpackDir(path.relative(src, dir), unpackDir, options.unpackDir))
+      }
+
+      regularFiles.push({
+        filename: file,
+        unpack: shouldUnpack
+      })
+      filesystem.insertFile(file, shouldUnpack, stat)
+    }
+    else if (stat.isDirectory()) {
+      let unpacked = autoUnpackDirs.has(file) || (unpackDir != null && isUnpackDir(path.relative(src, file), unpackDir, options.unpackDir))
+      if (!unpacked) {
+        for (let d of autoUnpackDirs) {
+          if (file.length > (d.length + 2) && file[d.length] === path.sep && file.startsWith(d)) {
+            unpacked = true
+            autoUnpackDirs.add(file)
+            break
+          }
+        }
+      }
+      filesystem.insertDirectory(file, unpacked)
+    }
+    else if (stat.isSymbolicLink()) {
+      filesystem.insertLink(file, stat)
+    }
+  }
+
+  await mkdirp(path.dirname(dest))
+  await writeFilesystem(dest, filesystem, regularFiles)
 }
 
 export async function checkFileInPackage(asarFile: string, relativeFile: string) {

src/packager/dirPackager.ts

@@ -17,9 +17,9 @@ export interface ElectronPackagerOptions {
 
   appInfo: AppInfo
 
-  icon?: string;
+  icon?: string
 
-  "helper-bundle-id"?: string | null;
+  "helper-bundle-id"?: string | null
 
   ignore?: any
 }

src/platformPackager.ts

@@ -295,14 +295,15 @@ export abstract class PlatformPackager<DC extends PlatformSpecificBuildOptions>
     }
 
     if (result == null || result === true) {
-      return {
+      result = {
         unpack: buildMetadata["asar-unpack"],
         unpackDir: buildMetadata["asar-unpack-dir"]
       }
     }
-    else {
-      return result
-    }
+
+    return deepAssign(result, {
+
+    })
   }
 
   private expandPattern(pattern: string, arch: Arch): string {

src/util/log.ts

@@ -91,7 +91,7 @@ class Logger {
     cursor.hide()
 
     let out = ""
-    let firstPendingLineIndex = 0;
+    let firstPendingLineIndex = 0
     while (firstPendingLineIndex < prevLineCount) {
       let line = this.lines[firstPendingLineIndex]
       if (line.promise == null || !line.promise.isPending()) {

src/util/util.ts

@@ -42,7 +42,7 @@ export function spawnNpmProduction(command: string, appDir: string, env?: any):
 
   return spawn(npmExecPath, npmExecArgs, {
     cwd: appDir,
-    stdio: "inherit",
+    stdio: ["ignore", "pipe", process.stderr],
     env: env || process.env
   })
 }
@@ -107,9 +107,7 @@ export function doSpawn(command: string, args: Array<string>, options?: SpawnOpt
 
 export function spawn(command: string, args?: Array<string> | null, options?: SpawnOptions): BluebirdPromise<any> {
   return new BluebirdPromise<any>((resolve, reject) => {
-    const notNullArgs = args || []
-    const childProcess = doSpawn(command, notNullArgs, options)
-    handleProcess("close", childProcess, command, resolve, reject)
+    handleProcess("close", doSpawn(command, args || [], options), command, resolve, reject)
   })
 }
 

test/src/helpers/avaEx.ts

@@ -4,15 +4,15 @@ import test from "ava-tf"
 
 declare module "ava-tf" {
   namespace test {
-    export const ifNotWindows: typeof test;
-    export const ifOsx: typeof test;
-    export const ifNotCi: typeof test;
-    export const ifCi: typeof test;
-    export const ifNotCiOsx: typeof test;
-    export const ifDevOrWinCi: typeof test;
-    export const ifWinCi: typeof test;
-    export const ifDevOrLinuxCi: typeof test;
-    export const ifNotTravis: typeof test;
+    export const ifNotWindows: typeof test
+    export const ifOsx: typeof test
+    export const ifNotCi: typeof test
+    export const ifCi: typeof test
+    export const ifNotCiOsx: typeof test
+    export const ifDevOrWinCi: typeof test
+    export const ifWinCi: typeof test
+    export const ifDevOrLinuxCi: typeof test
+    export const ifNotTravis: typeof test
   }
 }
 

tslint.json

@@ -37,7 +37,7 @@
       "ban-keywords"
     ],
     "ordered-imports": false,
-    "semicolon": "never",
+    "semicolon": [true, "never"],
     "trailing-comma": false,
     "object-literal-sort-keys": false,
     "no-var-requires": false,

typings/asar.d.ts

@@ -6,21 +6,18 @@ declare module "asar-electron-builder" {
     size: number
   }
 
-  interface AsarFileMetadata {
-    type: "file" | "directory" | "link"
-    stat?: Stats
-  }
-
   interface AsarOptions {
     unpack?: string
     unpackDir?: string
     dot?: boolean
+
+    ordering?: string | null
   }
 
   export function listPackage(archive: string): Array<string>
 
   // followLinks defaults to true
   export function statFile(archive: string, filename: string, followLinks?: boolean): AsarFileInfo | null
 
-  export function createPackageFromFiles(src: string, dest: string, filenames: Array<string>, metadata: { [key: string]: AsarFileMetadata;}, options: AsarOptions, callback: (error?: Error) => void): void
+  // export function createPackageFromFiles(src: string, dest: string, filenames: Array<string>, metadata: { [key: string]: AsarFileMetadata;}, options: AsarOptions, callback: (error?: Error) => void): void
 }