Merge master into stable for release 2.8.0

.github/CODEOWNERS

@@ -0,0 +1,9 @@
+# These owners will be the default owners for everything in
+# the repo. Unless a later match takes precedence,
+# they will be requested for review when someone opens a 
+# pull request.
+
+*       @bgklein  @cRui861  @dpwatrous  @paterasMSFT  @gingi
+
+# Builds
+/.vsts/ @bgklein  @cRui861  @dpwatrous

.gitignore

@@ -12,6 +12,7 @@ coverage/
 .nyc_output/
 coverage.lcov
 documentation/
+.python-version
 
 # Ignore the english compiled file
 i18n/resources.en.json

.vscode/launch.json

@@ -2,23 +2,22 @@
   "version": "0.2.0",
   "configurations": [
     {
-      "name": "Launch",
+      "name": "Debug: Main",
       "type": "node",
       "request": "launch",
       "protocol": "inspector",
       "program": "${workspaceRoot}/src/client/main.ts",
       "stopOnEntry": false,
-      "args": [],
+      "args": [
+        "--remote-debugging-port=9222",
+        "--nolazy"
+      ],
       "cwd": "${workspaceRoot}",
       "preLaunchTask": "build-client",
       "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/electron",
       "windows": {
         "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/electron.cmd"
       },
-      "runtimeArgs": [
-        "--remote-debugging-port=9222",
-        "--nolazy"
-      ],
       "env": {
         "NODE_ENV": "development",
         "HOT": "1"
@@ -28,17 +27,62 @@
       "sourceMaps": true,
       "outFiles": [
         "${workspaceRoot}/build/**/*.js"
-      ]
+      ],
+      "presentation": {
+          "hidden": false,
+          "group": "Debug",
+          "order": 20
+      }
     },
     {
-      "name": "Attach to Process",
+      "name": "Attach: Renderer",
       "type": "chrome",
       "request": "attach",
       "port": 9222,
       "sourceMaps": true,
       "trace": "verbose",
       "url": "file://*",
-      "webRoot": "${workspaceRoot}/build"
+      "webRoot": "${workspaceRoot}/build",
+      "timeout": 30000,
+      "presentation": {
+        "hidden": false,
+        "group": "Debug",
+        "order": 30
+      }
+    },
+    {
+      "name": "Test: Client",
+      "type": "node",
+      "request": "launch",
+      "protocol": "inspector",
+      "program": "${workspaceRoot}/test/client/run-jasmine.js",
+      "cwd": "${workspaceRoot}",
+      "runtimeExecutable": "node",
+      "runtimeArgs": [
+        "--nolazy"
+      ],
+      "env": {
+        "NODE_ENV": "test"
+      },
+      "presentation": {
+        "hidden": false,
+        "group": "Test",
+        "order": 10
+      }
+    }
+  ],
+  "compounds": [
+    {
+        "name": "Debug: All",
+        "configurations": [
+            "Debug: Main",
+            "Attach: Renderer"
+        ],
+        "presentation": {
+            "hidden": false,
+            "group": "Debug",
+            "order": 10
+        }
     }
   ]
 }

.vsts/darwin/ci.yml

@@ -1,6 +1,9 @@
 steps:
   - template: ./darwin-dependencies.yml
-  - script: npm run build-and-pack
+  - script: |
+      set -e
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      npm run build-and-pack
     displayName: Build and pack
   - template: ../common/publish-artifacts.yml
     parameters:

.vsts/darwin/darwin-dependencies.yml

@@ -1,17 +1,12 @@
 steps:
   - template: ../dependencies.yml
-  - script: npm config set python $(which python)
-    displayName: Set npm python2 path
   - template: ../pyenv.yml
   - script: |
       set -e
-      node --version
-      npm config list
-      npm install -g npm@latest
-      npm --version
-      npm install -g node-gyp node-pre-gyp
-      npm i --save-dev node@8.15.0
-      npm install
-      pip install --upgrade pip
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      python --version
       pip install -r python/requirements.txt
-    displayName: Install dependencies
+      echo "Node.js version $(node --version)"
+      echo "NPM version $(npm --version)"
+      npm ci
+    displayName: Install MacOS dependencies

.vsts/darwin/distribution.yml

@@ -1,6 +1,8 @@
 steps:
   - template: ./darwin-dependencies.yml
   - script: |
+      set -e
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
       npm run -s build:prod
       npm run -s build-python
       npm run -s package darwin-app
@@ -14,6 +16,8 @@ steps:
 
   - template: ./sign.yml
   - script: |
+      set -e
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
       unzip ./release/BatchExplorer*.zip -d ./release/mac
       ls ./release/mac
       rm -f ./release/mac/*.pkg

.vsts/dependencies.yml

@@ -1,7 +1,7 @@
 steps:
-  - powershell: ./scripts/azpipelines/build-type.ps1 "$(Build.SourceBranch)"
+  - powershell: ./scripts/azpipelines/build-type.ps1 "$(Build.SourceBranch)" "$(Build.BuildNumber)"
     displayName: Resolve build type
 
   - task: NodeTool@0
     inputs:
-      versionSpec: '10.11.x'
+      versionSpec: '12.18.x'

.vsts/distribution.yml

@@ -2,13 +2,13 @@
 jobs:
   - job: Linux
     pool:
-      vmImage: ubuntu-16.04
+      vmImage: ubuntu-18.04
     steps:
        - template: ./linux/distribution.yml
 
   - job: MacOS
     pool:
-      vmImage: macOS-10.14
+      vmImage: macOS-10.15
       demands: xcode
     steps:
        - template: ./darwin/distribution.yml

.vsts/linux/ci.yml

@@ -6,7 +6,7 @@ steps:
 
   - script: |
       export DISPLAY=:99
-      xvfb-run npm run test
+      xvfb-run --server-args '-screen 0 1024x768x24' npm run test
     displayName: Run tests
 
   - task: PublishTestResults@2
@@ -22,7 +22,10 @@ steps:
   - script: codecov -t $(codecov.uploadToken)
     displayName: Upload code coverage to codecov
 
-  - script: npm run build-and-pack
+  - script: |
+      set -e
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      npm run build-and-pack
     displayName: Build and pack
 
   - script: npm run lint

.vsts/linux/distribution.yml

@@ -1,8 +1,14 @@
 steps:
   - template: ./linux-dependencies.yml
-  - script: npm run build-and-pack
+  - script: |
+      set -e
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      npm run build-and-pack
     displayName: Build and pack
-  - script: npm run package linux-manifest
+  - script: |
+      set -e
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      npm run package linux-manifest
     displayName: Create manifest
   - template: ../common/publish-artifacts.yml
     parameters:

.vsts/linux/linux-dependencies.yml

@@ -2,15 +2,20 @@ steps:
   - script: |
       set -e
       sudo apt-get update
+      echo "Installing pyenv dependencies (see https://github.com/pyenv/pyenv/wiki/Common-build-problems#prerequisites)"
+      sudo apt-get install -y build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libffi-dev liblzma-dev python-openssl git
+      echo "Installing additional dependencies"
       sudo apt-get install -y libxkbfile-dev pkg-config libsecret-1-dev libxss1 libgconf-2-4 dbus xvfb libgtk-3-0 rpm
     displayName: Install Linux packages
   - template: ../dependencies.yml
   - template: ../pyenv.yml
   - script: |
       set -e
-      python --version
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      echo "Python version $(python --version)"
+      pip install -r python/requirements.txt
+      echo "Node.js version $(node --version)"
+      echo "NPM version $(npm --version)"
       npm install -g codecov
-      npm install -g npm
-      npm ci
-      pip3 install -r python/requirements.txt
-    displayName: Install dependencies
+      npm install
+    displayName: Install Linux dependencies

.vsts/pipelines.yml

@@ -2,20 +2,20 @@ name: $(SourceBranch)$(Rev:.r)
 jobs:
   - job: Linux
     pool:
-      vmImage: ubuntu-16.04
+      vmImage: ubuntu-18.04
     steps:
        - template: ./linux/ci.yml
 
   - job: MacOS
     pool:
-      vmImage: macOS-10.14
+      vmImage: macOS-10.15
       demands: xcode
     steps:
        - template: ./darwin/ci.yml
 
   - job: Windows
     pool:
-      vmImage: vs2017-win2016
+      vmImage: windows-2019
     steps:
       - template: ./win/ci.yml
 

.vsts/pyenv.yml

@@ -2,13 +2,21 @@ steps:
   - script: |
       set -e
       export PYENV_ROOT="$(Agent.WorkFolder)/.pyenv"
-      git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
+      mkdir -p "$PYENV_ROOT"
+      curl -L "https://github.com/pyenv/pyenv/archive/v1.2.20.tar.gz" > "$PYENV_ROOT/pyenv.tar.gz"
+      echo "5ecd10d3ec502ce9b7d8109fbe8cb2e4a8af8b73ce5c216b8e268452724a65f3  $PYENV_ROOT/pyenv.tar.gz" | shasum -c
+      tar xzf "$PYENV_ROOT/pyenv.tar.gz" -C "$PYENV_ROOT" --strip-components=1
       export PATH="$PYENV_ROOT/bin:$PATH"
       pyenv --version
-      PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.6.6
-      pyenv global 3.6.6
-      python --version
+      env PYTHON_CONFIGURE_OPTS="--enable-shared" pyenv install 3.8.5
+      pyenv global 3.8.5
+      eval "$(pyenv init -)"
+      echo "Setting up Python virtual environment..."
+      python -m venv "$(Agent.WorkFolder)/.venv/batchexplorer"
+      . "$(Agent.WorkFolder)/.venv/batchexplorer/bin/activate"
+      echo "Updating pip..."
+      pip install --upgrade pip
       echo "Path is $PATH"
       echo "##vso[task.prependpath]$PYENV_ROOT/bin"
       echo "##vso[task.prependpath]$PYENV_ROOT/shims"
-    displayName: Install python
+    displayName: Install Python and set up venv

.vsts/update-en-loc.yml

@@ -6,7 +6,7 @@ pool:
 steps:
   - task: NodeTool@0
     inputs:
-      versionSpec: '10.8.x'
+      versionSpec: '12.18.x'
 
   - script: npm ci
     displayName: Install dependencies
@@ -55,6 +55,6 @@ steps:
       today=`date +%Y-%m-%d`
       git -c user.email=batch-ex@microsoft.com -c user.name='Batch Explorer Automated' commit -am "Updating english translations $today"
       git status -s
-      git push https://timotheeguerin:$(GH_TOKEN)@github.com/Azure/BatchExplorer loc
+      git push https://dpwatrous:$(GH_TOKEN)@github.com/Azure/BatchExplorer loc
     workingDirectory: $(locdir)
     displayName: Commit and push changes if any

.vsts/win/distribution.yml

@@ -3,9 +3,11 @@ steps:
   - powershell: ./scripts/azpipelines/update-build-name.ps1
     displayName: Resolve version
   - powershell: |
-      npm run build:prod
-      npm run build-python
-      npm run package win-exe
+      . .vsts/win/exec.ps1
+      $ErrorActionPreference = "Stop"
+      exec { npm run build:prod }
+      exec { npm run build-python }
+      exec { npm run package win-exe }
     displayName: Build executable
 
   - template: ./sign.yml
@@ -23,7 +25,7 @@ steps:
       name: "Sign installer"
       pattern: |
         **/BatchExplorer*Setup*.exe
-  - script: npm run package win-manifest
+  - powershell: npm run package win-manifest
     displayName: Create manifest
   - template: ../common/publish-artifacts.yml
     parameters:

.vsts/win/win-dependencies.yml

@@ -1,27 +1,20 @@
 steps:
   - template: ../dependencies.yml
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '2.7.x'
-    displayName: Install python 2.7 for node-gyp
-
-  - powershell: |
-      $pythonPath = Get-Command python | Select-Object -ExpandProperty Definition
-      echo "Python2 $pythonPath"
-      npm config set python $pythonPath
-    displayName: Configure python2 for node-gyp
 
   - task: UsePythonVersion@0
     inputs:
-      versionSpec: '3.6.x'
-    displayName: Install python 3.6 for BatchExplorer
+      versionSpec: '3.8.x'
+    displayName: Install python 3.8 for BatchExplorer
 
+  # KLUDGE: Use `npm i` instead of `npm ci` due to this issue:
+  #         https://github.com/npm/cli/issues/558#issuecomment-580392554
   - powershell: |
       . .vsts/win/exec.ps1
       $ErrorActionPreference = "Stop"
-      node --version
-      exec { npm i -g npm }
-      exec { npm install }
       exec { python -m pip install --upgrade pip }
+      exec { python --version }
       exec { pip install -r python/requirements.txt }
-    displayName: Install dependencies
+      exec { Write-Host "Node.js version" $(node --version) }
+      exec { Write-Host "NPM version" $(npm --version) }
+      exec { npm install --loglevel=error }
+    displayName: Install Windows dependencies