From 871685732704d1547cb365885f88b8b3a1bc01ff Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Fri, 15 Jun 2018 09:40:48 -0400 Subject: [PATCH 1/6] jenkins: linter refactor & fix for 6.x * break down directive into single commands * don't fail if `lint-md-build` doesn't exist * replace `sed` with `awk` --- jenkins/pipelines/node-linter.jenkinsfile | 67 +++++++++++------------ 1 file changed, 33 insertions(+), 34 deletions(-) diff --git a/jenkins/pipelines/node-linter.jenkinsfile b/jenkins/pipelines/node-linter.jenkinsfile index 69de9f336..120a0d91b 100644 --- a/jenkins/pipelines/node-linter.jenkinsfile +++ b/jenkins/pipelines/node-linter.jenkinsfile @@ -1,5 +1,3 @@ -#!/usr/bin/env groovy - pipeline { agent { label 'linter' } parameters{ @@ -8,11 +6,9 @@ pipeline { string(name: 'GIT_REMOTE_REF', defaultValue: 'refs/heads/master', description: 'The remote portion of the Git refspec to fetch and test') string(name: 'REBASE_ONTO', defaultValue: '', description: 'Optionally, rebase onto the given ref before testing. Leave blank to skip rebasing.') string(name: 'POST_REBASE_SHA1_CHECK', defaultValue: '', description: 'After rebasing, check that the resulting commit sha1 matches the given one. If left blank, no check is performed.') - choice(name: 'GIT_ORIGIN_SCHEME', choices: "https://github.com/\ngit@github.com:", description: '') - string(name: 'POST_STATUS_TO_PR', defaultValue: '', description: 'Posts build status updates to a nodejs/node PR.') string(name: 'CONFIG_FLAGS', defaultValue: '', description: 'Add arguments to ./configure.') } - + stages { stage("Setup repository") { steps { @@ -22,49 +18,60 @@ pipeline { name: 'refs/heads/_jenkins_local_branch' ]], userRemoteConfigs: [[ - credentialsId: "96d5f81c-e9ad-45f7-ba5d-bc8107c0ae2c", - url: "${params.GIT_ORIGIN_SCHEME}${params.GITHUB_ORG}/${params.REPO_NAME}", - refspec: "+refs/heads/*:refs/remotes/origin/* +${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" + url: "https://github.com/${params.GITHUB_ORG}/${params.REPO_NAME}", + refspec: "+${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" ]] ]) } } - - stage('Preflight') { + + stage('Pre-flight') { steps { - sh "curl https://raw.githubusercontent.com/nodejs/build/master/jenkins/scripts/node-test-commit-pre.sh -s | bash -xe" + sh "curl -L -s https://raw.githubusercontent.com/nodejs/build/master/jenkins/scripts/node-test-commit-pre.sh -s | bash -xe" sendBuildStatus("pending", env) + checkMake() + sh 'node --version' } } - + + stage('Build linting tools') { + steps { + sh(returnStatus: true, script: "${env.MAKE} lint-md-build") + } + } + stage('Run tests') { steps { - checkMake() - checkSed() - sh """ - # this job does not build node, so we symlink the system's node - which node #&& ln -s ${sh(script: "which node", returnStdout: true).trim()} - node --version - - ${env.MAKE} lint-md-build || true - # If lint-ci fails, print all the interesting lines to the console. - ${env.MAKE} lint-ci || { cat test-eslint.tap | grep -v '^ok\\|^TAP version 13\\|^1\\.\\.' | ${env.SED} '/^/\\s*\$/d' && exit 1; } - """ + // If lint-ci fails, print all the interesting lines to the console. + sh "NODE=node ${env.MAKE} lint-ci" } } } - + post { + always { + tap2JUnit() + } + success { sendBuildStatus("success", env) } failure { + sh("cat test-eslint.tap | grep -v '^ok\\|^TAP version 13\\|^1\\.\\.|^' | awk 'NF'") sendBuildStatus("failure", env) } } } +def tap2JUnit() { + fileOperations([folderCreateOperation('out/junit')]) + def status = sh(returnStatus: true, script: 'tap2junit -i test-eslint.tap -o out/junit/test-eslint.xml') + if (status == 0) { + junit(allowEmptyResults: true, testResults: 'out/junit/*.xml') + } +} + def checkMake() { def status = sh(returnStatus: true, script: "which gmake") if (status != 0) { @@ -74,19 +81,11 @@ def checkMake() { } } -def checkSed() { - def status = sh(returnStatus: true, script: "which gsed") - if (status != 0) { - env.SED = 'sed' - } else { - env.SED = 'gsed' - } -} - def sendBuildStatus(status, env) { + return build job: 'post-build-status-update', parameters: [ string(name: 'REPO', value: 'node'), - string(name: 'IDENTIFIER', value: 'linter'), + string(name: 'IDENTIFIER', value: 'node-test-linter'), string(name: 'URL', value: env.BUILD_URL), string(name: 'COMMIT', value: sh(script: 'git rev-parse HEAD', returnStdout: true).trim()), string(name: 'REF', value: env.GIT_REMOTE_REF), From b362b92312a95b03a61693bd8cd2105503f7e072 Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Fri, 15 Jun 2018 09:44:33 -0400 Subject: [PATCH 2/6] fixup! --- jenkins/pipelines/node-linter.jenkinsfile | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/jenkins/pipelines/node-linter.jenkinsfile b/jenkins/pipelines/node-linter.jenkinsfile index 120a0d91b..2fff6a829 100644 --- a/jenkins/pipelines/node-linter.jenkinsfile +++ b/jenkins/pipelines/node-linter.jenkinsfile @@ -18,7 +18,8 @@ pipeline { name: 'refs/heads/_jenkins_local_branch' ]], userRemoteConfigs: [[ - url: "https://github.com/${params.GITHUB_ORG}/${params.REPO_NAME}", + credentialsId: "96d5f81c-e9ad-45f7-ba5d-bc8107c0ae2c", + url: "git@github.com:${params.GITHUB_ORG}/${params.REPO_NAME}", refspec: "+${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" ]] ]) @@ -36,13 +37,13 @@ pipeline { stage('Build linting tools') { steps { + // this target might not exist sh(returnStatus: true, script: "${env.MAKE} lint-md-build") } } stage('Run tests') { steps { - // If lint-ci fails, print all the interesting lines to the console. sh "NODE=node ${env.MAKE} lint-ci" } } @@ -50,7 +51,8 @@ pipeline { post { always { - tap2JUnit() + // Until we fix the TAP generator + // tap2JUnit() } success { @@ -58,6 +60,7 @@ pipeline { } failure { + // If lint-ci fails, print all the interesting lines to the console. sh("cat test-eslint.tap | grep -v '^ok\\|^TAP version 13\\|^1\\.\\.|^' | awk 'NF'") sendBuildStatus("failure", env) } From d139b9146c7836fe8ed9f40e1f076fef0f66d58d Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Fri, 15 Jun 2018 17:50:30 -0400 Subject: [PATCH 3/6] fixup! comments & bugs --- jenkins/pipelines/node-linter.jenkinsfile | 32 +++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/jenkins/pipelines/node-linter.jenkinsfile b/jenkins/pipelines/node-linter.jenkinsfile index 2fff6a829..614bf6776 100644 --- a/jenkins/pipelines/node-linter.jenkinsfile +++ b/jenkins/pipelines/node-linter.jenkinsfile @@ -1,3 +1,5 @@ +#!/usr/bin/env groovy + pipeline { agent { label 'linter' } parameters{ @@ -8,7 +10,7 @@ pipeline { string(name: 'POST_REBASE_SHA1_CHECK', defaultValue: '', description: 'After rebasing, check that the resulting commit sha1 matches the given one. If left blank, no check is performed.') string(name: 'CONFIG_FLAGS', defaultValue: '', description: 'Add arguments to ./configure.') } - + stages { stage("Setup repository") { steps { @@ -19,49 +21,47 @@ pipeline { ]], userRemoteConfigs: [[ credentialsId: "96d5f81c-e9ad-45f7-ba5d-bc8107c0ae2c", - url: "git@github.com:${params.GITHUB_ORG}/${params.REPO_NAME}", + url: "git@github.com:{params.GITHUB_ORG}/${params.REPO_NAME}", refspec: "+${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" ]] ]) } } - + stage('Pre-flight') { steps { sh "curl -L -s https://raw.githubusercontent.com/nodejs/build/master/jenkins/scripts/node-test-commit-pre.sh -s | bash -xe" sendBuildStatus("pending", env) checkMake() + checkSed() + // Make sure we have a node binary in the path sh 'node --version' } } - + stage('Build linting tools') { steps { - // this target might not exist - sh(returnStatus: true, script: "${env.MAKE} lint-md-build") + // Calling with `returnStatus` suppresses automatic failures + sh(script: "${env.MAKE} lint-md-build", returnStatus: true) } } stage('Run tests') { steps { + // this job does not build node, so we use the system's node sh "NODE=node ${env.MAKE} lint-ci" } } } - - post { - always { - // Until we fix the TAP generator - // tap2JUnit() - } + post { success { sendBuildStatus("success", env) } failure { - // If lint-ci fails, print all the interesting lines to the console. - sh("cat test-eslint.tap | grep -v '^ok\\|^TAP version 13\\|^1\\.\\.|^' | awk 'NF'") + // If lint-ci fails, print just the __interesting__ lines to the console. + sh("cat test-eslint.tap | grep -v '^ok\\|^TAP version 13\\|^1\\.\\. | awk 'NF'") sendBuildStatus("failure", env) } } @@ -84,11 +84,11 @@ def checkMake() { } } + def sendBuildStatus(status, env) { - return build job: 'post-build-status-update', parameters: [ string(name: 'REPO', value: 'node'), - string(name: 'IDENTIFIER', value: 'node-test-linter'), + string(name: 'IDENTIFIER', value: 'linter'), string(name: 'URL', value: env.BUILD_URL), string(name: 'COMMIT', value: sh(script: 'git rev-parse HEAD', returnStdout: true).trim()), string(name: 'REF', value: env.GIT_REMOTE_REF), From 8563e6c3005af6e18cd44e53c9f349a86621997e Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Fri, 15 Jun 2018 17:52:00 -0400 Subject: [PATCH 4/6] fixup! one more --- jenkins/pipelines/node-linter.jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/pipelines/node-linter.jenkinsfile b/jenkins/pipelines/node-linter.jenkinsfile index 614bf6776..5247f4bc8 100644 --- a/jenkins/pipelines/node-linter.jenkinsfile +++ b/jenkins/pipelines/node-linter.jenkinsfile @@ -22,7 +22,7 @@ pipeline { userRemoteConfigs: [[ credentialsId: "96d5f81c-e9ad-45f7-ba5d-bc8107c0ae2c", url: "git@github.com:{params.GITHUB_ORG}/${params.REPO_NAME}", - refspec: "+${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" + refspec: "+refs/heads/*:refs/remotes/origin/* +${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" ]] ]) } From f30d71b269a87876556f047b23e11f5d59d75a09 Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Fri, 15 Jun 2018 18:09:05 -0400 Subject: [PATCH 5/6] fixup! typo --- jenkins/pipelines/node-linter.jenkinsfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/jenkins/pipelines/node-linter.jenkinsfile b/jenkins/pipelines/node-linter.jenkinsfile index 5247f4bc8..2486744cb 100644 --- a/jenkins/pipelines/node-linter.jenkinsfile +++ b/jenkins/pipelines/node-linter.jenkinsfile @@ -21,7 +21,7 @@ pipeline { ]], userRemoteConfigs: [[ credentialsId: "96d5f81c-e9ad-45f7-ba5d-bc8107c0ae2c", - url: "git@github.com:{params.GITHUB_ORG}/${params.REPO_NAME}", + url: "git@github.com:${params.GITHUB_ORG}/${params.REPO_NAME}", refspec: "+refs/heads/*:refs/remotes/origin/* +${params.GIT_REMOTE_REF}:refs/remotes/origin/_jenkins_local_branch" ]] ]) @@ -33,7 +33,6 @@ pipeline { sh "curl -L -s https://raw.githubusercontent.com/nodejs/build/master/jenkins/scripts/node-test-commit-pre.sh -s | bash -xe" sendBuildStatus("pending", env) checkMake() - checkSed() // Make sure we have a node binary in the path sh 'node --version' } From 08dd226c81af43f94f4a92236416c6870c2f136d Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Fri, 15 Jun 2018 21:34:18 -0400 Subject: [PATCH 6/6] better way to show errors --- jenkins/pipelines/node-linter.jenkinsfile | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/jenkins/pipelines/node-linter.jenkinsfile b/jenkins/pipelines/node-linter.jenkinsfile index 2486744cb..fdd9fdd6f 100644 --- a/jenkins/pipelines/node-linter.jenkinsfile +++ b/jenkins/pipelines/node-linter.jenkinsfile @@ -47,8 +47,14 @@ pipeline { stage('Run tests') { steps { - // this job does not build node, so we use the system's node - sh "NODE=node ${env.MAKE} lint-ci" + script { + // this job does not build node, so we use the system's node + def ret = sh(script: "NODE=node ${env.MAKE} lint-ci", returnStatus: true) + if (ret != 0) { + echo(extractErrors()) + error('lint failed - open above section for details') + } + } } } } @@ -59,13 +65,20 @@ pipeline { } failure { - // If lint-ci fails, print just the __interesting__ lines to the console. - sh("cat test-eslint.tap | grep -v '^ok\\|^TAP version 13\\|^1\\.\\. | awk 'NF'") sendBuildStatus("failure", env) } } } +def extractErrors() { + def tap = readFile('test-eslint.tap') + tap = tap.replaceAll('(?m)^ok.*', '') + tap = tap.replaceAll('(?m)^TAP version 13.*', '') + tap = tap.replaceAll('(?m)^1\\.\\..*', '') + tap = tap.replaceAll('(?m)^\\s+$', '') + return tap +} + def tap2JUnit() { fileOperations([folderCreateOperation('out/junit')]) def status = sh(returnStatus: true, script: 'tap2junit -i test-eslint.tap -o out/junit/test-eslint.xml')