Skip to content

Commit f2ce44e

Browse files
pshevchepshevche
committed
Fix tests for older Gradle versions
Signed-off-by: Pavlo Shevchenko <[email protected]>
1 parent 4416228 commit f2ce44e

File tree

3 files changed

+117
-73
lines changed

3 files changed

+117
-73
lines changed

‎plugin/src/test/groovy/org/gradle/testretry/testframework/BaseTestNGFuncTest.groovy‎

Lines changed: 46 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,13 @@ import spock.lang.Issue
2121
import javax.annotation.Nullable
2222
import java.util.regex.Pattern
2323

24+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS
25+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD
26+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST
27+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS
28+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD
29+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST
30+
2431
abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
2532
@Override
2633
String getLanguagePlugin() {
@@ -36,13 +43,30 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
3643
"""
3744
}
3845

39-
abstract String reportedLifecycleMethodName(String methodName)
46+
enum TestNGLifecycleType {
47+
BEFORE_SUITE('BeforeSuite'),
48+
BEFORE_TEST('BeforeTest'),
49+
BEFORE_CLASS('BeforeClass'),
50+
BEFORE_METHOD('BeforeMethod'),
51+
AFTER_METHOD('AfterMethod'),
52+
AFTER_CLASS('AfterClass'),
53+
AFTER_TEST('AfterTest'),
54+
AFTER_SUITE('AfterSuite')
55+
56+
final String annotation
57+
58+
TestNGLifecycleType(String annotation) {
59+
this.annotation = annotation
60+
}
61+
}
4062

41-
abstract String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation)
63+
abstract String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName)
4264

43-
abstract boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType)
65+
abstract String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation)
4466

45-
def "handles failure in #lifecycle (gradle version #gradleVersion)"() {
67+
abstract boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType)
68+
69+
def "handles failure in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) {
4670
given:
4771
buildFile << """
4872
test.retry.maxRetries = 1
@@ -52,8 +76,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
5276
package acme;
5377
5478
public class SuccessfulTests {
55-
@org.testng.annotations.${lifecycle}
56-
public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() {
79+
@org.testng.annotations.${lifecycle.annotation}
80+
public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() {
5781
${flakyAssert()}
5882
}
5983
@@ -67,20 +91,20 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
6791

6892
then:
6993
with(result.output) {
70-
it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 1
71-
it.count("${reportedLifecycleMethodName('lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0)
94+
it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 1
95+
it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} PASSED") == (reportsSuccessfulLifecycleExecutions(lifecycle) ? 1 : 0)
7296
!it.contains("The following test methods could not be retried")
7397
}
7498

7599
where:
76100
[gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [
77101
GRADLE_VERSIONS_UNDER_TEST,
78-
['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest']
102+
[BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST]
79103
])
80104
// Note: we don't handle BeforeSuite AfterSuite
81105
}
82106

83-
def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"() {
107+
def "correctly reports exhausted retries on failures in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) {
84108
given:
85109
buildFile << """
86110
test.retry.maxRetries = 1
@@ -90,8 +114,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
90114
package acme;
91115
92116
public class AlwaysFailingLifecycle {
93-
@org.testng.annotations.${lifecycle}
94-
public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() {
117+
@org.testng.annotations.${lifecycle.annotation}
118+
public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() {
95119
throw new RuntimeException("Lifecycle goes boom!");
96120
}
97121
@@ -106,52 +130,19 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
106130
then:
107131
with(result.output) {
108132
// if BeforeTest fails, then methods won't be executed
109-
it.count('successTest SKIPPED') == (lifecycle.contains('Before') ? 2 : 0)
110-
it.count('successTest PASSED') == (lifecycle.contains('Before') ? 0 : 2)
111-
it.count("${reportedLifecycleMethodName('lifecycle')} FAILED") == 2
133+
it.count('successTest SKIPPED') == (lifecycle.annotation.contains('Before') ? 2 : 0)
134+
it.count('successTest PASSED') == (lifecycle.annotation.contains('Before') ? 0 : 2)
135+
it.count("${reportedLifecycleMethodName(gradleVersion, lifecycle, 'lifecycle')} FAILED") == 2
112136
!it.contains("The following test methods could not be retried")
113137
}
114138

115139
where:
116140
[gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [
117141
GRADLE_VERSIONS_UNDER_TEST,
118-
['BeforeTest', 'BeforeClass', 'BeforeMethod', 'AfterMethod', 'AfterClass', 'AfterTest']
142+
[BEFORE_TEST, BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD, AFTER_CLASS, AFTER_TEST]
119143
])
120144
}
121145

122-
def "does not handle flaky static initializers (gradle version #gradleVersion)"() {
123-
given:
124-
buildFile << """
125-
test.retry.maxRetries = 1
126-
"""
127-
128-
writeJavaTestSource """
129-
package acme;
130-
131-
public class SomeTests {
132-
133-
static {
134-
${flakyAssert()}
135-
}
136-
137-
@org.testng.annotations.Test
138-
public void someTest() {}
139-
}
140-
"""
141-
142-
when:
143-
def result = gradleRunner(gradleVersion as String).buildAndFail()
144-
145-
then:
146-
with(result.output) {
147-
it.contains('There were failing tests. See the report')
148-
!it.contains('The following test methods could not be retried')
149-
}
150-
151-
where:
152-
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
153-
}
154-
155146
def "handles parameterized test in super class (gradle version #gradleVersion)"() {
156147
given:
157148
buildFile << """
@@ -191,8 +182,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
191182
then:
192183
// we can't rerun just the failed parameter
193184
with(result.output) {
194-
it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2
195-
it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2
185+
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2
186+
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2
196187
}
197188
198189
where:
@@ -316,8 +307,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
316307
then:
317308
// we can't rerun just the failed parameter
318309
with(result.output) {
319-
it.count("${reportedParameterizedMethodName('test', 'int', 0, '0')} PASSED") == 2
320-
it.count("${reportedParameterizedMethodName('test', 'int', 1, '1')} FAILED") == 2
310+
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 0, '0')} PASSED") == 2
311+
it.count("${reportedParameterizedMethodName(gradleVersion, 'test', 'int', 1, '1')} FAILED") == 2
321312
}
322313
323314
where:
@@ -369,8 +360,8 @@ abstract class BaseTestNGFuncTest extends AbstractFrameworkFuncTest {
369360
then:
370361
// we can't rerun just the failed parameter
371362
with(result.output.readLines()) {
372-
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2
373-
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName('test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2
363+
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 0, ''))}.* PASSED/) }.size() == 2
364+
it.findAll { line -> line.matches(/.*${Pattern.quote(reportedParameterizedMethodName(gradleVersion, 'test', 'acme.ParameterTest$Foo', 1, ''))}.* FAILED/) }.size() == 2
374365
}
375366
376367
where:

‎plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGPlainFuncTest.groovy‎

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,17 +19,53 @@ import javax.annotation.Nullable
1919

2020
class TestNGPlainFuncTest extends BaseTestNGFuncTest {
2121
@Override
22-
String reportedLifecycleMethodName(String methodName) {
22+
String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) {
2323
methodName
2424
}
2525

2626
@Override
27-
String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
27+
String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
2828
"${methodName}[${invocationNumber}]${paramValueRepresentation ? "(${paramValueRepresentation})" : ""}"
2929
}
3030

3131
@Override
32-
boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) {
32+
boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) {
3333
true
3434
}
35+
36+
/**
37+
* If JUnit's TestNG engine is used, then tests won't even run and the failure is silently swallowed.
38+
*/
39+
def "does not handle flaky static initializers (gradle version #gradleVersion)"() {
40+
given:
41+
buildFile << """
42+
test.retry.maxRetries = 1
43+
"""
44+
45+
writeJavaTestSource """
46+
package acme;
47+
48+
public class SomeTests {
49+
50+
static {
51+
${flakyAssert()}
52+
}
53+
54+
@org.testng.annotations.Test
55+
public void someTest() {}
56+
}
57+
"""
58+
59+
when:
60+
def result = gradleRunner(gradleVersion as String).buildAndFail()
61+
62+
then:
63+
with(result.output) {
64+
it.contains('There were failing tests. See the report')
65+
!it.contains('The following test methods could not be retried')
66+
}
67+
68+
where:
69+
gradleVersion << GRADLE_VERSIONS_UNDER_TEST
70+
}
3571
}

‎plugin/src/test/groovy/org/gradle/testretry/testframework/TestNGViaJUnitEngineFuncTest.groovy‎

Lines changed: 32 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,24 @@
1515
*/
1616
package org.gradle.testretry.testframework
1717

18+
import org.gradle.util.GradleVersion
19+
1820
import javax.annotation.Nullable
1921

22+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_CLASS
23+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_METHOD
24+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.AFTER_TEST
25+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_CLASS
26+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_METHOD
27+
import static org.gradle.testretry.testframework.BaseTestNGFuncTest.TestNGLifecycleType.BEFORE_TEST
28+
2029
class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
2130

22-
private static final Set<String> UNREPORTED_LIFECYCLE_METHODS = ['BeforeTest', 'AfterTest', 'AfterClass']
31+
private static final EnumSet<TestNGLifecycleType> UNREPORTED_LIFECYCLE_METHODS = EnumSet.of(BEFORE_TEST, AFTER_TEST, AFTER_CLASS)
32+
private static final EnumSet<TestNGLifecycleType> CLASS_LIFECYCLE_METHODS = EnumSet.of(BEFORE_CLASS, BEFORE_METHOD, AFTER_METHOD)
33+
34+
private static final GradleVersion GRADLE_5_0 = GradleVersion.version("5.0")
35+
private static final GradleVersion GRADLE_5_4_1 = GradleVersion.version("5.4.1")
2336

2437
def setup() {
2538
buildFile << """
@@ -34,21 +47,25 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
3447
}
3548

3649
@Override
37-
String reportedLifecycleMethodName(String methodName) {
38-
"executionError"
50+
String reportedLifecycleMethodName(String gradleVersion, TestNGLifecycleType lifecycleType, String methodName) {
51+
GradleVersion.version(gradleVersion) > GRADLE_5_0
52+
? "executionError"
53+
: CLASS_LIFECYCLE_METHODS.contains(lifecycleType) ? "classMethod" : "initializationError"
3954
}
4055

4156
@Override
42-
String reportedParameterizedMethodName(String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
43-
"${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}"
57+
String reportedParameterizedMethodName(String gradleVersion, String methodName, String paramType, int invocationNumber, @Nullable String paramValueRepresentation) {
58+
GradleVersion.version(gradleVersion) > GRADLE_5_4_1
59+
? "${methodName}(${paramType}) > [${invocationNumber}] ${paramValueRepresentation ?: ''}"
60+
: "${methodName}(${paramType})[${invocationNumber}]"
4461
}
4562

4663
@Override
47-
boolean reportsSuccessfulLifecycleExecutions(String lifecycleMethodType) {
48-
!UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleMethodType)
64+
boolean reportsSuccessfulLifecycleExecutions(TestNGLifecycleType lifecycleType) {
65+
!UNREPORTED_LIFECYCLE_METHODS.contains(lifecycleType)
4966
}
5067

51-
def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"() {
68+
def "retries all classes if failure occurs in #lifecycle (gradle version #gradleVersion)"(String gradleVersion, TestNGLifecycleType lifecycle) {
5269
given:
5370
buildFile << """
5471
test.retry.maxRetries = 1
@@ -58,8 +75,8 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
5875
package acme;
5976
6077
public class SuccessfulTestsWithFailingLifecycle {
61-
@org.testng.annotations.${lifecycle}
62-
public ${lifecycle.contains('Class') ? 'static ' : ''}void lifecycle() {
78+
@org.testng.annotations.${lifecycle.annotation}
79+
public ${lifecycle.annotation.contains('Class') ? 'static ' : ''}void lifecycle() {
6380
${flakyAssert()}
6481
}
6582
@@ -83,18 +100,18 @@ class TestNGViaJUnitEngineFuncTest extends BaseTestNGFuncTest {
83100
then:
84101
with(result.output) {
85102
// if BeforeTest fails, then methods won't be executed
86-
it.count('successTest SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0)
87-
it.count('successTestWithLifecycle SKIPPED') == (lifecycle == 'BeforeTest' ? 1 : 0)
103+
it.count('successTest SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0)
104+
it.count('successTestWithLifecycle SKIPPED') == (lifecycle == BEFORE_TEST ? 1 : 0)
88105

89-
it.count('successTest PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2)
90-
it.count('successTestWithLifecycle PASSED') == (lifecycle == 'BeforeTest' ? 1 : 2)
106+
it.count('successTest PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2)
107+
it.count('successTestWithLifecycle PASSED') == (lifecycle == BEFORE_TEST ? 1 : 2)
91108
!it.contains("The following test methods could not be retried")
92109
}
93110

94111
where:
95112
[gradleVersion, lifecycle] << GroovyCollections.combinations((Iterable) [
96113
GRADLE_VERSIONS_UNDER_TEST,
97-
['BeforeTest', 'AfterClass', 'AfterTest']
114+
UNREPORTED_LIFECYCLE_METHODS
98115
])
99116
}
100117
}

0 commit comments

Comments
 (0)