Skip to content

Commit 9966995

Browse files
nikita-tkachenko-datadognikita-tkachenko-datadog
authored
Add support for skip-EFD tagging (#8487)
1 parent a5047d5 commit 9966995

File tree

82 files changed

+1794
-168
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

82 files changed

+1794
-168
lines changed

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
66
import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
77
import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
8+
import java.util.Collection;
89
import javax.annotation.Nonnull;
910
import javax.annotation.Nullable;
1011

@@ -47,7 +48,8 @@ TestSuiteImpl testSuiteStart(
4748
SkipReason skipReason(TestIdentifier test);
4849

4950
@Nonnull
50-
TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource);
51+
TestExecutionPolicy executionPolicy(
52+
TestIdentifier test, TestSourceData testSource, Collection<String> testTags);
5153

5254
/**
5355
* Returns the priority of the test execution that can be used for ordering tests. The higher the

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,8 +127,9 @@ public SkipReason skipReason(TestIdentifier test) {
127127

128128
@Override
129129
@Nonnull
130-
public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
131-
return executionStrategy.executionPolicy(test, testSource);
130+
public TestExecutionPolicy executionPolicy(
131+
TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
132+
return executionStrategy.executionPolicy(test, testSource, testTags);
132133
}
133134

134135
@Override

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import datadog.trace.civisibility.test.ExecutionResults;
2828
import datadog.trace.civisibility.test.ExecutionStrategy;
2929
import datadog.trace.civisibility.utils.SpanUtils;
30+
import java.util.Collection;
3031
import java.util.function.Consumer;
3132
import javax.annotation.Nonnull;
3233
import javax.annotation.Nullable;
@@ -112,8 +113,9 @@ public SkipReason skipReason(TestIdentifier test) {
112113

113114
@Override
114115
@Nonnull
115-
public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
116-
return executionStrategy.executionPolicy(test, testSource);
116+
public TestExecutionPolicy executionPolicy(
117+
TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
118+
return executionStrategy.executionPolicy(test, testSource, testTags);
117119
}
118120

119121
@Override

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/NoOpTestEventsHandler.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,8 @@ public SkipReason skipReason(TestIdentifier test) {
100100

101101
@NotNull
102102
@Override
103-
public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData source) {
103+
public TestExecutionPolicy executionPolicy(
104+
TestIdentifier test, TestSourceData source, Collection<String> testTags) {
104105
return Regular.INSTANCE;
105106
}
106107

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,9 @@
22

33
import datadog.json.JsonWriter;
44
import datadog.trace.api.DisableTestTrace;
5+
import datadog.trace.api.civisibility.CIConstants;
56
import datadog.trace.api.civisibility.DDTest;
67
import datadog.trace.api.civisibility.DDTestSuite;
7-
import datadog.trace.api.civisibility.InstrumentationBridge;
88
import datadog.trace.api.civisibility.config.TestIdentifier;
99
import datadog.trace.api.civisibility.config.TestSourceData;
1010
import datadog.trace.api.civisibility.events.TestEventsHandler;
@@ -206,7 +206,7 @@ public void onTestStart(
206206
test.setTag(Tags.TEST_TRAITS, getTestTraits(categories));
207207

208208
for (String category : categories) {
209-
if (category.endsWith(InstrumentationBridge.ITR_UNSKIPPABLE_TAG)) {
209+
if (category.endsWith(CIConstants.Tags.ITR_UNSKIPPABLE_TAG)) {
210210
test.setTag(Tags.TEST_ITR_UNSKIPPABLE, true);
211211
metricCollector.add(CiVisibilityCountMetric.ITR_UNSKIPPABLE, 1, EventType.TEST);
212212

@@ -295,8 +295,9 @@ public void onTestIgnore(
295295

296296
@Override
297297
@Nonnull
298-
public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
299-
return testModule.executionPolicy(test, testSource);
298+
public TestExecutionPolicy executionPolicy(
299+
TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
300+
return testModule.executionPolicy(test, testSource, testTags);
300301
}
301302

302303
@Override

‎dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package datadog.trace.civisibility.test;
22

33
import datadog.trace.api.Config;
4+
import datadog.trace.api.civisibility.CIConstants;
45
import datadog.trace.api.civisibility.config.TestIdentifier;
56
import datadog.trace.api.civisibility.config.TestMetadata;
67
import datadog.trace.api.civisibility.config.TestSourceData;
@@ -18,6 +19,7 @@
1819
import datadog.trace.civisibility.source.LinesResolver;
1920
import datadog.trace.civisibility.source.SourcePathResolver;
2021
import java.lang.reflect.Method;
22+
import java.util.Collection;
2123
import java.util.Map;
2224
import java.util.concurrent.atomic.AtomicInteger;
2325
import javax.annotation.Nonnull;
@@ -117,7 +119,8 @@ public SkipReason skipReason(TestIdentifier test) {
117119
}
118120

119121
@Nonnull
120-
public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
122+
public TestExecutionPolicy executionPolicy(
123+
TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
121124
if (test == null) {
122125
return Regular.INSTANCE;
123126
}
@@ -129,7 +132,7 @@ public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData t
129132
RetryReason.attemptToFix);
130133
}
131134

132-
if (isEFDApplicable(test, testSource)) {
135+
if (isEFDApplicable(test, testSource, testTags)) {
133136
// check-then-act with "earlyFlakeDetectionsUsed" is not atomic here,
134137
// but we don't care if we go "a bit" over the limit, it does not have to be precise
135138
earlyFlakeDetectionsUsed.incrementAndGet();
@@ -163,11 +166,14 @@ private boolean isAutoRetryApplicable(TestIdentifier test) {
163166
&& autoRetriesUsed.get() < config.getCiVisibilityTotalFlakyRetryCount();
164167
}
165168

166-
private boolean isEFDApplicable(@Nonnull TestIdentifier test, TestSourceData testSource) {
169+
private boolean isEFDApplicable(
170+
@Nonnull TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
167171
EarlyFlakeDetectionSettings efdSettings = executionSettings.getEarlyFlakeDetectionSettings();
168172
return efdSettings.isEnabled()
169173
&& !isEFDLimitReached()
170-
&& (isNew(test) || isModified(testSource));
174+
&& (isNew(test) || isModified(testSource))
175+
// endsWith matching is needed for JUnit4-based frameworks, where tags are classes
176+
&& testTags.stream().noneMatch(t -> t.endsWith(CIConstants.Tags.EFD_DISABLE_TAG));
171177
}
172178

173179
public boolean isEFDLimitReached() {

‎dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/buildsystem/ProxyTestModuleTest.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,21 +55,21 @@ class ProxyTestModuleTest extends DDSpecification {
5555
)
5656

5757
when:
58-
def retryPolicy1 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN)
58+
def retryPolicy1 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN, [])
5959

6060
then:
6161
retryPolicy1.retry(false, 1L) // 2nd test execution, 1st retry globally
6262
!retryPolicy1.retry(false, 1L) // asking for 3rd test execution - local limit reached
6363

6464
when:
65-
def retryPolicy2 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN)
65+
def retryPolicy2 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN, [])
6666

6767
then:
6868
retryPolicy2.retry(false, 1L) // 2nd test execution, 2nd retry globally (since previous test was retried too)
6969
!retryPolicy2.retry(false, 1L) // asking for 3rd test execution - local limit reached
7070

7171
when:
72-
def retryPolicy3 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN)
72+
def retryPolicy3 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN, [])
7373

7474
then:
7575
!retryPolicy3.retry(false, 1L) // asking for 3rd retry globally - global limit reached

‎dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestModuleTest.groovy‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,21 +21,21 @@ class HeadlessTestModuleTest extends SpanWriterTest {
2121
def headlessTestModule = givenAHeadlessTestModule()
2222

2323
when:
24-
def retryPolicy1 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN)
24+
def retryPolicy1 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN, [])
2525

2626
then:
2727
retryPolicy1.retry(false, 1L) // 2nd test execution, 1st retry globally
2828
!retryPolicy1.retry(false, 1L) // asking for 3rd test execution - local limit reached
2929

3030
when:
31-
def retryPolicy2 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN)
31+
def retryPolicy2 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN, [])
3232

3333
then:
3434
retryPolicy2.retry(false, 1L) // 2nd test execution, 2nd retry globally (since previous test was retried too)
3535
!retryPolicy2.retry(false, 1L) // asking for 3rd test execution - local limit reached
3636

3737
when:
38-
def retryPolicy3 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN)
38+
def retryPolicy3 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN, [])
3939

4040
then:
4141
!retryPolicy3.retry(false, 1L) // asking for 3rd retry globally - global limit reached

‎dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/test/ExecutionStrategyTest.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ class ExecutionStrategyTest extends Specification {
7575
def strategy = givenAnExecutionStrategy(executionSettings)
7676

7777
expect:
78-
strategy.executionPolicy(testID, TestSourceData.UNKNOWN).class == RunNTimes
78+
strategy.executionPolicy(testID, TestSourceData.UNKNOWN, []).class == RunNTimes
7979
}
8080

8181
def "test attempt to fix + efd"() {
@@ -98,7 +98,7 @@ class ExecutionStrategyTest extends Specification {
9898
executionSettings.isKnown(testFQN) >> false
9999

100100
def strategy = givenAnExecutionStrategy(executionSettings)
101-
def policy = strategy.executionPolicy(testID, TestSourceData.UNKNOWN)
101+
def policy = strategy.executionPolicy(testID, TestSourceData.UNKNOWN, [])
102102

103103
// retry once to get the retry reason
104104
policy.retry(true, 0)

‎dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy‎

Lines changed: 21 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -354,18 +354,27 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
354354
"content.meta.['test.toolchain']" : "${instrumentedLibraryName()}:${instrumentedLibraryVersion()}"
355355
] + replacements
356356

357-
// uncomment to generate expected data templates
358-
// def clazz = this.getClass()
359-
// def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
360-
// def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
361-
// def searchIndex = classfilePath.indexOf("/build/classes/groovy")
362-
// def modulePath = classfilePath.substring(0, searchIndex)
363-
// def submoduleName = classfilePath.substring(searchIndex + "/build/classes/groovy".length()).split("/")[1]
364-
// def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName
365-
// CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
366-
// return [:]
367-
368-
return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
357+
if (System.getenv().get("GENERATE_TEST_FIXTURES") != null) {
358+
return generateTestFixtures(testcaseName, events, coverages, additionalReplacements)
359+
} else {
360+
return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
361+
}
362+
}
363+
364+
def generateTestFixtures(testcaseName, events, coverages, additionalReplacements) {
365+
def clazz = this.getClass()
366+
def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
367+
def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
368+
def searchIndex = classfilePath.indexOf("/build/classes/groovy")
369+
def modulePath = classfilePath.substring(0, searchIndex)
370+
def submoduleName = classfilePath.substring(searchIndex + "/build/classes/groovy".length()).split("/")[1]
371+
if (!Files.exists(Paths.get(modulePath + "/src/" + submoduleName + "/resources/"))) {
372+
// probably running a "latestDepTest" that uses fixtures from "test"
373+
submoduleName = "test"
374+
}
375+
def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName
376+
CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
377+
return [:]
369378
}
370379

371380
def assertTestsOrder(List<TestFQN> expectedOrder) {

0 commit comments

Comments
 (0)