Add support for skip-EFD tagging

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestFrameworkModule.java

@@ -5,6 +5,7 @@
 import datadog.trace.api.civisibility.execution.TestExecutionPolicy;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
+import java.util.Collection;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
@@ -47,7 +48,8 @@ TestSuiteImpl testSuiteStart(
   SkipReason skipReason(TestIdentifier test);
 
   @Nonnull
-  TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource);
+  TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags);
 
   /**
    * Returns the priority of the test execution that can be used for ordering tests. The higher the

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/ProxyTestModule.java

@@ -127,8 +127,9 @@ public SkipReason skipReason(TestIdentifier test) {
 
   @Override
   @Nonnull
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
-    return executionStrategy.executionPolicy(test, testSource);
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
+    return executionStrategy.executionPolicy(test, testSource, testTags);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/headless/HeadlessTestModule.java

@@ -27,6 +27,7 @@
 import datadog.trace.civisibility.test.ExecutionResults;
 import datadog.trace.civisibility.test.ExecutionStrategy;
 import datadog.trace.civisibility.utils.SpanUtils;
+import java.util.Collection;
 import java.util.function.Consumer;
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
@@ -112,8 +113,9 @@ public SkipReason skipReason(TestIdentifier test) {
 
   @Override
   @Nonnull
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
-    return executionStrategy.executionPolicy(test, testSource);
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
+    return executionStrategy.executionPolicy(test, testSource, testTags);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/NoOpTestEventsHandler.java

@@ -100,7 +100,8 @@ public SkipReason skipReason(TestIdentifier test) {
 
   @NotNull
   @Override
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData source) {
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData source, Collection<String> testTags) {
     return Regular.INSTANCE;
   }
 

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/events/TestEventsHandlerImpl.java

@@ -2,9 +2,9 @@
 
 import datadog.json.JsonWriter;
 import datadog.trace.api.DisableTestTrace;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.DDTest;
 import datadog.trace.api.civisibility.DDTestSuite;
-import datadog.trace.api.civisibility.InstrumentationBridge;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestSourceData;
 import datadog.trace.api.civisibility.events.TestEventsHandler;
@@ -206,7 +206,7 @@ public void onTestStart(
       test.setTag(Tags.TEST_TRAITS, getTestTraits(categories));
 
       for (String category : categories) {
-        if (category.endsWith(InstrumentationBridge.ITR_UNSKIPPABLE_TAG)) {
+        if (category.endsWith(CIConstants.Tags.ITR_UNSKIPPABLE_TAG)) {
           test.setTag(Tags.TEST_ITR_UNSKIPPABLE, true);
           metricCollector.add(CiVisibilityCountMetric.ITR_UNSKIPPABLE, 1, EventType.TEST);
 
@@ -295,8 +295,9 @@ public void onTestIgnore(
 
   @Override
   @Nonnull
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
-    return testModule.executionPolicy(test, testSource);
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
+    return testModule.executionPolicy(test, testSource, testTags);
   }
 
   @Override

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/test/ExecutionStrategy.java

@@ -1,6 +1,7 @@
 package datadog.trace.civisibility.test;
 
 import datadog.trace.api.Config;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.config.TestMetadata;
 import datadog.trace.api.civisibility.config.TestSourceData;
@@ -18,6 +19,7 @@
 import datadog.trace.civisibility.source.LinesResolver;
 import datadog.trace.civisibility.source.SourcePathResolver;
 import java.lang.reflect.Method;
+import java.util.Collection;
 import java.util.Map;
 import java.util.concurrent.atomic.AtomicInteger;
 import javax.annotation.Nonnull;
@@ -117,7 +119,8 @@ public SkipReason skipReason(TestIdentifier test) {
   }
 
   @Nonnull
-  public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData testSource) {
+  public TestExecutionPolicy executionPolicy(
+      TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
     if (test == null) {
       return Regular.INSTANCE;
     }
@@ -129,7 +132,7 @@ public TestExecutionPolicy executionPolicy(TestIdentifier test, TestSourceData t
           RetryReason.attemptToFix);
     }
 
-    if (isEFDApplicable(test, testSource)) {
+    if (isEFDApplicable(test, testSource, testTags)) {
       // check-then-act with "earlyFlakeDetectionsUsed" is not atomic here,
       // but we don't care if we go "a bit" over the limit, it does not have to be precise
       earlyFlakeDetectionsUsed.incrementAndGet();
@@ -163,11 +166,14 @@ private boolean isAutoRetryApplicable(TestIdentifier test) {
         && autoRetriesUsed.get() < config.getCiVisibilityTotalFlakyRetryCount();
   }
 
-  private boolean isEFDApplicable(@Nonnull TestIdentifier test, TestSourceData testSource) {
+  private boolean isEFDApplicable(
+      @Nonnull TestIdentifier test, TestSourceData testSource, Collection<String> testTags) {
     EarlyFlakeDetectionSettings efdSettings = executionSettings.getEarlyFlakeDetectionSettings();
     return efdSettings.isEnabled()
         && !isEFDLimitReached()
-        && (isNew(test) || isModified(testSource));
+        && (isNew(test) || isModified(testSource))
+        // endsWith matching is needed for JUnit4-based frameworks, where tags are classes
+        && testTags.stream().noneMatch(t -> t.endsWith(CIConstants.Tags.EFD_DISABLE_TAG));
   }
 
   public boolean isEFDLimitReached() {

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/buildsystem/ProxyTestModuleTest.groovy

@@ -55,21 +55,21 @@ class ProxyTestModuleTest extends DDSpecification {
       )
 
     when:
-    def retryPolicy1 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN)
+    def retryPolicy1 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN, [])
 
     then:
     retryPolicy1.retry(false, 1L) // 2nd test execution, 1st retry globally
     !retryPolicy1.retry(false, 1L) // asking for 3rd test execution - local limit reached
 
     when:
-    def retryPolicy2 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN)
+    def retryPolicy2 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN, [])
 
     then:
     retryPolicy2.retry(false, 1L) // 2nd test execution, 2nd retry globally (since previous test was retried too)
     !retryPolicy2.retry(false, 1L) // asking for 3rd test execution - local limit reached
 
     when:
-    def retryPolicy3 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN)
+    def retryPolicy3 = proxyTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN, [])
 
     then:
     !retryPolicy3.retry(false, 1L) // asking for 3rd retry globally - global limit reached

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/domain/headless/HeadlessTestModuleTest.groovy

@@ -21,21 +21,21 @@ class HeadlessTestModuleTest extends SpanWriterTest {
     def headlessTestModule = givenAHeadlessTestModule()
 
     when:
-    def retryPolicy1 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN)
+    def retryPolicy1 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-1", null), TestSourceData.UNKNOWN, [])
 
     then:
     retryPolicy1.retry(false, 1L) // 2nd test execution, 1st retry globally
     !retryPolicy1.retry(false, 1L) // asking for 3rd test execution - local limit reached
 
     when:
-    def retryPolicy2 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN)
+    def retryPolicy2 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-2", null), TestSourceData.UNKNOWN, [])
 
     then:
     retryPolicy2.retry(false, 1L) // 2nd test execution, 2nd retry globally (since previous test was retried too)
     !retryPolicy2.retry(false, 1L) // asking for 3rd test execution - local limit reached
 
     when:
-    def retryPolicy3 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN)
+    def retryPolicy3 = headlessTestModule.executionPolicy(new TestIdentifier("suite", "test-3", null), TestSourceData.UNKNOWN, [])
 
     then:
     !retryPolicy3.retry(false, 1L) // asking for 3rd retry globally - global limit reached

dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/test/ExecutionStrategyTest.groovy

@@ -75,7 +75,7 @@ class ExecutionStrategyTest extends Specification {
     def strategy = givenAnExecutionStrategy(executionSettings)
 
     expect:
-    strategy.executionPolicy(testID, TestSourceData.UNKNOWN).class == RunNTimes
+    strategy.executionPolicy(testID, TestSourceData.UNKNOWN, []).class == RunNTimes
   }
 
   def "test attempt to fix + efd"() {
@@ -98,7 +98,7 @@ class ExecutionStrategyTest extends Specification {
     executionSettings.isKnown(testFQN) >> false
 
     def strategy = givenAnExecutionStrategy(executionSettings)
-    def policy = strategy.executionPolicy(testID, TestSourceData.UNKNOWN)
+    def policy = strategy.executionPolicy(testID, TestSourceData.UNKNOWN, [])
 
     // retry once to get the retry reason
     policy.retry(true, 0)

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

@@ -354,18 +354,27 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
       "content.meta.['test.toolchain']"        : "${instrumentedLibraryName()}:${instrumentedLibraryVersion()}"
     ] + replacements
 
-    // uncomment to generate expected data templates
-    //          def clazz = this.getClass()
-    //          def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
-    //          def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
-    //          def searchIndex = classfilePath.indexOf("/build/classes/groovy")
-    //          def modulePath = classfilePath.substring(0, searchIndex)
-    //          def submoduleName = classfilePath.substring(searchIndex + "/build/classes/groovy".length()).split("/")[1]
-    //          def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName
-    //          CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
-    //          return [:]
-
-    return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
+    if (System.getenv().get("GENERATE_TEST_FIXTURES") != null) {
+      return generateTestFixtures(testcaseName, events, coverages, additionalReplacements)
+    } else {
+      return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
+    }
+  }
+
+  def generateTestFixtures(testcaseName, events, coverages, additionalReplacements) {
+    def clazz = this.getClass()
+    def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
+    def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
+    def searchIndex = classfilePath.indexOf("/build/classes/groovy")
+    def modulePath = classfilePath.substring(0, searchIndex)
+    def submoduleName = classfilePath.substring(searchIndex + "/build/classes/groovy".length()).split("/")[1]
+    if (!Files.exists(Paths.get(modulePath + "/src/" + submoduleName + "/resources/"))) {
+      // probably running a "latestDepTest" that uses fixtures from "test"
+      submoduleName = "test"
+    }
+    def baseTemplatesPath = modulePath + "/src/" + submoduleName + "/resources/" + testcaseName
+    CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
+    return [:]
   }
 
   def assertTestsOrder(List<TestFQN> expectedOrder) {

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy

@@ -8,11 +8,12 @@ abstract class CiVisibilitySmokeTest extends Specification {
   protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, List<Map<String, Object>> events, List<Map<String, Object>> coverages) {
     def additionalReplacements = ["content.meta.['test.toolchain']": "$toolchain:$toolchainVersion"]
 
-    // uncomment to generate expected data templates
-    //    def baseTemplatesPath = CiVisibilitySmokeTest.classLoader.getResource(projectName).toURI().schemeSpecificPart.replace('build/resources/test', 'src/test/resources')
-    //    CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
-
-    CiVisibilityTestUtils.assertData(projectName, events, coverages, additionalReplacements)
+    if (System.getenv().get("GENERATE_TEST_FIXTURES") != null) {
+      def baseTemplatesPath = CiVisibilitySmokeTest.classLoader.getResource(projectName).toURI().schemeSpecificPart.replace('build/resources/test', 'src/test/resources')
+      CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
+    } else {
+      CiVisibilityTestUtils.assertData(projectName, events, coverages, additionalReplacements)
+    }
   }
 
   /**

dd-java-agent/build.gradle

@@ -289,7 +289,7 @@ tasks.register('checkAgentJarSize').configure {
   doLast {
     // Arbitrary limit to prevent unintentional increases to the agent jar size
     // Raise or lower as required
-    assert shadowJar.archiveFile.get().getAsFile().length() <= 30 * 1024 * 1024
+    assert shadowJar.archiveFile.get().getAsFile().length() <= 31 * 1024 * 1024
   }
 
   dependsOn "shadowJar"

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberTracingListener.java

@@ -9,7 +9,6 @@
 import datadog.trace.bootstrap.ContextStore;
 import io.cucumber.core.gherkin.Pickle;
 import java.net.URI;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
@@ -206,11 +205,6 @@ private static boolean isFeature(final Description description) {
 
   private List<String> getCategories(Description description) {
     Pickle pickle = pickleById.get(JUnit4Utils.getUniqueId(description));
-    List<String> pickleTags = pickle.getTags();
-    List<String> categories = new ArrayList<>(pickleTags.size());
-    for (String tag : pickleTags) {
-      categories.add(tag.substring(1)); // remove leading "@"
-    }
-    return categories;
+    return CucumberUtils.getCategories(pickle);
   }
 }

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/CucumberUtils.java

@@ -12,6 +12,8 @@
 import java.io.InputStream;
 import java.lang.invoke.MethodHandle;
 import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -43,6 +45,11 @@ public static String getVersion() {
     return "unknown";
   }
 
+  private static final String NO_STEP_PICKLE_RUNNER_CLASSNAME =
+      "io.cucumber.junit.PickleRunners$NoStepDescriptions";
+  private static final String WITH_STEP_PICKLE_RUNNER_CLASSNAME =
+      "io.cucumber.junit.PickleRunners$WithStepDescriptions";
+
   private static final MethodHandles REFLECTION = new MethodHandles(CUCUMBER_CLASS_LOADER);
   private static final MethodHandle FEATURE_GETTER =
       REFLECTION.privateFieldGetter("io.cucumber.junit.FeatureRunner", "feature");
@@ -54,6 +61,10 @@ public static String getVersion() {
       REFLECTION.privateFieldGetter("io.cucumber.junit.PickleRunners$PickleId", "pickleLine");
   private static final MethodHandle PICKLE_RUNNER_GET_DESCRIPTION =
       REFLECTION.method("io.cucumber.junit.PickleRunners$PickleRunner", "getDescription");
+  private static final MethodHandle PICKLE_RUNNER_NO_STEP_GET_PICKLE =
+      REFLECTION.privateFieldGetter(NO_STEP_PICKLE_RUNNER_CLASSNAME, "pickle");
+  private static final MethodHandle PICKLE_RUNNER_WITH_STEP_GET_PICKLE =
+      REFLECTION.privateFieldGetter(WITH_STEP_PICKLE_RUNNER_CLASSNAME, "pickle");
 
   private CucumberUtils() {}
 
@@ -178,6 +189,32 @@ public static TestSuiteDescriptor toSuiteDescriptor(Description description) {
     return new TestSuiteDescriptor(testSuiteName, null);
   }
 
+  public static Collection<String> getPickleRunnerTags(Object pickleRunner) {
+    Pickle pickle = getPickle(pickleRunner);
+    return getCategories(pickle);
+  }
+
+  private static Pickle getPickle(Object pickleRunner) {
+    try {
+      if (pickleRunner.getClass().getName().equals(NO_STEP_PICKLE_RUNNER_CLASSNAME)) {
+        return REFLECTION.invoke(PICKLE_RUNNER_NO_STEP_GET_PICKLE, pickleRunner);
+      } else {
+        return REFLECTION.invoke(PICKLE_RUNNER_WITH_STEP_GET_PICKLE, pickleRunner);
+      }
+    } catch (Exception e) {
+      return null;
+    }
+  }
+
+  public static List<String> getCategories(Pickle pickle) {
+    List<String> pickleTags = pickle.getTags();
+    List<String> categories = new ArrayList<>(pickleTags.size());
+    for (String tag : pickleTags) {
+      categories.add(tag.substring(1)); // remove leading "@"
+    }
+    return categories;
+  }
+
   public static final class MuzzleHelper {
     public static Reference[] additionalMuzzleReferences() {
       return new Reference[] {

dd-java-agent/instrumentation/junit-4.10/cucumber-junit-4/src/main/java/datadog/trace/instrumentation/junit4/JUnit4CucumberSkipInstrumentation.java

@@ -9,7 +9,7 @@
 import datadog.trace.agent.tooling.InstrumenterModule;
 import datadog.trace.agent.tooling.muzzle.Reference;
 import datadog.trace.api.Config;
-import datadog.trace.api.civisibility.InstrumentationBridge;
+import datadog.trace.api.civisibility.CIConstants;
 import datadog.trace.api.civisibility.config.TestIdentifier;
 import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
 import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
@@ -95,7 +95,7 @@ public static Boolean run(
       if (skipReason == SkipReason.ITR) {
         List<String> tags = pickle.getTags();
         for (String tag : tags) {
-          if (tag.endsWith(InstrumentationBridge.ITR_UNSKIPPABLE_TAG)) {
+          if (tag.endsWith(CIConstants.Tags.ITR_UNSKIPPABLE_TAG)) {
             return null;
           }
         }