Implement test visibility for Karate v2

.github/CODEOWNERS

@@ -110,7 +110,7 @@
 /dd-java-agent/instrumentation/cucumber-5.4/        @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/jacoco-0.8.9/        @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/junit                @DataDog/ci-app-libraries
-/dd-java-agent/instrumentation/karate-1.0/          @DataDog/ci-app-libraries
+/dd-java-agent/instrumentation/karate/              @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/scalatest-3.0.8/     @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/selenium-3.13/       @DataDog/ci-app-libraries
 /dd-java-agent/instrumentation/testng/              @DataDog/ci-app-libraries

dd-java-agent/instrumentation/karate/karate-2.0/build.gradle

@@ -0,0 +1,52 @@
+plugins {
+  id 'dd-trace-java.instrumentation.testing-framework-tests'
+}
+
+apply from: "$rootDir/gradle/java.gradle"
+
+// Karate 2.x is compiled with Java 21 (uses virtual threads for parallelism)
+tracerJava {
+  addSourceSetFor(JavaVersion.VERSION_21)
+}
+
+muzzle {
+  pass {
+    group = 'io.karatelabs'
+    module = 'karate-core'
+    versions = '[2.0.0,)'
+    javaVersion = '21'
+  }
+}
+
+["compileMain_java21Java", "compileTestJava"].each {
+  tasks.named(it, JavaCompile) {
+    configureCompiler(it, 21, JavaVersion.VERSION_21)
+  }
+}
+
+tasks.withType(GroovyCompile).configureEach {
+  configureCompiler(it, 21)
+}
+
+dependencies {
+  main_java21CompileOnly group: 'io.karatelabs', name: 'karate-core', version: '2.0.9'
+
+  testImplementation project(':dd-java-agent:agent-ci-visibility:civisibility-instrumentation-test-fixtures')
+  testImplementation group: 'org.junit.platform', name: 'junit-platform-launcher', version: libs.versions.junit.platform.get()
+  testImplementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: libs.versions.junit5.get()
+
+  testImplementation(group: 'io.karatelabs', name: 'karate-core', version: '2.0.9') {
+    // excluding logback to avoid conflicts with libs.bundles.test.logging
+    exclude group: 'ch.qos.logback', module: 'logback-classic'
+  }
+}
+
+// Using recommended Karate project layout where Karate feature files sit in same /test/java folders as their java counterparts
+sourceSets {
+  test {
+    resources {
+      srcDir file('src/test/java')
+      exclude '**/*.java'
+    }
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java/datadog/trace/instrumentation/karate2/KarateInstrumentation.java

@@ -0,0 +1,44 @@
+package datadog.trace.instrumentation.karate2;
+
+import static net.bytebuddy.matcher.ElementMatchers.isConstructor;
+
+import com.google.auto.service.AutoService;
+import datadog.trace.agent.tooling.Instrumenter;
+import datadog.trace.agent.tooling.InstrumenterModule;
+
+/**
+ * Registers a {@code io.karatelabs.core.RunListener} on every {@code
+ * io.karatelabs.core.Runner.Builder}.
+ *
+ * <p>This module is compiled for Java 8 so the agent can enumerate it on any JVM; the advice and
+ * helper classes that reference the Java 21 {@code io.karatelabs} API live in the {@code java21}
+ * source set and are referenced by name.
+ */
+@AutoService(InstrumenterModule.class)
+public class KarateInstrumentation extends InstrumenterModule.CiVisibility
+    implements Instrumenter.ForSingleType, Instrumenter.HasMethodAdvice {
+
+  public KarateInstrumentation() {
+    super("ci-visibility", "karate");
+  }
+
+  @Override
+  public String instrumentedType() {
+    return "io.karatelabs.core.Runner$Builder";
+  }
+
+  @Override
+  public String[] helperClassNames() {
+    return new String[] {
+      packageName + ".KarateUtils",
+      packageName + ".TestEventsHandlerHolder",
+      packageName + ".KarateTracingListener",
+      packageName + ".KarateBuilderAdvice"
+    };
+  }
+
+  @Override
+  public void methodAdvice(MethodTransformer transformer) {
+    transformer.applyAdvice(isConstructor(), packageName + ".KarateBuilderAdvice");
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateBuilderAdvice.java

@@ -0,0 +1,20 @@
+package datadog.trace.instrumentation.karate2;
+
+import io.karatelabs.core.RunEvent;
+import io.karatelabs.core.RunListener;
+import io.karatelabs.core.Runner;
+import net.bytebuddy.asm.Advice;
+
+/** Advice for the {@code io.karatelabs.core.Runner.Builder} constructor. */
+public class KarateBuilderAdvice {
+
+  @Advice.OnMethodExit
+  public static void onRunnerBuilderConstructorExit(@Advice.This Runner.Builder builder) {
+    builder.listener(new KarateTracingListener());
+  }
+
+  // Karate 2.0.0 and above
+  public static void muzzleCheck(RunListener runListener) {
+    runListener.onEvent((RunEvent) null);
+  }
+}

dd-java-agent/instrumentation/karate/karate-2.0/src/main/java21/datadog/trace/instrumentation/karate2/KarateTracingListener.java

@@ -0,0 +1,240 @@
+package datadog.trace.instrumentation.karate2;
+
+import datadog.trace.api.Config;
+import datadog.trace.api.civisibility.CIConstants;
+import datadog.trace.api.civisibility.config.TestIdentifier;
+import datadog.trace.api.civisibility.config.TestSourceData;
+import datadog.trace.api.civisibility.events.TestDescriptor;
+import datadog.trace.api.civisibility.events.TestSuiteDescriptor;
+import datadog.trace.api.civisibility.telemetry.tag.SkipReason;
+import datadog.trace.api.civisibility.telemetry.tag.TestFrameworkInstrumentation;
+import datadog.trace.bootstrap.instrumentation.api.AgentSpan;
+import datadog.trace.bootstrap.instrumentation.api.AgentTracer;
+import datadog.trace.bootstrap.instrumentation.api.Tags;
+import io.karatelabs.core.FeatureResult;
+import io.karatelabs.core.FeatureRunEvent;
+import io.karatelabs.core.FeatureRuntime;
+import io.karatelabs.core.RunEvent;
+import io.karatelabs.core.RunEventType;
+import io.karatelabs.core.RunListener;
+import io.karatelabs.core.ScenarioResult;
+import io.karatelabs.core.ScenarioRunEvent;
+import io.karatelabs.core.ScenarioRuntime;
+import io.karatelabs.core.StepResult;
+import io.karatelabs.core.StepRunEvent;
+import io.karatelabs.gherkin.Feature;
+import io.karatelabs.gherkin.Scenario;
+import io.karatelabs.gherkin.Step;
+import java.util.Collection;
+import java.util.List;
+
+public class KarateTracingListener implements RunListener {
+
+  private static final String FRAMEWORK_NAME = "karate";
+  public static final String FRAMEWORK_VERSION = KarateUtils.getKarateVersion();
+  public static final String KARATE_STEP_SPAN_NAME = "karate.step";
+
+  @Override
+  public boolean onEvent(RunEvent event) {
+    RunEventType type = event.getType();
+    if (type == RunEventType.FEATURE_ENTER) {
+      return beforeFeature((FeatureRunEvent) event);
+    } else if (type == RunEventType.FEATURE_EXIT) {
+      afterFeature((FeatureRunEvent) event);
+    } else if (type == RunEventType.SCENARIO_ENTER) {
+      return beforeScenario((ScenarioRunEvent) event);
+    } else if (type == RunEventType.SCENARIO_EXIT) {
+      afterScenario((ScenarioRunEvent) event);
+    } else if (type == RunEventType.STEP_ENTER) {
+      beforeStep((StepRunEvent) event);
+    } else if (type == RunEventType.STEP_EXIT) {
+      afterStep((StepRunEvent) event);
+    }
+    return true;
+  }
+
+  private boolean beforeFeature(FeatureRunEvent event) {
+    FeatureRuntime fr = event.source();
+    if (skipTracking(fr)) {
+      return true;
+    }
+    TestSuiteDescriptor suiteDescriptor = KarateUtils.toSuiteDescriptor(fr);
+    Feature feature = fr.getFeature();
+    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteStart(
+        suiteDescriptor,
+        KarateUtils.getFeatureNameForReport(feature),
+        FRAMEWORK_NAME,
+        FRAMEWORK_VERSION,
+        null,
+        KarateUtils.getCategories(feature.getTags()),
+        isParallel(fr),
+        TestFrameworkInstrumentation.KARATE,
+        null);
+    return true;
+  }
+
+  private void afterFeature(FeatureRunEvent event) {
+    FeatureRuntime fr = event.source();
+    if (skipTracking(fr)) {
+      return;
+    }
+    TestSuiteDescriptor suiteDescriptor = KarateUtils.toSuiteDescriptor(fr);
+    FeatureResult result = event.result();
+    if (result != null && result.isFailed()) {
+      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFailure(
+          suiteDescriptor, suiteThrowable(result));
+    } else if (result != null && result.isEmpty()) {
+      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteSkip(suiteDescriptor, null);
+    }
+    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSuiteFinish(suiteDescriptor, null);
+  }
+
+  private boolean beforeScenario(ScenarioRunEvent event) {
+    ScenarioRuntime sr = event.source();
+    if (skipTracking(sr)) {
+      return true;
+    }
+    Scenario scenario = sr.getScenario();
+    TestSuiteDescriptor suiteDescriptor = KarateUtils.toSuiteDescriptor(sr.getFeatureRuntime());
+    TestDescriptor testDescriptor = KarateUtils.toTestDescriptor(sr);
+    String scenarioName = KarateUtils.getScenarioName(scenario);
+    String parameters = KarateUtils.getParameters(scenario);
+    Collection<String> categories = KarateUtils.getCategories(scenario.getTagsEffective());
+
+    if (Config.get().isCiVisibilityTestSkippingEnabled()
+        || Config.get().isCiVisibilityTestManagementEnabled()) {
+      TestIdentifier skippableTest = KarateUtils.toTestIdentifier(scenario);
+      SkipReason skipReason = TestEventsHandlerHolder.TEST_EVENTS_HANDLER.skipReason(skippableTest);
+
+      if (skipReason != null
+          && !(skipReason == SkipReason.ITR
+              && categories.contains(CIConstants.Tags.ITR_UNSKIPPABLE_TAG))) {
+        TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestIgnore(
+            suiteDescriptor,
+            testDescriptor,
+            scenarioName,
+            FRAMEWORK_NAME,
+            FRAMEWORK_VERSION,
+            parameters,
+            categories,
+            TestSourceData.UNKNOWN,
+            skipReason.getDescription(),
+            null);
+        return false;
+      }
+    }
+
+    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestStart(
+        suiteDescriptor,
+        testDescriptor,
+        scenarioName,
+        FRAMEWORK_NAME,
+        FRAMEWORK_VERSION,
+        parameters,
+        categories,
+        TestSourceData.UNKNOWN,
+        null,
+        null);
+    return true;
+  }
+
+  private void afterScenario(ScenarioRunEvent event) {
+    ScenarioRuntime sr = event.source();
+    if (skipTracking(sr)) {
+      return;
+    }
+    ScenarioResult result = event.result();
+    TestDescriptor testDescriptor = KarateUtils.toTestDescriptor(sr);
+
+    Throwable failedReason = getFailedReason(result);
+    if ((result != null && result.isFailed()) || failedReason != null) {
+      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFailure(testDescriptor, failedReason);
+    } else if (result == null || result.getStepResults().isEmpty()) {
+      TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestSkip(testDescriptor, null);
+    }
+
+    TestEventsHandlerHolder.TEST_EVENTS_HANDLER.onTestFinish(testDescriptor, null, null);
+  }
+
+  private void beforeStep(StepRunEvent event) {
+    Step step = event.step();
+    if (skipTracking(step)) {
+      return;
+    }
+    AgentSpan span = AgentTracer.startSpan("karate", KARATE_STEP_SPAN_NAME);
+    AgentTracer.activateSpanWithoutScope(span);
+    String stepName = step.getPrefix() + " " + step.getText();
+    span.setResourceName(stepName);
+    span.setTag(Tags.COMPONENT, "karate");
+    span.spanContext().setIntegrationName("karate");
+    span.setTag("step.name", stepName);
+    span.setTag("step.startLine", step.getLine());
+    span.setTag("step.endLine", step.getEndLine());
+    span.setTag("step.docString", step.getDocString());
+  }
+
+  private void afterStep(StepRunEvent event) {
+    if (skipTracking(event.step())) {
+      return;
+    }
+
+    AgentSpan span = AgentTracer.activeSpan();
+    if (span == null) {
+      return;
+    }
+
+    AgentTracer.closeActive();
+    span.finish();
+  }
+
+  private static Throwable getFailedReason(ScenarioResult result) {
+    if (result == null) {
+      return null;
+    }
+    Throwable error = result.getError();
+    if (error != null) {
+      return error;
+    }
+    for (StepResult stepResult : result.getStepResults()) {
+      if (stepResult.getError() != null) {
+        return stepResult.getError();
+      }
+    }
+    return null;
+  }
+
+  private static Throwable suiteThrowable(FeatureResult result) {
+    List<ScenarioResult> scenarioResults = result.getScenarioResults();
+    if (scenarioResults != null) {
+      for (ScenarioResult scenarioResult : scenarioResults) {
+        Throwable error = scenarioResult.getError();
+        if (error != null) {
+          return error;
+        }
+      }
+    }
+    return new RuntimeException(result.getFailureMessage());
+  }
+
+  private static boolean isParallel(FeatureRuntime fr) {
+    return fr.getSuite() != null && fr.getSuite().parallel;
+  }
+
+  private static boolean skipTracking(FeatureRuntime fr) {
+    // do not track nested (called) feature runs
+    return fr.getCaller() != null;
+  }
+
+  private static boolean skipTracking(ScenarioRuntime sr) {
+    // do not track nested (called) scenario runs and setup scenarios
+    return sr.getFeatureRuntime().getCaller() != null || sr.getScenario().isSetup();
+  }
+
+  private static boolean skipTracking(Step step) {
+    // do not track steps that are not children of a tracked scenario or another tracked step
+    AgentSpan activeSpan = AgentTracer.activeSpan();
+    return activeSpan == null
+        || (!KARATE_STEP_SPAN_NAME.contentEquals(activeSpan.getSpanName())
+            && !Tags.SPAN_KIND_TEST.contentEquals(activeSpan.getSpanType()));
+  }
+}