Merge branch 'master' into datadog/trace-agent-sampling-feedback-opti…

…mization

.circleci/collect_results.sh

@@ -4,7 +4,7 @@
 # This folder will be saved by circleci and available after test runs.
 
 set -e
-#Enable '**' support
+# Enable '**' support
 shopt -s globstar
 
 TEST_RESULTS_DIR=results
@@ -19,13 +19,36 @@ if [[ ${#TEST_RESULT_DIRS[@]} -eq 0 ]]; then
   exit 0
 fi
 
+function get_source_file () {
+  file_path="${RESULT_XML_FILE%%"/build"*}"
+  file_path="${file_path/#"$WORKSPACE_DIR"\//}/src"
+  if ! [[ $RESULT_XML_FILE == *"#"* ]]; then
+    class="${RESULT_XML_FILE%.xml}"
+    class="${class##*"TEST-"}"
+    class="${class##*"."}"
+    common_root=$(grep -rl "class $class" "$file_path" | head -n 1)
+    while IFS= read -r line; do
+      while [[ $line != "$common_root"* ]]; do
+        common_root=$(dirname "$common_root")
+        if [[ "$common_root" == "$common_root/.." ]]; then
+          break
+        fi
+      done
+    done < <(grep -rl "class $class" "$file_path")
+    file_path="$common_root"
+  fi
+}
+
 echo "Saving test results:"
 while IFS= read -r -d '' RESULT_XML_FILE
 do
   echo -n "- $RESULT_XML_FILE"
   AGGREGATED_FILE_NAME=$(echo "$RESULT_XML_FILE" | rev | cut -d "/" -f 1,2,5 | rev | tr "/" "_")
   echo -n " as $AGGREGATED_FILE_NAME"
   cp "$RESULT_XML_FILE" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
+  # Insert file attribute to testcase XML nodes
+  get_source_file
+  sed -i "/<testcase/ s|\(time=\"[^\"]*\"\)|\1 file=\"$file_path\"|g" "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
   # Replace Java Object hashCode by marker in testcase XML nodes to get stable test names
   sed -i '/<testcase/ s/@[0-9a-f]\{5,\}/@HASHCODE/g' "$TEST_RESULTS_DIR/$AGGREGATED_FILE_NAME"
   # Replace random port numbers by marker in testcase XML nodes to get stable test names

components/context/src/main/java/datadog/context/Context.java

@@ -114,7 +114,7 @@ static Context detachFrom(Object carrier) {
   /**
    * Creates a copy of this context with the given key-value set.
    *
-   * <p>Existing value with the given key will be replaced, and mapping to a {@code null} value will
+   * <p>Existing value with the given key will be replaced. Mapping to a {@code null} value will
    * remove the key-value from the context copy.
    *
    * @param <T> the type of the value.
@@ -124,6 +124,28 @@ static Context detachFrom(Object carrier) {
    */
   <T> Context with(ContextKey<T> key, @Nullable T value);
 
+  /**
+   * Creates a copy of this context with the given pair of key-values.
+   *
+   * <p>Existing values with the given keys will be replaced. Mapping to a {@code null} value will
+   * remove the key-value from the context copy.
+   *
+   * @param <T> the type of the first value.
+   * @param <U> the type of the second value.
+   * @param firstKey the first key to store the first value.
+   * @param firstValue the first value to store.
+   * @param secondKey the second key to store the second value.
+   * @param secondValue the second value to store.
+   * @return a new context with the pair of key-values set.
+   */
+  default <T, U> Context with(
+      ContextKey<T> firstKey,
+      @Nullable T firstValue,
+      ContextKey<U> secondKey,
+      @Nullable U secondValue) {
+    return with(firstKey, firstValue).with(secondKey, secondValue);
+  }
+
   /**
    * Creates a copy of this context with the implicit key is mapped to the value.
    *

components/context/src/test/java/datadog/context/ContextTest.java

@@ -63,6 +63,41 @@ void testWith(Context context) {
     assertDoesNotThrow(() -> context.with(null), "Null implicitly keyed value not throw exception");
   }
 
+  @ParameterizedTest
+  @MethodSource("contextImplementations")
+  void testWithPair(Context context) {
+    // Test retrieving value
+    String stringValue = "value";
+    Context context1 = context.with(BOOLEAN_KEY, false, STRING_KEY, stringValue);
+    assertEquals(stringValue, context1.get(STRING_KEY));
+    assertEquals(false, context1.get(BOOLEAN_KEY));
+    // Test overriding value
+    String stringValue2 = "value2";
+    Context context2 = context1.with(STRING_KEY, stringValue2, BOOLEAN_KEY, true);
+    assertEquals(stringValue2, context2.get(STRING_KEY));
+    assertEquals(true, context2.get(BOOLEAN_KEY));
+    // Test clearing value
+    Context context3 = context2.with(BOOLEAN_KEY, null, STRING_KEY, null);
+    assertNull(context3.get(STRING_KEY));
+    assertNull(context3.get(BOOLEAN_KEY));
+    // Test null key handling
+    assertThrows(
+        NullPointerException.class,
+        () -> context.with(null, "test", STRING_KEY, "test"),
+        "Context forbids null keys");
+    assertThrows(
+        NullPointerException.class,
+        () -> context.with(STRING_KEY, "test", null, "test"),
+        "Context forbids null keys");
+    // Test null value handling
+    assertDoesNotThrow(
+        () -> context.with(BOOLEAN_KEY, null, STRING_KEY, "test"),
+        "Null value should not throw exception");
+    assertDoesNotThrow(
+        () -> context.with(STRING_KEY, "test", BOOLEAN_KEY, null),
+        "Null value should not throw exception");
+  }
+
   @ParameterizedTest
   @MethodSource("contextImplementations")
   void testGet(Context original) {

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Constants.java

@@ -15,6 +15,7 @@ public final class Constants {
    */
   public static final String[] BOOTSTRAP_PACKAGE_PREFIXES = {
     "datadog.slf4j",
+    "datadog.context",
     "datadog.appsec.api",
     "datadog.trace.api",
     "datadog.trace.bootstrap",

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityRepoServices.java

@@ -8,6 +8,7 @@
 import datadog.trace.civisibility.ci.CIInfo;
 import datadog.trace.civisibility.ci.CIProviderInfo;
 import datadog.trace.civisibility.ci.CITagsProvider;
+import datadog.trace.civisibility.ci.PullRequestInfo;
 import datadog.trace.civisibility.codeowners.Codeowners;
 import datadog.trace.civisibility.codeowners.CodeownersProvider;
 import datadog.trace.civisibility.codeowners.NoCodeowners;
@@ -22,6 +23,7 @@
 import datadog.trace.civisibility.git.tree.GitDataApi;
 import datadog.trace.civisibility.git.tree.GitDataUploader;
 import datadog.trace.civisibility.git.tree.GitDataUploaderImpl;
+import datadog.trace.civisibility.git.tree.GitRepoUnshallow;
 import datadog.trace.civisibility.ipc.ExecutionSettingsRequest;
 import datadog.trace.civisibility.ipc.ExecutionSettingsResponse;
 import datadog.trace.civisibility.ipc.SignalClient;
@@ -31,6 +33,8 @@
 import datadog.trace.civisibility.source.SourcePathResolver;
 import datadog.trace.civisibility.source.index.RepoIndexProvider;
 import datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver;
+import datadog.trace.util.Strings;
+import java.io.File;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Map;
@@ -59,16 +63,26 @@ public class CiVisibilityRepoServices {
     ciProvider = ciProviderInfo.getProvider();
 
     CIInfo ciInfo = ciProviderInfo.buildCIInfo();
-    repoRoot = ciInfo.getNormalizedCiWorkspace();
-    moduleName = getModuleName(services.config, path, ciInfo);
-    ciTags = new CITagsProvider().getCiTags(ciInfo);
+    PullRequestInfo pullRequestInfo = ciProviderInfo.buildPullRequestInfo();
+
+    if (pullRequestInfo.isNotEmpty()) {
+      LOGGER.info("PR detected: {}", pullRequestInfo);
+    }
+
+    repoRoot = appendSlashIfNeeded(getRepoRoot(ciInfo, services.gitClientFactory));
+    moduleName = getModuleName(services.config, repoRoot, path);
+    ciTags = new CITagsProvider().getCiTags(ciInfo, pullRequestInfo);
+
+    GitClient gitClient = services.gitClientFactory.create(repoRoot);
+    GitRepoUnshallow gitRepoUnshallow = new GitRepoUnshallow(services.config, gitClient);
 
     gitDataUploader =
         buildGitDataUploader(
             services.config,
             services.metricCollector,
             services.gitInfoProvider,
-            services.gitClientFactory,
+            gitClient,
+            gitRepoUnshallow,
             services.backendApi,
             repoRoot);
     repoIndexProvider = services.repoIndexProviderFactory.create(repoRoot);
@@ -84,18 +98,49 @@ public class CiVisibilityRepoServices {
               services.config,
               services.metricCollector,
               services.backendApi,
+              gitClient,
+              gitRepoUnshallow,
               gitDataUploader,
+              pullRequestInfo,
               repoRoot);
     }
   }
 
-  static String getModuleName(Config config, Path path, CIInfo ciInfo) {
+  private static String getRepoRoot(CIInfo ciInfo, GitClient.Factory gitClientFactory) {
+    String ciWorkspace = ciInfo.getNormalizedCiWorkspace();
+    if (Strings.isNotBlank(ciWorkspace)) {
+      return ciWorkspace;
+
+    } else {
+      try {
+        return gitClientFactory.create(".").getRepoRoot();
+
+      } catch (InterruptedException e) {
+        Thread.currentThread().interrupt();
+        LOGGER.error("Interrupted while getting repo root", e);
+        return null;
+
+      } catch (Exception e) {
+        LOGGER.error("Error while getting repo root", e);
+        return null;
+      }
+    }
+  }
+
+  private static String appendSlashIfNeeded(String repoRoot) {
+    if (repoRoot != null && !repoRoot.endsWith(File.separator)) {
+      return repoRoot + File.separator;
+    } else {
+      return repoRoot;
+    }
+  }
+
+  static String getModuleName(Config config, String repoRoot, Path path) {
     // if parent process is instrumented, it will provide build system's module name
     String parentModuleName = config.getCiVisibilityModuleName();
     if (parentModuleName != null) {
       return parentModuleName;
     }
-    String repoRoot = ciInfo.getNormalizedCiWorkspace();
     if (repoRoot != null && path.startsWith(repoRoot)) {
       String relativePath = Paths.get(repoRoot).relativize(path).toString();
       if (!relativePath.isEmpty()) {
@@ -126,7 +171,10 @@ private static ExecutionSettingsFactory buildExecutionSettingsFactory(
       Config config,
       CiVisibilityMetricCollector metricCollector,
       BackendApi backendApi,
+      GitClient gitClient,
+      GitRepoUnshallow gitRepoUnshallow,
       GitDataUploader gitDataUploader,
+      PullRequestInfo pullRequestInfo,
       String repoRoot) {
     ConfigurationApi configurationApi;
     if (backendApi == null) {
@@ -138,7 +186,14 @@ private static ExecutionSettingsFactory buildExecutionSettingsFactory(
     }
 
     ExecutionSettingsFactoryImpl factory =
-        new ExecutionSettingsFactoryImpl(config, configurationApi, gitDataUploader, repoRoot);
+        new ExecutionSettingsFactoryImpl(
+            config,
+            configurationApi,
+            gitClient,
+            gitRepoUnshallow,
+            gitDataUploader,
+            pullRequestInfo,
+            repoRoot);
     if (processHierarchy.isHeadless()) {
       return factory;
     } else {
@@ -150,7 +205,8 @@ private static GitDataUploader buildGitDataUploader(
       Config config,
       CiVisibilityMetricCollector metricCollector,
       GitInfoProvider gitInfoProvider,
-      GitClient.Factory gitClientFactory,
+      GitClient gitClient,
+      GitRepoUnshallow gitRepoUnshallow,
       BackendApi backendApi,
       String repoRoot) {
     if (!config.isCiVisibilityGitUploadEnabled()) {
@@ -171,9 +227,15 @@ private static GitDataUploader buildGitDataUploader(
 
     String remoteName = config.getCiVisibilityGitRemoteName();
     GitDataApi gitDataApi = new GitDataApi(backendApi, metricCollector);
-    GitClient gitClient = gitClientFactory.create(repoRoot);
     return new GitDataUploaderImpl(
-        config, metricCollector, gitDataApi, gitClient, gitInfoProvider, repoRoot, remoteName);
+        config,
+        metricCollector,
+        gitDataApi,
+        gitClient,
+        gitRepoUnshallow,
+        gitInfoProvider,
+        repoRoot,
+        remoteName);
   }
 
   private static SourcePathResolver buildSourcePathResolver(

dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilityServices.java

@@ -8,8 +8,11 @@
 import datadog.communication.ddagent.SharedCommunicationObjects;
 import datadog.communication.http.HttpRetryPolicy;
 import datadog.communication.http.OkHttpUtils;
+import datadog.communication.util.IOUtils;
 import datadog.trace.api.Config;
+import datadog.trace.api.civisibility.telemetry.CiVisibilityCountMetric;
 import datadog.trace.api.civisibility.telemetry.CiVisibilityMetricCollector;
+import datadog.trace.api.civisibility.telemetry.tag.Command;
 import datadog.trace.api.git.GitInfoProvider;
 import datadog.trace.civisibility.ci.CIProviderInfoFactory;
 import datadog.trace.civisibility.ci.env.CiEnvironment;
@@ -22,24 +25,28 @@
 import datadog.trace.civisibility.git.CIProviderGitInfoBuilder;
 import datadog.trace.civisibility.git.GitClientGitInfoBuilder;
 import datadog.trace.civisibility.git.tree.GitClient;
+import datadog.trace.civisibility.git.tree.NoOpGitClient;
+import datadog.trace.civisibility.git.tree.ShellGitClient;
 import datadog.trace.civisibility.ipc.SignalClient;
 import datadog.trace.civisibility.source.BestEffortLinesResolver;
 import datadog.trace.civisibility.source.ByteCodeLinesResolver;
 import datadog.trace.civisibility.source.CompilerAidedLinesResolver;
 import datadog.trace.civisibility.source.LinesResolver;
 import datadog.trace.civisibility.source.index.*;
+import datadog.trace.civisibility.utils.ShellCommandExecutor;
+import java.io.File;
 import java.lang.reflect.Type;
 import java.net.InetSocketAddress;
 import java.nio.file.FileSystem;
 import java.nio.file.FileSystems;
 import java.nio.file.Path;
 import java.util.Collections;
 import java.util.Map;
+import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
-import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -78,7 +85,7 @@ public class CiVisibilityServices {
     this.backendApi =
         new BackendApiFactory(config, sco).createBackendApi(BackendApiFactory.Intake.API);
     this.jvmInfoFactory = new CachingJvmInfoFactory(config, new JvmInfoFactoryImpl());
-    this.gitClientFactory = new GitClient.Factory(config, metricCollector);
+    this.gitClientFactory = buildGitClientFactory(config, metricCollector);
 
     CiEnvironment environment = buildCiEnvironment(config, sco);
     this.ciProviderInfoFactory = new CIProviderInfoFactory(config, environment);
@@ -111,7 +118,30 @@ public class CiVisibilityServices {
     }
   }
 
-  @NotNull
+  private static GitClient.Factory buildGitClientFactory(
+      Config config, CiVisibilityMetricCollector metricCollector) {
+    if (!config.isCiVisibilityGitClientEnabled()) {
+      return r -> NoOpGitClient.INSTANCE;
+    }
+    try {
+      ShellCommandExecutor shellCommandExecutor =
+          new ShellCommandExecutor(new File("."), config.getCiVisibilityGitCommandTimeoutMillis());
+      String gitVersion = shellCommandExecutor.executeCommand(IOUtils::readFully, "git", "version");
+      logger.debug("Detected git executable version {}", gitVersion);
+      return new ShellGitClient.Factory(config, metricCollector);
+
+    } catch (Exception e) {
+      metricCollector.add(
+          CiVisibilityCountMetric.GIT_COMMAND_ERRORS,
+          1,
+          Command.OTHER,
+          ShellCommandExecutor.getExitCode(e));
+      logger.info("No git executable detected, some features will not be available");
+      return r -> NoOpGitClient.INSTANCE;
+    }
+  }
+
+  @Nonnull
   private static CiEnvironment buildCiEnvironment(Config config, SharedCommunicationObjects sco) {
     String remoteEnvVarsProviderUrl = config.getCiVisibilityRemoteEnvVarsProviderUrl();
     if (remoteEnvVarsProviderUrl != null) {