Nessie: Support ApiV2 for Nessie client (#6712)

apache · Jun 15, 2023 · d29f392 · d29f392
1 parent c115f49
commit d29f392
Show file tree

Hide file tree

Showing 6 changed files with 55 additions and 13 deletions.
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java
@@ -44,6 +44,7 @@
 import org.projectnessie.client.NessieClientBuilder;
 import org.projectnessie.client.NessieConfigConstants;
 import org.projectnessie.client.api.NessieApiV1;
+import org.projectnessie.client.api.NessieApiV2;
 import org.projectnessie.client.http.HttpClientBuilder;
 import org.projectnessie.model.ContentKey;
 import org.projectnessie.model.TableReference;
@@ -88,11 +89,28 @@ public void initialize(String name, Map<String, String> options) {
         options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF));
     String requestedHash =
         options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF_HASH));
-    NessieApiV1 api =
+
+    NessieClientBuilder<?> nessieClientBuilder =
         createNessieClientBuilder(
                 options.get(NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL))
-            .fromConfig(x -> options.get(removePrefix.apply(x)))
-            .build(NessieApiV1.class);
+            .fromConfig(x -> options.get(removePrefix.apply(x)));
+    // default version is set to v1.
+    final String apiVersion =
+        options.getOrDefault(removePrefix.apply(NessieUtil.CLIENT_API_VERSION), "1");
+    NessieApiV1 api;
+    switch (apiVersion) {
+      case "1":
+        api = nessieClientBuilder.build(NessieApiV1.class);
+        break;
+      case "2":
+        api = nessieClientBuilder.build(NessieApiV2.class);
+        break;
+      default:
+        throw new IllegalArgumentException(
+            String.format(
+                "Unsupported %s: %s. Can only be 1 or 2",
+                removePrefix.apply(NessieUtil.CLIENT_API_VERSION), apiVersion));
+    }
 
     initialize(
         name,

diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
@@ -37,6 +37,8 @@ public final class NessieUtil {
   public static final String NESSIE_CONFIG_PREFIX = "nessie.";
   static final String APPLICATION_TYPE = "application-type";
 
+  public static final String CLIENT_API_VERSION = "nessie.client-api-version";
+
   private NessieUtil() {}
 
   static TableIdentifier removeCatalogName(TableIdentifier to, String name) {

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java
@@ -73,7 +73,7 @@
 @ExtendWith(DatabaseAdapterExtension.class)
 @NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
 @NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
-@NessieApiVersions(versions = NessieApiVersion.V1)
+@NessieApiVersions // test all versions
 public abstract class BaseTestIceberg {
 
   @NessieDbAdapter static DatabaseAdapter databaseAdapter;
@@ -88,6 +88,7 @@ public abstract class BaseTestIceberg {
 
   protected NessieCatalog catalog;
   protected NessieApiV1 api;
+  protected String apiVersion;
   protected Configuration hadoopConfig;
   protected final String branch;
   private String initialHashOfDefaultBranch;
@@ -122,6 +123,7 @@ public void beforeEach(NessieClientFactory clientFactory, @NessieClientUri URI n
       throws IOException {
     this.uri = nessieUri.toASCIIString();
     this.api = clientFactory.make();
+    this.apiVersion = clientFactory.apiVersion() == NessieApiVersion.V2 ? "2" : "1";
 
     Branch defaultBranch = api.getDefaultBranch();
     initialHashOfDefaultBranch = defaultBranch.getHash();
@@ -145,7 +147,8 @@ NessieCatalog initCatalog(String ref, String hash) {
             .put("ref", ref)
             .put(CatalogProperties.URI, uri)
             .put("auth-type", "NONE")
-            .put(CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString());
+            .put(CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString())
+            .put("client-api-version", apiVersion);
     if (null != hash) {
       options.put("ref.hash", hash);
     }

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java
@@ -48,7 +48,9 @@ public void testNoCustomClient() {
             CatalogProperties.WAREHOUSE_LOCATION,
             temp.toUri().toString(),
             CatalogProperties.URI,
-            uri));
+            uri,
+            "client-api-version",
+            apiVersion));
   }
 
   @Test
@@ -62,7 +64,9 @@ public void testUnnecessaryDefaultCustomClient() {
             CatalogProperties.URI,
             uri,
             NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL,
-            HttpClientBuilder.class.getName()));
+            HttpClientBuilder.class.getName(),
+            "client-api-version",
+            apiVersion));
   }
 
   @Test

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
@@ -54,7 +54,7 @@
 @ExtendWith(DatabaseAdapterExtension.class)
 @NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
 @NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
-@NessieApiVersions(versions = NessieApiVersion.V1)
+@NessieApiVersions // test all versions
 public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
 
   @NessieDbAdapter static DatabaseAdapter databaseAdapter;
@@ -67,6 +67,7 @@ public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
 
   private NessieCatalog catalog;
   private NessieApiV1 api;
+  private NessieApiVersion apiVersion;
   private Configuration hadoopConfig;
   private String initialHashOfDefaultBranch;
   private String uri;
@@ -75,6 +76,7 @@ public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
   public void setUp(NessieClientFactory clientFactory, @NessieClientUri URI nessieUri)
       throws NessieNotFoundException {
     api = clientFactory.make();
+    apiVersion = clientFactory.apiVersion();
     initialHashOfDefaultBranch = api.getDefaultBranch().getHash();
     uri = nessieUri.toASCIIString();
     hadoopConfig = new Configuration();
@@ -115,17 +117,17 @@ private void resetData() throws NessieConflictException, NessieNotFoundException
   private NessieCatalog initNessieCatalog(String ref) {
     NessieCatalog newCatalog = new NessieCatalog();
     newCatalog.setConf(hadoopConfig);
-    newCatalog.initialize(
-        "nessie",
+    ImmutableMap<String, String> options =
         ImmutableMap.of(
             "ref",
             ref,
             CatalogProperties.URI,
             uri,
-            "auth-type",
-            "NONE",
             CatalogProperties.WAREHOUSE_LOCATION,
-            temp.toUri().toString()));
+            temp.toUri().toString(),
+            "client-api-version",
+            apiVersion == NessieApiVersion.V2 ? "2" : "1");
+    newCatalog.initialize("nessie", options);
     return newCatalog;
   }
 

diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java
@@ -18,6 +18,7 @@
  */
 package org.apache.iceberg.nessie;
 
+import java.io.IOException;
 import org.apache.iceberg.catalog.Namespace;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.assertj.core.api.Assertions;
@@ -89,4 +90,16 @@ public void testWithReferenceAfterRecreatingBranch()
         .isEqualTo(ref);
     Assertions.assertThat(client.withReference(branch, null)).isNotEqualTo(client);
   }
+
+  @Test
+  public void testInvalidClientApiVersion() throws IOException {
+    try (NessieCatalog newCatalog = new NessieCatalog()) {
+      newCatalog.setConf(hadoopConfig);
+      ImmutableMap.Builder<String, String> options =
+          ImmutableMap.<String, String>builder().put("client-api-version", "3");
+      Assertions.assertThatThrownBy(() -> newCatalog.initialize("nessie", options.buildOrThrow()))
+          .isInstanceOf(IllegalArgumentException.class)
+          .hasMessage("Unsupported client-api-version: 3. Can only be 1 or 2");
+    }
+  }
 }