From d29f392e09733b4d75c24f49bbd036d536a6c3e7 Mon Sep 17 00:00:00 2001 From: Ajantha Bhat Date: Thu, 15 Jun 2023 16:58:23 +0530 Subject: [PATCH] Nessie: Support ApiV2 for Nessie client (#6712) --- .../apache/iceberg/nessie/NessieCatalog.java | 24 ++++++++++++++++--- .../org/apache/iceberg/nessie/NessieUtil.java | 2 ++ .../iceberg/nessie/BaseTestIceberg.java | 7 ++++-- .../nessie/TestCustomNessieClient.java | 8 +++++-- .../iceberg/nessie/TestNessieCatalog.java | 14 ++++++----- .../nessie/TestNessieIcebergClient.java | 13 ++++++++++ 6 files changed, 55 insertions(+), 13 deletions(-) diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java index 0c6f2d4c5911..0c91cfb1dbe1 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieCatalog.java @@ -44,6 +44,7 @@ import org.projectnessie.client.NessieClientBuilder; import org.projectnessie.client.NessieConfigConstants; import org.projectnessie.client.api.NessieApiV1; +import org.projectnessie.client.api.NessieApiV2; import org.projectnessie.client.http.HttpClientBuilder; import org.projectnessie.model.ContentKey; import org.projectnessie.model.TableReference; @@ -88,11 +89,28 @@ public void initialize(String name, Map options) { options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF)); String requestedHash = options.get(removePrefix.apply(NessieConfigConstants.CONF_NESSIE_REF_HASH)); - NessieApiV1 api = + + NessieClientBuilder nessieClientBuilder = createNessieClientBuilder( options.get(NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL)) - .fromConfig(x -> options.get(removePrefix.apply(x))) - .build(NessieApiV1.class); + .fromConfig(x -> options.get(removePrefix.apply(x))); + // default version is set to v1. + final String apiVersion = + options.getOrDefault(removePrefix.apply(NessieUtil.CLIENT_API_VERSION), "1"); + NessieApiV1 api; + switch (apiVersion) { + case "1": + api = nessieClientBuilder.build(NessieApiV1.class); + break; + case "2": + api = nessieClientBuilder.build(NessieApiV2.class); + break; + default: + throw new IllegalArgumentException( + String.format( + "Unsupported %s: %s. Can only be 1 or 2", + removePrefix.apply(NessieUtil.CLIENT_API_VERSION), apiVersion)); + } initialize( name, diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java index 0e57d8ccc365..0420597945d1 100644 --- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java +++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java @@ -37,6 +37,8 @@ public final class NessieUtil { public static final String NESSIE_CONFIG_PREFIX = "nessie."; static final String APPLICATION_TYPE = "application-type"; + public static final String CLIENT_API_VERSION = "nessie.client-api-version"; + private NessieUtil() {} static TableIdentifier removeCatalogName(TableIdentifier to, String name) { diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java index ae4947a98cb7..5a9bbc273837 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/BaseTestIceberg.java @@ -73,7 +73,7 @@ @ExtendWith(DatabaseAdapterExtension.class) @NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME) @NessieExternalDatabase(InmemoryTestConnectionProviderSource.class) -@NessieApiVersions(versions = NessieApiVersion.V1) +@NessieApiVersions // test all versions public abstract class BaseTestIceberg { @NessieDbAdapter static DatabaseAdapter databaseAdapter; @@ -88,6 +88,7 @@ public abstract class BaseTestIceberg { protected NessieCatalog catalog; protected NessieApiV1 api; + protected String apiVersion; protected Configuration hadoopConfig; protected final String branch; private String initialHashOfDefaultBranch; @@ -122,6 +123,7 @@ public void beforeEach(NessieClientFactory clientFactory, @NessieClientUri URI n throws IOException { this.uri = nessieUri.toASCIIString(); this.api = clientFactory.make(); + this.apiVersion = clientFactory.apiVersion() == NessieApiVersion.V2 ? "2" : "1"; Branch defaultBranch = api.getDefaultBranch(); initialHashOfDefaultBranch = defaultBranch.getHash(); @@ -145,7 +147,8 @@ NessieCatalog initCatalog(String ref, String hash) { .put("ref", ref) .put(CatalogProperties.URI, uri) .put("auth-type", "NONE") - .put(CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString()); + .put(CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString()) + .put("client-api-version", apiVersion); if (null != hash) { options.put("ref.hash", hash); } diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java index 2a3c0c85e000..40341358073f 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestCustomNessieClient.java @@ -48,7 +48,9 @@ public void testNoCustomClient() { CatalogProperties.WAREHOUSE_LOCATION, temp.toUri().toString(), CatalogProperties.URI, - uri)); + uri, + "client-api-version", + apiVersion)); } @Test @@ -62,7 +64,9 @@ public void testUnnecessaryDefaultCustomClient() { CatalogProperties.URI, uri, NessieConfigConstants.CONF_NESSIE_CLIENT_BUILDER_IMPL, - HttpClientBuilder.class.getName())); + HttpClientBuilder.class.getName(), + "client-api-version", + apiVersion)); } @Test diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java index f8ed3987c1ea..8ff052712bcd 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java @@ -54,7 +54,7 @@ @ExtendWith(DatabaseAdapterExtension.class) @NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME) @NessieExternalDatabase(InmemoryTestConnectionProviderSource.class) -@NessieApiVersions(versions = NessieApiVersion.V1) +@NessieApiVersions // test all versions public class TestNessieCatalog extends CatalogTests { @NessieDbAdapter static DatabaseAdapter databaseAdapter; @@ -67,6 +67,7 @@ public class TestNessieCatalog extends CatalogTests { private NessieCatalog catalog; private NessieApiV1 api; + private NessieApiVersion apiVersion; private Configuration hadoopConfig; private String initialHashOfDefaultBranch; private String uri; @@ -75,6 +76,7 @@ public class TestNessieCatalog extends CatalogTests { public void setUp(NessieClientFactory clientFactory, @NessieClientUri URI nessieUri) throws NessieNotFoundException { api = clientFactory.make(); + apiVersion = clientFactory.apiVersion(); initialHashOfDefaultBranch = api.getDefaultBranch().getHash(); uri = nessieUri.toASCIIString(); hadoopConfig = new Configuration(); @@ -115,17 +117,17 @@ private void resetData() throws NessieConflictException, NessieNotFoundException private NessieCatalog initNessieCatalog(String ref) { NessieCatalog newCatalog = new NessieCatalog(); newCatalog.setConf(hadoopConfig); - newCatalog.initialize( - "nessie", + ImmutableMap options = ImmutableMap.of( "ref", ref, CatalogProperties.URI, uri, - "auth-type", - "NONE", CatalogProperties.WAREHOUSE_LOCATION, - temp.toUri().toString())); + temp.toUri().toString(), + "client-api-version", + apiVersion == NessieApiVersion.V2 ? "2" : "1"); + newCatalog.initialize("nessie", options); return newCatalog; } diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java index d6643be91e24..12fa807dcfb6 100644 --- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java +++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieIcebergClient.java @@ -18,6 +18,7 @@ */ package org.apache.iceberg.nessie; +import java.io.IOException; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; import org.assertj.core.api.Assertions; @@ -89,4 +90,16 @@ public void testWithReferenceAfterRecreatingBranch() .isEqualTo(ref); Assertions.assertThat(client.withReference(branch, null)).isNotEqualTo(client); } + + @Test + public void testInvalidClientApiVersion() throws IOException { + try (NessieCatalog newCatalog = new NessieCatalog()) { + newCatalog.setConf(hadoopConfig); + ImmutableMap.Builder options = + ImmutableMap.builder().put("client-api-version", "3"); + Assertions.assertThatThrownBy(() -> newCatalog.initialize("nessie", options.buildOrThrow())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("Unsupported client-api-version: 3. Can only be 1 or 2"); + } + } }