From 28e6cc95702bd75344224ee46314588369e23592 Mon Sep 17 00:00:00 2001 From: Christian Bush Date: Wed, 11 Mar 2026 15:48:06 -0700 Subject: [PATCH 1/6] Add Iceberg REST controller, serde, and exception handling Implement the read-only Iceberg REST Catalog surface on top of the generated OpenAPI interfaces from PR1: Controller (IcebergRestCatalogController): - Implements CatalogApiApi + ConfigurationApiApi generated interfaces - GET /v1/config returns prefix override for route isolation - GET /v1/{prefix}/namespaces/{ns}/tables lists tables via TablesService - GET /v1/{prefix}/namespaces/{ns}/tables/{t} loads table via CatalogHandlers - HEAD /v1/{prefix}/namespaces/{ns}/tables/{t} checks table existence - All unimplemented endpoints return 501 via generated defaults Serialization: - IcebergRestHttpMessageConverter for Iceberg REST types (kebab-case JSON) - IcebergRestSerde with Iceberg RESTSerializers + kebab-case ObjectMapper - IcebergRestSerdeConfig registers converter without affecting existing Jackson Error handling: - IcebergRestExceptionHandler scoped to controller, returns Iceberg ErrorResponse Service: - TablesService.searchTables(databaseId, actingPrincipal) overload for auth --- .../IcebergRestCatalogController.java | 140 ++++++++++++++++++ .../IcebergRestExceptionHandler.java | 64 ++++++++ .../IcebergRestHttpMessageConverter.java | 45 ++++++ .../tables/controller/IcebergRestSerde.java | 33 +++++ .../controller/IcebergRestSerdeConfig.java | 20 +++ .../tables/services/TablesService.java | 9 ++ .../tables/services/TablesServiceImpl.java | 7 + 7 files changed, 318 insertions(+) create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java new file mode 100644 index 000000000..d58e2253f --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java @@ -0,0 +1,140 @@ +package com.linkedin.openhouse.tables.controller; + +import static com.linkedin.openhouse.common.security.AuthenticationUtils.extractAuthenticatedUserPrincipal; + +import com.linkedin.openhouse.common.exception.NoSuchUserTableException; +import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; +import com.linkedin.openhouse.tables.api.validator.TablesApiValidator; +import com.linkedin.openhouse.tables.generated.iceberg.api.CatalogApiApi; +import com.linkedin.openhouse.tables.generated.iceberg.api.ConfigurationApiApi; +import com.linkedin.openhouse.tables.services.TablesService; +import io.swagger.v3.oas.annotations.Hidden; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.responses.ConfigResponse; +import org.apache.iceberg.rest.responses.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.context.request.NativeWebRequest; + +/** + * Read-only Iceberg REST Catalog surface. + * + *

Implements the generated {@link CatalogApiApi} and {@link ConfigurationApiApi} interfaces from + * the upstream Iceberg REST OpenAPI spec. Only config/list/load/exists are overridden; all other + * endpoints inherit the generated 501 (Not Implemented) default. + * + *

The {@code /v1/config} endpoint returns a {@code prefix} override so that the Iceberg REST + * client addresses all subsequent requests via {@code /v1/{prefix}/namespaces/...}, keeping them + * separate from the existing OpenHouse API routes under {@code /v1/databases/...}. + * + *

Serialization of Iceberg REST types is handled by {@link IcebergRestHttpMessageConverter}, + * registered in {@link IcebergRestSerdeConfig}. + */ +@Hidden // Exclude from SpringDoc OpenAPI spec to avoid operationId clashes with OpenHouse API +@RestController +public class IcebergRestCatalogController implements CatalogApiApi, ConfigurationApiApi { + + /** Prefix returned by {@code /v1/config} and used in all Iceberg REST routes. */ + public static final String ICEBERG_REST_PREFIX = "iceberg"; + + private final OpenHouseInternalCatalog openHouseInternalCatalog; + + private final TablesService tablesService; + + private final TablesApiValidator tablesApiValidator; + + public IcebergRestCatalogController( + OpenHouseInternalCatalog openHouseInternalCatalog, + TablesService tablesService, + TablesApiValidator tablesApiValidator) { + this.openHouseInternalCatalog = openHouseInternalCatalog; + this.tablesService = tablesService; + this.tablesApiValidator = tablesApiValidator; + } + + /** Resolves the diamond-inherited {@code getRequest()} from both interfaces. */ + @Override + public Optional getRequest() { + return Optional.empty(); + } + + @Override + public ResponseEntity getConfig(String warehouse) { + ConfigResponse response = + ConfigResponse.builder().withOverride("prefix", ICEBERG_REST_PREFIX).build(); + return ResponseEntity.ok(response); + } + + @Override + public ResponseEntity listTables( + String prefix, String namespace, String pageToken, Integer pageSize) { + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + tablesApiValidator.validateSearchTables(databaseId); + + List tableIdentifiers = + tablesService.searchTables(databaseId, extractAuthenticatedUserPrincipal()).stream() + .map(table -> TableIdentifier.of(icebergNamespace, table.getTableId())) + .collect(Collectors.toList()); + ListTablesResponse response = ListTablesResponse.builder().addAll(tableIdentifiers).build(); + return ResponseEntity.ok(response); + } + + @Override + public ResponseEntity loadTable( + String prefix, + String namespace, + String table, + String xIcebergAccessDelegation, + String ifNoneMatch, + String snapshots) { + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + tablesApiValidator.validateGetTable(databaseId, table); + + // Reuse the existing table-read authorization and lock visibility checks. + try { + tablesService.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); + } catch (NoSuchUserTableException e) { + throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); + } + + LoadTableResponse response = + CatalogHandlers.loadTable( + openHouseInternalCatalog, TableIdentifier.of(icebergNamespace, table)); + return ResponseEntity.ok(response); + } + + @Override + public ResponseEntity tableExists(String prefix, String namespace, String table) { + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + tablesApiValidator.validateGetTable(databaseId, table); + + try { + tablesService.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); + } catch (NoSuchUserTableException e) { + throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); + } + + return ResponseEntity.noContent().build(); + } + + private Namespace decodeSingleLevelNamespace(String encodedNamespace) { + Namespace namespace = RESTUtil.decodeNamespace(encodedNamespace); + if (namespace.isEmpty() || namespace.levels().length != 1) { + throw new NoSuchNamespaceException("Invalid namespace: %s", namespace); + } + + return namespace; + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java new file mode 100644 index 000000000..0d56a9a88 --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java @@ -0,0 +1,64 @@ +package com.linkedin.openhouse.tables.controller; + +import com.linkedin.openhouse.common.exception.RequestValidationFailureException; +import org.apache.iceberg.exceptions.ForbiddenException; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.responses.ErrorResponse; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; + +/** Scoped exception mapper for Iceberg REST endpoints. */ +@Order(Ordered.HIGHEST_PRECEDENCE) +@RestControllerAdvice(assignableTypes = IcebergRestCatalogController.class) +public class IcebergRestExceptionHandler { + + @ExceptionHandler(NoSuchTableException.class) + public ResponseEntity handleNoSuchTable(NoSuchTableException e) { + return errorResponse(404, e.getMessage(), NoSuchTableException.class.getSimpleName(), e); + } + + @ExceptionHandler(NoSuchNamespaceException.class) + public ResponseEntity handleNoSuchNamespace(NoSuchNamespaceException e) { + return errorResponse(404, e.getMessage(), NoSuchNamespaceException.class.getSimpleName(), e); + } + + @ExceptionHandler({RequestValidationFailureException.class, IllegalArgumentException.class}) + public ResponseEntity handleBadRequest(Exception e) { + return errorResponse(400, e.getMessage(), IllegalArgumentException.class.getSimpleName(), e); + } + + @ExceptionHandler(AccessDeniedException.class) + public ResponseEntity handleForbidden(AccessDeniedException e) { + return errorResponse(403, e.getMessage(), ForbiddenException.class.getSimpleName(), e); + } + + @ExceptionHandler(UnsupportedOperationException.class) + public ResponseEntity handleNotImplemented(UnsupportedOperationException e) { + return errorResponse( + 501, e.getMessage(), UnsupportedOperationException.class.getSimpleName(), e); + } + + @ExceptionHandler(Exception.class) + public ResponseEntity handleDefault(Exception e) { + return errorResponse(500, e.getMessage(), e.getClass().getSimpleName(), e); + } + + private ResponseEntity errorResponse( + int statusCode, String message, String type, Throwable throwable) { + ErrorResponse response = + ErrorResponse.builder() + .responseCode(statusCode) + .withMessage(message) + .withType(type) + .build(); + return ResponseEntity.status(statusCode) + .contentType(MediaType.APPLICATION_JSON) + .body(IcebergRestSerde.toJson(response)); + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java new file mode 100644 index 000000000..e55f106fe --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java @@ -0,0 +1,45 @@ +package com.linkedin.openhouse.tables.controller; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import org.apache.iceberg.rest.RESTResponse; +import org.springframework.http.HttpInputMessage; +import org.springframework.http.HttpOutputMessage; +import org.springframework.http.MediaType; +import org.springframework.http.converter.AbstractHttpMessageConverter; + +/** + * Spring {@link org.springframework.http.converter.HttpMessageConverter} for Iceberg {@link + * RESTResponse} types. Uses {@link IcebergRestSerde} (Iceberg custom serializers, kebab-case) to + * write JSON because Iceberg REST types do not follow JavaBean conventions and cannot be serialized + * by Spring's default Jackson converter. + * + *

This converter only handles writes (responses). Deserialization is not supported because we do + * not accept Iceberg REST request bodies through Spring MVC. + */ +public class IcebergRestHttpMessageConverter extends AbstractHttpMessageConverter { + + public IcebergRestHttpMessageConverter() { + super(MediaType.APPLICATION_JSON); + } + + @Override + protected boolean supports(Class clazz) { + return RESTResponse.class.isAssignableFrom(clazz); + } + + @Override + protected RESTResponse readInternal( + Class clazz, HttpInputMessage inputMessage) { + throw new UnsupportedOperationException("Iceberg REST request deserialization not supported"); + } + + @Override + protected void writeInternal(RESTResponse response, HttpOutputMessage outputMessage) + throws IOException { + OutputStream body = outputMessage.getBody(); + body.write(IcebergRestSerde.toJson(response).getBytes(StandardCharsets.UTF_8)); + body.flush(); + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java new file mode 100644 index 000000000..e57b20df4 --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerde.java @@ -0,0 +1,33 @@ +package com.linkedin.openhouse.tables.controller; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import org.apache.iceberg.rest.RESTSerializers; + +/** Serde helper for Iceberg REST payloads that require kebab-case and Iceberg serializers. */ +final class IcebergRestSerde { + + private static final ObjectMapper MAPPER = new ObjectMapper(new JsonFactory()); + + static { + MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + MAPPER.setPropertyNamingStrategy(new PropertyNamingStrategies.KebabCaseStrategy()); + RESTSerializers.registerAll(MAPPER); + } + + private IcebergRestSerde() {} + + static String toJson(Object payload) { + try { + return MAPPER.writeValueAsString(payload); + } catch (JsonProcessingException e) { + throw new IllegalStateException("Unable to serialize Iceberg REST response payload", e); + } + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java new file mode 100644 index 000000000..ce9cd8b8e --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java @@ -0,0 +1,20 @@ +package com.linkedin.openhouse.tables.controller; + +import java.util.List; +import org.springframework.context.annotation.Configuration; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +/** + * Registers the {@link IcebergRestHttpMessageConverter} so that Spring MVC can serialize typed + * Iceberg REST responses ({@code ConfigResponse}, {@code LoadTableResponse}, etc.) returned by + * {@link IcebergRestCatalogController}. + */ +@Configuration +public class IcebergRestSerdeConfig implements WebMvcConfigurer { + + @Override + public void extendMessageConverters(List> converters) { + converters.add(0, new IcebergRestHttpMessageConverter()); + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java index 363ec0cdd..abbe4f5cf 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java @@ -32,6 +32,15 @@ public interface TablesService { */ List searchTables(String databaseId); + /** + * Given a databaseId, prepare list of {@link TableDto}s if actingPrincipal has list permission. + * + * @param databaseId + * @param actingPrincipal + * @return list of {@link TableDto} + */ + List searchTables(String databaseId, String actingPrincipal); + /** * Given a databaseId, prepare list of {@link TableDto}s. * diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java index 050e6635f..9500e6273 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java @@ -79,6 +79,13 @@ public List searchTables(String databaseId) { return openHouseInternalRepository.searchTables(databaseId); } + @Override + public List searchTables(String databaseId, String actingPrincipal) { + authorizationUtils.checkDatabasePrivilege( + databaseId, actingPrincipal, Privileges.GET_TABLE_METADATA); + return searchTables(databaseId); + } + @Override public Page searchTables(String databaseId, int page, int size, String sortBy) { Pageable pageable = createPageable(page, size, sortBy, null); From 34ced608ca8e2a8d98f1c809c6fd090da82701e6 Mon Sep 17 00:00:00 2001 From: Christian Bush Date: Thu, 20 Aug 2026 22:30:12 -0700 Subject: [PATCH 2/6] Refine the Iceberg REST runtime facade Keep the generated controller transport-only while delegating through a protocol handler that preserves existing table behavior, errors, auditing, and rollout controls. --- .../cluster/configs/ClusterProperties.java | 3 + docs/iceberg-rest-catalog.md | 61 ++++++ .../docker-compose/oh-only/docker-compose.yml | 2 + .../api/handler/IcebergRestApiHandler.java | 26 +++ .../impl/OpenHouseIcebergRestApiHandler.java | 177 ++++++++++++++++++ .../IcebergRestCatalogController.java | 112 ++--------- .../IcebergRestExceptionHandler.java | 38 ++-- .../IcebergRestHttpMessageConverter.java | 25 ++- .../controller/IcebergRestSerdeConfig.java | 5 +- .../tables/services/TablesService.java | 9 - .../tables/services/TablesServiceImpl.java | 7 - .../src/main/resources/application.properties | 1 + .../OpenHouseIcebergRestApiHandlerTest.java | 142 ++++++++++++++ .../IcebergRestCatalogControllerTest.java | 172 +++++++++++++++++ .../IcebergRestFeatureFlagTest.java | 41 ++++ 15 files changed, 683 insertions(+), 138 deletions(-) create mode 100644 docs/iceberg-rest-catalog.md create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java create mode 100644 services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java create mode 100644 services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java create mode 100644 services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java create mode 100644 services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestFeatureFlagTest.java diff --git a/cluster/configs/src/main/java/com/linkedin/openhouse/cluster/configs/ClusterProperties.java b/cluster/configs/src/main/java/com/linkedin/openhouse/cluster/configs/ClusterProperties.java index 9d0c81876..ba894488b 100644 --- a/cluster/configs/src/main/java/com/linkedin/openhouse/cluster/configs/ClusterProperties.java +++ b/cluster/configs/src/main/java/com/linkedin/openhouse/cluster/configs/ClusterProperties.java @@ -96,4 +96,7 @@ public class ClusterProperties { // string @Value("${cluster.tables.allowed-client-name-values:}") private List allowedClientNameValues; + + @Value("${cluster.tables.iceberg-rest.enabled:false}") + private boolean clusterTablesIcebergRestEnabled; } diff --git a/docs/iceberg-rest-catalog.md b/docs/iceberg-rest-catalog.md new file mode 100644 index 000000000..7662e8c63 --- /dev/null +++ b/docs/iceberg-rest-catalog.md @@ -0,0 +1,61 @@ +# Iceberg REST catalog + +OpenHouse exposes a read-only Apache Iceberg REST Catalog facade for new clients while preserving +the existing OpenHouse APIs and business behavior. + +## Enablement + +The facade is disabled by default. Enable it with: + +```properties +cluster.tables.iceberg-rest.enabled=true +``` + +`GET /v1/config` returns the `iceberg` route prefix and advertises only the implemented endpoints: + +- `GET /v1/{prefix}/namespaces/{namespace}/tables` +- `GET /v1/{prefix}/namespaces/{namespace}/tables/{table}` +- `HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}` + +## Architecture + +The OpenAPI-generated interfaces own the HTTP contract. `IcebergRestCatalogController` is a thin +Spring MVC adapter, and `IcebergRestApiHandler` translates the Iceberg protocol to existing +`TablesApiHandler` and `OpenHouseInternalCatalog` behavior. The facade does not add business rules +or change the existing OpenHouse endpoints. + +Iceberg response types use a narrowly scoped Spring `HttpMessageConverter`. Errors are translated +by controller-scoped advice into the standard Iceberg error envelope. + +## Compatibility and limitations + +- Only single-level namespaces are supported. +- The optional `warehouse` configuration hint does not select a different OpenHouse warehouse. +- List responses support opaque continuation tokens and page sizes from 1 through 1000. +- Table loads return all snapshots. The `snapshots=refs` projection is explicitly unsupported. +- Access delegation may be requested, but this read-only version does not vend credentials. +- Conditional ETag responses are not currently emitted. +- Namespace, table-write, view, transaction, credential, and OAuth endpoints are not advertised. + +Existing OpenHouse APIs remain supported. Client migrations can therefore be incremental. + +## Observability and audit + +Spring Boot records the facade through the standard `http.server.requests` metrics, including URI, +status, and latency. Table reads delegate through `TablesApiHandler`, retaining existing +authorization, lock visibility, and table-read audit behavior. + +## Contract maintenance + +`spec/iceberg-rest-catalog-open-api.yaml` is the byte-pinned upstream Iceberg 1.10 specification. +`spec/iceberg-rest-catalog-readonly-open-api.yaml` is generated from it: + +```bash +python3 spec/generate-iceberg-rest-readonly-spec.py \ + spec/iceberg-rest-catalog-open-api.yaml \ + spec/iceberg-rest-catalog-readonly-open-api.yaml +``` + +The Gradle build verifies checksums locally and generates only the four Phase 1 operations. When +updating the upstream specification, regenerate the read-only profile and update both pinned +checksums in `services/tables/build.gradle`. diff --git a/infra/recipes/docker-compose/oh-only/docker-compose.yml b/infra/recipes/docker-compose/oh-only/docker-compose.yml index 823a21131..0ce12ecb4 100644 --- a/infra/recipes/docker-compose/oh-only/docker-compose.yml +++ b/infra/recipes/docker-compose/oh-only/docker-compose.yml @@ -4,6 +4,8 @@ services: extends: file: ../common/oh-services.yml service: openhouse-tables + environment: + CLUSTER_TABLES_ICEBERGREST_ENABLED: "true" volumes: - ./:/var/config/ depends_on: diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java new file mode 100644 index 000000000..85f348ce8 --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java @@ -0,0 +1,26 @@ +package com.linkedin.openhouse.tables.api.handler; + +import com.linkedin.openhouse.tables.generated.iceberg.model.CatalogConfig; +import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; +import org.apache.iceberg.rest.responses.LoadTableResponse; + +/** Protocol adapter between the generated Iceberg REST API and existing OpenHouse behavior. */ +public interface IcebergRestApiHandler { + + String ICEBERG_REST_PREFIX = "iceberg"; + + CatalogConfig getConfig(String warehouse); + + ListTablesResponse listTables( + String prefix, String namespace, String pageToken, Integer pageSize); + + LoadTableResponse loadTable( + String prefix, + String namespace, + String table, + String accessDelegation, + String ifNoneMatch, + String snapshots); + + void tableExists(String prefix, String namespace, String table); +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java new file mode 100644 index 000000000..78cdbc513 --- /dev/null +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java @@ -0,0 +1,177 @@ +package com.linkedin.openhouse.tables.api.handler.impl; + +import static com.linkedin.openhouse.common.security.AuthenticationUtils.extractAuthenticatedUserPrincipal; + +import com.linkedin.openhouse.common.api.spec.ApiResponse; +import com.linkedin.openhouse.common.exception.NoSuchUserTableException; +import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; +import com.linkedin.openhouse.tables.api.handler.IcebergRestApiHandler; +import com.linkedin.openhouse.tables.api.handler.TablesApiHandler; +import com.linkedin.openhouse.tables.api.spec.v0.response.GetAllTablesResponseBody; +import com.linkedin.openhouse.tables.api.spec.v0.response.GetTableResponseBody; +import com.linkedin.openhouse.tables.generated.iceberg.model.CatalogConfig; +import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.stream.Collectors; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.RESTUtil; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.data.domain.Page; +import org.springframework.stereotype.Component; + +/** Default Iceberg REST adapter backed by existing OpenHouse API handlers and catalog behavior. */ +@Component +@ConditionalOnProperty(value = "cluster.tables.iceberg-rest.enabled", havingValue = "true") +public class OpenHouseIcebergRestApiHandler implements IcebergRestApiHandler { + + static final int DEFAULT_PAGE_SIZE = 100; + static final int MAX_PAGE_SIZE = 1000; + private static final String PAGE_TOKEN_VERSION = "v1"; + private static final List SUPPORTED_ENDPOINTS = + Collections.unmodifiableList( + Arrays.asList( + "GET /v1/{prefix}/namespaces/{namespace}/tables", + "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}")); + + private final TablesApiHandler tablesApiHandler; + private final OpenHouseInternalCatalog openHouseInternalCatalog; + + public OpenHouseIcebergRestApiHandler( + TablesApiHandler tablesApiHandler, OpenHouseInternalCatalog openHouseInternalCatalog) { + this.tablesApiHandler = tablesApiHandler; + this.openHouseInternalCatalog = openHouseInternalCatalog; + } + + @Override + public CatalogConfig getConfig(String warehouse) { + return new CatalogConfig( + Collections.singletonMap("prefix", ICEBERG_REST_PREFIX), Collections.emptyMap()) + .endpoints(SUPPORTED_ENDPOINTS); + } + + @Override + public ListTablesResponse listTables( + String prefix, String namespace, String pageToken, Integer pageSize) { + validatePrefix(prefix); + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + PageCursor cursor = decodePageToken(pageToken, pageSize); + ApiResponse response = + tablesApiHandler.searchTables( + icebergNamespace.level(0), cursor.getPage(), cursor.getPageSize(), "tableId"); + Page page = response.getResponseBody().getPageResults(); + LinkedHashSet identifiers = + page.getContent().stream() + .map(table -> TableIdentifier.of(icebergNamespace, table.getTableId())) + .collect(Collectors.toCollection(LinkedHashSet::new)); + String nextPageToken = + page.hasNext() ? encodePageToken(cursor.getPage() + 1, cursor.getPageSize()) : null; + return new ListTablesResponse().identifiers(identifiers).nextPageToken(nextPageToken); + } + + @Override + public LoadTableResponse loadTable( + String prefix, + String namespace, + String table, + String accessDelegation, + String ifNoneMatch, + String snapshots) { + validatePrefix(prefix); + if (snapshots != null && !"all".equals(snapshots)) { + throw new UnsupportedOperationException( + "The snapshots=refs projection is not supported by this catalog"); + } + + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + try { + tablesApiHandler.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); + } catch (NoSuchUserTableException e) { + throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); + } + + return CatalogHandlers.loadTable( + openHouseInternalCatalog, TableIdentifier.of(icebergNamespace, table)); + } + + @Override + public void tableExists(String prefix, String namespace, String table) { + validatePrefix(prefix); + Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); + String databaseId = icebergNamespace.level(0); + try { + tablesApiHandler.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); + } catch (NoSuchUserTableException e) { + throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); + } + } + + private static void validatePrefix(String prefix) { + if (!ICEBERG_REST_PREFIX.equals(prefix)) { + throw new IllegalArgumentException("Unsupported Iceberg REST prefix"); + } + } + + private static Namespace decodeSingleLevelNamespace(String encodedNamespace) { + Namespace namespace = RESTUtil.decodeNamespace(encodedNamespace); + if (namespace.isEmpty() || namespace.levels().length != 1) { + throw new NoSuchNamespaceException("Only single-level namespaces are supported"); + } + return namespace; + } + + private static PageCursor decodePageToken(String pageToken, Integer requestedPageSize) { + if (pageToken == null) { + return new PageCursor(0, validatePageSize(requestedPageSize)); + } + + try { + String decoded = new String(Base64.getUrlDecoder().decode(pageToken), StandardCharsets.UTF_8); + String[] parts = decoded.split(":", -1); + if (parts.length != 3 || !PAGE_TOKEN_VERSION.equals(parts[0])) { + throw new IllegalArgumentException("Invalid Iceberg REST page token"); + } + int page = Integer.parseInt(parts[1]); + int pageSize = validatePageSize(Integer.parseInt(parts[2])); + if (page < 1 || (requestedPageSize != null && requestedPageSize != pageSize)) { + throw new IllegalArgumentException("Invalid Iceberg REST page token"); + } + return new PageCursor(page, pageSize); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid Iceberg REST page token", e); + } + } + + private static int validatePageSize(Integer requestedPageSize) { + int pageSize = requestedPageSize == null ? DEFAULT_PAGE_SIZE : requestedPageSize; + if (pageSize < 1 || pageSize > MAX_PAGE_SIZE) { + throw new IllegalArgumentException( + String.format("page-size must be between 1 and %s", MAX_PAGE_SIZE)); + } + return pageSize; + } + + private static String encodePageToken(int page, int pageSize) { + String value = String.format("%s:%s:%s", PAGE_TOKEN_VERSION, page, pageSize); + return Base64.getUrlEncoder() + .withoutPadding() + .encodeToString(value.getBytes(StandardCharsets.UTF_8)); + } + + @lombok.Value + private static class PageCursor { + int page; + int pageSize; + } +} diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java index d58e2253f..a450b2e7a 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java @@ -1,92 +1,50 @@ package com.linkedin.openhouse.tables.controller; -import static com.linkedin.openhouse.common.security.AuthenticationUtils.extractAuthenticatedUserPrincipal; - -import com.linkedin.openhouse.common.exception.NoSuchUserTableException; -import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; -import com.linkedin.openhouse.tables.api.validator.TablesApiValidator; +import com.linkedin.openhouse.tables.api.handler.IcebergRestApiHandler; import com.linkedin.openhouse.tables.generated.iceberg.api.CatalogApiApi; import com.linkedin.openhouse.tables.generated.iceberg.api.ConfigurationApiApi; -import com.linkedin.openhouse.tables.services.TablesService; +import com.linkedin.openhouse.tables.generated.iceberg.model.CatalogConfig; +import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; import io.swagger.v3.oas.annotations.Hidden; -import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; -import org.apache.iceberg.catalog.Namespace; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.exceptions.NoSuchNamespaceException; -import org.apache.iceberg.exceptions.NoSuchTableException; -import org.apache.iceberg.rest.CatalogHandlers; -import org.apache.iceberg.rest.RESTUtil; -import org.apache.iceberg.rest.responses.ConfigResponse; -import org.apache.iceberg.rest.responses.ListTablesResponse; import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.context.request.NativeWebRequest; /** - * Read-only Iceberg REST Catalog surface. - * - *

Implements the generated {@link CatalogApiApi} and {@link ConfigurationApiApi} interfaces from - * the upstream Iceberg REST OpenAPI spec. Only config/list/load/exists are overridden; all other - * endpoints inherit the generated 501 (Not Implemented) default. + * Thin Spring MVC adapter for the generated read-only Iceberg REST contract. * - *

The {@code /v1/config} endpoint returns a {@code prefix} override so that the Iceberg REST - * client addresses all subsequent requests via {@code /v1/{prefix}/namespaces/...}, keeping them - * separate from the existing OpenHouse API routes under {@code /v1/databases/...}. - * - *

Serialization of Iceberg REST types is handled by {@link IcebergRestHttpMessageConverter}, - * registered in {@link IcebergRestSerdeConfig}. + *

Protocol translation and orchestration live in {@link IcebergRestApiHandler}; existing + * OpenHouse handlers and services remain the source of business behavior. */ -@Hidden // Exclude from SpringDoc OpenAPI spec to avoid operationId clashes with OpenHouse API +@Hidden @RestController +@ConditionalOnProperty(value = "cluster.tables.iceberg-rest.enabled", havingValue = "true") public class IcebergRestCatalogController implements CatalogApiApi, ConfigurationApiApi { - /** Prefix returned by {@code /v1/config} and used in all Iceberg REST routes. */ - public static final String ICEBERG_REST_PREFIX = "iceberg"; - - private final OpenHouseInternalCatalog openHouseInternalCatalog; + private final IcebergRestApiHandler icebergRestApiHandler; - private final TablesService tablesService; - - private final TablesApiValidator tablesApiValidator; - - public IcebergRestCatalogController( - OpenHouseInternalCatalog openHouseInternalCatalog, - TablesService tablesService, - TablesApiValidator tablesApiValidator) { - this.openHouseInternalCatalog = openHouseInternalCatalog; - this.tablesService = tablesService; - this.tablesApiValidator = tablesApiValidator; + public IcebergRestCatalogController(IcebergRestApiHandler icebergRestApiHandler) { + this.icebergRestApiHandler = icebergRestApiHandler; } - /** Resolves the diamond-inherited {@code getRequest()} from both interfaces. */ @Override public Optional getRequest() { return Optional.empty(); } @Override - public ResponseEntity getConfig(String warehouse) { - ConfigResponse response = - ConfigResponse.builder().withOverride("prefix", ICEBERG_REST_PREFIX).build(); - return ResponseEntity.ok(response); + public ResponseEntity getConfig(String warehouse) { + return ResponseEntity.ok(icebergRestApiHandler.getConfig(warehouse)); } @Override public ResponseEntity listTables( String prefix, String namespace, String pageToken, Integer pageSize) { - Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); - String databaseId = icebergNamespace.level(0); - tablesApiValidator.validateSearchTables(databaseId); - - List tableIdentifiers = - tablesService.searchTables(databaseId, extractAuthenticatedUserPrincipal()).stream() - .map(table -> TableIdentifier.of(icebergNamespace, table.getTableId())) - .collect(Collectors.toList()); - ListTablesResponse response = ListTablesResponse.builder().addAll(tableIdentifiers).build(); - return ResponseEntity.ok(response); + return ResponseEntity.ok( + icebergRestApiHandler.listTables(prefix, namespace, pageToken, pageSize)); } @Override @@ -97,44 +55,14 @@ public ResponseEntity loadTable( String xIcebergAccessDelegation, String ifNoneMatch, String snapshots) { - Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); - String databaseId = icebergNamespace.level(0); - tablesApiValidator.validateGetTable(databaseId, table); - - // Reuse the existing table-read authorization and lock visibility checks. - try { - tablesService.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); - } catch (NoSuchUserTableException e) { - throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); - } - - LoadTableResponse response = - CatalogHandlers.loadTable( - openHouseInternalCatalog, TableIdentifier.of(icebergNamespace, table)); - return ResponseEntity.ok(response); + return ResponseEntity.ok( + icebergRestApiHandler.loadTable( + prefix, namespace, table, xIcebergAccessDelegation, ifNoneMatch, snapshots)); } @Override public ResponseEntity tableExists(String prefix, String namespace, String table) { - Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); - String databaseId = icebergNamespace.level(0); - tablesApiValidator.validateGetTable(databaseId, table); - - try { - tablesService.getTable(databaseId, table, extractAuthenticatedUserPrincipal()); - } catch (NoSuchUserTableException e) { - throw new NoSuchTableException("Table does not exist: %s.%s", databaseId, table); - } - + icebergRestApiHandler.tableExists(prefix, namespace, table); return ResponseEntity.noContent().build(); } - - private Namespace decodeSingleLevelNamespace(String encodedNamespace) { - Namespace namespace = RESTUtil.decodeNamespace(encodedNamespace); - if (namespace.isEmpty() || namespace.levels().length != 1) { - throw new NoSuchNamespaceException("Invalid namespace: %s", namespace); - } - - return namespace; - } } diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java index 0d56a9a88..103609bf5 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestExceptionHandler.java @@ -1,13 +1,14 @@ package com.linkedin.openhouse.tables.controller; import com.linkedin.openhouse.common.exception.RequestValidationFailureException; +import lombok.extern.slf4j.Slf4j; import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.rest.responses.ErrorResponse; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.core.Ordered; import org.springframework.core.annotation.Order; -import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.AccessDeniedException; import org.springframework.web.bind.annotation.ExceptionHandler; @@ -16,49 +17,48 @@ /** Scoped exception mapper for Iceberg REST endpoints. */ @Order(Ordered.HIGHEST_PRECEDENCE) @RestControllerAdvice(assignableTypes = IcebergRestCatalogController.class) +@ConditionalOnProperty(value = "cluster.tables.iceberg-rest.enabled", havingValue = "true") +@Slf4j public class IcebergRestExceptionHandler { @ExceptionHandler(NoSuchTableException.class) - public ResponseEntity handleNoSuchTable(NoSuchTableException e) { - return errorResponse(404, e.getMessage(), NoSuchTableException.class.getSimpleName(), e); + public ResponseEntity handleNoSuchTable(NoSuchTableException e) { + return errorResponse(404, e.getMessage(), NoSuchTableException.class.getSimpleName()); } @ExceptionHandler(NoSuchNamespaceException.class) - public ResponseEntity handleNoSuchNamespace(NoSuchNamespaceException e) { - return errorResponse(404, e.getMessage(), NoSuchNamespaceException.class.getSimpleName(), e); + public ResponseEntity handleNoSuchNamespace(NoSuchNamespaceException e) { + return errorResponse(404, e.getMessage(), NoSuchNamespaceException.class.getSimpleName()); } @ExceptionHandler({RequestValidationFailureException.class, IllegalArgumentException.class}) - public ResponseEntity handleBadRequest(Exception e) { - return errorResponse(400, e.getMessage(), IllegalArgumentException.class.getSimpleName(), e); + public ResponseEntity handleBadRequest(Exception e) { + return errorResponse(400, e.getMessage(), IllegalArgumentException.class.getSimpleName()); } @ExceptionHandler(AccessDeniedException.class) - public ResponseEntity handleForbidden(AccessDeniedException e) { - return errorResponse(403, e.getMessage(), ForbiddenException.class.getSimpleName(), e); + public ResponseEntity handleForbidden(AccessDeniedException e) { + return errorResponse(403, "Access denied", ForbiddenException.class.getSimpleName()); } @ExceptionHandler(UnsupportedOperationException.class) - public ResponseEntity handleNotImplemented(UnsupportedOperationException e) { - return errorResponse( - 501, e.getMessage(), UnsupportedOperationException.class.getSimpleName(), e); + public ResponseEntity handleNotImplemented(UnsupportedOperationException e) { + return errorResponse(501, e.getMessage(), UnsupportedOperationException.class.getSimpleName()); } @ExceptionHandler(Exception.class) - public ResponseEntity handleDefault(Exception e) { - return errorResponse(500, e.getMessage(), e.getClass().getSimpleName(), e); + public ResponseEntity handleDefault(Exception e) { + log.error("Unhandled Iceberg REST request failure", e); + return errorResponse(500, "Internal server error", "InternalServerError"); } - private ResponseEntity errorResponse( - int statusCode, String message, String type, Throwable throwable) { + private ResponseEntity errorResponse(int statusCode, String message, String type) { ErrorResponse response = ErrorResponse.builder() .responseCode(statusCode) .withMessage(message) .withType(type) .build(); - return ResponseEntity.status(statusCode) - .contentType(MediaType.APPLICATION_JSON) - .body(IcebergRestSerde.toJson(response)); + return ResponseEntity.status(statusCode).body(response); } } diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java index e55f106fe..1ad538d14 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestHttpMessageConverter.java @@ -1,5 +1,7 @@ package com.linkedin.openhouse.tables.controller; +import com.linkedin.openhouse.tables.generated.iceberg.model.CatalogConfig; +import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; @@ -10,15 +12,14 @@ import org.springframework.http.converter.AbstractHttpMessageConverter; /** - * Spring {@link org.springframework.http.converter.HttpMessageConverter} for Iceberg {@link - * RESTResponse} types. Uses {@link IcebergRestSerde} (Iceberg custom serializers, kebab-case) to - * write JSON because Iceberg REST types do not follow JavaBean conventions and cannot be serialized - * by Spring's default Jackson converter. + * Spring {@link org.springframework.http.converter.HttpMessageConverter} for generated and runtime + * Iceberg response types. Uses {@link IcebergRestSerde} (Iceberg custom serializers, kebab-case) to + * write JSON because runtime Iceberg types do not follow JavaBean conventions. * *

This converter only handles writes (responses). Deserialization is not supported because we do * not accept Iceberg REST request bodies through Spring MVC. */ -public class IcebergRestHttpMessageConverter extends AbstractHttpMessageConverter { +public class IcebergRestHttpMessageConverter extends AbstractHttpMessageConverter { public IcebergRestHttpMessageConverter() { super(MediaType.APPLICATION_JSON); @@ -26,17 +27,23 @@ public IcebergRestHttpMessageConverter() { @Override protected boolean supports(Class clazz) { - return RESTResponse.class.isAssignableFrom(clazz); + return RESTResponse.class.isAssignableFrom(clazz) + || CatalogConfig.class.equals(clazz) + || ListTablesResponse.class.equals(clazz); } @Override - protected RESTResponse readInternal( - Class clazz, HttpInputMessage inputMessage) { + public boolean canRead(Class clazz, MediaType mediaType) { + return false; + } + + @Override + protected Object readInternal(Class clazz, HttpInputMessage inputMessage) { throw new UnsupportedOperationException("Iceberg REST request deserialization not supported"); } @Override - protected void writeInternal(RESTResponse response, HttpOutputMessage outputMessage) + protected void writeInternal(Object response, HttpOutputMessage outputMessage) throws IOException { OutputStream body = outputMessage.getBody(); body.write(IcebergRestSerde.toJson(response).getBytes(StandardCharsets.UTF_8)); diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java index ce9cd8b8e..a8295ac76 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestSerdeConfig.java @@ -1,16 +1,17 @@ package com.linkedin.openhouse.tables.controller; import java.util.List; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.context.annotation.Configuration; import org.springframework.http.converter.HttpMessageConverter; import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; /** * Registers the {@link IcebergRestHttpMessageConverter} so that Spring MVC can serialize typed - * Iceberg REST responses ({@code ConfigResponse}, {@code LoadTableResponse}, etc.) returned by - * {@link IcebergRestCatalogController}. + * Iceberg REST responses returned by {@link IcebergRestCatalogController}. */ @Configuration +@ConditionalOnProperty(value = "cluster.tables.iceberg-rest.enabled", havingValue = "true") public class IcebergRestSerdeConfig implements WebMvcConfigurer { @Override diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java index abbe4f5cf..363ec0cdd 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesService.java @@ -32,15 +32,6 @@ public interface TablesService { */ List searchTables(String databaseId); - /** - * Given a databaseId, prepare list of {@link TableDto}s if actingPrincipal has list permission. - * - * @param databaseId - * @param actingPrincipal - * @return list of {@link TableDto} - */ - List searchTables(String databaseId, String actingPrincipal); - /** * Given a databaseId, prepare list of {@link TableDto}s. * diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java index 9500e6273..050e6635f 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/services/TablesServiceImpl.java @@ -79,13 +79,6 @@ public List searchTables(String databaseId) { return openHouseInternalRepository.searchTables(databaseId); } - @Override - public List searchTables(String databaseId, String actingPrincipal) { - authorizationUtils.checkDatabasePrivilege( - databaseId, actingPrincipal, Privileges.GET_TABLE_METADATA); - return searchTables(databaseId); - } - @Override public Page searchTables(String databaseId, int page, int size, String sortBy) { Pageable pageable = createPageable(page, size, sortBy, null); diff --git a/services/tables/src/main/resources/application.properties b/services/tables/src/main/resources/application.properties index d6cfaa5d5..222be5d55 100644 --- a/services/tables/src/main/resources/application.properties +++ b/services/tables/src/main/resources/application.properties @@ -7,6 +7,7 @@ springdoc.swagger-ui.disable-swagger-default-url=true springdoc.swagger-ui.filter=true springdoc.swagger-ui.path=/tables/api-docs springdoc.swagger-ui.operationsSorter=method +cluster.tables.iceberg-rest.enabled=false server.tomcat.basedir=tomcat server.tomcat.accesslog.enabled=true server.tomcat.accesslog.rename-on-rotate=false diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java new file mode 100644 index 000000000..66c459f5d --- /dev/null +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java @@ -0,0 +1,142 @@ +package com.linkedin.openhouse.tables.mock.api.handler.impl; + +import static com.linkedin.openhouse.tables.api.handler.IcebergRestApiHandler.ICEBERG_REST_PREFIX; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import com.linkedin.openhouse.common.api.spec.ApiResponse; +import com.linkedin.openhouse.internal.catalog.OpenHouseInternalCatalog; +import com.linkedin.openhouse.tables.api.handler.TablesApiHandler; +import com.linkedin.openhouse.tables.api.handler.impl.OpenHouseIcebergRestApiHandler; +import com.linkedin.openhouse.tables.api.spec.v0.response.GetAllTablesResponseBody; +import com.linkedin.openhouse.tables.api.spec.v0.response.GetTableResponseBody; +import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; +import java.util.Collections; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.data.domain.PageImpl; +import org.springframework.data.domain.PageRequest; +import org.springframework.http.HttpStatus; + +@ExtendWith(MockitoExtension.class) +public class OpenHouseIcebergRestApiHandlerTest { + + @Mock private TablesApiHandler tablesApiHandler; + @Mock private OpenHouseInternalCatalog openHouseInternalCatalog; + + private OpenHouseIcebergRestApiHandler handler; + + @BeforeEach + void setUp() { + handler = new OpenHouseIcebergRestApiHandler(tablesApiHandler, openHouseInternalCatalog); + } + + @Test + void configAdvertisesOnlyImplementedEndpoints() { + assertThat(handler.getConfig("openhouse").getOverrides()) + .containsEntry("prefix", ICEBERG_REST_PREFIX); + assertThat(handler.getConfig("openhouse").getEndpoints()) + .containsExactly( + "GET /v1/{prefix}/namespaces/{namespace}/tables", + "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}"); + } + + @Test + void listTablesUsesOpaquePaginationToken() { + when(tablesApiHandler.searchTables("db", 0, 1, "tableId")) + .thenReturn(pageResponse("db", "t1", 0, 1, 2)); + + ListTablesResponse firstPage = handler.listTables(ICEBERG_REST_PREFIX, "db", null, 1); + + assertThat(firstPage.getIdentifiers()).containsExactly(TableIdentifier.of("db", "t1")); + assertThat(firstPage.getNextPageToken()).isNotBlank(); + + when(tablesApiHandler.searchTables("db", 1, 1, "tableId")) + .thenReturn(pageResponse("db", "t2", 1, 1, 2)); + ListTablesResponse secondPage = + handler.listTables(ICEBERG_REST_PREFIX, "db", firstPage.getNextPageToken(), null); + + assertThat(secondPage.getIdentifiers()).containsExactly(TableIdentifier.of("db", "t2")); + assertThat(secondPage.getNextPageToken()).isNull(); + } + + @Test + void rejectsInvalidPrefixAndPageToken() { + assertThatThrownBy(() -> handler.listTables("other", "db", null, null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("prefix"); + assertThatThrownBy(() -> handler.listTables(ICEBERG_REST_PREFIX, "db", "invalid", null)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("page token"); + } + + @Test + void rejectsUnsupportedSnapshotProjection() { + assertThatThrownBy(() -> handler.loadTable(ICEBERG_REST_PREFIX, "db", "t1", null, null, "refs")) + .isInstanceOf(UnsupportedOperationException.class) + .hasMessageContaining("snapshots=refs"); + } + + @Test + void loadTableReusesExistingReadHandlerBeforeCatalogLoad() { + when(tablesApiHandler.getTable(eq("db"), eq("t1"), eq("undefined"))) + .thenReturn( + ApiResponse.builder() + .httpStatus(HttpStatus.OK) + .responseBody(GetTableResponseBody.builder().databaseId("db").tableId("t1").build()) + .build()); + TableMetadata metadata = testMetadata("hdfs://warehouse/db/t1"); + TableOperations operations = org.mockito.Mockito.mock(TableOperations.class); + when(operations.current()).thenReturn(metadata); + when(openHouseInternalCatalog.loadTable(TableIdentifier.of("db", "t1"))) + .thenReturn(new BaseTable(operations, "openhouse.db.t1")); + + assertThat( + handler + .loadTable(ICEBERG_REST_PREFIX, "db", "t1", null, null, "all") + .tableMetadata() + .location()) + .isEqualTo(metadata.location()); + verify(tablesApiHandler).getTable("db", "t1", "undefined"); + } + + private static ApiResponse pageResponse( + String databaseId, String tableId, int page, int size, int total) { + GetTableResponseBody table = + GetTableResponseBody.builder().databaseId(databaseId).tableId(tableId).build(); + return ApiResponse.builder() + .httpStatus(HttpStatus.OK) + .responseBody( + GetAllTablesResponseBody.builder() + .pageResults( + new PageImpl<>( + Collections.singletonList(table), PageRequest.of(page, size), total)) + .build()) + .build(); + } + + private static TableMetadata testMetadata(String location) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get())); + return TableMetadata.newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + location, + Collections.emptyMap()); + } +} diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java new file mode 100644 index 000000000..e631bb28a --- /dev/null +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java @@ -0,0 +1,172 @@ +package com.linkedin.openhouse.tables.mock.controller; + +import static com.linkedin.openhouse.tables.api.handler.IcebergRestApiHandler.ICEBERG_REST_PREFIX; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.nullable; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; +import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; + +import com.linkedin.openhouse.tables.api.handler.IcebergRestApiHandler; +import com.linkedin.openhouse.tables.controller.IcebergRestCatalogController; +import com.linkedin.openhouse.tables.controller.IcebergRestExceptionHandler; +import com.linkedin.openhouse.tables.controller.IcebergRestHttpMessageConverter; +import com.linkedin.openhouse.tables.generated.iceberg.model.CatalogConfig; +import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashSet; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.exceptions.NoSuchTableException; +import org.apache.iceberg.rest.responses.LoadTableResponse; +import org.apache.iceberg.types.Types; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.springframework.http.converter.StringHttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.test.web.servlet.MockMvc; +import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; +import org.springframework.test.web.servlet.setup.MockMvcBuilders; + +@ExtendWith(MockitoExtension.class) +public class IcebergRestCatalogControllerTest { + + private MockMvc mvc; + + @Mock private IcebergRestApiHandler icebergRestApiHandler; + + @BeforeEach + public void setup() { + mvc = + MockMvcBuilders.standaloneSetup(new IcebergRestCatalogController(icebergRestApiHandler)) + .setControllerAdvice(new IcebergRestExceptionHandler()) + .setMessageConverters( + new IcebergRestHttpMessageConverter(), + new MappingJackson2HttpMessageConverter(), + new StringHttpMessageConverter()) + .build(); + } + + @Test + public void testConfigAdvertisesSupportedEndpoints() throws Exception { + when(icebergRestApiHandler.getConfig(nullable(String.class))) + .thenReturn( + new CatalogConfig( + Collections.singletonMap("prefix", ICEBERG_REST_PREFIX), Collections.emptyMap()) + .endpoints( + Collections.singletonList("GET /v1/{prefix}/namespaces/{namespace}/tables"))); + + mvc.perform(MockMvcRequestBuilders.get("/v1/config")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.overrides.prefix").value(ICEBERG_REST_PREFIX)) + .andExpect(jsonPath("$.endpoints[0]").exists()); + } + + @Test + public void testListTablesDelegatesTypedResponse() throws Exception { + when(icebergRestApiHandler.listTables( + eq(ICEBERG_REST_PREFIX), eq("db"), nullable(String.class), nullable(Integer.class))) + .thenReturn( + new ListTablesResponse() + .identifiers( + new LinkedHashSet<>( + Arrays.asList( + TableIdentifier.of("db", "tb1"), TableIdentifier.of("db", "tb2")))) + .nextPageToken("next")); + + mvc.perform( + MockMvcRequestBuilders.get("/v1/{prefix}/namespaces/db/tables", ICEBERG_REST_PREFIX) + .param("pageSize", "2")) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.identifiers[0].namespace[0]").value("db")) + .andExpect(jsonPath("$.identifiers[1].name").value("tb2")) + .andExpect(jsonPath("$.next-page-token").value("next")); + } + + @Test + public void testLoadTableDelegatesTypedResponse() throws Exception { + TableMetadata metadata = testMetadata("hdfs://warehouse/db/tb1"); + when(icebergRestApiHandler.loadTable( + eq(ICEBERG_REST_PREFIX), + eq("db"), + eq("tb1"), + nullable(String.class), + nullable(String.class), + nullable(String.class))) + .thenReturn(LoadTableResponse.builder().withTableMetadata(metadata).build()); + + mvc.perform( + MockMvcRequestBuilders.get( + "/v1/{prefix}/namespaces/db/tables/tb1", ICEBERG_REST_PREFIX)) + .andExpect(status().isOk()) + .andExpect(jsonPath("$.metadata-location").value(metadata.metadataFileLocation())) + .andExpect(jsonPath("$.metadata").exists()); + } + + @Test + public void testTypedNotFoundError() throws Exception { + when(icebergRestApiHandler.loadTable( + eq(ICEBERG_REST_PREFIX), + eq("db"), + eq("missing"), + nullable(String.class), + nullable(String.class), + nullable(String.class))) + .thenThrow(new NoSuchTableException("Table does not exist")); + + mvc.perform( + MockMvcRequestBuilders.get( + "/v1/{prefix}/namespaces/db/tables/missing", ICEBERG_REST_PREFIX)) + .andExpect(status().isNotFound()) + .andExpect(jsonPath("$.error.code").value(404)) + .andExpect(jsonPath("$.error.type").value("NoSuchTableException")); + } + + @Test + public void testForbiddenErrorIsSanitized() throws Exception { + when(icebergRestApiHandler.loadTable( + eq(ICEBERG_REST_PREFIX), + eq("db"), + eq("private"), + nullable(String.class), + nullable(String.class), + nullable(String.class))) + .thenThrow(new AccessDeniedException("sensitive policy details")); + + mvc.perform( + MockMvcRequestBuilders.get( + "/v1/{prefix}/namespaces/db/tables/private", ICEBERG_REST_PREFIX)) + .andExpect(status().isForbidden()) + .andExpect(jsonPath("$.error.message").value("Access denied")) + .andExpect(jsonPath("$.error.type").value("ForbiddenException")); + } + + @Test + public void testHeadDelegates() throws Exception { + mvc.perform( + MockMvcRequestBuilders.head( + "/v1/{prefix}/namespaces/db/tables/tb1", ICEBERG_REST_PREFIX)) + .andExpect(status().isNoContent()); + + verify(icebergRestApiHandler).tableExists(ICEBERG_REST_PREFIX, "db", "tb1"); + } + + private static TableMetadata testMetadata(String location) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.LongType.get())); + return TableMetadata.newTableMetadata( + schema, + PartitionSpec.unpartitioned(), + SortOrder.unsorted(), + location, + Collections.emptyMap()); + } +} diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestFeatureFlagTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestFeatureFlagTest.java new file mode 100644 index 000000000..8a0de4815 --- /dev/null +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestFeatureFlagTest.java @@ -0,0 +1,41 @@ +package com.linkedin.openhouse.tables.mock.controller; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; + +import com.linkedin.openhouse.tables.api.handler.IcebergRestApiHandler; +import com.linkedin.openhouse.tables.controller.IcebergRestCatalogController; +import org.junit.jupiter.api.Test; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +public class IcebergRestFeatureFlagTest { + + private final ApplicationContextRunner contextRunner = + new ApplicationContextRunner().withUserConfiguration(TestConfiguration.class); + + @Test + void controllerIsDisabledByDefault() { + contextRunner.run( + context -> assertThat(context).doesNotHaveBean(IcebergRestCatalogController.class)); + } + + @Test + void controllerCanBeEnabled() { + contextRunner + .withPropertyValues("cluster.tables.iceberg-rest.enabled=true") + .run(context -> assertThat(context).hasSingleBean(IcebergRestCatalogController.class)); + } + + @Configuration(proxyBeanMethods = false) + @Import(IcebergRestCatalogController.class) + static class TestConfiguration { + + @Bean + IcebergRestApiHandler icebergRestApiHandler() { + return mock(IcebergRestApiHandler.class); + } + } +} From 5b1a7089bf09fe9b1f5b23c5be08fe1202694a2f Mon Sep 17 00:00:00 2001 From: Christian Bush Date: Fri, 21 Aug 2026 09:22:57 -0700 Subject: [PATCH 3/6] Accept Iceberg 1.11 loadTable referenced-by on the REST facade. Keep Phase 1 behavior unchanged by ignoring the new optional query parameter while matching the generated 1.11 contract signatures. --- docs/iceberg-rest-catalog.md | 19 +++++++------------ .../api/handler/IcebergRestApiHandler.java | 3 ++- .../impl/OpenHouseIcebergRestApiHandler.java | 4 +++- .../IcebergRestCatalogController.java | 11 +++++++++-- .../OpenHouseIcebergRestApiHandlerTest.java | 5 +++-- .../IcebergRestCatalogControllerTest.java | 3 +++ 6 files changed, 27 insertions(+), 18 deletions(-) diff --git a/docs/iceberg-rest-catalog.md b/docs/iceberg-rest-catalog.md index 7662e8c63..9f76689fb 100644 --- a/docs/iceberg-rest-catalog.md +++ b/docs/iceberg-rest-catalog.md @@ -33,6 +33,7 @@ by controller-scoped advice into the standard Iceberg error envelope. - The optional `warehouse` configuration hint does not select a different OpenHouse warehouse. - List responses support opaque continuation tokens and page sizes from 1 through 1000. - Table loads return all snapshots. The `snapshots=refs` projection is explicitly unsupported. +- The Iceberg 1.11 `referenced-by` query parameter is accepted and ignored. - Access delegation may be requested, but this read-only version does not vend credentials. - Conditional ETag responses are not currently emitted. - Namespace, table-write, view, transaction, credential, and OAuth endpoints are not advertised. @@ -47,15 +48,9 @@ authorization, lock visibility, and table-read audit behavior. ## Contract maintenance -`spec/iceberg-rest-catalog-open-api.yaml` is the byte-pinned upstream Iceberg 1.10 specification. -`spec/iceberg-rest-catalog-readonly-open-api.yaml` is generated from it: - -```bash -python3 spec/generate-iceberg-rest-readonly-spec.py \ - spec/iceberg-rest-catalog-open-api.yaml \ - spec/iceberg-rest-catalog-readonly-open-api.yaml -``` - -The Gradle build verifies checksums locally and generates only the four Phase 1 operations. When -updating the upstream specification, regenerate the read-only profile and update both pinned -checksums in `services/tables/build.gradle`. +`spec/iceberg-rest-catalog-open-api.yaml` is the checked-in Phase 1 read-only profile derived from +Apache Iceberg `apache-iceberg-1.11.0`. The file is held constant and is not regenerated by the +build. Its header comments document upstream provenance and how to diff against the full Iceberg +OpenAPI. Gradle verifies the file checksum and generates Spring interfaces for the four Phase 1 +operations. When bumping the contract, update the YAML and the pinned checksum in +`services/tables/build.gradle` together. diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java index 85f348ce8..1a082ada7 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/IcebergRestApiHandler.java @@ -20,7 +20,8 @@ LoadTableResponse loadTable( String table, String accessDelegation, String ifNoneMatch, - String snapshots); + String snapshots, + String referencedBy); void tableExists(String prefix, String namespace, String table); } diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java index 78cdbc513..fe0e559b8 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java @@ -86,12 +86,14 @@ public LoadTableResponse loadTable( String table, String accessDelegation, String ifNoneMatch, - String snapshots) { + String snapshots, + String referencedBy) { validatePrefix(prefix); if (snapshots != null && !"all".equals(snapshots)) { throw new UnsupportedOperationException( "The snapshots=refs projection is not supported by this catalog"); } + // Iceberg 1.11 loadTable may send referenced-by for view-load chains; Phase 1 ignores it. Namespace icebergNamespace = decodeSingleLevelNamespace(namespace); String databaseId = icebergNamespace.level(0); diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java index a450b2e7a..06bbe107e 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/controller/IcebergRestCatalogController.java @@ -54,10 +54,17 @@ public ResponseEntity loadTable( String table, String xIcebergAccessDelegation, String ifNoneMatch, - String snapshots) { + String snapshots, + String referencedBy) { return ResponseEntity.ok( icebergRestApiHandler.loadTable( - prefix, namespace, table, xIcebergAccessDelegation, ifNoneMatch, snapshots)); + prefix, + namespace, + table, + xIcebergAccessDelegation, + ifNoneMatch, + snapshots, + referencedBy)); } @Override diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java index 66c459f5d..047bdd52e 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/api/handler/impl/OpenHouseIcebergRestApiHandlerTest.java @@ -87,7 +87,8 @@ void rejectsInvalidPrefixAndPageToken() { @Test void rejectsUnsupportedSnapshotProjection() { - assertThatThrownBy(() -> handler.loadTable(ICEBERG_REST_PREFIX, "db", "t1", null, null, "refs")) + assertThatThrownBy( + () -> handler.loadTable(ICEBERG_REST_PREFIX, "db", "t1", null, null, "refs", null)) .isInstanceOf(UnsupportedOperationException.class) .hasMessageContaining("snapshots=refs"); } @@ -108,7 +109,7 @@ void loadTableReusesExistingReadHandlerBeforeCatalogLoad() { assertThat( handler - .loadTable(ICEBERG_REST_PREFIX, "db", "t1", null, null, "all") + .loadTable(ICEBERG_REST_PREFIX, "db", "t1", null, null, "all", null) .tableMetadata() .location()) .isEqualTo(metadata.location()); diff --git a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java index e631bb28a..af30fd547 100644 --- a/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java +++ b/services/tables/src/test/java/com/linkedin/openhouse/tables/mock/controller/IcebergRestCatalogControllerTest.java @@ -101,6 +101,7 @@ public void testLoadTableDelegatesTypedResponse() throws Exception { eq("tb1"), nullable(String.class), nullable(String.class), + nullable(String.class), nullable(String.class))) .thenReturn(LoadTableResponse.builder().withTableMetadata(metadata).build()); @@ -120,6 +121,7 @@ public void testTypedNotFoundError() throws Exception { eq("missing"), nullable(String.class), nullable(String.class), + nullable(String.class), nullable(String.class))) .thenThrow(new NoSuchTableException("Table does not exist")); @@ -139,6 +141,7 @@ public void testForbiddenErrorIsSanitized() throws Exception { eq("private"), nullable(String.class), nullable(String.class), + nullable(String.class), nullable(String.class))) .thenThrow(new AccessDeniedException("sensitive policy details")); From fef259346e3e2baf3a875890e48105add07790c6 Mon Sep 17 00:00:00 2001 From: Christian Bush Date: Fri, 21 Aug 2026 11:27:27 -0700 Subject: [PATCH 4/6] Document automated Iceberg REST OpenAPI profile upgrades. --- docs/iceberg-rest-catalog.md | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/iceberg-rest-catalog.md b/docs/iceberg-rest-catalog.md index 9f76689fb..e5507ce13 100644 --- a/docs/iceberg-rest-catalog.md +++ b/docs/iceberg-rest-catalog.md @@ -48,9 +48,25 @@ authorization, lock visibility, and table-read audit behavior. ## Contract maintenance -`spec/iceberg-rest-catalog-open-api.yaml` is the checked-in Phase 1 read-only profile derived from -Apache Iceberg `apache-iceberg-1.11.0`. The file is held constant and is not regenerated by the -build. Its header comments document upstream provenance and how to diff against the full Iceberg -OpenAPI. Gradle verifies the file checksum and generates Spring interfaces for the four Phase 1 -operations. When bumping the contract, update the YAML and the pinned checksum in -`services/tables/build.gradle` together. +`spec/iceberg-rest-catalog-open-api.yaml` is the checked-in Phase 1 read-only profile. The Gradle +build verifies its checksum and generates Spring interfaces from it; it does not regenerate the +YAML. + +To bump Iceberg OpenAPI (same Phase 1 allowlist): + +```bash +# either +python3 spec/upgrade_iceberg_rest_profile.py --tag apache-iceberg-1.12.0 +# or +./gradlew :services:tables:upgradeIcebergRestProfile -PicebergRestTag=apache-iceberg-1.12.0 + +./gradlew :services:tables:icebergRestValidateSpec :services:tables:compileJava +``` + +The upgrade tool downloads upstream, keeps only the allowlisted operations in +`spec/upgrade_iceberg_rest_profile.py` (`KEEP_OPERATIONS`), rewrites the checked-in YAML header, and +updates `icebergRestSpecSha256` in `services/tables/build.gradle`. Review generated signature +drift, then keep `SUPPORTED_ENDPOINTS` aligned with implemented routes. + +To add a new resource, extend `KEEP_OPERATIONS`, re-run the upgrade, implement the handler, and +advertise the endpoint. From f00c99c5d1a6ad61f91a1541d0b892fb70d34c19 Mon Sep 17 00:00:00 2001 From: Christian Bush Date: Fri, 21 Aug 2026 15:45:32 -0700 Subject: [PATCH 5/6] Wire /v1/config endpoints from annotated OpenAPI support. Advertise IcebergRestOpenHouseSupport.SUPPORTED_ENDPOINTS generated from x-openhouse-support markers so YAML remains the source of truth. --- docs/iceberg-rest-catalog.md | 30 ++++++++----------- .../impl/OpenHouseIcebergRestApiHandler.java | 11 ++----- 2 files changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/iceberg-rest-catalog.md b/docs/iceberg-rest-catalog.md index e5507ce13..1dafb2417 100644 --- a/docs/iceberg-rest-catalog.md +++ b/docs/iceberg-rest-catalog.md @@ -48,25 +48,19 @@ authorization, lock visibility, and table-read audit behavior. ## Contract maintenance -`spec/iceberg-rest-catalog-open-api.yaml` is the checked-in Phase 1 read-only profile. The Gradle -build verifies its checksum and generates Spring interfaces from it; it does not regenerate the -YAML. +`spec/iceberg-rest-catalog-open-api.yaml` is the full Apache Iceberg REST OpenAPI with OpenHouse +support declared per operation: -To bump Iceberg OpenAPI (same Phase 1 allowlist): - -```bash -# either -python3 spec/upgrade_iceberg_rest_profile.py --tag apache-iceberg-1.12.0 -# or -./gradlew :services:tables:upgradeIcebergRestProfile -PicebergRestTag=apache-iceberg-1.12.0 - -./gradlew :services:tables:icebergRestValidateSpec :services:tables:compileJava +```yaml +operationId: listTables +x-openhouse-support: supported # or unsupported ``` -The upgrade tool downloads upstream, keeps only the allowlisted operations in -`spec/upgrade_iceberg_rest_profile.py` (`KEEP_OPERATIONS`), rewrites the checked-in YAML header, and -updates `icebergRestSpecSha256` in `services/tables/build.gradle`. Review generated signature -drift, then keep `SUPPORTED_ENDPOINTS` aligned with implemented routes. +The build asserts every operation is annotated, codegens Spring interfaces only for `supported` +operations, and generates `IcebergRestOpenHouseSupport.SUPPORTED_ENDPOINTS` for `/v1/config`. +Marking a new operation `supported` (or changing a supported signature) fails compilation until the +facade implements it. New upstream operations without annotations fail the build. -To add a new resource, extend `KEEP_OPERATIONS`, re-run the upgrade, implement the handler, and -advertise the endpoint. +To upgrade Iceberg OpenAPI: merge the newer upstream YAML into the checked-in file, keep/adjust +`x-openhouse-support` markers, update the pinned checksum in `services/tables/build.gradle`, then +compile. diff --git a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java index fe0e559b8..af1791dfc 100644 --- a/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java +++ b/services/tables/src/main/java/com/linkedin/openhouse/tables/api/handler/impl/OpenHouseIcebergRestApiHandler.java @@ -9,14 +9,13 @@ import com.linkedin.openhouse.tables.api.handler.TablesApiHandler; import com.linkedin.openhouse.tables.api.spec.v0.response.GetAllTablesResponseBody; import com.linkedin.openhouse.tables.api.spec.v0.response.GetTableResponseBody; +import com.linkedin.openhouse.tables.generated.iceberg.IcebergRestOpenHouseSupport; import com.linkedin.openhouse.tables.generated.iceberg.model.CatalogConfig; import com.linkedin.openhouse.tables.generated.iceberg.model.ListTablesResponse; import java.nio.charset.StandardCharsets; -import java.util.Arrays; import java.util.Base64; import java.util.Collections; import java.util.LinkedHashSet; -import java.util.List; import java.util.stream.Collectors; import org.apache.iceberg.catalog.Namespace; import org.apache.iceberg.catalog.TableIdentifier; @@ -37,12 +36,6 @@ public class OpenHouseIcebergRestApiHandler implements IcebergRestApiHandler { static final int DEFAULT_PAGE_SIZE = 100; static final int MAX_PAGE_SIZE = 1000; private static final String PAGE_TOKEN_VERSION = "v1"; - private static final List SUPPORTED_ENDPOINTS = - Collections.unmodifiableList( - Arrays.asList( - "GET /v1/{prefix}/namespaces/{namespace}/tables", - "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}", - "HEAD /v1/{prefix}/namespaces/{namespace}/tables/{table}")); private final TablesApiHandler tablesApiHandler; private final OpenHouseInternalCatalog openHouseInternalCatalog; @@ -57,7 +50,7 @@ public OpenHouseIcebergRestApiHandler( public CatalogConfig getConfig(String warehouse) { return new CatalogConfig( Collections.singletonMap("prefix", ICEBERG_REST_PREFIX), Collections.emptyMap()) - .endpoints(SUPPORTED_ENDPOINTS); + .endpoints(IcebergRestOpenHouseSupport.SUPPORTED_ENDPOINTS); } @Override From 29e853d3d9bdabb88b66e8b0ecbfc6f950f401cd Mon Sep 17 00:00:00 2001 From: Christian Bush Date: Fri, 21 Aug 2026 17:12:52 -0700 Subject: [PATCH 6/6] Document opt-in Iceberg REST OpenAPI support annotations. --- docs/iceberg-rest-catalog.md | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/docs/iceberg-rest-catalog.md b/docs/iceberg-rest-catalog.md index 1dafb2417..e0b4ab641 100644 --- a/docs/iceberg-rest-catalog.md +++ b/docs/iceberg-rest-catalog.md @@ -48,19 +48,18 @@ authorization, lock visibility, and table-read audit behavior. ## Contract maintenance -`spec/iceberg-rest-catalog-open-api.yaml` is the full Apache Iceberg REST OpenAPI with OpenHouse -support declared per operation: +`spec/iceberg-rest-catalog-open-api.yaml` is the full Apache Iceberg REST OpenAPI. OpenHouse support +is opt-in: ```yaml operationId: listTables -x-openhouse-support: supported # or unsupported +x-openhouse-support: supported ``` -The build asserts every operation is annotated, codegens Spring interfaces only for `supported` -operations, and generates `IcebergRestOpenHouseSupport.SUPPORTED_ENDPOINTS` for `/v1/config`. -Marking a new operation `supported` (or changing a supported signature) fails compilation until the -facade implements it. New upstream operations without annotations fail the build. +Operations without the annotation are unsupported. The build codegens Spring interfaces only for +`supported` operations and generates `IcebergRestOpenHouseSupport.SUPPORTED_ENDPOINTS` for +`/v1/config`. Marking a new operation `supported` (or changing a supported signature) fails +compilation until the facade implements it. -To upgrade Iceberg OpenAPI: merge the newer upstream YAML into the checked-in file, keep/adjust -`x-openhouse-support` markers, update the pinned checksum in `services/tables/build.gradle`, then -compile. +To upgrade Iceberg OpenAPI: merge the newer upstream YAML into the checked-in file, add +`x-openhouse-support: supported` where needed, then compile.