From c27891e8b71f69eb91f8840a1985032a2a4058b5 Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Wed, 15 Oct 2025 16:27:47 +0800 Subject: [PATCH 1/3] feat: Implement JSON serialization for REST Catalog API types --- src/iceberg/catalog/rest/CMakeLists.txt | 28 +- src/iceberg/catalog/rest/json_internal.cc | 347 ++++++++++++++++ src/iceberg/catalog/rest/json_internal.h | 165 ++++++++ src/iceberg/catalog/rest/rest_catalog.h | 34 +- src/iceberg/catalog/rest/types.h | 99 +++++ src/iceberg/test/CMakeLists.txt | 3 +- src/iceberg/test/rest_json_internal_test.cc | 418 ++++++++++++++++++++ 7 files changed, 1068 insertions(+), 26 deletions(-) create mode 100644 src/iceberg/catalog/rest/json_internal.cc create mode 100644 src/iceberg/catalog/rest/json_internal.h create mode 100644 src/iceberg/catalog/rest/types.h create mode 100644 src/iceberg/test/rest_json_internal_test.cc diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index 2f9c2f05..f7bdf9c3 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -15,25 +15,35 @@ # specific language governing permissions and limitations # under the License. -set(ICEBERG_REST_SOURCES rest_catalog.cc) +set(ICEBERG_REST_SOURCES rest_catalog.cc json_internal.cc) set(ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS) set(ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS) set(ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS) set(ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS) -list(APPEND ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS - "$,iceberg_static,iceberg_shared>" cpr::cpr) -list(APPEND ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS - "$,iceberg_shared,iceberg_static>" cpr::cpr) +list(APPEND + ICEBERG_REST_STATIC_BUILD_INTERFACE_LIBS + "$,iceberg_static,iceberg_shared>" + cpr::cpr + nlohmann_json::nlohmann_json) +list(APPEND + ICEBERG_REST_SHARED_BUILD_INTERFACE_LIBS + "$,iceberg_shared,iceberg_static>" + cpr::cpr + nlohmann_json::nlohmann_json) list(APPEND ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS - "$,iceberg::iceberg_static,iceberg::iceberg_shared>" - "$,iceberg::cpr,cpr::cpr>") + "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" + "$,Iceberg::cpr,cpr::cpr>" + "$,Iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" +) list(APPEND ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS - "$,iceberg::iceberg_shared,iceberg::iceberg_static>" - "$,iceberg::cpr,cpr::cpr>") + "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" + "$,Iceberg::cpr,cpr::cpr>" + "$,Iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" +) add_iceberg_lib(iceberg_rest SOURCES diff --git a/src/iceberg/catalog/rest/json_internal.cc b/src/iceberg/catalog/rest/json_internal.cc new file mode 100644 index 00000000..22dfbcbd --- /dev/null +++ b/src/iceberg/catalog/rest/json_internal.cc @@ -0,0 +1,347 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "iceberg/catalog/rest/json_internal.h" + +#include +#include +#include + +#include + +#include "iceberg/json_internal.h" +#include "iceberg/result.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" +#include "iceberg/util/json_util_internal.h" +#include "iceberg/util/macros.h" + +namespace iceberg::rest { + +namespace { + +// REST API JSON field constants +constexpr std::string_view kNamespaces = "namespaces"; +constexpr std::string_view kRemovals = "removals"; +constexpr std::string_view kUpdates = "updates"; +constexpr std::string_view kUpdated = "updated"; +constexpr std::string_view kRemoved = "removed"; +constexpr std::string_view kMissing = "missing"; +constexpr std::string_view kIdentifiers = "identifiers"; +constexpr std::string_view kSource = "source"; +constexpr std::string_view kDestination = "destination"; +constexpr std::string_view kMetadataLocation = "metadata-location"; +constexpr std::string_view kMetadata = "metadata"; +constexpr std::string_view kConfig = "config"; +constexpr std::string_view kName = "name"; +constexpr std::string_view kLocation = "location"; +constexpr std::string_view kSchema = "schema"; +constexpr std::string_view kPartitionSpec = "partition-spec"; +constexpr std::string_view kWriteOrder = "write-order"; +constexpr std::string_view kStageCreate = "stage-create"; +constexpr std::string_view kProperties = "properties"; +constexpr std::string_view kOverwrite = "overwrite"; +constexpr std::string_view kNamespace = "namespace"; + +/// Helper function to convert TableIdentifier to JSON +nlohmann::json TableIdentifierToJson(const TableIdentifier& identifier) { + nlohmann::json json; + json[kNamespace] = identifier.ns.levels; + json[kName] = identifier.name; + return json; +} + +/// Helper function to parse TableIdentifier from JSON +Result TableIdentifierFromJson(const nlohmann::json& json) { + TableIdentifier identifier; + + ICEBERG_ASSIGN_OR_RAISE(identifier.ns.levels, + GetJsonValue>(json, kNamespace)); + ICEBERG_ASSIGN_OR_RAISE(identifier.name, GetJsonValue(json, kName)); + + return identifier; +} + +} // namespace + +nlohmann::json ToJson(const ListNamespaceResponse& response) { + nlohmann::json json; + json[kNamespaces] = response.namespaces; + return json; +} + +Result ListNamespaceResponseFromJson(const nlohmann::json& json) { + ListNamespaceResponse response; + + ICEBERG_ASSIGN_OR_RAISE( + response.namespaces, + GetJsonValue>>(json, kNamespaces)); + return response; +} + +nlohmann::json ToJson(const CreateNamespaceRequest& request) { + nlohmann::json json; + json[kNamespaces] = request.namespaces; + SetOptionalField(json, kProperties, request.properties); + return json; +} + +Result CreateNamespaceRequestFromJson( + const nlohmann::json& json) { + CreateNamespaceRequest request; + + ICEBERG_ASSIGN_OR_RAISE(request.namespaces, + GetJsonValue>(json, kNamespaces)); + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(request.properties, + GetJsonValueOptional(json, kProperties)); + + return request; +} + +nlohmann::json ToJson(const CreateNamespaceResponse& response) { + nlohmann::json json; + json[kNamespaces] = response.namespaces; + SetOptionalField(json, kProperties, response.properties); + return json; +} + +Result CreateNamespaceResponseFromJson( + const nlohmann::json& json) { + CreateNamespaceResponse response; + + ICEBERG_ASSIGN_OR_RAISE(response.namespaces, + GetJsonValue>(json, kNamespaces)); + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(response.properties, + GetJsonValueOptional(json, kProperties)); + + return response; +} + +nlohmann::json ToJson(const GetNamespacePropertiesResponse& response) { + nlohmann::json json; + json[kNamespaces] = response.namespaces; + json[kProperties] = response.properties; + return json; +} + +Result GetNamespacePropertiesResponseFromJson( + const nlohmann::json& json) { + GetNamespacePropertiesResponse response; + + ICEBERG_ASSIGN_OR_RAISE(response.namespaces, + GetJsonValue>(json, kNamespaces)); + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(response.properties, GetJsonValue(json, kProperties)); + + return response; +} + +nlohmann::json ToJson(const UpdateNamespacePropsRequest& request) { + nlohmann::json json; + SetOptionalField(json, kRemovals, request.removals); + SetOptionalField(json, kUpdates, request.updates); + return json; +} + +Result UpdateNamespacePropsRequestFromJson( + const nlohmann::json& json) { + UpdateNamespacePropsRequest request; + + ICEBERG_ASSIGN_OR_RAISE( + request.removals, GetJsonValueOptional>(json, kRemovals)); + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(request.updates, GetJsonValueOptional(json, kUpdates)); + + return request; +} + +nlohmann::json ToJson(const UpdateNamespacePropsResponse& response) { + nlohmann::json json; + json[kUpdated] = response.updated; + json[kRemoved] = response.removed; + SetOptionalField(json, kMissing, response.missing); + return json; +} + +Result UpdateNamespacePropsResponseFromJson( + const nlohmann::json& json) { + UpdateNamespacePropsResponse response; + + ICEBERG_ASSIGN_OR_RAISE(response.updated, + GetJsonValue>(json, kUpdated)); + ICEBERG_ASSIGN_OR_RAISE(response.removed, + GetJsonValue>(json, kRemoved)); + ICEBERG_ASSIGN_OR_RAISE(response.missing, + GetJsonValueOptional>(json, kMissing)); + + return response; +} + +nlohmann::json ToJson(const ListTableResponse& response) { + nlohmann::json json; + + nlohmann::json identifiers_json = nlohmann::json::array(); + for (const auto& identifier : response.identifiers) { + identifiers_json.push_back(TableIdentifierToJson(identifier)); + } + json[kIdentifiers] = identifiers_json; + return json; +} + +Result ListTableResponseFromJson(const nlohmann::json& json) { + ListTableResponse response; + + for (const auto& id_json : json[kIdentifiers]) { + ICEBERG_ASSIGN_OR_RAISE(auto identifier, TableIdentifierFromJson(id_json)); + response.identifiers.push_back(std::move(identifier)); + } + return response; +} + +nlohmann::json ToJson(const CreateTableRequest& request) { + nlohmann::json json; + json[kName] = request.name; + SetOptionalField(json, kLocation, request.location); + json[kSchema] = ToJson(*request.schema); + + if (request.partition_spec) { + json[kPartitionSpec] = ToJson(*request.partition_spec); + } + + if (request.write_order) { + json[kWriteOrder] = ToJson(*request.write_order); + } + + SetOptionalField(json, kStageCreate, request.stage_create); + SetOptionalField(json, kProperties, request.properties); + return json; +} + +Result CreateTableRequestFromJson(const nlohmann::json& json) { + CreateTableRequest request; + + ICEBERG_ASSIGN_OR_RAISE(request.name, GetJsonValue(json, kName)); + + ICEBERG_ASSIGN_OR_RAISE(request.location, + GetJsonValueOptional(json, kLocation)); + + ICEBERG_ASSIGN_OR_RAISE(auto schema_json, GetJsonValue(json, kSchema)); + ICEBERG_ASSIGN_OR_RAISE(auto schema_ptr, iceberg::SchemaFromJson(schema_json)); + request.schema = std::move(schema_ptr); + + if (json.contains(kPartitionSpec)) { + ICEBERG_ASSIGN_OR_RAISE(auto partition_spec_json, + GetJsonValue(json, kPartitionSpec)); + ICEBERG_ASSIGN_OR_RAISE( + request.partition_spec, + iceberg::PartitionSpecFromJson(request.schema, partition_spec_json)); + } else { + request.partition_spec = nullptr; + } + + if (json.contains(kWriteOrder)) { + ICEBERG_ASSIGN_OR_RAISE(auto write_order_json, + GetJsonValue(json, kWriteOrder)); + ICEBERG_ASSIGN_OR_RAISE(request.write_order, + iceberg::SortOrderFromJson(write_order_json)); + } else { + request.write_order = nullptr; + } + + ICEBERG_ASSIGN_OR_RAISE(request.stage_create, + GetJsonValueOptional(json, kStageCreate)); + + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(request.properties, + GetJsonValueOptional(json, kProperties)); + + return request; +} + +nlohmann::json ToJson(const RegisterTableRequest& request) { + nlohmann::json json; + json[kName] = request.name; + json[kMetadataLocation] = request.metadata_location; + SetOptionalField(json, kOverwrite, request.overwrite); + return json; +} + +Result RegisterTableRequestFromJson(const nlohmann::json& json) { + RegisterTableRequest request; + + ICEBERG_ASSIGN_OR_RAISE(request.name, GetJsonValue(json, kName)); + ICEBERG_ASSIGN_OR_RAISE(request.metadata_location, + GetJsonValue(json, kMetadataLocation)); + ICEBERG_ASSIGN_OR_RAISE(request.overwrite, + GetJsonValueOptional(json, kOverwrite)); + + return request; +} + +nlohmann::json ToJson(const RenameTableRequest& request) { + nlohmann::json json; + json[kSource] = TableIdentifierToJson(request.source); + json[kDestination] = TableIdentifierToJson(request.destination); + return json; +} + +Result RenameTableRequestFromJson(const nlohmann::json& json) { + RenameTableRequest request; + + ICEBERG_ASSIGN_OR_RAISE(auto source_json, GetJsonValue(json, kSource)); + ICEBERG_ASSIGN_OR_RAISE(request.source, TableIdentifierFromJson(source_json)); + + ICEBERG_ASSIGN_OR_RAISE(auto dest_json, + GetJsonValue(json, kDestination)); + ICEBERG_ASSIGN_OR_RAISE(request.destination, TableIdentifierFromJson(dest_json)); + + return request; +} + +nlohmann::json ToJson(const LoadTableResponse& response) { + nlohmann::json json; + + SetOptionalField(json, kMetadataLocation, response.metadata_location); + json[kMetadata] = iceberg::ToJson(response.metadata); + SetOptionalField(json, kConfig, response.config); + + return json; +} + +Result LoadTableResponseFromJson(const nlohmann::json& json) { + LoadTableResponse response; + + ICEBERG_ASSIGN_OR_RAISE(response.metadata_location, + GetJsonValueOptional(json, kMetadataLocation)); + + ICEBERG_ASSIGN_OR_RAISE(auto metadata_json, + GetJsonValue(json, kMetadata)); + ICEBERG_ASSIGN_OR_RAISE(auto metadata_ptr, + iceberg::TableMetadataFromJson(metadata_json)); + response.metadata = std::move(*metadata_ptr); + + using MapType = std::unordered_map; + ICEBERG_ASSIGN_OR_RAISE(response.config, GetJsonValueOptional(json, kConfig)); + + return response; +} + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/json_internal.h b/src/iceberg/catalog/rest/json_internal.h new file mode 100644 index 00000000..0c537918 --- /dev/null +++ b/src/iceberg/catalog/rest/json_internal.h @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include + +#include "iceberg/catalog/rest/types.h" +#include "iceberg/result.h" + +namespace iceberg::rest { + +/// \brief Serializes a `ListNamespaceResponse` object to JSON. +/// +/// \param response The `ListNamespaceResponse` object to be serialized. +/// \return A JSON object representing the `ListNamespaceResponse`. +nlohmann::json ToJson(const ListNamespaceResponse& response); + +/// \brief Deserializes a JSON object into a `ListNamespaceResponse` object. +/// +/// \param json The JSON object representing a `ListNamespaceResponse`. +/// \return A `ListNamespaceResponse` object or an error if the conversion fails. +Result ListNamespaceResponseFromJson(const nlohmann::json& json); + +/// \brief Serializes a `CreateNamespaceRequest` object to JSON. +/// +/// \param request The `CreateNamespaceRequest` object to be serialized. +/// \return A JSON object representing the `CreateNamespaceRequest`. +nlohmann::json ToJson(const CreateNamespaceRequest& request); + +/// \brief Deserializes a JSON object into a `CreateNamespaceRequest` object. +/// +/// \param json The JSON object representing a `CreateNamespaceRequest`. +/// \return A `CreateNamespaceRequest` object or an error if the conversion fails. +Result CreateNamespaceRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes a `CreateNamespaceResponse` object to JSON. +/// +/// \param response The `CreateNamespaceResponse` object to be serialized. +/// \return A JSON object representing the `CreateNamespaceResponse`. +nlohmann::json ToJson(const CreateNamespaceResponse& response); + +/// \brief Deserializes a JSON object into a `CreateNamespaceResponse` object. +/// +/// \param json The JSON object representing a `CreateNamespaceResponse`. +/// \return A `CreateNamespaceResponse` object or an error if the conversion fails. +Result CreateNamespaceResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `GetNamespacePropertiesResponse` object to JSON. +/// +/// \param response The `GetNamespacePropertiesResponse` object to be serialized. +/// \return A JSON object representing the `GetNamespacePropertiesResponse`. +nlohmann::json ToJson(const GetNamespacePropertiesResponse& response); + +/// \brief Deserializes a JSON object into a `GetNamespacePropertiesResponse` object. +/// +/// \param json The JSON object representing a `GetNamespacePropertiesResponse`. +/// \return A `GetNamespacePropertiesResponse` object or an error if the conversion fails. +Result GetNamespacePropertiesResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes an `UpdateNamespacePropsRequest` object to JSON. +/// +/// \param request The `UpdateNamespacePropsRequest` object to be serialized. +/// \return A JSON object representing the `UpdateNamespacePropsRequest`. +nlohmann::json ToJson(const UpdateNamespacePropsRequest& request); + +/// \brief Deserializes a JSON object into an `UpdateNamespacePropsRequest` object. +/// +/// \param json The JSON object representing an `UpdateNamespacePropsRequest`. +/// \return An `UpdateNamespacePropsRequest` object or an error if the conversion fails. +Result UpdateNamespacePropsRequestFromJson( + const nlohmann::json& json); + +/// \brief Serializes an `UpdateNamespacePropsResponse` object to JSON. +/// +/// \param response The `UpdateNamespacePropsResponse` object to be serialized. +/// \return A JSON object representing the `UpdateNamespacePropsResponse`. +nlohmann::json ToJson(const UpdateNamespacePropsResponse& response); + +/// \brief Deserializes a JSON object into an `UpdateNamespacePropsResponse` object. +/// +/// \param json The JSON object representing an `UpdateNamespacePropsResponse`. +/// \return An `UpdateNamespacePropsResponse` object or an error if the conversion fails. +Result UpdateNamespacePropsResponseFromJson( + const nlohmann::json& json); + +/// \brief Serializes a `ListTableResponse` object to JSON. +/// +/// \param response The `ListTableResponse` object to be serialized. +/// \return A JSON object representing the `ListTableResponse`. +nlohmann::json ToJson(const ListTableResponse& response); + +/// \brief Deserializes a JSON object into a `ListTableResponse` object. +/// +/// \param json The JSON object representing a `ListTableResponse`. +/// \return A `ListTableResponse` object or an error if the conversion fails. +Result ListTableResponseFromJson(const nlohmann::json& json); + +/// \brief Serializes a `CreateTableRequest` object to JSON. +/// +/// \param request The `CreateTableRequest` object to be serialized. +/// \return A JSON object representing the `CreateTableRequest`. +nlohmann::json ToJson(const CreateTableRequest& request); + +/// \brief Deserializes a JSON object into a `CreateTableRequest` object. +/// +/// \param json The JSON object representing a `CreateTableRequest`. +/// \return A `CreateTableRequest` object or an error if the conversion fails. +Result CreateTableRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes a `RegisterTableRequest` object to JSON. +/// +/// \param request The `RegisterTableRequest` object to be serialized. +/// \return A JSON object representing the `RegisterTableRequest`. +nlohmann::json ToJson(const RegisterTableRequest& request); + +/// \brief Deserializes a JSON object into a `RegisterTableRequest` object. +/// +/// \param json The JSON object representing a `RegisterTableRequest`. +/// \return A `RegisterTableRequest` object or an error if the conversion fails. +Result RegisterTableRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes a `RenameTableRequest` object to JSON. +/// +/// \param request The `RenameTableRequest` object to be serialized. +/// \return A JSON object representing the `RenameTableRequest`. +nlohmann::json ToJson(const RenameTableRequest& request); + +/// \brief Deserializes a JSON object into a `RenameTableRequest` object. +/// +/// \param json The JSON object representing a `RenameTableRequest`. +/// \return A `RenameTableRequest` object or an error if the conversion fails. +Result RenameTableRequestFromJson(const nlohmann::json& json); + +/// \brief Serializes a `LoadTableResponse` object to JSON. +/// +/// \param response The `LoadTableResponse` object to be serialized. +/// \return A JSON object representing the `LoadTableResponse`. +nlohmann::json ToJson(const LoadTableResponse& response); + +/// \brief Deserializes a JSON object into a `LoadTableResponse` object. +/// +/// \param json The JSON object representing a `LoadTableResponse`. +/// \return A `LoadTableResponse` object or an error if the conversion fails. +Result LoadTableResponseFromJson(const nlohmann::json& json); + +} // namespace iceberg::rest diff --git a/src/iceberg/catalog/rest/rest_catalog.h b/src/iceberg/catalog/rest/rest_catalog.h index 7b3e205c..4f4b5625 100644 --- a/src/iceberg/catalog/rest/rest_catalog.h +++ b/src/iceberg/catalog/rest/rest_catalog.h @@ -1,19 +1,21 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ #pragma once diff --git a/src/iceberg/catalog/rest/types.h b/src/iceberg/catalog/rest/types.h new file mode 100644 index 00000000..cf0e7eb9 --- /dev/null +++ b/src/iceberg/catalog/rest/types.h @@ -0,0 +1,99 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_identifier.h" +#include "iceberg/table_metadata.h" + +namespace iceberg::rest { + +struct ListNamespaceResponse { + std::vector> namespaces; +}; + +struct CreateNamespaceRequest { + std::vector namespaces; + std::optional> properties; +}; + +struct CreateNamespaceResponse { + std::vector namespaces; + std::optional> properties; +}; + +struct GetNamespacePropertiesResponse { + std::vector namespaces; + std::unordered_map properties; +}; + +struct UpdateNamespacePropsRequest { + std::optional> removals; + std::optional> updates; +}; + +struct UpdateNamespacePropsResponse { + std::vector updated; + std::vector removed; + std::optional> missing; +}; + +struct ListTableResponse { + std::vector identifiers; +}; + +struct CreateTableRequest { + std::string name; + std::optional location; + std::shared_ptr schema; + std::shared_ptr partition_spec; // optional + std::shared_ptr write_order; // optional + std::optional stage_create; + std::optional> properties; +}; + +struct RegisterTableRequest { + std::string name; + std::string metadata_location; + std::optional overwrite; +}; + +struct RenameTableRequest { + TableIdentifier source; + TableIdentifier destination; +}; + +// This is also used for CreateTableResponse and RegisterTableResponse +struct LoadTableResponse { + std::optional metadata_location; + TableMetadata metadata; + std::optional> config; +}; + +// TODO(Li Feiyang): Add UpdateTable request and response + +} // namespace iceberg::rest diff --git a/src/iceberg/test/CMakeLists.txt b/src/iceberg/test/CMakeLists.txt index e687f97c..71540e1b 100644 --- a/src/iceberg/test/CMakeLists.txt +++ b/src/iceberg/test/CMakeLists.txt @@ -148,7 +148,8 @@ if(ICEBERG_BUILD_BUNDLE) endif() if(ICEBERG_BUILD_REST) - add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc) + add_iceberg_test(rest_catalog_test SOURCES rest_catalog_test.cc + rest_json_internal_test.cc) target_link_libraries(rest_catalog_test PRIVATE iceberg_rest_static) target_include_directories(rest_catalog_test PRIVATE ${cpp-httplib_SOURCE_DIR}) endif() diff --git a/src/iceberg/test/rest_json_internal_test.cc b/src/iceberg/test/rest_json_internal_test.cc new file mode 100644 index 00000000..5fc8ad05 --- /dev/null +++ b/src/iceberg/test/rest_json_internal_test.cc @@ -0,0 +1,418 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include +#include + +#include "iceberg/catalog/rest/json_internal.h" +#include "iceberg/partition_spec.h" +#include "iceberg/schema.h" +#include "iceberg/snapshot.h" +#include "iceberg/sort_order.h" +#include "iceberg/table_identifier.h" +#include "iceberg/transform.h" +#include "matchers.h" + +namespace iceberg::rest { + +TEST(RestJsonInternalTest, ListNamespaceResponse) { + ListNamespaceResponse response; + response.namespaces = {{"db1"}, {"db2", "schema1"}}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "namespaces": [["db1"], ["db2", "schema1"]] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = ListNamespaceResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.namespaces, parsed_result->namespaces); +} + +TEST(RestJsonInternalTest, CreateNamespaceRequest) { + CreateNamespaceRequest request; + request.namespaces = {"db1", "schema1"}; + request.properties = std::unordered_map{{"key1", "value1"}, + {"key2", "value2"}}; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "namespaces": ["db1", "schema1"], + "properties": {"key1": "value1", "key2": "value2"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateNamespaceRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(request.namespaces, parsed_result->namespaces); + EXPECT_EQ(*request.properties, *parsed_result->properties); +} + +TEST(RestJsonInternalTest, CreateNamespaceRequestWithoutProperties) { + CreateNamespaceRequest request; + request.namespaces = {"db1"}; + request.properties = std::nullopt; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "namespaces": ["db1"] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateNamespaceRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(request.namespaces, parsed_result->namespaces); + EXPECT_FALSE(parsed_result->properties.has_value()); +} + +TEST(RestJsonInternalTest, CreateNamespaceResponse) { + CreateNamespaceResponse response; + response.namespaces = {"db1", "schema1"}; + response.properties = std::unordered_map{{"created", "true"}}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "namespaces": ["db1", "schema1"], + "properties": {"created": "true"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateNamespaceResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.namespaces, parsed_result->namespaces); + EXPECT_EQ(*response.properties, *parsed_result->properties); +} + +TEST(RestJsonInternalTest, CreateNamespaceResponseWithoutProperties) { + CreateNamespaceResponse response; + response.namespaces = {"db1"}; + response.properties = std::nullopt; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "namespaces": ["db1"] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateNamespaceResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.namespaces, parsed_result->namespaces); + EXPECT_FALSE(parsed_result->properties.has_value()); +} + +TEST(RestJsonInternalTest, GetNamespacePropertiesResponse) { + GetNamespacePropertiesResponse response; + response.namespaces = {"db1", "schema1"}; + response.properties = {{"prop1", "value1"}, {"prop2", "value2"}}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "namespaces": ["db1", "schema1"], + "properties": {"prop1": "value1", "prop2": "value2"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = GetNamespacePropertiesResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.namespaces, parsed_result->namespaces); + EXPECT_EQ(response.properties, parsed_result->properties); +} + +TEST(RestJsonInternalTest, UpdateNamespacePropsRequest) { + UpdateNamespacePropsRequest request; + request.removals = std::vector{"key1", "key2"}; + request.updates = std::unordered_map{{"key3", "value3"}}; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "removals": ["key1", "key2"], + "updates": {"key3": "value3"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = UpdateNamespacePropsRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(*request.removals, *parsed_result->removals); + EXPECT_EQ(*request.updates, *parsed_result->updates); +} + +TEST(RestJsonInternalTest, UpdateNamespacePropsResponse) { + UpdateNamespacePropsResponse response; + response.updated = {"key1", "key2"}; + response.removed = {"key3"}; + response.missing = std::vector{"key4"}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "updated": ["key1", "key2"], + "removed": ["key3"], + "missing": ["key4"] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = UpdateNamespacePropsResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.updated, parsed_result->updated); + EXPECT_EQ(response.removed, parsed_result->removed); + EXPECT_EQ(*response.missing, *parsed_result->missing); +} + +TEST(RestJsonInternalTest, ListTableResponse) { + ListTableResponse response; + response.identifiers = { + TableIdentifier{.ns = Namespace{{"db1"}}, .name = "table1"}, + TableIdentifier{.ns = Namespace{{"db2", "schema1"}}, .name = "table2"}}; + + auto json = ToJson(response); + nlohmann::json expected_json = R"({ + "identifiers": [ + {"namespace": ["db1"], "name": "table1"}, + {"namespace": ["db2", "schema1"], "name": "table2"} + ] + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = ListTableResponseFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(response.identifiers.size(), parsed_result->identifiers.size()); + EXPECT_EQ(response.identifiers[0].name, parsed_result->identifiers[0].name); + EXPECT_EQ(response.identifiers[1].name, parsed_result->identifiers[1].name); +} + +TEST(RestJsonInternalTest, CreateTableRequestBasic) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "data", iceberg::string(), true)}, + 0); + + CreateTableRequest request; + request.name = "test_table"; + request.location = "/tmp/test_location"; + request.schema = schema; + request.partition_spec = nullptr; + request.write_order = nullptr; + request.stage_create = false; + request.properties = std::unordered_map{{"key1", "value1"}}; + + auto json = ToJson(request); + + nlohmann::json expected_json = R"({ + "name": "test_table", + "location": "/tmp/test_location", + "schema": { + "type": "struct", + "schema-id": 0, + "fields": [ + {"id": 1, "name": "id", "required": true, "type": "long"}, + {"id": 2, "name": "data", "required": false, "type": "string"} + ] + }, + "stage-create": false, + "properties": {"key1": "value1"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = CreateTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(*request.location, *parsed_result->location); + EXPECT_EQ(*request.schema, *parsed_result->schema); + EXPECT_EQ(request.partition_spec == nullptr, parsed_result->partition_spec == nullptr); + EXPECT_EQ(request.write_order == nullptr, parsed_result->write_order == nullptr); + EXPECT_EQ(*request.stage_create, *parsed_result->stage_create); + EXPECT_EQ(*request.properties, *parsed_result->properties); +} + +TEST(RestJsonInternalTest, CreateTableRequest) { + auto schema = std::make_shared( + std::vector{SchemaField(1, "id", iceberg::int64(), false), + SchemaField(2, "ts", iceberg::timestamp(), false), + SchemaField(3, "data", iceberg::string(), true)}, + 0); + + auto identity_transform = Transform::Identity(); + auto partition_spec = + std::make_shared(schema, 1, + std::vector{PartitionField( + 2, 1000, "ts_partition", identity_transform)}); + + SortField sort_field(1, identity_transform, SortDirection::kAscending, + NullOrder::kFirst); + auto write_order = std::make_shared(1, std::vector{sort_field}); + + CreateTableRequest request; + request.name = "complete_table"; + request.location = "/tmp/complete"; + request.schema = schema; + request.partition_spec = partition_spec; + request.write_order = write_order; + request.stage_create = true; + request.properties = std::unordered_map{{"key1", "value1"}, + {"key2", "value2"}}; + + auto json = ToJson(request); + + auto parsed_result = CreateTableRequestFromJson(json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(*request.location, *parsed_result->location); + EXPECT_EQ(*request.schema, *parsed_result->schema); + ASSERT_NE(parsed_result->partition_spec, nullptr); + EXPECT_EQ(*request.partition_spec, *parsed_result->partition_spec); + ASSERT_NE(parsed_result->write_order, nullptr); + EXPECT_EQ(*request.write_order, *parsed_result->write_order); + EXPECT_EQ(*request.stage_create, *parsed_result->stage_create); + EXPECT_EQ(*request.properties, *parsed_result->properties); +} + +TEST(RestJsonInternalTest, CreateTableRequestMissingRequiredFields) { + nlohmann::json invalid_json = R"({ + "location": "/tmp/test" + })"_json; + + auto result = CreateTableRequestFromJson(invalid_json); + EXPECT_FALSE(result.has_value()); + EXPECT_THAT(result, IsError(ErrorKind::kJsonParseError)); + EXPECT_THAT(result, HasErrorMessage("Missing 'name'")); +} + +TEST(RestJsonInternalTest, RegisterTableRequest) { + RegisterTableRequest request; + request.name = "registered_table"; + request.metadata_location = "/tmp/metadata.json"; + request.overwrite = true; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "name": "registered_table", + "metadata-location": "/tmp/metadata.json", + "overwrite": true + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = RegisterTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(request.name, parsed_result->name); + EXPECT_EQ(request.metadata_location, parsed_result->metadata_location); + EXPECT_EQ(*request.overwrite, *parsed_result->overwrite); +} + +TEST(RestJsonInternalTest, RenameTableRequest) { + RenameTableRequest request; + request.source = TableIdentifier{.ns = Namespace{{"db1"}}, .name = "old_table"}; + request.destination = TableIdentifier{.ns = Namespace{{"db2"}}, .name = "new_table"}; + + auto json = ToJson(request); + nlohmann::json expected_json = R"({ + "source": {"namespace": ["db1"], "name": "old_table"}, + "destination": {"namespace": ["db2"], "name": "new_table"} + })"_json; + + EXPECT_EQ(json, expected_json); + + auto parsed_result = RenameTableRequestFromJson(expected_json); + ASSERT_TRUE(parsed_result.has_value()); + EXPECT_EQ(request.source.name, parsed_result->source.name); + EXPECT_EQ(request.destination.name, parsed_result->destination.name); +} + +TEST(RestJsonInternalTest, LoadTableResponseComplete) { + std::vector schema_fields; + schema_fields.emplace_back(/*field_id=*/1, "x", iceberg::int64(), /*optional=*/false); + auto schema = std::make_shared(std::move(schema_fields), /*schema_id=*/1); + + auto partition_spec = + std::make_shared(schema, 0, std::vector{}); + auto sort_order = std::make_shared(0, std::vector{}); + + TableMetadata metadata{.format_version = 1, + .table_uuid = "test-uuid-load-table", + .location = "s3://bucket/path", + .last_sequence_number = 0, + .last_updated_ms = TimePointMsFromUnixMs(1602638573874).value(), + .last_column_id = 1, + .schemas = {schema}, + .current_schema_id = 1, + .partition_specs = {partition_spec}, + .default_spec_id = 0, + .last_partition_id = 0, + .properties = {{"key", "value"}}, + .current_snapshot_id = 3051729675574597004, + .snapshots = {std::make_shared(Snapshot{ + .snapshot_id = 3051729675574597004, + .sequence_number = 0, + .timestamp_ms = TimePointMsFromUnixMs(1515100955770).value(), + .manifest_list = "s3://a/b/1.avro", + .summary = {{"operation", "append"}}, + })}, + .snapshot_log = {}, + .metadata_log = {}, + .sort_orders = {sort_order}, + .default_sort_order_id = 0, + .refs = {}, + .statistics = {}, + .partition_statistics = {}, + .next_row_id = 0}; + + LoadTableResponse response{ + .metadata_location = "/tmp/warehouse/test_table/metadata/v1.metadata.json", + .metadata = metadata, + .config = std::unordered_map{ + {"warehouse", "/tmp/warehouse"}, + {"catalog-impl", "org.apache.iceberg.rest.RESTCatalog"}}}; + + auto json = ToJson(response); + + EXPECT_TRUE(json.contains("metadata-location")); + EXPECT_TRUE(json.contains("metadata")); + EXPECT_TRUE(json.contains("config")); + + auto parsed_result = LoadTableResponseFromJson(json); + ASSERT_TRUE(parsed_result.has_value()) << parsed_result.error().message; + + EXPECT_EQ(*response.metadata_location, *parsed_result->metadata_location); + EXPECT_EQ(response.metadata.table_uuid, parsed_result->metadata.table_uuid); + EXPECT_EQ(response.metadata.location, parsed_result->metadata.location); + EXPECT_EQ(response.metadata.format_version, parsed_result->metadata.format_version); + EXPECT_EQ(response.metadata.last_column_id, parsed_result->metadata.last_column_id); + EXPECT_EQ(response.metadata.current_schema_id, + parsed_result->metadata.current_schema_id); + EXPECT_EQ(response.metadata.current_snapshot_id, + parsed_result->metadata.current_snapshot_id); + EXPECT_EQ(*response.config, *parsed_result->config); +} + +} // namespace iceberg::rest From 94d892423d80980f574557134c9cc133016c5c8b Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Wed, 15 Oct 2025 17:01:06 +0800 Subject: [PATCH 2/3] fix --- src/iceberg/catalog/rest/CMakeLists.txt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/iceberg/catalog/rest/CMakeLists.txt b/src/iceberg/catalog/rest/CMakeLists.txt index f7bdf9c3..60cd6d91 100644 --- a/src/iceberg/catalog/rest/CMakeLists.txt +++ b/src/iceberg/catalog/rest/CMakeLists.txt @@ -34,15 +34,15 @@ list(APPEND nlohmann_json::nlohmann_json) list(APPEND ICEBERG_REST_STATIC_INSTALL_INTERFACE_LIBS - "$,Iceberg::iceberg_static,Iceberg::iceberg_shared>" - "$,Iceberg::cpr,cpr::cpr>" - "$,Iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" + "$,iceberg::iceberg_static,iceberg::iceberg_shared>" + "$,iceberg::cpr,cpr::cpr>" + "$,iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" ) list(APPEND ICEBERG_REST_SHARED_INSTALL_INTERFACE_LIBS - "$,Iceberg::iceberg_shared,Iceberg::iceberg_static>" - "$,Iceberg::cpr,cpr::cpr>" - "$,Iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" + "$,iceberg::iceberg_shared,iceberg::iceberg_static>" + "$,iceberg::cpr,cpr::cpr>" + "$,iceberg::nlohmann_json,nlohmann_json::nlohmann_json>" ) add_iceberg_lib(iceberg_rest From 5af1ab9cacc1a64f226d20028df93dc3fe48e206 Mon Sep 17 00:00:00 2001 From: Li Feiyang Date: Wed, 15 Oct 2025 17:41:30 +0800 Subject: [PATCH 3/3] remove install headers in cmakelists --- src/iceberg/catalog/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/iceberg/catalog/CMakeLists.txt b/src/iceberg/catalog/CMakeLists.txt index 71675b5c..13cdb094 100644 --- a/src/iceberg/catalog/CMakeLists.txt +++ b/src/iceberg/catalog/CMakeLists.txt @@ -15,8 +15,6 @@ # specific language governing permissions and limitations # under the License. -iceberg_install_all_headers(iceberg/catalog) - add_subdirectory(memory) if(ICEBERG_BUILD_REST)