From a20a91e8cdf4e9821737120aadd1c6dc0733b2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20=C3=85hl=C3=A9n?= Date: Tue, 22 Sep 2026 15:05:40 +0200 Subject: [PATCH 1/7] Added support for OData --- CHANGELOG.md | 2 + datacontract/command_import.py | 51 + datacontract/imports/importer.py | 1 + datacontract/imports/importer_factory.py | 5 + datacontract/imports/odata_importer.py | 513 +++++++ docs/docs/imports/index.md | 4 + docs/docs/imports/odata.md | 275 ++++ docs/docs/imports/odcs.md | 2 +- docs/docs/imports/oracle.md | 2 +- docs/docs/imports/parquet.md | 2 +- docs/docs/imports/postgres.md | 2 +- docs/docs/imports/powerbi.md | 2 +- docs/docs/imports/protobuf.md | 2 +- docs/docs/imports/pydantic-model.md | 2 +- docs/docs/imports/snowflake.md | 2 +- docs/docs/imports/spark.md | 2 +- docs/docs/imports/sql.md | 2 +- docs/docs/imports/sqlserver.md | 2 +- docs/docs/imports/trino.md | 2 +- examples/imports/odata/datacontract.odcs.yaml | 162 +++ tests/fixtures/odata/README.md | 16 + tests/fixtures/odata/products.json | 32 + tests/fixtures/odata/products.xml | 29 + tests/fixtures/odata/service-document.json | 10 + tests/fixtures/odata/who-contract.yaml | 162 +++ tests/fixtures/odata/who-metadata.xml | 330 +++++ tests/test_import_odata.py | 1253 +++++++++++++++++ 27 files changed, 2857 insertions(+), 12 deletions(-) create mode 100644 datacontract/imports/odata_importer.py create mode 100644 docs/docs/imports/odata.md create mode 100644 examples/imports/odata/datacontract.odcs.yaml create mode 100644 tests/fixtures/odata/README.md create mode 100644 tests/fixtures/odata/products.json create mode 100644 tests/fixtures/odata/products.xml create mode 100644 tests/fixtures/odata/service-document.json create mode 100644 tests/fixtures/odata/who-contract.yaml create mode 100644 tests/fixtures/odata/who-metadata.xml create mode 100644 tests/test_import_odata.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bbe6ab83d..a75f2a17a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,10 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Highlights - Support for SAP HANA +- Support for OData ### Added - `datacontract test` supports SAP HANA Cloud and SAP Datasphere through the optional `hana` extra (#1332) - `datacontract export excel` and `datacontract import excel` now support all versions of the Excel template (ODCS v3.0.2, v3.1.0, v3.2.0) +- `datacontract import odata` creates a datacontract from OData 4 metadata at an URL or from a local file. ### Fixed - `datacontract import s3`, `gcs` and `adls` accept `--format` again, so Delta tables can be imported (#1628) diff --git a/datacontract/command_import.py b/datacontract/command_import.py index df145de1a..982e3440d 100644 --- a/datacontract/command_import.py +++ b/datacontract/command_import.py @@ -909,3 +909,54 @@ def import_athena( id=id, ) _write_result(result, output) + + +@import_app.command( + name="odata", + epilog="Example: datacontract import odata --service-root-url https://example.com/odata/ --entity-set Products --output datacontract.yaml", +) +def import_odata( + service_root_url: Annotated[str, typer.Option(help="HTTP(S) root URL of the OData service.")], + service_root_file: Annotated[ + Optional[Path], + typer.Option(help="Local JSON service document. Ignored when --entity-set is supplied."), + ] = None, + entity_set: Annotated[ + Optional[List[str]], + typer.Option(help="EntitySet to import (repeat for multiple sets). If omitted, use the service document."), + ] = None, + metadata_url: Annotated[ + Optional[str], + typer.Option( + "--metadata-url", + help="CSDL XML or JSON URL. Defaults to SERVICE_ROOT_URL/$metadata unless --metadata-file is supplied.", + ), + ] = None, + metadata_file: Annotated[ + Optional[Path], + typer.Option( + "--metadata-file", + help="Path to a local OData CSDL XML or JSON file. Use either --metadata-file or --metadata-url.", + ), + ] = None, + output: output_option = None, + owner: owner_option = None, + id: id_option = None, + debug: debug_option = None, +): + """Import a data contract from OData 4.x CSDL XML or JSON metadata, using a URL or local file.""" + if metadata_url is not None and metadata_file is not None: + raise typer.BadParameter("--metadata-url and --metadata-file are mutually exclusive.") + enable_debug_logging(debug) + result = DataContract.import_from_source( + config=cli_config(), + format="odata", + source=service_root_url, + odata_service_root_file=service_root_file, + odata_entity_set=entity_set, + odata_metadata_url=metadata_url, + odata_metadata_file=metadata_file, + owner=owner, + id=id, + ) + _write_result(result, output) diff --git a/datacontract/imports/importer.py b/datacontract/imports/importer.py index 415c24908..5c20f47d5 100644 --- a/datacontract/imports/importer.py +++ b/datacontract/imports/importer.py @@ -40,6 +40,7 @@ class ImportFormat(str, Enum): json = "json" bigquery = "bigquery" odcs = "odcs" + odata = "odata" unity = "unity" databricks = "databricks" spark = "spark" diff --git a/datacontract/imports/importer_factory.py b/datacontract/imports/importer_factory.py index adb3e673c..a4bb12cc9 100644 --- a/datacontract/imports/importer_factory.py +++ b/datacontract/imports/importer_factory.py @@ -187,3 +187,8 @@ def load_module_class(module_path, class_name): module_path="datacontract.imports.json_importer", class_name="JsonImporter", ) +importer_factory.register_lazy_importer( + name=ImportFormat.odata, + module_path="datacontract.imports.odata_importer", + class_name="ODataImporter", +) diff --git a/datacontract/imports/odata_importer.py b/datacontract/imports/odata_importer.py new file mode 100644 index 000000000..b04fcb1af --- /dev/null +++ b/datacontract/imports/odata_importer.py @@ -0,0 +1,513 @@ +"""Import structural properties of OData 4.x EntitySets from CSDL XML or JSON.""" + +import json +import logging +import re +from pathlib import Path +from urllib.parse import quote, urljoin, urlsplit +from xml.etree import ElementTree +from xml.etree.ElementTree import ParseError, TreeBuilder + +import requests +from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard, SchemaObject + +from datacontract.config import Config +from datacontract.imports.importer import Importer +from datacontract.imports.odcs_helper import create_odcs, create_property, create_schema_object, create_server +from datacontract.model.exceptions import DataContractException + +logger = logging.getLogger(__name__) + +EDMX = "{http://docs.oasis-open.org/odata/ns/edmx}" +EDM = "{http://docs.oasis-open.org/odata/ns/edm}" +ODATA_4_TYPES = { + "Edm.String": "string", + "Edm.Guid": "string", + "Edm.Byte": "integer", + "Edm.SByte": "integer", + "Edm.Int16": "integer", + "Edm.Int32": "integer", + "Edm.Int64": "integer", + "Edm.Decimal": "number", + "Edm.Single": "number", + "Edm.Double": "number", + "Edm.Boolean": "boolean", + "Edm.Date": "date", + "Edm.DateTimeOffset": "timestamp", + "Edm.TimeOfDay": "time", +} + + +def _schema_error(reason: str) -> DataContractException: + return DataContractException(type="schema", name="Import OData metadata", reason=reason) + + +class _MetadataTreeBuilder(TreeBuilder): + """Reject DTDs before the parser processes their entity declarations.""" + + def doctype(self, name: str, pubid: str | None, system: str | None) -> None: + raise ParseError("DTD declarations are not allowed in OData metadata.") + + +class ODataImporter(Importer): + def import_source(self, source: str, import_args: dict, config: Config | None = None) -> OpenDataContractStandard: + config = Config.resolve(config) + metadata_url = import_args.get("odata_metadata_url") + metadata_file = import_args.get("odata_metadata_file") + service_file = import_args.get("odata_service_root_file") + selected_names = import_args.get("odata_entity_set") + if metadata_url is not None and metadata_file is not None: + raise _schema_error("--metadata-url and --metadata-file are mutually exclusive.") + _validate_http_url(source, "service-root-url", root=True) + if selected_names is not None and ( + not isinstance(selected_names, list) + or not selected_names + or any(not isinstance(name, str) or not name.strip() for name in selected_names) + ): + raise _schema_error("--entity-set must be a non-empty list of EntitySet names.") + root_url = source.rstrip("/") + "/" + header_version = None + if metadata_file is not None: + content = _read_file(metadata_file, "metadata") + else: + if metadata_url is None: + metadata_url = root_url + "$metadata" + _validate_http_url(metadata_url, "metadata-url") + response = _fetch_document(metadata_url, "metadata", "application/xml, application/json;q=0.9", config) + content = response.content + header_version = response.headers.get("OData-Version") + + # Inspect the first non-whitespace character; leave XML's encoding declaration to its parser. + prefix = content.decode(json.detect_encoding(content), errors="ignore").lstrip("\ufeff \t\r\n") + document = _read_xml(content) if prefix.startswith("<") else _read_json(content) + version = _odata_version(document["version"], header_version) + if selected_names is not None: + selections = [(name, None) for name in selected_names] + else: + document_base = root_url + if service_file is not None: + content = _read_file(service_file, "service document") + else: + response = _fetch_document(root_url, "service document", "application/json", config) + content = response.content + # Redirects can change the base for relative context URLs. + document_base = response.url + service_version = response.headers.get("OData-Version") + if service_version is not None: + _odata_version(service_version.strip(), service_version) + selections = _read_service_document(content, document_base) + if not selections: + raise _schema_error("No EntitySets selected: the service document contains no EntitySets.") + schemas = [] + seen = set() + for name, entity_url in selections: + schema = _odata_4_schema(document, name) + if schema.name in seen: + if selected_names is not None: + continue + raise _schema_error(f"Duplicate EntitySet {schema.name!r} in the service document.") + seen.add(schema.name) + schema.customProperties = [ + CustomProperty(property="odataEntitySet", value=schema.name), + CustomProperty( + property="odataEntitySetUrl", value=entity_url or root_url + quote(schema.name, safe="") + ), + ] + schemas.append(schema) + contract = create_odcs(name=schemas[0].name if len(schemas) == 1 else document["container_name"]) + contract.schema_ = schemas + server = create_server(name="source", server_type="api", location=root_url) + server.customProperties = [ + CustomProperty(property="apiType", value="odata"), + CustomProperty(property="odataVersion", value=version), + CustomProperty(property="odataMetadataFile", value=str(metadata_file)) + if metadata_file is not None + else CustomProperty(property="odataMetadataUrl", value=metadata_url), + # ODCS API servers do not allow a top-level format field. + CustomProperty(property="format", value="json"), + ] + contract.servers = [server] + return contract + + +def _validate_http_url(url: str, label: str, *, root: bool = False) -> None: + try: + parsed = urlsplit(url) if isinstance(url, str) else None + valid = ( + parsed is not None + and parsed.scheme in ("http", "https") + and bool(parsed.hostname) + and parsed.username is None + and parsed.password is None + and not any(char.isspace() for char in url) + ) + if valid: + parsed.port # Validate a supplied port as well as the hostname. + if root and valid and ("?" in url or "#" in url): + valid = False + except ValueError: + valid = False + if not valid: + raise _schema_error( + f"Invalid {label}: expected a public HTTP(S) URL without credentials. " + "The service root must not contain a query or fragment." + ) + + +def _read_file(path: str | Path, label: str) -> bytes: + try: + return Path(path).read_bytes() + except (OSError, ValueError) as exc: + raise _schema_error(f"Failed to read OData {label} file {path}: {exc}") from exc + + +class _ODataSession(requests.Session): + """Keep Requests' redirect policy without introducing .netrc credentials.""" + + def rebuild_auth(self, prepared_request: requests.PreparedRequest, response: requests.Response) -> None: + if self.should_strip_auth(response.request.url, prepared_request.url): + prepared_request.headers.pop("Authorization", None) + + +def _fetch_document(url: str, label: str, accept: str, config: Config) -> requests.Response: + headers = {"Accept": accept} + authorization = config.get_api_header_authorization() + if authorization is not None: + headers["Authorization"] = authorization + try: + with _ODataSession() as session: + response = session.get( + url, + headers=headers, + timeout=30, + # Prevent .netrc from replacing the header on the initial request. + auth=lambda request: request, + ) + response.raise_for_status() + return response + except requests.RequestException as exc: + detail = type(exc).__name__ + if exc.response is not None: + detail += f" (HTTP {exc.response.status_code})" + # InvalidHeader and other transport errors may include credentials in their text. + raise DataContractException( + type="connection", + name=f"Fetch OData {label}", + reason=f"Failed to fetch OData {label}: {detail}.", + ) from None + + +def _context_base(obj: dict, base: str) -> str: + context = obj.get("@odata.context", obj.get("@context")) + if "@odata.context" not in obj and "@context" not in obj: + return base + if not isinstance(context, str) or not context: + raise _schema_error("Invalid context URL in OData service document.") + if "@odata.context" in obj and "@context" in obj and obj["@odata.context"] != obj["@context"]: + raise _schema_error("Conflicting context URLs in OData service document.") + try: + result = urljoin(base, context) + except ValueError as exc: + raise _schema_error(f"Invalid context URL in OData service document: {exc}") from exc + _validate_http_url(result, "service document context") + return result + + +def _read_service_document(content: bytes, base: str) -> list[tuple[str, str]]: + document = _json_object(_load_json(content, "service document"), "the service document") + if "@odata.context" not in document and "@context" not in document: + raise _schema_error("Expected an OData service document with a context URL and a value array.") + base = _context_base(document, base) + entries = document.get("value") + if not isinstance(entries, list): + raise _schema_error("Expected a value array in the OData service document.") + selections, names = [], set() + for entry in entries: + entry = _json_object(entry, "service document entry") + kind = entry.get("kind", "EntitySet") + if not isinstance(kind, str): + raise _schema_error("Invalid kind in OData service document: expected a string.") + if kind != "EntitySet": + continue # Singletons, operations and linked services are not imported. + name, url = entry.get("name"), entry.get("url") + if not isinstance(name, str) or not name.strip() or not isinstance(url, str) or not url.strip(): + raise _schema_error("Each service document EntitySet must have a non-empty string name and url.") + if name in names: + raise _schema_error(f"Duplicate EntitySet {name!r} in the service document.") + names.add(name) + try: + entity_url = urljoin(_context_base(entry, base), url) + except ValueError as exc: + raise _schema_error(f"Invalid service document URL for EntitySet {name!r}: {exc}") from exc + _validate_http_url(entity_url, f"EntitySet {name!r} URL") + selections.append((name, entity_url)) + return selections + + +def _odata_version(document_version: object, header: str | None) -> str: + """Select the supported protocol family independently of its schema mapping.""" + version = header.strip() if header is not None else document_version + if any( + not isinstance(value, str) or re.fullmatch(r"4\.[0-9]+", value) is None for value in (document_version, version) + ): + raise _schema_error( + f"Unsupported OData version (header={header!r}, document={document_version!r}); " + "expected an OData 4.x version in the form 4.[0-9]+." + ) + if version != document_version: + raise _schema_error(f"Conflicting OData versions: header={version}, document={document_version}.") + return version + + +def _read_xml(content: bytes) -> dict: + try: + root = ElementTree.fromstring(content, parser=ElementTree.XMLParser(target=_MetadataTreeBuilder())) + except (ParseError, LookupError, ValueError) as exc: + raise _schema_error(f"Invalid or unsafe OData XML metadata: {exc}") from exc + if root.tag != f"{EDMX}Edmx": + raise _schema_error("Expected an OData 4 Edmx document with the OData 4 XML namespace.") + containers = root.findall(f"{EDMX}DataServices/{EDM}Schema/{EDM}EntityContainer") + if len(containers) != 1 or not containers[0].get("Name"): + raise _schema_error( + "The XML service EntityContainer is not found or ambiguous; expected exactly one named container." + ) + document = { + "version": root.get("Version"), + "container_name": containers[0].get("Name"), + "entity_sets": [], + "types": [], + } + for schema in root.findall(f"{EDMX}DataServices/{EDM}Schema"): + for container in schema.findall(f"{EDM}EntityContainer"): + for entity_set in container.findall(f"{EDM}EntitySet"): + document["entity_sets"].append( + { + "name": entity_set.get("Name"), + "type": entity_set.get("EntityType"), + "extends": container.get("Extends"), + } + ) + for entity_type in schema.findall(f"{EDM}EntityType"): + fields = [] + for field in entity_type.findall(f"{EDM}Property"): + facets = {} + for key in ("MaxLength", "Precision", "Scale"): + value = field.get(key) + if value is not None: + try: + value = int(value) + except ValueError: + pass # Preserve symbolic values for validation during mapping. + facets[key] = value + if field.get("Type") == "Edm.Decimal": + facets.setdefault("Scale", 0) + fields.append( + { + "name": field.get("Name"), + "type": field.get("Type"), + "nullable": {"true": True, "false": False}.get( + field.get("Nullable", "true"), field.get("Nullable") + ), + "collection": False, + "facets": facets, + } + ) + document["types"].append( + { + "names": { + f"{prefix}.{entity_type.get('Name')}" + for prefix in (schema.get("Namespace"), schema.get("Alias")) + if prefix + }, + "base_type": entity_type.get("BaseType"), + "keys": [key.get("Name") for key in entity_type.findall(f"{EDM}Key/{EDM}PropertyRef")], + "fields": fields, + "navigation": [prop.get("Name") for prop in entity_type.findall(f"{EDM}NavigationProperty")], + } + ) + return document + + +def _json_pairs(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise _schema_error(f"Duplicate JSON key {key!r} in OData JSON document.") + result[key] = value + return result + + +def _json_object(value: object, context: str) -> dict: + if not isinstance(value, dict): + raise _schema_error(f"Expected a JSON object for {context} in OData metadata.") + return value + + +def _load_json(content: bytes, label: str) -> object: + def reject_constant(value: str): + raise _schema_error(f"Invalid JSON constant {value!r} in OData metadata.") + + try: + return json.loads(content, object_pairs_hook=_json_pairs, parse_constant=reject_constant) + except (ValueError, UnicodeError) as exc: + raise _schema_error(f"Invalid or unsafe OData JSON {label}: {exc}") from exc + + +def _read_json(content: bytes) -> dict: + root = _json_object(_load_json(content, "metadata"), "the CSDL document") + if "$Version" not in root and "value" in root and ("@odata.context" in root or "@context" in root): + raise _schema_error( + "This is an OData service/data document, not CSDL metadata. Supply the $metadata document with field definitions." + ) + container_name = root.get("$EntityContainer") + if not isinstance(container_name, str) or "." not in container_name: + raise _schema_error("CSDL JSON metadata requires a namespace-qualified $EntityContainer.") + namespace, _, local_name = container_name.rpartition(".") + container_schema = _json_object(root.get(namespace), f"schema {namespace!r}") + container = _json_object(container_schema.get(local_name), f"$EntityContainer {container_name!r}") + if container.get("$Kind") != "EntityContainer": + raise _schema_error(f"{container_name!r} must have $Kind EntityContainer.") + if "$Extends" in container: + raise _schema_error(f"EntityContainer {container_name!r} uses unsupported inheritance ($Extends).") + document = {"version": root.get("$Version"), "container_name": local_name, "entity_sets": [], "types": []} + for name, member in container.items(): + if name.startswith("$") or "@" in name: + continue + member = _json_object(member, f"container member {name!r}") + collection = member.get("$Collection", False) + if type(collection) is not bool: + raise _schema_error(f"Invalid $Collection for container member {name!r}: expected a boolean.") + if collection: + document["entity_sets"].append({"name": name, "type": member.get("$Type"), "extends": None}) + for namespace, schema in root.items(): + if namespace.startswith("$") or "@" in namespace: + continue + schema = _json_object(schema, f"schema {namespace!r}") + alias = schema.get("$Alias") + if "$Alias" in schema and not isinstance(alias, str): + raise _schema_error(f"Invalid $Alias in schema {namespace!r}: expected a string.") + for type_name, entity_type in schema.items(): + if type_name.startswith("$") or "@" in type_name: + continue + # Action/function overloads are arrays; only entity types are needed. + if not isinstance(entity_type, dict) or entity_type.get("$Kind") != "EntityType": + continue + if "$BaseType" in entity_type and not isinstance(entity_type["$BaseType"], str): + raise _schema_error(f"Invalid $BaseType for {namespace}.{type_name}: expected a qualified type name.") + fields, navigation = [], [] + for field_name, field in entity_type.items(): + if field_name.startswith("$") or "@" in field_name: + continue + field = _json_object(field, f"field {namespace}.{type_name}.{field_name}") + kind = field.get("$Kind", "Property") + if kind == "NavigationProperty": + navigation.append(field_name) + continue + facets = {key: field[f"${key}"] for key in ("MaxLength", "Precision", "Scale") if f"${key}" in field} + if field.get("$Type") == "Edm.Decimal": + facets.setdefault("Scale", "variable") + fields.append( + { + "name": field_name, + "type": field.get("$Type", "Edm.String"), + "nullable": field.get("$Nullable", False), + "collection": field.get("$Collection", False), + "facets": facets, + "kind": kind, + } + ) + document["types"].append( + { + "names": {f"{prefix}.{type_name}" for prefix in (namespace, alias) if prefix}, + "base_type": entity_type.get("$BaseType"), + "keys": entity_type.get("$Key", []), + "fields": fields, + "navigation": navigation, + } + ) + return document + + +def _odata_4_schema(document: dict, requested_name: str) -> SchemaObject: + entity_sets = document["entity_sets"] + matches = [entity_set for entity_set in entity_sets if entity_set["name"] == requested_name] + if not matches: + matches = [ + entity_set + for entity_set in entity_sets + if (entity_set["name"] or "").casefold() == requested_name.casefold() + ] + if len(matches) != 1: + raise _schema_error( + f"EntitySet {requested_name!r} is {'ambiguous' if matches else 'not found'} in the metadata." + ) + entity_set = matches[0] + name, type_name = entity_set["name"], entity_set["type"] + if entity_set["extends"]: + raise _schema_error(f"EntitySet {name!r} uses unsupported container inheritance (Extends).") + if not isinstance(type_name, str): + raise _schema_error(f"Invalid EntityType for EntitySet {name!r}: expected a qualified type name.") + types = [entity_type for entity_type in document["types"] if type_name in entity_type["names"]] + if len(types) != 1: + raise _schema_error( + f"EntityType {type_name!r} for EntitySet {name!r} must resolve uniquely within this metadata document. " + "External metadata references are not fetched." + ) + entity_type = types[0] + if entity_type["base_type"] is not None: + raise _schema_error(f"EntityType {type_name!r} uses unsupported inheritance (BaseType).") + key_names, fields = entity_type["keys"], entity_type["fields"] + field_names = [field["name"] for field in fields] + if any(not name for name in field_names) or len(set(field_names)) != len(field_names): + raise _schema_error(f"EntityType {type_name!r} has missing or duplicate property names.") + if ( + not isinstance(key_names, list) + or any(not isinstance(key, str) for key in key_names) + or len(set(key_names)) != len(key_names) + or any(key not in field_names for key in key_names) + ): + raise _schema_error(f"EntityType {type_name!r} has invalid or unsupported key references: {key_names}.") + properties = [] + for field in fields: + field_name, field_type = field["name"], field["type"] + if field.get("kind", "Property") != "Property": + raise _schema_error(f"Unsupported $Kind for field {name}.{field_name}: {field['kind']!r}.") + if type(field["collection"]) is not bool: + raise _schema_error(f"Invalid $Collection for field {name}.{field_name}: expected a boolean.") + if not isinstance(field_type, str) or field_type not in ODATA_4_TYPES or field["collection"]: + raise _schema_error( + f"Unsupported OData type {field_type!r} for field {name}.{field_name}. " + "Only supported primitive types can be imported; complex, collection, enum types and type definitions " + "are not supported." + ) + nullable = field["nullable"] + if type(nullable) is not bool: + raise _schema_error(f"Invalid Nullable value {nullable!r} for field {name}.{field_name}.") + is_key = field_name in key_names + if is_key and nullable: + raise _schema_error(f"Key field {name}.{field_name} must declare Nullable=false.") + facets = {} + for facet, value in field["facets"].items(): + if facet == "MaxLength" and value == "max": + continue + if facet == "Scale" and isinstance(value, str) and value.lower() in ("variable", "floating"): + facets["custom_properties"] = {"scale": value.lower()} + continue + if type(value) is not int or value < 0 or (facet == "MaxLength" and value == 0): + raise _schema_error(f"Invalid {facet} value {value!r} for field {name}.{field_name}.") + facets[{"MaxLength": "max_length", "Precision": "precision", "Scale": "scale"}[facet]] = value + properties.append( + create_property( + name=field_name, + logical_type=ODATA_4_TYPES[field_type], + physical_type=field_type, + required=not nullable, + primary_key=is_key, + primary_key_position=key_names.index(field_name) + 1 if is_key else None, + format="uuid" if field_type == "Edm.Guid" else None, + **facets, + ) + ) + for navigation in entity_type["navigation"]: + logger.warning("Omitting OData navigation property %s.%s from the imported schema.", name, navigation) + return create_schema_object(name=name, physical_type="object", properties=properties) diff --git a/docs/docs/imports/index.md b/docs/docs/imports/index.md index f10838560..43c6598d8 100644 --- a/docs/docs/imports/index.md +++ b/docs/docs/imports/index.md @@ -88,6 +88,10 @@ Each import page shows a runnable example: a small source file under [`examples/ mysqlA MySQL database. + + + odataA public OData 4 service. + odcsAn ODCS data contract file. diff --git a/docs/docs/imports/odata.md b/docs/docs/imports/odata.md new file mode 100644 index 000000000..847ab924a --- /dev/null +++ b/docs/docs/imports/odata.md @@ -0,0 +1,275 @@ +--- +sidebar_position: 18 +title: "Import: OData" +description: "Create a data contract from one or more OData 4.x EntitySets using CSDL XML or JSON." +--- + +# Import: OData + +Import one or more EntitySets from **OData 4.x** into an ODCS 3.2.0 contract. + +- **Select specific EntitySets:** Use `--entity-set Products` for one EntitySet, or repeat the option to select several. +- **Import all automatically:** Omit `--entity-set` to import every EntitySet listed in the service document into one contract. + +CSDL XML or JSON provides field types and constraints. The service document can be fetched from the service root +or read from a local file with `--service-root-file`. No data records are downloaded. + +## Parameters + +| Parameter | Required | Description | +|---|---|---| +| `--service-root-url` | Yes | HTTP(S) root URL of the OData service. Stored with exactly one trailing `/` as the server's `location`, including for offline imports. | +| `--service-root-file` | No | Path to a local JSON service document listing the available EntitySets. Without this option, the service document is fetched from the root URL. Ignored when `--entity-set` is supplied. | +| `--entity-set` | No | Name of an EntitySet to import. Repeat to select several. If omitted, import all EntitySets listed in the service document. Explicit selection skips reading the service document. | +| `--metadata-url` | No | HTTP(S) URL of the CSDL XML or JSON metadata. Defaults to `SERVICE_ROOT_URL/$metadata` when neither metadata option is supplied. | +| `--metadata-file` | No | Path to a local OData CSDL XML or JSON metadata file. | +| `--output` | No | Path to the generated data contract YAML file. If omitted, print the contract to stdout. | +| `--owner` | No | Owner or team name to assign to the generated contract. | +| `--id` | No | Identifier to assign to the generated contract. | +| `--debug` / `--no-debug` | No | Control debug logging; use `--debug` for diagnostic output. | +| `--help` | No | Show the command's options and exit. | + +`--metadata-url` and `--metadata-file` cannot be combined. For an import without network access, use +`--metadata-file` together with either `--entity-set` or `--service-root-file`. + +## Example: WHO countries + +The WHO [country endpoint](https://xmart-api-public-uat.who.int/refmart/ref_country) is described by its +[metadata document](https://xmart-api-public-uat.who.int/refmart/$metadata). + +```bash +datacontract import odata \ + --service-root-url 'https://xmart-api-public-uat.who.int/refmart/' \ + --entity-set ref_country \ + --output datacontract.yaml +``` + +`--service-root-url` is required and is stored with exactly one trailing `/` as the server's `location`. +A trailing slash is optional in the input. + +If neither `--metadata-file` nor `--metadata-url` is supplied, metadata is fetched from the service root +with `$metadata` appended. Use `--metadata-url` to override that address; quote URLs containing `$metadata` +with single quotes to prevent shell expansion. The two metadata options are mutually exclusive. + +The importer matches EntitySet names exactly first. If there is no exact match, it accepts a case-insensitive +match only when unique. Ambiguous or missing matches produce an error. For WHO, `ref_country` matches +`REF_COUNTRY`; the schema keeps the metadata name. + +Omit `--output` to print YAML to stdout. Use `--owner` and `--id` to override the contract's owner and identifier. + +## Multiple EntitySets + +Repeat `--entity-set` to select several EntitySets: + +```bash +datacontract import odata \ + --service-root-url 'https://example.com/odata/' \ + --entity-set Products \ + --entity-set Orders \ + --output datacontract.yaml +``` + +Explicit selection reads only CSDL. It does not request the service root or read `--service-root-file`, +even if that option is supplied. Argument order is preserved; repeated selections of the same EntitySet +produce one schema object. EntitySets sharing an EntityType still produce separate schema objects. + +Without `--entity-set`, the importer reads the JSON service document from `--service-root-file`, if supplied, +or from `--service-root-url`. It imports the EntitySets advertised there, in document order. It does not +fall back to importing all EntitySets from CSDL. Singletons, function imports and linked services are ignored. +An omitted `kind` in a service document entry means `EntitySet`. + +One EntitySet gives the contract its name. For multiple EntitySets, the contract uses the service container's name. +Missing or ambiguous EntitySets, duplicate entries in the service document and empty selections produce errors. +If any selected schema uses unsupported types, the whole import fails before writing the output file. + +## Offline import + +Use a local CSDL file and an explicit selection to import without any network requests: + +```bash +datacontract import odata \ + --service-root-url 'https://example.com/odata/' \ + --metadata-file metadata.xml \ + --entity-set Products \ + --entity-set Orders \ + --output datacontract.yaml +``` + +Alternatively, provide both documents locally to import every advertised EntitySet: + +```bash +datacontract import odata \ + --service-root-url 'https://example.com/odata/' \ + --service-root-file service-document.json \ + --metadata-file metadata.json \ + --output datacontract.yaml +``` + +CSDL XML and JSON are detected from content, regardless of file extension or HTTP `Content-Type`. +Both formats also work with `--metadata-url`. Relative file paths are resolved from the current working directory. +Metadata saved from an authenticated service can be imported offline. + +`--service-root-file` alone does not guarantee offline import: CSDL must also be supplied through +`--metadata-file`. Conversely, `--metadata-file` without an explicit selection or a local service document +still causes a request to the service root. + +## Generated contract + +The server describes the service, and each schema object identifies its own EntitySet: + +```yaml +servers: + - server: source + type: api + location: https://example.com/odata/ + customProperties: + - property: apiType + value: odata + - property: odataVersion + value: "4.01" + - property: odataMetadataUrl + value: https://example.com/odata/$metadata + - property: format + value: json +schema: + - name: Products + physicalName: Products + logicalType: object + physicalType: object + customProperties: + - property: odataEntitySet + value: Products + - property: odataEntitySetUrl + value: https://example.com/odata/Products + # properties: ... +``` + +These custom properties are Data Contract CLI conventions for future OData-aware consumers. +`odataVersion` records the CSDL response version, or the document version for file imports. +`odataMetadataFile` replaces `odataMetadataUrl` for local CSDL files. +`format: json` describes the data response, independently of the metadata format; ODCS API servers do not +permit a top-level `format` field. + +`odataEntitySet` now belongs to each schema object, including for single-EntitySet imports. +`odataEntitySetUrl` records the advertised address when reading a service document. Relative addresses are +resolved using OData's context URL rules, with the service root as the document base for local files. +With explicit selection, the URL is constructed from the service root and the EntitySet name in CSDL. +Context links and EntitySet addresses are never fetched, and context links do not override the metadata source. + +## Supported schema + +| OData type | ODCS logical type | +|---|---| +| `Edm.String`, `Edm.Guid` | `string` (`uuid` format for Guid) | +| `Edm.Byte`, `Edm.SByte`, `Edm.Int16`, `Edm.Int32`, `Edm.Int64` | `integer` | +| `Edm.Decimal`, `Edm.Single`, `Edm.Double` | `number` | +| `Edm.Boolean` | `boolean` | +| `Edm.Date` | `date` | +| `Edm.DateTimeOffset` | `timestamp` | +| `Edm.TimeOfDay` | `time` | + +Original types are preserved in `physicalType`. The importer copies nullability, declared primary keys and their +positions, numeric maximum lengths, precision and scale. Precision and scale use property-level custom properties; +symbolic scale values are preserved. No primary keys are inferred when metadata omits them, as in the WHO example. + +Qualified type names and local schema aliases work across schemas. JSON selects the service container with +`$EntityContainer`; XML must identify one unambiguous, named EntityContainer. +Omitted `Nullable` means nullable in XML; omitted `$Nullable` means required in JSON. +In JSON, omitted `$Type` means `Edm.String`, and a decimal's omitted `$Scale` means `variable`. +An omitted `$Precision` introduces no precision constraint. + +## Current limitations + +- The service root must not contain credentials, query parameters or a fragment. +- Service documents must be JSON; CSDL supports both XML and JSON. +- All version strings of the form `4.x` are accepted, with no upper minor-version limit. + Requests do not send `OData-MaxVersion`. Other major versions, malformed versions and conflicting + CSDL document/header versions are rejected. Later 4.x versions use the same supported schema constructs; + accepting their version numbers does not add support for new OData features. +- Complex types, collection-valued fields, entity or container inheritance, enum types, type definitions and + types outside the table produce errors when used by selected schemas. +- Navigation properties are omitted with a warning. External metadata references are not downloaded. +- Malformed documents, duplicate JSON keys and XML DTD declarations are rejected. +- This feature implements **import only**. + +## Planned improvements + +- **`datacontract test`:** Validate the contract's schema against the service's `$metadata` document. + Testing will only read metadata and will not fetch EntitySets or data records. + +This capability is planned and is not yet implemented. + +## Authentication + +For authenticated services, set `DATACONTRACT_API_HEADER_AUTHORIZATION` to the complete `Authorization` +header value. For example, use a Bearer token supplied by your service: + +```bash +export DATACONTRACT_API_HEADER_AUTHORIZATION="Bearer ${ODATA_ACCESS_TOKEN}" +datacontract import odata \ + --service-root-url 'https://example.com/odata/' \ + --output datacontract.yaml +``` + +A precomputed `Basic ` header works too. The importer does not obtain +or refresh tokens. Without this setting, requests are anonymous; `.netrc` credentials are not used. + +The same header is sent to both the metadata URL and the service root when those documents are fetched, +including an explicitly supplied `--metadata-url` on another host. Requests' redirect rules remove the +header when redirecting to another host or downgrading from HTTPS to HTTP. Once removed, it is not +restored later in that redirect chain. Credentials are not written to the generated contract. + +You can also use the existing global `--config-file` option: + +```yaml +# datacontract-config.yaml +api_header_authorization: "Bearer ${ODATA_ACCESS_TOKEN}" +``` + +```bash +datacontract --config-file datacontract-config.yaml import odata \ + --service-root-url 'https://example.com/odata/' \ + --output datacontract.yaml +``` + +Explicit configuration takes precedence over the environment. Fully offline imports do not use the header +or make network requests. See [Configuration](../configuration.md) for all supported configuration sources. + +## Python + +```python +from datacontract.data_contract import DataContract + +contract = DataContract.import_from_source( + "odata", + source="https://example.com/odata/", + odata_metadata_file="metadata.json", + odata_entity_set=["Products", "Orders"], +) +print(contract.to_yaml()) +``` + +`source` is now the service root URL. Use `odata_metadata_url` for an explicit metadata URL, +or omit both metadata arguments to derive the URL from `source`. For offline import of all advertised sets, +replace `odata_entity_set` with `odata_service_root_file="service-document.json"`. +File arguments accept strings or `Path` objects; `odata_entity_set` accepts a non-empty list of names. + +For an authenticated URL import, pass the existing `Config` option: + +```python +import os + +from datacontract import Config +from datacontract.data_contract import DataContract + +contract = DataContract.import_from_source( + "odata", + source="https://example.com/odata/", + odata_entity_set=["Products"], + config=Config(api_header_authorization=f"Bearer {os.environ['ODATA_ACCESS_TOKEN']}"), +) +``` + +The `config` argument also accepts a dictionary keyed by `DATACONTRACT_API_HEADER_AUTHORIZATION`. + +All options: **[`datacontract import odata`](../commands/import/odata.md)**. diff --git a/docs/docs/imports/odcs.md b/docs/docs/imports/odcs.md index f0327d463..75567dcff 100644 --- a/docs/docs/imports/odcs.md +++ b/docs/docs/imports/odcs.md @@ -1,5 +1,5 @@ --- -sidebar_position: 18 +sidebar_position: 19 title: "Import: ODCS" description: "Create a data contract from an existing ODCS file." --- diff --git a/docs/docs/imports/oracle.md b/docs/docs/imports/oracle.md index d01dc0c5d..8a4f40568 100644 --- a/docs/docs/imports/oracle.md +++ b/docs/docs/imports/oracle.md @@ -1,5 +1,5 @@ --- -sidebar_position: 19 +sidebar_position: 20 title: "Import: Oracle" description: "Create a data contract from an Oracle database." --- diff --git a/docs/docs/imports/parquet.md b/docs/docs/imports/parquet.md index 14e7e3e28..db496a481 100644 --- a/docs/docs/imports/parquet.md +++ b/docs/docs/imports/parquet.md @@ -1,5 +1,5 @@ --- -sidebar_position: 20 +sidebar_position: 21 title: "Import: Parquet" description: "Create a data contract from a Parquet file." --- diff --git a/docs/docs/imports/postgres.md b/docs/docs/imports/postgres.md index e0ecbb97e..ac41bbd19 100644 --- a/docs/docs/imports/postgres.md +++ b/docs/docs/imports/postgres.md @@ -1,5 +1,5 @@ --- -sidebar_position: 21 +sidebar_position: 22 title: "Import: Postgres" description: "Create a data contract from a Postgres schema." --- diff --git a/docs/docs/imports/powerbi.md b/docs/docs/imports/powerbi.md index 81db13d25..8c852bc8b 100644 --- a/docs/docs/imports/powerbi.md +++ b/docs/docs/imports/powerbi.md @@ -1,5 +1,5 @@ --- -sidebar_position: 22 +sidebar_position: 23 title: "Import: Power BI" description: "Create a data contract from a Power BI semantic model (.pbit, .bim, or .json)." --- diff --git a/docs/docs/imports/protobuf.md b/docs/docs/imports/protobuf.md index cb4716017..40fd9b5ec 100644 --- a/docs/docs/imports/protobuf.md +++ b/docs/docs/imports/protobuf.md @@ -1,5 +1,5 @@ --- -sidebar_position: 23 +sidebar_position: 24 title: "Import: Protobuf" description: "Create a data contract from a Protobuf schema file." --- diff --git a/docs/docs/imports/pydantic-model.md b/docs/docs/imports/pydantic-model.md index 38384de23..a45dd55a5 100644 --- a/docs/docs/imports/pydantic-model.md +++ b/docs/docs/imports/pydantic-model.md @@ -1,5 +1,5 @@ --- -sidebar_position: 24 +sidebar_position: 25 title: "Import: Pydantic Model" description: "Create a data contract from Pydantic models." --- diff --git a/docs/docs/imports/snowflake.md b/docs/docs/imports/snowflake.md index ebcbf4509..4259ffb28 100644 --- a/docs/docs/imports/snowflake.md +++ b/docs/docs/imports/snowflake.md @@ -1,5 +1,5 @@ --- -sidebar_position: 25 +sidebar_position: 26 title: "Import: Snowflake" description: "Create a data contract from a Snowflake workspace." --- diff --git a/docs/docs/imports/spark.md b/docs/docs/imports/spark.md index 2c379a59d..c5dcaf7bc 100644 --- a/docs/docs/imports/spark.md +++ b/docs/docs/imports/spark.md @@ -1,5 +1,5 @@ --- -sidebar_position: 26 +sidebar_position: 27 title: "Import: Spark" description: "Create a data contract from Spark tables or DataFrames (programmatic)." --- diff --git a/docs/docs/imports/sql.md b/docs/docs/imports/sql.md index 6895d5b78..dc8e5f619 100644 --- a/docs/docs/imports/sql.md +++ b/docs/docs/imports/sql.md @@ -1,5 +1,5 @@ --- -sidebar_position: 27 +sidebar_position: 28 title: "Import: SQL DDL" description: "Create a data contract from a SQL DDL file." --- diff --git a/docs/docs/imports/sqlserver.md b/docs/docs/imports/sqlserver.md index 3df1b5415..a1a0e71dc 100644 --- a/docs/docs/imports/sqlserver.md +++ b/docs/docs/imports/sqlserver.md @@ -1,5 +1,5 @@ --- -sidebar_position: 28 +sidebar_position: 29 title: "Import: SQL Server" description: "Create a data contract from a SQL Server database." --- diff --git a/docs/docs/imports/trino.md b/docs/docs/imports/trino.md index 228247bb4..dedf04c81 100644 --- a/docs/docs/imports/trino.md +++ b/docs/docs/imports/trino.md @@ -1,5 +1,5 @@ --- -sidebar_position: 29 +sidebar_position: 30 title: "Import: Trino" description: "Create a data contract from a Trino catalog." --- diff --git a/examples/imports/odata/datacontract.odcs.yaml b/examples/imports/odata/datacontract.odcs.yaml new file mode 100644 index 000000000..7a83eb39f --- /dev/null +++ b/examples/imports/odata/datacontract.odcs.yaml @@ -0,0 +1,162 @@ +version: 1.0.0 +kind: DataContract +apiVersion: v3.2.0 +id: my-data-contract +name: REF_COUNTRY +status: draft +servers: +- server: source + type: api + customProperties: + - property: apiType + value: odata + - property: odataVersion + value: '4.0' + - property: odataMetadataUrl + value: https://xmart-api-public-uat.who.int/refmart/$metadata + - property: format + value: json + location: https://xmart-api-public-uat.who.int/refmart/ +schema: +- name: REF_COUNTRY + physicalType: object + customProperties: + - property: odataEntitySet + value: REF_COUNTRY + - property: odataEntitySetUrl + value: https://xmart-api-public-uat.who.int/refmart/REF_COUNTRY + logicalType: object + physicalName: REF_COUNTRY + properties: + - name: GEO_M49_CODE + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_ISO_2 + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_ISO_3 + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_WHO + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_ISO_NUMERIC + physicalType: Edm.Int32 + logicalType: integer + required: false + - name: NAME_SHORT_EN + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_EN + physicalType: Edm.String + logicalType: string + required: false + - name: CAPITAL_CITY + physicalType: Edm.String + logicalType: string + required: false + - name: ADJECTIVE_PEOPLE + physicalType: Edm.String + logicalType: string + required: false + - name: GEO_SMALL_POP_FLAG + physicalType: Edm.Boolean + logicalType: boolean + required: false + - name: GEO_SOVEREIGN + physicalType: Edm.String + logicalType: string + required: false + - name: SOVEREIGN_ISO_3 + physicalType: Edm.String + logicalType: string + required: false + - name: GRP_WB_INCOME + physicalType: Edm.String + logicalType: string + required: false + - name: GRP_WHO_REGION + physicalType: Edm.String + logicalType: string + required: false + - name: WHO_LEGAL_STATUS + physicalType: Edm.String + logicalType: string + required: false + - name: WHO_LEGAL_STATUS_TITLE + physicalType: Edm.String + logicalType: string + required: false + - name: DATE_START + physicalType: Edm.Date + logicalType: date + required: false + - name: NAME_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: ISO_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: CAPITAL_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: STATISTICAL_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: GEO_PRECEDED_BY + physicalType: Edm.String + logicalType: string + required: false + - name: GEO_SUCCEEDED_BY + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_AR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_AR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_ES + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_ES + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_FR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_FR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_RU + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_RU + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_ZH + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_ZH + physicalType: Edm.String + logicalType: string + required: false diff --git a/tests/fixtures/odata/README.md b/tests/fixtures/odata/README.md new file mode 100644 index 000000000..8a9eefbff --- /dev/null +++ b/tests/fixtures/odata/README.md @@ -0,0 +1,16 @@ +`who-metadata.xml` was retrieved from +https://xmart-api-public-uat.who.int/refmart/$metadata on 2026-09-18. +The response declared `OData-Version: 4.0`. XML whitespace was formatted for readability. + +`who-contract.yaml` describes the expected import of +https://xmart-api-public-uat.who.int/refmart/ref_country. +All network access is mocked in the tests. + +`products.xml` and `products.json` are independently authored synthetic CSDL examples +of the same catalog model. Entity types and the service container live in separate +schemas and use an alias. They cover composite keys, ordered properties and facets; +they contain no data or schema from a private service. + +`service-document.json` advertises Products and Orders from the synthetic model, +with relative and absolute addresses, plus other resource kinds to ignore. +ArchivedProducts shares the Product type but is available only through explicit selection. diff --git a/tests/fixtures/odata/products.json b/tests/fixtures/odata/products.json new file mode 100644 index 000000000..a102a7988 --- /dev/null +++ b/tests/fixtures/odata/products.json @@ -0,0 +1,32 @@ +{ + "$Version": "4.01", + "$EntityContainer": "Catalog.Service.Store", + "Catalog.Model": { + "$Alias": "Catalog", + "Product": { + "$Kind": "EntityType", + "$Key": ["Revision", "Sku"], + "Sku": {"$MaxLength": 32}, + "Revision": {"$Type": "Edm.Int32"}, + "Price": {"$Type": "Edm.Decimal", "$Nullable": true, "$Precision": 10, "$Scale": 2}, + "PublishedAt": {"$Type": "Edm.DateTimeOffset", "$Nullable": true, "$Precision": 3}, + "Active": {"$Type": "Edm.Boolean"}, + "TrackingId": {"$Type": "Edm.Guid", "$Nullable": true}, + "Description": {"$Kind": "Property", "$Nullable": true, "$MaxLength": "max"} + }, + "Order": { + "$Kind": "EntityType", + "$Key": ["OrderId"], + "OrderId": {"$Type": "Edm.Guid"}, + "Total": {"$Type": "Edm.Decimal", "$Precision": 12, "$Scale": 2} + } + }, + "Catalog.Service": { + "Store": { + "$Kind": "EntityContainer", + "Products": {"$Collection": true, "$Type": "Catalog.Product"}, + "Orders": {"$Collection": true, "$Type": "Catalog.Order"}, + "ArchivedProducts": {"$Collection": true, "$Type": "Catalog.Product", "$IncludeInServiceDocument": false} + } + } +} diff --git a/tests/fixtures/odata/products.xml b/tests/fixtures/odata/products.xml new file mode 100644 index 000000000..a173fd6d3 --- /dev/null +++ b/tests/fixtures/odata/products.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/odata/service-document.json b/tests/fixtures/odata/service-document.json new file mode 100644 index 000000000..d144f069e --- /dev/null +++ b/tests/fixtures/odata/service-document.json @@ -0,0 +1,10 @@ +{ + "@odata.context": "$metadata", + "value": [ + {"name": "Products", "kind": "EntitySet", "url": "products"}, + {"name": "Orders", "url": "https://example.com/fulfilment/orders"}, + {"name": "FeaturedProduct", "kind": "Singleton", "url": "FeaturedProduct"}, + {"name": "Search", "kind": "FunctionImport", "url": "Search"}, + {"name": "RelatedService", "kind": "ServiceDocument", "url": "https://example.com/other/"} + ] +} diff --git a/tests/fixtures/odata/who-contract.yaml b/tests/fixtures/odata/who-contract.yaml new file mode 100644 index 000000000..2e17cd2c9 --- /dev/null +++ b/tests/fixtures/odata/who-contract.yaml @@ -0,0 +1,162 @@ +apiVersion: v3.2.0 +kind: DataContract +id: my-data-contract +name: REF_COUNTRY +version: 1.0.0 +status: draft +servers: +- server: source + type: api + location: https://xmart-api-public-uat.who.int/refmart/ + customProperties: + - property: apiType + value: odata + - property: odataVersion + value: '4.0' + - property: odataMetadataUrl + value: https://xmart-api-public-uat.who.int/refmart/$metadata + - property: format + value: json +schema: +- name: REF_COUNTRY + physicalName: REF_COUNTRY + logicalType: object + physicalType: object + customProperties: + - property: odataEntitySet + value: REF_COUNTRY + - property: odataEntitySetUrl + value: https://xmart-api-public-uat.who.int/refmart/REF_COUNTRY + properties: + - name: GEO_M49_CODE + logicalType: string + physicalType: Edm.String + required: false + - name: CODE_ISO_2 + logicalType: string + physicalType: Edm.String + required: false + - name: CODE_ISO_3 + logicalType: string + physicalType: Edm.String + required: false + - name: CODE_WHO + logicalType: string + physicalType: Edm.String + required: false + - name: CODE_ISO_NUMERIC + logicalType: integer + physicalType: Edm.Int32 + required: false + - name: NAME_SHORT_EN + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_FORMAL_EN + logicalType: string + physicalType: Edm.String + required: false + - name: CAPITAL_CITY + logicalType: string + physicalType: Edm.String + required: false + - name: ADJECTIVE_PEOPLE + logicalType: string + physicalType: Edm.String + required: false + - name: GEO_SMALL_POP_FLAG + logicalType: boolean + physicalType: Edm.Boolean + required: false + - name: GEO_SOVEREIGN + logicalType: string + physicalType: Edm.String + required: false + - name: SOVEREIGN_ISO_3 + logicalType: string + physicalType: Edm.String + required: false + - name: GRP_WB_INCOME + logicalType: string + physicalType: Edm.String + required: false + - name: GRP_WHO_REGION + logicalType: string + physicalType: Edm.String + required: false + - name: WHO_LEGAL_STATUS + logicalType: string + physicalType: Edm.String + required: false + - name: WHO_LEGAL_STATUS_TITLE + logicalType: string + physicalType: Edm.String + required: false + - name: DATE_START + logicalType: date + physicalType: Edm.Date + required: false + - name: NAME_CHANGE + logicalType: string + physicalType: Edm.String + required: false + - name: ISO_CHANGE + logicalType: string + physicalType: Edm.String + required: false + - name: CAPITAL_CHANGE + logicalType: string + physicalType: Edm.String + required: false + - name: STATISTICAL_CHANGE + logicalType: string + physicalType: Edm.String + required: false + - name: GEO_PRECEDED_BY + logicalType: string + physicalType: Edm.String + required: false + - name: GEO_SUCCEEDED_BY + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_SHORT_AR + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_FORMAL_AR + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_SHORT_ES + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_FORMAL_ES + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_SHORT_FR + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_FORMAL_FR + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_SHORT_RU + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_FORMAL_RU + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_SHORT_ZH + logicalType: string + physicalType: Edm.String + required: false + - name: NAME_FORMAL_ZH + logicalType: string + physicalType: Edm.String + required: false diff --git a/tests/fixtures/odata/who-metadata.xml b/tests/fixtures/odata/who-metadata.xml new file mode 100644 index 000000000..2fa4cced1 --- /dev/null +++ b/tests/fixtures/odata/who-metadata.xml @@ -0,0 +1,330 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_import_odata.py b/tests/test_import_odata.py new file mode 100644 index 000000000..2d58219f8 --- /dev/null +++ b/tests/test_import_odata.py @@ -0,0 +1,1253 @@ +import json +import re +import traceback +from io import BytesIO +from pathlib import Path +from unittest.mock import Mock + +import jsonschema +import pytest +import requests +import yaml +from typer.testing import CliRunner + +from datacontract import Config +from datacontract.cli import app +from datacontract.config import set_cli_config +from datacontract.data_contract import DataContract +from datacontract.model.exceptions import DataContractException + +ROOT = "https://xmart-api-public-uat.who.int/refmart/" +METADATA = "https://xmart-api-public-uat.who.int/refmart/$metadata" +FIXTURES = Path(__file__).parent / "fixtures/odata" +ODATA_VERSIONS = ["4.0", "4.01", "4.02", "4.1", "4.123"] + + +@pytest.fixture(autouse=True) +def isolate_authentication(monkeypatch): + monkeypatch.delenv("DATACONTRACT_API_HEADER_AUTHORIZATION", raising=False) + set_cli_config(None) + yield + set_cli_config(None) + + +@pytest.fixture +def metadata_response(monkeypatch): + response = Mock() + response.content = (FIXTURES / "who-metadata.xml").read_bytes() + response.headers = {"OData-Version": "4.0"} + get = Mock(return_value=response) + monkeypatch.setattr("datacontract.imports.odata_importer._ODataSession.get", get) + return response, get + + +def import_contract(entity_set="ref_country", *, source=ROOT, **kwargs): + return DataContract.import_from_source( + "odata", source=source, odata_entity_set=[entity_set], odata_metadata_url=METADATA, **kwargs + ) + + +def metadata_xml(properties="", *, version="4.01", entity_attributes="", extra_schema="", extra_sets="", keys=""): + return f''' + + + {keys}{properties} + + {extra_sets} + {extra_schema} + + + '''.encode() + + +def test_who_contract_and_transport(metadata_response): + response, get = metadata_response + contract = import_contract() + actual = yaml.safe_load(contract.to_yaml()) + assert actual == yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) + schema = json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) + jsonschema.validate(actual, schema) + get.assert_called_once() + assert get.call_args.args == (METADATA,) + assert get.call_args.kwargs["headers"] == { + "Accept": "application/xml, application/json;q=0.9", + } + assert get.call_args.kwargs["timeout"] == 30 + response.raise_for_status.assert_called_once() + # The explicit auth handler must prevent implicit .netrc credentials. + request = requests.Request("GET", METADATA).prepare() + assert get.call_args.kwargs["auth"](request).headers.get("Authorization") is None + + +@pytest.mark.parametrize("version", ODATA_VERSIONS) +@pytest.mark.parametrize("header", [True, False]) +def test_versions_and_header_fallback(metadata_response, version, header): + response, _ = metadata_response + response.content = metadata_xml('', version=version) + response.headers = {"OData-Version": version} if header else {} + contract = import_contract() + assert contract.servers[0].customProperties[1].value == version + assert contract.schema_[0].properties[0].name == "Code" + + +@pytest.mark.parametrize( + "edm_type, logical_type", + [ + ("String", "string"), + ("Guid", "string"), + ("Byte", "integer"), + ("SByte", "integer"), + ("Int16", "integer"), + ("Int32", "integer"), + ("Int64", "integer"), + ("Decimal", "number"), + ("Single", "number"), + ("Double", "number"), + ("Boolean", "boolean"), + ("Date", "date"), + ("DateTimeOffset", "timestamp"), + ("TimeOfDay", "time"), + ], +) +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_primitive_types(metadata_response, edm_type, logical_type, format): + response, _ = metadata_response + response.headers = {} + if format == "xml": + response.content = metadata_xml(f'') + prop = import_contract().schema_[0].properties[0] + else: + document = json.loads((FIXTURES / "products.json").read_text()) + document["Catalog.Model"]["Product"] = { + "$Kind": "EntityType", + "Value": {"$Type": f"Edm.{edm_type}", "$Nullable": True}, + } + response.content = json.dumps(document).encode() + prop = import_contract("Products").schema_[0].properties[0] + assert prop.logicalType == logical_type + assert prop.physicalType == f"Edm.{edm_type}" + assert prop.required is False + if edm_type == "Guid": + assert prop.logicalTypeOptions == {"format": "uuid"} + facets = {p.property: p.value for p in prop.customProperties or []} + if edm_type == "Decimal": + assert facets == {"scale": 0 if format == "xml" else "variable"} + else: + assert "scale" not in facets + + +def test_keys_nullability_and_facets(metadata_response): + response, _ = metadata_response + response.headers = {} + response.content = metadata_xml( + """ + + + + + """, + keys='', + ) + code, year, amount, variable, floating, text = import_contract().schema_[0].properties + assert code.required is True and year.required is True + assert code.primaryKey is True and year.primaryKey is True + assert (code.primaryKeyPosition, year.primaryKeyPosition) == (2, 1) + assert not code.unique and not year.unique + assert code.logicalTypeOptions == {"maxLength": 3} + assert amount.required is False and not amount.primaryKey + assert {p.property: p.value for p in amount.customProperties} == {"precision": 12, "scale": 2} + assert variable.customProperties[0].value == "variable" + assert floating.customProperties[0].value == "floating" + assert not text.logicalTypeOptions + + +@pytest.mark.parametrize("entity_set", ["REF_COUNTRY", "ref_country", "Ref_Country"]) +@pytest.mark.parametrize("root", ["https://example.com", "https://example.com/service", "https://example.com/service/"]) +def test_entity_set_matching_normalizes_root(metadata_response, entity_set, root): + contract = import_contract(entity_set, source=root) + assert contract.schema_[0].name == "REF_COUNTRY" + assert contract.servers[0].location == root.rstrip("/") + "/" + assert {p.property: p.value for p in contract.schema_[0].customProperties} == { + "odataEntitySet": "REF_COUNTRY", + "odataEntitySetUrl": root.rstrip("/") + "/REF_COUNTRY", + } + + +def test_exact_match_wins_and_casefold_ambiguity_fails(metadata_response): + response, _ = metadata_response + response.headers = {} + response.content = metadata_xml(extra_sets='') + assert import_contract().schema_[0].name == "ref_country" + with pytest.raises(DataContractException, match="ambiguous"): + import_contract("Ref_Country") + + +def test_navigation_and_unrelated_types_are_omitted(metadata_response, caplog): + response, _ = metadata_response + response.headers = {} + response.content = metadata_xml( + '', + extra_schema='', + ) + contract = import_contract() + assert [p.name for p in contract.schema_[0].properties] == ["Code"] + assert "Omitting OData navigation property REF_COUNTRY.Related" in caplog.text + + +@pytest.mark.parametrize( + "field_type", + ["D.Address", "Collection(Edm.String)", "Collection(D.Address)", "D.Status", "Edm.Binary", "Edm.Unknown"], +) +def test_unsupported_types_fail_with_field_and_type(metadata_response, field_type): + response, _ = metadata_response + response.headers = {} + response.content = metadata_xml(f'') + with pytest.raises(DataContractException) as error: + import_contract() + assert "REF_COUNTRY.Value" in str(error.value) + assert field_type in str(error.value) + + +@pytest.mark.parametrize( + "document, header, error", + [ + (metadata_xml(version="3.0"), "3.0", "Unsupported OData version"), + (metadata_xml(version="5.0"), "5.0", "Unsupported OData version"), + (metadata_xml(version="4.0"), "4.01", "Conflicting OData versions"), + (metadata_xml(entity_attributes='BaseType="Demo.Base"'), None, "inheritance"), + (metadata_xml().replace(b'Name="Service"', b'Name="Service" Extends="Demo.Base"'), None, "inheritance"), + (metadata_xml().replace(b"D.Country", b"Other.Country"), None, "must resolve uniquely"), + (metadata_xml().replace(b"REF_COUNTRY", b"OTHER"), None, "not found"), + (metadata_xml().replace(b'http://docs.oasis-open.org/odata/ns/edm"', b'urn:wrong"'), None, "not found"), + (b"", None, "Expected an OData 4 Edmx"), + (b"'), None, "Invalid Nullable"), + (metadata_xml(''), None, "Invalid MaxLength"), + (metadata_xml(keys=''), None, "key references"), + ], +) +def test_metadata_errors(metadata_response, document, header, error): + response, _ = metadata_response + response.content = document + response.headers = {"OData-Version": header} if header else {} + with pytest.raises(DataContractException, match=error): + import_contract() + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) +@pytest.mark.parametrize( + "doctype", + [ + "", + ']>', + ']>', + '', + ], +) +def test_metadata_rejects_dtd(metadata_response, encoding, doctype): + response, _ = metadata_response + response.headers = {} + document = metadata_xml().decode().replace('Name="REF_COUNTRY"', 'Name="&country;"') + response.content = (f'{doctype}{document}').encode(encoding) + with pytest.raises(DataContractException, match="DTD declarations are not allowed"): + import_contract() + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) +def test_metadata_encoding_and_escaped_text(metadata_response, encoding): + response, _ = metadata_response + response.headers = {} + document = metadata_xml('').decode() + response.content = ( + f'{document}' + ).encode(encoding) + assert import_contract().schema_[0].properties[0].name == "Code" + + +@pytest.mark.parametrize( + "location", + [ + None, + "", + "file:///tmp/source", + "https://example.com:bad/", + ROOT + "?$select=CODE_ISO_3", + ROOT + "#fragment", + "https://user:secret@example.com/Countries", + ], +) +def test_invalid_root_does_not_fetch_metadata(metadata_response, location): + _, get = metadata_response + with pytest.raises(DataContractException, match="Invalid service-root-url"): + import_contract(source=location) + get.assert_not_called() + + +@pytest.mark.parametrize( + "options", + [{"odata_metadata_url": METADATA, "odata_metadata_file": FIXTURES / "who-metadata.xml"}], +) +def test_metadata_sources_are_mutually_exclusive_for_python_api(metadata_response, options): + _, get = metadata_response + with pytest.raises(DataContractException, match="mutually exclusive"): + DataContract.import_from_source("odata", source=ROOT, odata_entity_set=["ref_country"], **options) + get.assert_not_called() + + +@pytest.mark.parametrize("error", [requests.Timeout("timed out"), requests.ConnectionError("connection failed")]) +def test_network_errors(metadata_response, error): + _, get = metadata_response + get.side_effect = error + with pytest.raises(DataContractException, match="Failed to fetch OData metadata") as exc: + import_contract() + assert exc.value.type == "connection" + + +@pytest.mark.parametrize("status", [401, 403, 404, 500]) +def test_http_errors(metadata_response, status): + response, _ = metadata_response + response.status_code = status + response.raise_for_status.side_effect = requests.HTTPError(f"HTTP {status}", response=response) + with pytest.raises(DataContractException, match=f"HTTP {status}"): + import_contract() + + +def test_cli_stdout_and_output(metadata_response, tmp_path): + runner = CliRunner() + args = ["import", "odata", "--service-root-url", ROOT, "--entity-set", "ref_country", "--metadata-url", METADATA] + stdout = runner.invoke(app, args) + assert stdout.exit_code == 0, stdout.output + assert yaml.safe_load(stdout.stdout) == yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) + output = tmp_path / "contract.yaml" + result = runner.invoke(app, args + ["--output", str(output), "--owner", "WHO", "--id", "country", "--debug"]) + assert result.exit_code == 0, result.output + contract = yaml.safe_load(output.read_text()) + assert contract["id"] == "country" + assert contract["team"]["name"] == "WHO" + + +@pytest.mark.parametrize( + "args", + [ + [], + ["--metadata-url", METADATA], + ["--metadata-file", str(FIXTURES / "who-metadata.xml")], + ["--service-root-url", ROOT, "--entity-set", "ref_country", "--metadataUrl", METADATA], + [ + "--service-root-url", + ROOT, + "--entity-set", + "ref_country", + "--metadata-url", + METADATA, + "--metadata-file", + str(FIXTURES / "who-metadata.xml"), + ], + ], +) +def test_cli_requires_options_and_rejects_alias(metadata_response, args): + _, get = metadata_response + result = CliRunner().invoke(app, ["import", "odata"] + args) + assert result.exit_code == 2 + get.assert_not_called() + + +def test_failed_import_preserves_existing_output(metadata_response, tmp_path): + response, _ = metadata_response + response.content = b"invalid XML" + output = tmp_path / "contract.yaml" + output.write_text("existing contract") + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + ROOT, + "--entity-set", + "ref_country", + "--metadata-url", + METADATA, + "--output", + str(output), + ], + ) + assert result.exit_code != 0 + assert output.read_text() == "existing contract" + + +@pytest.mark.parametrize("path_type", [str, Path]) +def test_local_who_metadata_matches_url_import(metadata_response, path_type): + _, get = metadata_response + path = path_type(FIXTURES / "who-metadata.xml") + contract = DataContract.import_from_source( + "odata", source=ROOT, odata_entity_set=["ref_country"], odata_metadata_file=path + ) + actual = yaml.safe_load(contract.to_yaml()) + expected = yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) + expected["servers"][0]["customProperties"][2] = {"property": "odataMetadataFile", "value": str(path)} + assert actual == expected + get.assert_not_called() + + +@pytest.mark.parametrize("version", ODATA_VERSIONS) +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) +def test_local_metadata_version_and_encoding(metadata_response, tmp_path, version, encoding): + _, get = metadata_response + path = tmp_path / "metadata.xml" + document = metadata_xml('', version=version).decode() + path.write_bytes((f'{document}').encode(encoding)) + contract = DataContract.import_from_source( + "odata", source=ROOT, odata_entity_set=["ref_country"], odata_metadata_file=path + ) + assert contract.servers[0].customProperties[1].value == version + assert contract.schema_[0].properties[0].name == "Code" + get.assert_not_called() + + +@pytest.mark.parametrize("kind", ["missing", "directory", "unreadable", "malformed", "dtd"]) +def test_local_metadata_errors_preserve_output(metadata_response, tmp_path, monkeypatch, kind): + _, get = metadata_response + path = tmp_path / "metadata.xml" + if kind == "directory": + path.mkdir() + elif kind == "unreadable": + monkeypatch.setattr(Path, "read_bytes", Mock(side_effect=PermissionError("Permission denied"))) + elif kind == "malformed": + path.write_bytes(b"not XML") + elif kind == "dtd": + path.write_bytes(b']>' + metadata_xml()) + output = tmp_path / "contract.yaml" + output.write_text("existing contract") + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + ROOT, + "--entity-set", + "ref_country", + "--metadata-file", + str(path), + "--output", + str(output), + ], + ) + assert result.exit_code != 0 + assert isinstance(result.exception, DataContractException) + message = "Invalid or unsafe" if kind in ("malformed", "dtd") else "Failed to read OData metadata file" + assert message in str(result.exception) + assert output.read_text() == "existing contract" + get.assert_not_called() + + +SERVICE_ROOT = "https://example.com/odata/" + + +@pytest.fixture +def json_metadata(metadata_response): + response, _ = metadata_response + response.headers = {} + return json.loads((FIXTURES / "products.json").read_text()) + + +@pytest.mark.parametrize("source", ["file", "url"]) +@pytest.mark.parametrize("version", ODATA_VERSIONS) +@pytest.mark.parametrize("scale", [None, 0, 2]) +def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source, version, scale): + response, get = metadata_response + response.headers = {"OData-Version": version, "Content-Type": "text/plain"} + contracts = [] + for format in ("xml", "json"): + content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode()) + if format == "xml": + content = content.replace(b' Scale="2"', b"" if scale is None else f' Scale="{scale}"'.encode(), 1) + else: + document = json.loads(content) + document["Catalog.Model"]["Product"]["Price"]["$Scale"] = 0 if scale is None else scale + content = json.dumps(document).encode() + if source == "file": + # Deliberately use an unrelated extension: content determines the format. + path = tmp_path / "metadata.txt" + path.write_bytes(content) + contract = DataContract.import_from_source( + "odata", source=SERVICE_ROOT, odata_entity_set=["Products"], odata_metadata_file=path + ) + get.assert_not_called() + else: + response.content = content + contract = import_contract("Products", source=SERVICE_ROOT) + actual = yaml.safe_load(contract.to_yaml()) + schema = json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) + jsonschema.validate(actual, schema) + contracts.append(actual) + assert contracts[0] == contracts[1] + properties = contracts[0]["schema"][0]["properties"] + assert [p["name"] for p in properties] == [ + "Sku", + "Revision", + "Price", + "PublishedAt", + "Active", + "TrackingId", + "Description", + ] + assert [p["primaryKeyPosition"] for p in properties[:2]] == [2, 1] + assert not any(p.get("unique") for p in properties) + assert properties[0]["required"] and not properties[2]["required"] + assert properties[0]["logicalTypeOptions"] == {"maxLength": 32} + assert properties[2]["customProperties"] == [ + {"property": "precision", "value": 10}, + {"property": "scale", "value": 0 if scale is None else scale}, + ] + + +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_cli_synthetic_metadata_file(metadata_response, tmp_path, monkeypatch, format): + _, get = metadata_response + content = (FIXTURES / f"products.{format}").read_bytes() + monkeypatch.chdir(tmp_path) + Path("metadata").write_bytes(content) + args = [ + "import", + "odata", + "--service-root-url", + SERVICE_ROOT, + "--entity-set", + "Products", + "--metadata-file", + "metadata", + "--owner", + "Catalog", + "--id", + "products", + ] + stdout = CliRunner().invoke(app, args) + assert stdout.exit_code == 0, stdout.output + actual = yaml.safe_load(stdout.stdout) + assert actual["name"] == "Products" + assert actual["id"] == "products" and actual["team"]["name"] == "Catalog" + assert {"property": "odataMetadataFile", "value": "metadata"} in actual["servers"][0]["customProperties"] + result = CliRunner().invoke(app, args + ["--output", "contract.yaml"]) + assert result.exit_code == 0, result.output + assert yaml.safe_load(Path("contract.yaml").read_text()) == actual + get.assert_not_called() + + +@pytest.mark.parametrize("source", ["file", "url"]) +@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig", "utf-16", "utf-32"]) +def test_json_encoding_and_leading_whitespace(metadata_response, tmp_path, source, encoding): + response, get = metadata_response + content = (" \n\t" * 100 + (FIXTURES / "products.json").read_text()).encode(encoding) + response.headers = {"Content-Type": "application/xml"} + if source == "file": + path = tmp_path / "metadata.xml" + path.write_bytes(content) + contract = DataContract.import_from_source( + "odata", source=SERVICE_ROOT, odata_entity_set=["Products"], odata_metadata_file=path + ) + get.assert_not_called() + else: + response.content = content + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + SERVICE_ROOT, + "--entity-set", + "Products", + "--metadata-url", + METADATA, + ], + ) + assert result.exit_code == 0, result.output + assert yaml.safe_load(result.stdout)["name"] == "Products" + get.assert_called_once_with(METADATA, **get.call_args.kwargs) + return + assert contract.name == "Products" + + +def test_json_defaults_and_annotations(metadata_response, json_metadata): + response, _ = metadata_response + json_metadata["Catalog.Model"]["Product"] = { + "$Kind": "EntityType", + "Label": {}, + "Amount": {"$Type": "Edm.Decimal"}, + "Optional": {"$Kind": "Property", "$Nullable": True}, + "@Example.Note": "ignored", + "Label@Example.Note": "also ignored", + } + response.content = json.dumps(json_metadata).encode() + label, amount, optional = import_contract("Products").schema_[0].properties + assert label.physicalType == "Edm.String" and label.required + assert amount.required + assert {p.property: p.value for p in amount.customProperties} == {"scale": "variable"} + assert len(amount.customProperties) == 1 # No inferred precision. + assert not optional.required + assert not any(p.primaryKey for p in (label, amount, optional)) + + +@pytest.mark.parametrize("type_name", ["Catalog.Product", "Catalog.Model.Product"]) +def test_json_alias_and_navigation(metadata_response, json_metadata, caplog, type_name): + response, get = metadata_response + json_metadata["Catalog.Service"]["Store"]["Products"]["$Type"] = type_name + json_metadata["Catalog.Model"]["Product"]["Orders"] = { + "$Kind": "NavigationProperty", + "$Type": "External.Order", + "$Collection": True, + } + json_metadata["Catalog.Model"]["Unused"] = {"$Kind": "ComplexType", "Nested": {"$Type": "External.Unknown"}} + json_metadata["$Reference"] = {"https://example.com/external/$metadata": {"$Include": [{"$Namespace": "External"}]}} + json_metadata["Catalog.Service"]["Store"]["Featured"] = {"$Type": "Catalog.Product"} # Singleton. + response.content = json.dumps(json_metadata).encode() + contract = import_contract("Products") + assert len(contract.schema_[0].properties) == 7 + assert "Omitting OData navigation property Products.Orders" in caplog.text + get.assert_called_once() + + +@pytest.mark.parametrize("entity_set", ["Products", "products", "PRODUCTS"]) +def test_json_name_matching(metadata_response, json_metadata, entity_set): + response, _ = metadata_response + response.content = json.dumps(json_metadata).encode() + contract = import_contract(entity_set, source=SERVICE_ROOT) + assert contract.name == "Products" and contract.servers[0].location == SERVICE_ROOT + + +def test_json_exact_match_and_ambiguous_names(metadata_response, json_metadata): + response, _ = metadata_response + json_metadata["Catalog.Service"]["Store"]["products"] = {"$Collection": True, "$Type": "Catalog.Product"} + response.content = json.dumps(json_metadata).encode() + assert import_contract("Products").name == "Products" + with pytest.raises(DataContractException, match="ambiguous"): + import_contract("PRODUCTS") + with pytest.raises(DataContractException, match="not found"): + import_contract("Missing") + + +@pytest.mark.parametrize( + "path,value,error", + [ + (("$Version",), "3.0", "Unsupported OData version"), + (("$Version",), 4.01, "Unsupported OData version"), + (("$EntityContainer",), [], "namespace-qualified"), + (("$EntityContainer",), "Missing.Store", "JSON object"), + (("Catalog.Model", "$Alias"), 42, "Invalid .*Alias"), + (("Catalog.Model", "$Alias"), None, "Invalid .*Alias"), + (("Catalog.Service", "Store", "$Kind"), "EntityType", "EntityContainer"), + (("Catalog.Service", "Store", "$Extends"), "Other.Store", "inheritance"), + (("Catalog.Service", "Store", "Products", "$Collection"), "true", "Invalid .*Collection"), + (("Catalog.Service", "Store", "Products", "$Collection"), False, "not found"), + (("Catalog.Service", "Store", "Products", "$Type"), [], "qualified type name"), + (("Catalog.Service", "Store", "Products", "$Type"), "External.Product", "External metadata references"), + (("Catalog.Model", "Product", "$BaseType"), "Catalog.Base", "inheritance"), + (("Catalog.Model", "Product", "$BaseType"), None, "Invalid .*BaseType"), + (("Catalog.Model", "Product", "$Key"), "Sku", "key references"), + (("Catalog.Model", "Product", "$Key"), ["Sku", "Sku"], "key references"), + (("Catalog.Model", "Product", "$Key"), ["Missing"], "key references"), + (("Catalog.Model", "Product", "$Key"), [{"Alias": "Address/Code"}], "key references"), + (("Catalog.Model", "Product", "Sku", "$Nullable"), True, "Key field"), + (("Catalog.Model", "Product", "Price"), [], "JSON object"), + (("Catalog.Model", "Product", "Price", "$Kind"), "Unexpected", "Unsupported .*Kind"), + (("Catalog.Model", "Product", "Price", "$Type"), None, "Unsupported OData type"), + (("Catalog.Model", "Product", "Price", "$Nullable"), "false", "Invalid Nullable"), + (("Catalog.Model", "Product", "Price", "$Collection"), 1, "Invalid .*Collection"), + (("Catalog.Model", "Product", "Price", "$Collection"), True, "collection"), + (("Catalog.Model", "Product", "Price", "$Precision"), "10", "Invalid Precision"), + (("Catalog.Model", "Product", "Price", "$Scale"), True, "Invalid Scale"), + (("Catalog.Model", "Product", "Sku", "$MaxLength"), -1, "Invalid MaxLength"), + (("Catalog.Model", "Product", "Sku", "$MaxLength"), None, "Invalid MaxLength"), + ], +) +def test_json_invalid_csdl(metadata_response, json_metadata, path, value, error): + response, _ = metadata_response + target = json_metadata + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + response.content = json.dumps(json_metadata).encode() + with pytest.raises(DataContractException, match=error): + import_contract("Products") + + +@pytest.mark.parametrize("kind", ["ComplexType", "EnumType", "TypeDefinition"]) +def test_json_unsupported_selected_type(metadata_response, json_metadata, kind): + response, _ = metadata_response + json_metadata["Catalog.Model"]["Special"] = {"$Kind": kind} + json_metadata["Catalog.Model"]["Product"]["Price"]["$Type"] = "Catalog.Special" + response.content = json.dumps(json_metadata).encode() + with pytest.raises(DataContractException, match="Catalog.Special.*Products.Price"): + import_contract("Products") + + +@pytest.mark.parametrize( + "content,error", + [ + (b'{"$Version": "4.01",', "Invalid or unsafe"), + (b'{"$Version":"4.01","$Version":"4.0"}', "Duplicate JSON key"), + (b'{"Schema":{"Field":{},"Field":{}}}', "Duplicate JSON key"), + (b'{"value":NaN}', "Invalid JSON constant"), + (b'{"value":Infinity}', "Invalid JSON constant"), + (b"[]", "JSON object"), + (b"null", "JSON object"), + (b'{"bad":"\xff"}', "Invalid or unsafe"), + ( + b'{"@odata.context":"https://example.com/odata/$metadata","value":[{"name":"Products","kind":"EntitySet","url":"Products"}]}', + "service/data document.*field definitions", + ), + ], +) +def test_json_invalid_documents_preserve_output(metadata_response, tmp_path, content, error): + response, get = metadata_response + response.headers = {} + response.content = content + output = tmp_path / "contract.yaml" + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + SERVICE_ROOT, + "--entity-set", + "Products", + "--metadata-url", + METADATA, + "--output", + str(output), + ], + ) + assert result.exit_code != 0 + assert isinstance(result.exception, DataContractException) + assert re.search(error, str(result.exception)) + assert not output.exists() + get.assert_called_once() + + +@pytest.mark.parametrize("document_version, header_version", [("4.01", "4.0"), ("4.123", "4.1"), ("4.01", "4.1")]) +def test_json_version_conflict(metadata_response, json_metadata, document_version, header_version): + response, _ = metadata_response + json_metadata["$Version"] = document_version + response.headers = {"OData-Version": header_version} + response.content = json.dumps(json_metadata).encode() + with pytest.raises(DataContractException, match="Conflicting OData versions"): + import_contract("Products") + + +@pytest.mark.parametrize("format", ["xml", "json"]) +@pytest.mark.parametrize("version", [None, "3.0", "5.0", "4", "4.", "4.x", "4.1.0", "4.01beta", "4.١"]) +def test_invalid_document_versions(metadata_response, format, version): + response, _ = metadata_response + response.headers = {"OData-Version": "4.01"} + if format == "json": + document = json.loads((FIXTURES / "products.json").read_text()) + if version is None: + document.pop("$Version") + else: + document["$Version"] = version + response.content = json.dumps(document).encode() + else: + response.content = (FIXTURES / "products.xml").read_bytes() + if version is None: + response.content = response.content.replace(b' Version="4.01"', b"") + else: + response.content = response.content.replace(b'Version="4.01"', f'Version="{version}"'.encode()) + with pytest.raises(DataContractException, match="Unsupported OData version"): + import_contract("Products") + + +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_version_header_whitespace_preserves_version_string(metadata_response, format): + response, _ = metadata_response + response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", b"4.00123") + response.headers = {"OData-Version": " 4.00123 "} + contract = import_contract("Products") + assert {p.property: p.value for p in contract.servers[0].customProperties}["odataVersion"] == "4.00123" + + +@pytest.mark.parametrize("prefix", [b" \t\n" * 100, b"\xef\xbb\xbf \t\n"]) +def test_xml_detection_with_whitespace_and_bom(metadata_response, prefix): + response, _ = metadata_response + response.headers = {"Content-Type": "application/json"} + response.content = prefix + metadata_xml('') + assert import_contract().schema_[0].properties[0].name == "Label" + + +def test_xml_encoding_declaration_is_respected(metadata_response): + response, _ = metadata_response + response.headers = {} + document = metadata_xml('').decode() + response.content = ('' + document).encode("iso-8859-1") + assert import_contract().schema_[0].properties[0].name == "Libellé" + + +@pytest.fixture +def service_response(metadata_response): + metadata, get = metadata_response + metadata.content = (FIXTURES / "products.json").read_bytes() + metadata.headers = {"OData-Version": "4.01"} + service = Mock( + content=(FIXTURES / "service-document.json").read_bytes(), + headers={"OData-Version": "4.01"}, + url=SERVICE_ROOT, + ) + documents = {SERVICE_ROOT + "$metadata": metadata, METADATA: metadata, SERVICE_ROOT: service} + get.side_effect = lambda url, **kwargs: documents[url] + return service, get + + +@pytest.mark.parametrize("format", ["xml", "json"]) +@pytest.mark.parametrize("metadata_source", ["file", "derived_url", "explicit_url"]) +@pytest.mark.parametrize("service_source", ["file", "url"]) +@pytest.mark.parametrize("root", [SERVICE_ROOT, SERVICE_ROOT.rstrip("/")]) +def test_import_all_advertised_sets(metadata_response, service_response, format, metadata_source, service_source, root): + metadata, _ = metadata_response + _, get = service_response + metadata.content = (FIXTURES / f"products.{format}").read_bytes() + options, expected_requests = {}, [] + if metadata_source == "file": + options["odata_metadata_file"] = FIXTURES / f"products.{format}" + elif metadata_source == "explicit_url": + options["odata_metadata_url"] = METADATA + expected_requests.append(METADATA) + else: + expected_requests.append(SERVICE_ROOT + "$metadata") + if service_source == "file": + options["odata_service_root_file"] = FIXTURES / "service-document.json" + else: + expected_requests.append(SERVICE_ROOT) + contract = DataContract.import_from_source("odata", source=root, **options) + assert contract.name == "Store" + assert contract.servers[0].location == SERVICE_ROOT + assert [s.name for s in contract.schema_] == ["Products", "Orders"] + assert [s.physicalName for s in contract.schema_] == ["Products", "Orders"] + assert all(s.logicalType == s.physicalType == "object" for s in contract.schema_) + assert [{p.property: p.value for p in s.customProperties} for s in contract.schema_] == [ + {"odataEntitySet": "Products", "odataEntitySetUrl": SERVICE_ROOT + "products"}, + {"odataEntitySet": "Orders", "odataEntitySetUrl": "https://example.com/fulfilment/orders"}, + ] + server_properties = {p.property: p.value for p in contract.servers[0].customProperties} + assert "odataEntitySet" not in server_properties + assert server_properties["odataVersion"] == "4.01" + if metadata_source != "file": + assert server_properties["odataMetadataUrl"] == expected_requests[0] + schema = json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) + jsonschema.validate(yaml.safe_load(contract.to_yaml()), schema) + assert [call.args[0] for call in get.call_args_list] == expected_requests + if service_source == "url": + assert get.call_args.kwargs["headers"] == {"Accept": "application/json"} + assert get.call_args.kwargs["timeout"] == 30 + request = requests.Request("GET", SERVICE_ROOT).prepare() + assert get.call_args.kwargs["auth"](request).headers.get("Authorization") is None + + +@pytest.mark.parametrize("format", ["xml", "json"]) +@pytest.mark.parametrize("version", ODATA_VERSIONS) +def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_path, format, version): + _, get = metadata_response + path = tmp_path / "metadata" + path.write_bytes((FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode())) + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + SERVICE_ROOT, + "--metadata-file", + str(path), + "--service-root-file", + str(tmp_path / "does-not-exist.json"), + "--entity-set", + "Orders", + "--entity-set", + "products", + "--entity-set", + "ArchivedProducts", + "--entity-set", + "Products", + "--owner", + "Catalog", + "--id", + "catalog", + "--debug", + ], + ) + assert result.exit_code == 0, result.output + contract = yaml.safe_load(result.stdout) + assert contract["name"] == "Store" and contract["id"] == "catalog" + assert contract["team"]["name"] == "Catalog" + schemas = contract["schema"] + assert [s["name"] for s in schemas] == ["Orders", "Products", "ArchivedProducts"] + assert schemas[1]["properties"] == schemas[2]["properties"] + assert schemas[1]["customProperties"] != schemas[2]["customProperties"] + get.assert_not_called() + + +@pytest.mark.parametrize("root", [SERVICE_ROOT, SERVICE_ROOT.rstrip("/")]) +def test_derived_metadata_url_with_explicit_selection(metadata_response, root): + metadata, get = metadata_response + metadata.content = (FIXTURES / "products.json").read_bytes() + metadata.headers = {} + contract = DataContract.import_from_source("odata", source=root, odata_entity_set=["Products", "Orders"]) + get.assert_called_once() + assert get.call_args.args == (SERVICE_ROOT + "$metadata",) + assert contract.servers[0].location == root.rstrip("/") + "/" + + +@pytest.mark.parametrize("selection", [[], "Products", [None], [""], [" "]]) +def test_invalid_explicit_selection_does_not_fetch(metadata_response, selection): + _, get = metadata_response + with pytest.raises(DataContractException, match="non-empty list"): + DataContract.import_from_source("odata", source=SERVICE_ROOT, odata_entity_set=selection) + get.assert_not_called() + + +@pytest.mark.parametrize("context_key", ["@odata.context", "@context"]) +@pytest.mark.parametrize("context", ["$metadata", "https://example.com/custom/$metadata"]) +def test_service_document_url_resolution(service_response, context_key, context): + service, get = service_response + service.content = json.dumps( + { + context_key: context, + "value": [ + {"name": "Products", "url": "items/products"}, + {"name": "Orders", "@context": "../shipping/$metadata", "url": "orders"}, + {"name": "Unknown", "kind": "FutureResource", "url": "ignored"}, + ], + } + ).encode() + contract = DataContract.import_from_source("odata", source=SERVICE_ROOT) + urls = [{p.property: p.value for p in s.customProperties}["odataEntitySetUrl"] for s in contract.schema_] + directory = "odata" if context == "$metadata" else "custom" + assert urls == [f"https://example.com/{directory}/items/products", "https://example.com/shipping/orders"] + assert [call.args[0] for call in get.call_args_list] == [SERVICE_ROOT + "$metadata", SERVICE_ROOT] + + +@pytest.mark.parametrize("url", ["https://example.com/redirected/", "https://example.com/redirected/service.json"]) +@pytest.mark.parametrize("root", [SERVICE_ROOT, SERVICE_ROOT.rstrip("/")]) +def test_redirected_service_document_base(service_response, url, root): + service, get = service_response + service.url = url + contract = DataContract.import_from_source("odata", source=root) + assert contract.schema_[0].customProperties[1].value == "https://example.com/redirected/products" + assert contract.servers[0].location == SERVICE_ROOT + assert [call.args[0] for call in get.call_args_list] == [SERVICE_ROOT + "$metadata", SERVICE_ROOT] + + +@pytest.mark.parametrize("version", [None, *ODATA_VERSIONS]) +def test_service_header_does_not_override_csdl_version(service_response, version): + service, _ = service_response + service.headers = {"OData-Version": version} if version else {} + contract = DataContract.import_from_source("odata", source=SERVICE_ROOT) + assert {p.property: p.value for p in contract.servers[0].customProperties}["odataVersion"] == "4.01" + + +@pytest.mark.parametrize("target", ["metadata", "service"]) +@pytest.mark.parametrize("version", ["", "3.0", "5.0", "4", "4.", "4.x", "4.1.0"]) +def test_invalid_http_versions(metadata_response, service_response, target, version): + metadata, _ = metadata_response + service, _ = service_response + response = metadata if target == "metadata" else service + response.headers = {"OData-Version": version} + with pytest.raises(DataContractException, match="Unsupported OData version"): + DataContract.import_from_source("odata", source=SERVICE_ROOT) + + +@pytest.mark.parametrize( + "document,error", + [ + ({}, "context URL"), + ({"@context": None, "value": []}, "Invalid context URL"), + ({"@context": "http://[", "value": []}, "Invalid context URL"), + ({"@context": "$metadata", "@odata.context": "other", "value": []}, "Conflicting context"), + ({"@context": "$metadata", "value": {}}, "value array"), + ({"@context": "$metadata", "value": []}, "No EntitySets"), + ({"@context": "$metadata", "value": [None]}, "JSON object"), + ({"@context": "$metadata", "value": [{"name": "Products", "url": "Products", "kind": 1}]}, "Invalid kind"), + ({"@context": "$metadata", "value": [{"name": "Products"}]}, "name and url"), + ({"@context": "$metadata", "value": [{"name": 1, "url": "Products"}]}, "name and url"), + ({"@context": "$metadata", "value": [{"name": "Products", "url": "file:///tmp/data"}]}, "Invalid .*URL"), + ({"@context": "$metadata", "value": [{"name": "Products", "url": "http://["}]}, "Invalid service document URL"), + ({"@context": "$metadata", "value": [{"name": "Absent", "url": "Absent"}]}, "not found"), + ({"@context": "$metadata", "value": [{"name": "Products", "url": "Products"}] * 2}, "Duplicate EntitySet"), + ( + { + "@context": "$metadata", + "value": [{"name": "products", "url": "Products"}, {"name": "Products", "url": "Products"}], + }, + "Duplicate EntitySet", + ), + ], +) +def test_invalid_service_documents(service_response, tmp_path, document, error): + service, _ = service_response + service.content = json.dumps(document).encode() + output = tmp_path / "contract.yaml" + output.write_text("existing contract") + result = CliRunner().invoke(app, ["import", "odata", "--service-root-url", SERVICE_ROOT, "--output", str(output)]) + assert result.exit_code != 0 and isinstance(result.exception, DataContractException) + assert re.search(error, str(result.exception)) + assert output.read_text() == "existing contract" + + +@pytest.mark.parametrize("content", [b"", b"{", b'{"value":[],"value":[]}']) +def test_service_document_invalid_json(service_response, content): + service, _ = service_response + service.content = content + with pytest.raises(DataContractException, match="Invalid or unsafe|Duplicate JSON key"): + DataContract.import_from_source("odata", source=SERVICE_ROOT) + + +@pytest.mark.parametrize("failure", ["http", "timeout", "version"]) +def test_service_document_transport_errors(service_response, failure): + service, get = service_response + if failure == "http": + service.raise_for_status.side_effect = requests.HTTPError("HTTP 403") + elif failure == "timeout": + previous = get.side_effect + + def fetch(url, **kwargs): + if url == SERVICE_ROOT: + raise requests.Timeout("timed out") + return previous(url, **kwargs) + + get.side_effect = fetch + else: + service.headers = {"OData-Version": "3.0"} + with pytest.raises(DataContractException, match="Failed to fetch OData service document|Unsupported OData version"): + DataContract.import_from_source("odata", source=SERVICE_ROOT) + + +def test_missing_service_file_does_not_fall_back_to_network(metadata_response, tmp_path): + _, get = metadata_response + with pytest.raises(DataContractException, match="Failed to read OData service document file"): + DataContract.import_from_source( + "odata", + source=SERVICE_ROOT, + odata_metadata_file=FIXTURES / "products.xml", + odata_service_root_file=tmp_path / "missing.json", + ) + get.assert_not_called() + + +def test_cli_offline_all_sets(metadata_response, tmp_path): + _, get = metadata_response + output = tmp_path / "contract.yaml" + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + SERVICE_ROOT, + "--metadata-file", + str(FIXTURES / "products.xml"), + "--service-root-file", + str(FIXTURES / "service-document.json"), + "--output", + str(output), + ], + ) + assert result.exit_code == 0, result.output + assert [s["name"] for s in yaml.safe_load(output.read_text())["schema"]] == ["Products", "Orders"] + get.assert_not_called() + + +def test_failure_in_later_selected_schema_preserves_output(metadata_response, json_metadata, tmp_path): + metadata, get = metadata_response + json_metadata["Catalog.Model"]["Order"]["Total"]["$Type"] = "Edm.Binary" + metadata.content = json.dumps(json_metadata).encode() + output = tmp_path / "contract.yaml" + output.write_text("existing contract") + result = CliRunner().invoke( + app, + [ + "import", + "odata", + "--service-root-url", + SERVICE_ROOT, + "--metadata-url", + METADATA, + "--entity-set", + "Products", + "--entity-set", + "Orders", + "--output", + str(output), + ], + ) + assert result.exit_code != 0 and isinstance(result.exception, DataContractException) + assert "Orders.Total" in str(result.exception) + assert output.read_text() == "existing contract" + get.assert_called_once() + + +def test_xml_requires_unambiguous_container(metadata_response): + metadata, _ = metadata_response + metadata.headers = {} + metadata.content = metadata_xml(extra_schema='') + with pytest.raises(DataContractException, match="EntityContainer.*ambiguous"): + import_contract() + + +@pytest.mark.parametrize("authorization", [None, "Bearer synthetic-token", "Basic dXNlcjpwYXNz"]) +@pytest.mark.parametrize("config_source", ["environment", "config", "dict"]) +def test_authentication_on_both_documents(service_response, monkeypatch, authorization, config_source): + _, get = service_response + config = None + if config_source == "environment": + if authorization is not None: + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", authorization) + elif authorization is not None: + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", "Bearer overridden-token") + config = ( + Config(api_header_authorization=authorization) + if config_source == "config" + else {"DATACONTRACT_API_HEADER_AUTHORIZATION": authorization} + ) + contract = DataContract.import_from_source("odata", source=SERVICE_ROOT, odata_metadata_url=METADATA, config=config) + assert [call.args[0] for call in get.call_args_list] == [METADATA, SERVICE_ROOT] + for call in get.call_args_list: + headers = call.kwargs["headers"] + assert headers.get("Authorization") == authorization + if authorization is None: + assert "Authorization" not in headers + if authorization is not None: + assert authorization not in contract.to_yaml() + + +def test_cli_authentication_from_config_file(service_response, tmp_path, monkeypatch): + _, get = service_response + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", "Bearer overridden-token") + monkeypatch.setenv("ODATA_TEST_AUTH", "Bearer config-file-token") + config_file = tmp_path / "config.yaml" + config_file.write_text('api_header_authorization: "${ODATA_TEST_AUTH}"\n') + result = CliRunner().invoke( + app, + ["--config-file", str(config_file), "import", "odata", "--service-root-url", SERVICE_ROOT], + ) + assert result.exit_code == 0, result.output + assert len(get.call_args_list) == 2 + assert all(call.kwargs["headers"]["Authorization"] == "Bearer config-file-token" for call in get.call_args_list) + assert "config-file-token" not in result.output + assert "overridden-token" not in result.output + + +@pytest.mark.parametrize("selection", [True, False]) +def test_authenticated_offline_import(metadata_response, selection): + _, get = metadata_response + options = ( + {"odata_entity_set": ["Products"]} + if selection + else {"odata_service_root_file": FIXTURES / "service-document.json"} + ) + contract = DataContract.import_from_source( + "odata", + source=SERVICE_ROOT, + odata_metadata_file=FIXTURES / "products.xml", + config=Config(api_header_authorization="Bearer offline-token"), + **options, + ) + get.assert_not_called() + assert "offline-token" not in contract.to_yaml() + + +@pytest.fixture +def http_transport(monkeypatch): + # Exercise real request preparation and redirects; only the adapter's I/O is mocked. + routes = { + SERVICE_ROOT + "$metadata": (200, {}, (FIXTURES / "products.json").read_bytes()), + SERVICE_ROOT: (200, {}, (FIXTURES / "service-document.json").read_bytes()), + } + sent = [] + + def send(adapter, request, **kwargs): + sent.append((request, kwargs)) + status, headers, content = routes[request.url] + response = requests.Response() + response.status_code = status + response.headers.update(headers) + response._content = content + response.raw = BytesIO(content) + response.request = request + response.url = request.url + return response + + monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send) + return routes, sent + + +@pytest.mark.parametrize("authorization", [None, "Bearer redirect-token"]) +@pytest.mark.parametrize("target", ["metadata", "service"]) +@pytest.mark.parametrize( + "redirect_url, retained", + [ + ("https://example.com/redirected/document", True), + ("https://other.example.com/document", False), + ("http://example.com/document", False), + ], +) +def test_redirect_authentication(http_transport, monkeypatch, authorization, target, redirect_url, retained): + routes, sent = http_transport + netrc = Mock(return_value=("unexpected-user", "unexpected-password")) + monkeypatch.setattr("requests.sessions.get_netrc_auth", netrc) + initial_url = SERVICE_ROOT + "$metadata" if target == "metadata" else SERVICE_ROOT + final_url = initial_url + "final" + document = routes[initial_url] + routes[initial_url] = (302, {"Location": redirect_url}, b"") + routes[redirect_url] = (302, {"Location": final_url}, b"") + routes[final_url] = document + DataContract.import_from_source("odata", source=SERVICE_ROOT, config=Config(api_header_authorization=authorization)) + requests_by_url = {request.url: request for request, _ in sent} + assert requests_by_url[initial_url].headers.get("Authorization") == authorization + expected = authorization if retained else None + assert requests_by_url[redirect_url].headers.get("Authorization") == expected + # Returning to the initial host must not restore credentials removed earlier. + assert requests_by_url[final_url].headers.get("Authorization") == expected + netrc.assert_not_called() + + +def test_authentication_preserves_proxy_and_certificate_environment(http_transport, monkeypatch): + _, sent = http_transport + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example.com:8080") + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setenv("REQUESTS_CA_BUNDLE", "/synthetic/ca.pem") + DataContract.import_from_source( + "odata", source=SERVICE_ROOT, config=Config(api_header_authorization="Bearer proxy-token") + ) + assert len(sent) == 2 + for request, options in sent: + assert request.headers["Authorization"] == "Bearer proxy-token" + assert options["proxies"]["https"] == "http://proxy.example.com:8080" + assert options["verify"] == "/synthetic/ca.pem" + + +@pytest.mark.parametrize("failure", [401, 403, "invalid_header"]) +@pytest.mark.parametrize("target", ["metadata", "service"]) +def test_authentication_errors_do_not_expose_secrets(http_transport, tmp_path, monkeypatch, caplog, failure, target): + routes, _ = http_transport + secret = "Bearer confidential-test-token" + if failure == "invalid_header": + secret += "\ninvalid" + else: + url = SERVICE_ROOT + "$metadata" if target == "metadata" else SERVICE_ROOT + routes[url] = (failure, {}, secret.encode()) + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", secret) + output = tmp_path / "contract.yaml" + output.write_text("existing contract") + args = ["import", "odata", "--service-root-url", SERVICE_ROOT, "--output", str(output), "--debug"] + if target == "service": + args += ["--metadata-file", str(FIXTURES / "products.json")] + result = CliRunner().invoke(app, args) + assert result.exit_code != 0 + assert isinstance(result.exception, DataContractException) + assert result.exception.type == "connection" + assert ("InvalidHeader" if failure == "invalid_header" else f"HTTP {failure}") in str(result.exception) + assert result.exception.original_exception is None + rendered = result.output + "".join(traceback.format_exception(result.exception)) + caplog.text + assert "confidential-test-token" not in rendered + assert output.read_text() == "existing contract" From 215f1b59ac3890b734c0372bf263507a21e576dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20=C3=85hl=C3=A9n?= Date: Tue, 22 Sep 2026 19:32:21 +0200 Subject: [PATCH 2/7] Simplified OData tests and added changelog entry --- CHANGELOG.md | 3 ++ tests/test_import_odata.py | 89 ++++++++++++++++++++++---------------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 87e4732bf..9efa18367 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `datacontract import odata` creates a datacontract from OData 4 metadata at an URL or from a local file. + ## [1.2.1] - 2026-09-22 ### Highlights diff --git a/tests/test_import_odata.py b/tests/test_import_odata.py index 2d58219f8..a73fddc05 100644 --- a/tests/test_import_odata.py +++ b/tests/test_import_odata.py @@ -81,13 +81,14 @@ def test_who_contract_and_transport(metadata_response): @pytest.mark.parametrize("version", ODATA_VERSIONS) @pytest.mark.parametrize("header", [True, False]) -def test_versions_and_header_fallback(metadata_response, version, header): +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_versions_and_header_fallback(metadata_response, version, header, format): response, _ = metadata_response - response.content = metadata_xml('', version=version) + response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode()) response.headers = {"OData-Version": version} if header else {} - contract = import_contract() + contract = import_contract("Products") assert contract.servers[0].customProperties[1].value == version - assert contract.schema_[0].properties[0].name == "Code" + assert contract.schema_[0].properties[0].name == "Sku" @pytest.mark.parametrize( @@ -161,6 +162,15 @@ def test_keys_nullability_and_facets(metadata_response): assert not text.logicalTypeOptions +@pytest.mark.parametrize("scale", ["", ' Scale="0"'], ids=["implicit-zero", "explicit-zero"]) +def test_xml_decimal_zero_scale(metadata_response, scale): + response, _ = metadata_response + response.headers = {} + response.content = metadata_xml(f'') + prop = import_contract().schema_[0].properties[0] + assert {p.property: p.value for p in prop.customProperties} == {"precision": 10, "scale": 0} + + @pytest.mark.parametrize("entity_set", ["REF_COUNTRY", "ref_country", "Ref_Country"]) @pytest.mark.parametrize("root", ["https://example.com", "https://example.com/service", "https://example.com/service/"]) def test_entity_set_matching_normalizes_root(metadata_response, entity_set, root): @@ -211,8 +221,6 @@ def test_unsupported_types_fail_with_field_and_type(metadata_response, field_typ @pytest.mark.parametrize( "document, header, error", [ - (metadata_xml(version="3.0"), "3.0", "Unsupported OData version"), - (metadata_xml(version="5.0"), "5.0", "Unsupported OData version"), (metadata_xml(version="4.0"), "4.01", "Conflicting OData versions"), (metadata_xml(entity_attributes='BaseType="Demo.Base"'), None, "inheritance"), (metadata_xml().replace(b'Name="Service"', b'Name="Service" Extends="Demo.Base"'), None, "inheritance"), @@ -283,14 +291,16 @@ def test_invalid_root_does_not_fetch_metadata(metadata_response, location): get.assert_not_called() -@pytest.mark.parametrize( - "options", - [{"odata_metadata_url": METADATA, "odata_metadata_file": FIXTURES / "who-metadata.xml"}], -) -def test_metadata_sources_are_mutually_exclusive_for_python_api(metadata_response, options): +def test_metadata_sources_are_mutually_exclusive_for_python_api(metadata_response): _, get = metadata_response with pytest.raises(DataContractException, match="mutually exclusive"): - DataContract.import_from_source("odata", source=ROOT, odata_entity_set=["ref_country"], **options) + DataContract.import_from_source( + "odata", + source=ROOT, + odata_entity_set=["ref_country"], + odata_metadata_url=METADATA, + odata_metadata_file=FIXTURES / "who-metadata.xml", + ) get.assert_not_called() @@ -390,17 +400,15 @@ def test_local_who_metadata_matches_url_import(metadata_response, path_type): get.assert_not_called() -@pytest.mark.parametrize("version", ODATA_VERSIONS) @pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) -def test_local_metadata_version_and_encoding(metadata_response, tmp_path, version, encoding): +def test_local_metadata_encoding(metadata_response, tmp_path, encoding): _, get = metadata_response path = tmp_path / "metadata.xml" - document = metadata_xml('', version=version).decode() + document = metadata_xml('').decode() path.write_bytes((f'{document}').encode(encoding)) contract = DataContract.import_from_source( "odata", source=ROOT, odata_entity_set=["ref_country"], odata_metadata_file=path ) - assert contract.servers[0].customProperties[1].value == version assert contract.schema_[0].properties[0].name == "Code" get.assert_not_called() @@ -453,20 +461,12 @@ def json_metadata(metadata_response): @pytest.mark.parametrize("source", ["file", "url"]) -@pytest.mark.parametrize("version", ODATA_VERSIONS) -@pytest.mark.parametrize("scale", [None, 0, 2]) -def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source, version, scale): +def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source): response, get = metadata_response - response.headers = {"OData-Version": version, "Content-Type": "text/plain"} + response.headers = {"OData-Version": "4.01", "Content-Type": "text/plain"} contracts = [] for format in ("xml", "json"): - content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode()) - if format == "xml": - content = content.replace(b' Scale="2"', b"" if scale is None else f' Scale="{scale}"'.encode(), 1) - else: - document = json.loads(content) - document["Catalog.Model"]["Product"]["Price"]["$Scale"] = 0 if scale is None else scale - content = json.dumps(document).encode() + content = (FIXTURES / f"products.{format}").read_bytes() if source == "file": # Deliberately use an unrelated extension: content determines the format. path = tmp_path / "metadata.txt" @@ -499,7 +499,7 @@ def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source, vers assert properties[0]["logicalTypeOptions"] == {"maxLength": 32} assert properties[2]["customProperties"] == [ {"property": "precision", "value": 10}, - {"property": "scale", "value": 0 if scale is None else scale}, + {"property": "scale", "value": 2}, ] @@ -631,7 +631,6 @@ def test_json_exact_match_and_ambiguous_names(metadata_response, json_metadata): @pytest.mark.parametrize( "path,value,error", [ - (("$Version",), "3.0", "Unsupported OData version"), (("$Version",), 4.01, "Unsupported OData version"), (("$EntityContainer",), [], "namespace-qualified"), (("$EntityContainer",), "Missing.Store", "JSON object"), @@ -799,10 +798,17 @@ def service_response(metadata_response): return service, get -@pytest.mark.parametrize("format", ["xml", "json"]) -@pytest.mark.parametrize("metadata_source", ["file", "derived_url", "explicit_url"]) -@pytest.mark.parametrize("service_source", ["file", "url"]) -@pytest.mark.parametrize("root", [SERVICE_ROOT, SERVICE_ROOT.rstrip("/")]) +@pytest.mark.parametrize( + "format, metadata_source, service_source, root", + [ + pytest.param("xml", "file", "file", SERVICE_ROOT, id="offline-xml"), + pytest.param("json", "file", "file", SERVICE_ROOT, id="offline-json"), + pytest.param("xml", "derived_url", "url", SERVICE_ROOT, id="http-xml"), + pytest.param("json", "derived_url", "url", SERVICE_ROOT.rstrip("/"), id="http-json-normalized-root"), + pytest.param("json", "explicit_url", "file", SERVICE_ROOT, id="explicit-metadata-url-local-service"), + pytest.param("xml", "file", "url", SERVICE_ROOT, id="local-metadata-http-service"), + ], +) def test_import_all_advertised_sets(metadata_response, service_response, format, metadata_source, service_source, root): metadata, _ = metadata_response _, get = service_response @@ -845,11 +851,10 @@ def test_import_all_advertised_sets(metadata_response, service_response, format, @pytest.mark.parametrize("format", ["xml", "json"]) -@pytest.mark.parametrize("version", ODATA_VERSIONS) -def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_path, format, version): +def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_path, format): _, get = metadata_response path = tmp_path / "metadata" - path.write_bytes((FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode())) + path.write_bytes((FIXTURES / f"products.{format}").read_bytes()) result = CliRunner().invoke( app, [ @@ -1094,8 +1099,16 @@ def test_xml_requires_unambiguous_container(metadata_response): import_contract() -@pytest.mark.parametrize("authorization", [None, "Bearer synthetic-token", "Basic dXNlcjpwYXNz"]) -@pytest.mark.parametrize("config_source", ["environment", "config", "dict"]) +@pytest.mark.parametrize( + "authorization, config_source", + [ + pytest.param(None, "environment", id="anonymous"), + pytest.param("Bearer synthetic-token", "environment", id="bearer-environment"), + pytest.param("Bearer synthetic-token", "config", id="bearer-config"), + pytest.param("Basic dXNlcjpwYXNz", "config", id="basic-config"), + pytest.param("Bearer synthetic-token", "dict", id="bearer-dict"), + ], +) def test_authentication_on_both_documents(service_response, monkeypatch, authorization, config_source): _, get = service_response config = None From 5ee1bc25dedb1f4ebeaa8da36ec22a1bf0fa637c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20=C3=85hl=C3=A9n?= Date: Tue, 22 Sep 2026 19:43:22 +0200 Subject: [PATCH 3/7] Minor text updates. --- datacontract/imports/odata_importer.py | 2 +- docs/docs/imports/index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/datacontract/imports/odata_importer.py b/datacontract/imports/odata_importer.py index b04fcb1af..85f6a46d7 100644 --- a/datacontract/imports/odata_importer.py +++ b/datacontract/imports/odata_importer.py @@ -149,7 +149,7 @@ def _validate_http_url(url: str, label: str, *, root: bool = False) -> None: valid = False if not valid: raise _schema_error( - f"Invalid {label}: expected a public HTTP(S) URL without credentials. " + f"Invalid {label}: expected an HTTP(S) URL without embedded credentials. " "The service root must not contain a query or fragment." ) diff --git a/docs/docs/imports/index.md b/docs/docs/imports/index.md index 43c6598d8..3a6a7a3ae 100644 --- a/docs/docs/imports/index.md +++ b/docs/docs/imports/index.md @@ -90,7 +90,7 @@ Each import page shows a runnable example: a small source file under [`examples/ - odataA public OData 4 service. + odataAn OData 4.x service. From db00b98e79e0c16b9ba4669ea4fa969cb9c87daa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20=C3=85hl=C3=A9n?= Date: Wed, 23 Sep 2026 20:29:32 +0200 Subject: [PATCH 4/7] Updated documentation and tests. --- datacontract/command_import.py | 4 +- docs/docs/imports/odata.md | 259 +------------- tests/test_import_odata.py | 631 +++++++-------------------------- 3 files changed, 136 insertions(+), 758 deletions(-) diff --git a/datacontract/command_import.py b/datacontract/command_import.py index 982e3440d..720ad9d7c 100644 --- a/datacontract/command_import.py +++ b/datacontract/command_import.py @@ -923,13 +923,13 @@ def import_odata( ] = None, entity_set: Annotated[ Optional[List[str]], - typer.Option(help="EntitySet to import (repeat for multiple sets). If omitted, use the service document."), + typer.Option(help="EntitySet to import (repeat for multiple sets). If omitted, will import all sets from the service document."), ] = None, metadata_url: Annotated[ Optional[str], typer.Option( "--metadata-url", - help="CSDL XML or JSON URL. Defaults to SERVICE_ROOT_URL/$metadata unless --metadata-file is supplied.", + help="CSDL XML or JSON URL. Defaults to SERVICE_ROOT_URL/$metadata.", ), ] = None, metadata_file: Annotated[ diff --git a/docs/docs/imports/odata.md b/docs/docs/imports/odata.md index 847ab924a..6359fa100 100644 --- a/docs/docs/imports/odata.md +++ b/docs/docs/imports/odata.md @@ -6,36 +6,7 @@ description: "Create a data contract from one or more OData 4.x EntitySets using # Import: OData -Import one or more EntitySets from **OData 4.x** into an ODCS 3.2.0 contract. - -- **Select specific EntitySets:** Use `--entity-set Products` for one EntitySet, or repeat the option to select several. -- **Import all automatically:** Omit `--entity-set` to import every EntitySet listed in the service document into one contract. - -CSDL XML or JSON provides field types and constraints. The service document can be fetched from the service root -or read from a local file with `--service-root-file`. No data records are downloaded. - -## Parameters - -| Parameter | Required | Description | -|---|---|---| -| `--service-root-url` | Yes | HTTP(S) root URL of the OData service. Stored with exactly one trailing `/` as the server's `location`, including for offline imports. | -| `--service-root-file` | No | Path to a local JSON service document listing the available EntitySets. Without this option, the service document is fetched from the root URL. Ignored when `--entity-set` is supplied. | -| `--entity-set` | No | Name of an EntitySet to import. Repeat to select several. If omitted, import all EntitySets listed in the service document. Explicit selection skips reading the service document. | -| `--metadata-url` | No | HTTP(S) URL of the CSDL XML or JSON metadata. Defaults to `SERVICE_ROOT_URL/$metadata` when neither metadata option is supplied. | -| `--metadata-file` | No | Path to a local OData CSDL XML or JSON metadata file. | -| `--output` | No | Path to the generated data contract YAML file. If omitted, print the contract to stdout. | -| `--owner` | No | Owner or team name to assign to the generated contract. | -| `--id` | No | Identifier to assign to the generated contract. | -| `--debug` / `--no-debug` | No | Control debug logging; use `--debug` for diagnostic output. | -| `--help` | No | Show the command's options and exit. | - -`--metadata-url` and `--metadata-file` cannot be combined. For an import without network access, use -`--metadata-file` together with either `--entity-set` or `--service-root-file`. - -## Example: WHO countries - -The WHO [country endpoint](https://xmart-api-public-uat.who.int/refmart/ref_country) is described by its -[metadata document](https://xmart-api-public-uat.who.int/refmart/$metadata). +Creates a data contract from OData 4.x CSDL XML or JSON metadata. Select individual EntitySets, or automatically import all EntitySets advertised in the service document. Both public and authenticated services are supported. ```bash datacontract import odata \ @@ -44,232 +15,16 @@ datacontract import odata \ --output datacontract.yaml ``` -`--service-root-url` is required and is stored with exactly one trailing `/` as the server's `location`. -A trailing slash is optional in the input. - -If neither `--metadata-file` nor `--metadata-url` is supplied, metadata is fetched from the service root -with `$metadata` appended. Use `--metadata-url` to override that address; quote URLs containing `$metadata` -with single quotes to prevent shell expansion. The two metadata options are mutually exclusive. - -The importer matches EntitySet names exactly first. If there is no exact match, it accepts a case-insensitive -match only when unique. Ambiguous or missing matches produce an error. For WHO, `ref_country` matches -`REF_COUNTRY`; the schema keeps the metadata name. - -Omit `--output` to print YAML to stdout. Use `--owner` and `--id` to override the contract's owner and identifier. - -## Multiple EntitySets - -Repeat `--entity-set` to select several EntitySets: - -```bash -datacontract import odata \ - --service-root-url 'https://example.com/odata/' \ - --entity-set Products \ - --entity-set Orders \ - --output datacontract.yaml -``` - -Explicit selection reads only CSDL. It does not request the service root or read `--service-root-file`, -even if that option is supplied. Argument order is preserved; repeated selections of the same EntitySet -produce one schema object. EntitySets sharing an EntityType still produce separate schema objects. - -Without `--entity-set`, the importer reads the JSON service document from `--service-root-file`, if supplied, -or from `--service-root-url`. It imports the EntitySets advertised there, in document order. It does not -fall back to importing all EntitySets from CSDL. Singletons, function imports and linked services are ignored. -An omitted `kind` in a service document entry means `EntitySet`. +This example imports the WHO [country endpoint](https://xmart-api-public-uat.who.int/refmart/ref_country). `--service-root-url` is always required and is stored with a trailing `/` as the server's `location`. Metadata is read from `$metadata` under that root unless you supply `--metadata-url` or `--metadata-file`. -One EntitySet gives the contract its name. For multiple EntitySets, the contract uses the service container's name. -Missing or ambiguous EntitySets, duplicate entries in the service document and empty selections produce errors. -If any selected schema uses unsupported types, the whole import fails before writing the output file. +Repeat `--entity-set` to select several EntitySets. An explicit selection skips the service document. Without `--entity-set`, the importer reads the JSON service document from the root URL and imports every EntitySet listed there into the same contract. -## Offline import - -Use a local CSDL file and an explicit selection to import without any network requests: - -```bash -datacontract import odata \ - --service-root-url 'https://example.com/odata/' \ - --metadata-file metadata.xml \ - --entity-set Products \ - --entity-set Orders \ - --output datacontract.yaml -``` - -Alternatively, provide both documents locally to import every advertised EntitySet: - -```bash -datacontract import odata \ - --service-root-url 'https://example.com/odata/' \ - --service-root-file service-document.json \ - --metadata-file metadata.json \ - --output datacontract.yaml -``` - -CSDL XML and JSON are detected from content, regardless of file extension or HTTP `Content-Type`. -Both formats also work with `--metadata-url`. Relative file paths are resolved from the current working directory. -Metadata saved from an authenticated service can be imported offline. - -`--service-root-file` alone does not guarantee offline import: CSDL must also be supplied through -`--metadata-file`. Conversely, `--metadata-file` without an explicit selection or a local service document -still causes a request to the service root. - -## Generated contract - -The server describes the service, and each schema object identifies its own EntitySet: - -```yaml -servers: - - server: source - type: api - location: https://example.com/odata/ - customProperties: - - property: apiType - value: odata - - property: odataVersion - value: "4.01" - - property: odataMetadataUrl - value: https://example.com/odata/$metadata - - property: format - value: json -schema: - - name: Products - physicalName: Products - logicalType: object - physicalType: object - customProperties: - - property: odataEntitySet - value: Products - - property: odataEntitySetUrl - value: https://example.com/odata/Products - # properties: ... -``` +For offline import, supply a local CSDL XML or JSON file through `--metadata-file`. Also supply `--service-root-file` or `--entity-set`. -These custom properties are Data Contract CLI conventions for future OData-aware consumers. -`odataVersion` records the CSDL response version, or the document version for file imports. -`odataMetadataFile` replaces `odataMetadataUrl` for local CSDL files. -`format: json` describes the data response, independently of the metadata format; ODCS API servers do not -permit a top-level `format` field. +For authentication, set `DATACONTRACT_API_HEADER_AUTHORIZATION` to the complete header value, such as `Bearer ` or a precomputed `Basic `. -`odataEntitySet` now belongs to each schema object, including for single-EntitySet imports. -`odataEntitySetUrl` records the advertised address when reading a service document. Relative addresses are -resolved using OData's context URL rules, with the service root as the document base for local files. -With explicit selection, the URL is constructed from the service root and the EntitySet name in CSDL. -Context links and EntitySet addresses are never fetched, and context links do not override the metadata source. - -## Supported schema - -| OData type | ODCS logical type | -|---|---| -| `Edm.String`, `Edm.Guid` | `string` (`uuid` format for Guid) | -| `Edm.Byte`, `Edm.SByte`, `Edm.Int16`, `Edm.Int32`, `Edm.Int64` | `integer` | -| `Edm.Decimal`, `Edm.Single`, `Edm.Double` | `number` | -| `Edm.Boolean` | `boolean` | -| `Edm.Date` | `date` | -| `Edm.DateTimeOffset` | `timestamp` | -| `Edm.TimeOfDay` | `time` | - -Original types are preserved in `physicalType`. The importer copies nullability, declared primary keys and their -positions, numeric maximum lengths, precision and scale. Precision and scale use property-level custom properties; -symbolic scale values are preserved. No primary keys are inferred when metadata omits them, as in the WHO example. - -Qualified type names and local schema aliases work across schemas. JSON selects the service container with -`$EntityContainer`; XML must identify one unambiguous, named EntityContainer. -Omitted `Nullable` means nullable in XML; omitted `$Nullable` means required in JSON. -In JSON, omitted `$Type` means `Edm.String`, and a decimal's omitted `$Scale` means `variable`. -An omitted `$Precision` introduces no precision constraint. - -## Current limitations - -- The service root must not contain credentials, query parameters or a fragment. -- Service documents must be JSON; CSDL supports both XML and JSON. -- All version strings of the form `4.x` are accepted, with no upper minor-version limit. - Requests do not send `OData-MaxVersion`. Other major versions, malformed versions and conflicting - CSDL document/header versions are rejected. Later 4.x versions use the same supported schema constructs; - accepting their version numbers does not add support for new OData features. -- Complex types, collection-valued fields, entity or container inheritance, enum types, type definitions and - types outside the table produce errors when used by selected schemas. -- Navigation properties are omitted with a warning. External metadata references are not downloaded. -- Malformed documents, duplicate JSON keys and XML DTD declarations are rejected. -- This feature implements **import only**. - -## Planned improvements - -- **`datacontract test`:** Validate the contract's schema against the service's `$metadata` document. - Testing will only read metadata and will not fetch EntitySets or data records. - -This capability is planned and is not yet implemented. - -## Authentication - -For authenticated services, set `DATACONTRACT_API_HEADER_AUTHORIZATION` to the complete `Authorization` -header value. For example, use a Bearer token supplied by your service: - -```bash -export DATACONTRACT_API_HEADER_AUTHORIZATION="Bearer ${ODATA_ACCESS_TOKEN}" -datacontract import odata \ - --service-root-url 'https://example.com/odata/' \ - --output datacontract.yaml -``` - -A precomputed `Basic ` header works too. The importer does not obtain -or refresh tokens. Without this setting, requests are anonymous; `.netrc` credentials are not used. - -The same header is sent to both the metadata URL and the service root when those documents are fetched, -including an explicitly supplied `--metadata-url` on another host. Requests' redirect rules remove the -header when redirecting to another host or downgrading from HTTPS to HTTP. Once removed, it is not -restored later in that redirect chain. Credentials are not written to the generated contract. - -You can also use the existing global `--config-file` option: - -```yaml -# datacontract-config.yaml -api_header_authorization: "Bearer ${ODATA_ACCESS_TOKEN}" -``` - -```bash -datacontract --config-file datacontract-config.yaml import odata \ - --service-root-url 'https://example.com/odata/' \ - --output datacontract.yaml -``` - -Explicit configuration takes precedence over the environment. Fully offline imports do not use the header -or make network requests. See [Configuration](../configuration.md) for all supported configuration sources. - -## Python - -```python -from datacontract.data_contract import DataContract - -contract = DataContract.import_from_source( - "odata", - source="https://example.com/odata/", - odata_metadata_file="metadata.json", - odata_entity_set=["Products", "Orders"], -) -print(contract.to_yaml()) -``` - -`source` is now the service root URL. Use `odata_metadata_url` for an explicit metadata URL, -or omit both metadata arguments to derive the URL from `source`. For offline import of all advertised sets, -replace `odata_entity_set` with `odata_service_root_file="service-document.json"`. -File arguments accept strings or `Path` objects; `odata_entity_set` accepts a non-empty list of names. - -For an authenticated URL import, pass the existing `Config` option: - -```python -import os - -from datacontract import Config -from datacontract.data_contract import DataContract - -contract = DataContract.import_from_source( - "odata", - source="https://example.com/odata/", - odata_entity_set=["Products"], - config=Config(api_header_authorization=f"Bearer {os.environ['ODATA_ACCESS_TOKEN']}"), -) -``` +The importer reads metadata only; no data records are fetched. It preserves supported primitive field types, nullability, declared keys and constraints. Complex types, collection-valued fields, enums, type definitions, inheritance and unsupported primitive types cause errors when used by a selected schema. Navigation properties are omitted, and external metadata references are not downloaded. -The `config` argument also accepts a dictionary keyed by `DATACONTRACT_API_HEADER_AUTHORIZATION`. +OData support currently covers import only. Support for `datacontract test` is planned to compare the contract's schema with `$metadata`, without fetching data records. All options: **[`datacontract import odata`](../commands/import/odata.md)**. diff --git a/tests/test_import_odata.py b/tests/test_import_odata.py index a73fddc05..b5f5fa170 100644 --- a/tests/test_import_odata.py +++ b/tests/test_import_odata.py @@ -1,5 +1,6 @@ +"""Tests for the OData importer, using local CSDL fixtures and mocked HTTP responses.""" + import json -import re import traceback from io import BytesIO from pathlib import Path @@ -15,12 +16,31 @@ from datacontract.cli import app from datacontract.config import set_cli_config from datacontract.data_contract import DataContract +from datacontract.imports.odata_importer import ( + _odata_4_schema, + _odata_version, + _read_json, + _read_xml, +) from datacontract.model.exceptions import DataContractException ROOT = "https://xmart-api-public-uat.who.int/refmart/" METADATA = "https://xmart-api-public-uat.who.int/refmart/$metadata" FIXTURES = Path(__file__).parent / "fixtures/odata" -ODATA_VERSIONS = ["4.0", "4.01", "4.02", "4.1", "4.123"] +ODATA_TEST_VERSIONS = ["4.0", "4.01", "4.123"] +SERVICE_ROOT = "https://example.com/odata/" + + +# Helper functions and shared fixtures + + +def run_cli(*args, source=SERVICE_ROOT): + return CliRunner().invoke(app, ["import", "odata", "--service-root-url", source, *map(str, args)]) + + +@pytest.fixture(scope="module") +def odcs_schema(): + return json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) @pytest.fixture(autouse=True) @@ -60,13 +80,19 @@ def metadata_xml(properties="", *, version="4.01", entity_attributes="", extra_s '''.encode() -def test_who_contract_and_transport(metadata_response): +def xml_schema(properties="", **kwargs): + return _odata_4_schema(_read_xml(metadata_xml(properties, **kwargs)), "REF_COUNTRY") + + +# Contract generation, type mapping and metadata validation + + +def test_who_contract_and_transport(metadata_response, odcs_schema): response, get = metadata_response contract = import_contract() actual = yaml.safe_load(contract.to_yaml()) assert actual == yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) - schema = json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) - jsonschema.validate(actual, schema) + jsonschema.validate(actual, odcs_schema) get.assert_called_once() assert get.call_args.args == (METADATA,) assert get.call_args.kwargs["headers"] == { @@ -79,7 +105,7 @@ def test_who_contract_and_transport(metadata_response): assert get.call_args.kwargs["auth"](request).headers.get("Authorization") is None -@pytest.mark.parametrize("version", ODATA_VERSIONS) +@pytest.mark.parametrize("version", ODATA_TEST_VERSIONS) @pytest.mark.parametrize("header", [True, False]) @pytest.mark.parametrize("format", ["xml", "json"]) def test_versions_and_header_fallback(metadata_response, version, header, format): @@ -111,20 +137,17 @@ def test_versions_and_header_fallback(metadata_response, version, header, format ], ) @pytest.mark.parametrize("format", ["xml", "json"]) -def test_primitive_types(metadata_response, edm_type, logical_type, format): - response, _ = metadata_response - response.headers = {} +def test_primitive_types(edm_type, logical_type, format): if format == "xml": - response.content = metadata_xml(f'') - prop = import_contract().schema_[0].properties[0] + schema = xml_schema(f'') else: document = json.loads((FIXTURES / "products.json").read_text()) document["Catalog.Model"]["Product"] = { "$Kind": "EntityType", "Value": {"$Type": f"Edm.{edm_type}", "$Nullable": True}, } - response.content = json.dumps(document).encode() - prop = import_contract("Products").schema_[0].properties[0] + schema = _odata_4_schema(_read_json(json.dumps(document).encode()), "Products") + prop = schema.properties[0] assert prop.logicalType == logical_type assert prop.physicalType == f"Edm.{edm_type}" assert prop.required is False @@ -137,10 +160,8 @@ def test_primitive_types(metadata_response, edm_type, logical_type, format): assert "scale" not in facets -def test_keys_nullability_and_facets(metadata_response): - response, _ = metadata_response - response.headers = {} - response.content = metadata_xml( +def test_keys_nullability_and_facets(): + schema = xml_schema( """ @@ -149,7 +170,7 @@ def test_keys_nullability_and_facets(metadata_response): """, keys='', ) - code, year, amount, variable, floating, text = import_contract().schema_[0].properties + code, year, amount, variable, floating, text = schema.properties assert code.required is True and year.required is True assert code.primaryKey is True and year.primaryKey is True assert (code.primaryKeyPosition, year.primaryKeyPosition) == (2, 1) @@ -162,15 +183,6 @@ def test_keys_nullability_and_facets(metadata_response): assert not text.logicalTypeOptions -@pytest.mark.parametrize("scale", ["", ' Scale="0"'], ids=["implicit-zero", "explicit-zero"]) -def test_xml_decimal_zero_scale(metadata_response, scale): - response, _ = metadata_response - response.headers = {} - response.content = metadata_xml(f'') - prop = import_contract().schema_[0].properties[0] - assert {p.property: p.value for p in prop.customProperties} == {"precision": 10, "scale": 0} - - @pytest.mark.parametrize("entity_set", ["REF_COUNTRY", "ref_country", "Ref_Country"]) @pytest.mark.parametrize("root", ["https://example.com", "https://example.com/service", "https://example.com/service/"]) def test_entity_set_matching_normalizes_root(metadata_response, entity_set, root): @@ -192,15 +204,12 @@ def test_exact_match_wins_and_casefold_ambiguity_fails(metadata_response): import_contract("Ref_Country") -def test_navigation_and_unrelated_types_are_omitted(metadata_response, caplog): - response, _ = metadata_response - response.headers = {} - response.content = metadata_xml( +def test_navigation_and_unrelated_types_are_omitted(caplog): + schema = xml_schema( '', extra_schema='', ) - contract = import_contract() - assert [p.name for p in contract.schema_[0].properties] == ["Code"] + assert [p.name for p in schema.properties] == ["Code"] assert "Omitting OData navigation property REF_COUNTRY.Related" in caplog.text @@ -208,12 +217,9 @@ def test_navigation_and_unrelated_types_are_omitted(metadata_response, caplog): "field_type", ["D.Address", "Collection(Edm.String)", "Collection(D.Address)", "D.Status", "Edm.Binary", "Edm.Unknown"], ) -def test_unsupported_types_fail_with_field_and_type(metadata_response, field_type): - response, _ = metadata_response - response.headers = {} - response.content = metadata_xml(f'') +def test_unsupported_types_fail_with_field_and_type(field_type): with pytest.raises(DataContractException) as error: - import_contract() + xml_schema(f'') assert "REF_COUNTRY.Value" in str(error.value) assert field_type in str(error.value) @@ -252,24 +258,11 @@ def test_metadata_errors(metadata_response, document, header, error): '', ], ) -def test_metadata_rejects_dtd(metadata_response, encoding, doctype): - response, _ = metadata_response - response.headers = {} +def test_metadata_rejects_dtd(encoding, doctype): document = metadata_xml().decode().replace('Name="REF_COUNTRY"', 'Name="&country;"') - response.content = (f'{doctype}{document}').encode(encoding) + content = (f'{doctype}{document}').encode(encoding) with pytest.raises(DataContractException, match="DTD declarations are not allowed"): - import_contract() - - -@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) -def test_metadata_encoding_and_escaped_text(metadata_response, encoding): - response, _ = metadata_response - response.headers = {} - document = metadata_xml('').decode() - response.content = ( - f'{document}' - ).encode(encoding) - assert import_contract().schema_[0].properties[0].name == "Code" + _read_xml(content) @pytest.mark.parametrize( @@ -322,68 +315,46 @@ def test_http_errors(metadata_response, status): import_contract() +# CLI options and output handling + + def test_cli_stdout_and_output(metadata_response, tmp_path): - runner = CliRunner() - args = ["import", "odata", "--service-root-url", ROOT, "--entity-set", "ref_country", "--metadata-url", METADATA] - stdout = runner.invoke(app, args) + args = ["--entity-set", "ref_country", "--metadata-url", METADATA] + stdout = run_cli(*args, source=ROOT) assert stdout.exit_code == 0, stdout.output assert yaml.safe_load(stdout.stdout) == yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) output = tmp_path / "contract.yaml" - result = runner.invoke(app, args + ["--output", str(output), "--owner", "WHO", "--id", "country", "--debug"]) + result = run_cli(*args, "--output", output, "--owner", "WHO", "--id", "country", "--debug", source=ROOT) assert result.exit_code == 0, result.output contract = yaml.safe_load(output.read_text()) assert contract["id"] == "country" assert contract["team"]["name"] == "WHO" -@pytest.mark.parametrize( - "args", - [ - [], - ["--metadata-url", METADATA], - ["--metadata-file", str(FIXTURES / "who-metadata.xml")], - ["--service-root-url", ROOT, "--entity-set", "ref_country", "--metadataUrl", METADATA], - [ - "--service-root-url", - ROOT, - "--entity-set", - "ref_country", - "--metadata-url", - METADATA, - "--metadata-file", - str(FIXTURES / "who-metadata.xml"), - ], - ], -) -def test_cli_requires_options_and_rejects_alias(metadata_response, args): - _, get = metadata_response - result = CliRunner().invoke(app, ["import", "odata"] + args) - assert result.exit_code == 2 - get.assert_not_called() - - -def test_failed_import_preserves_existing_output(metadata_response, tmp_path): +@pytest.mark.parametrize("existing", [False, True], ids=["no-new-file", "preserve-existing-file"]) +@pytest.mark.parametrize("source", ["file", "url"]) +def test_failed_import_preserves_output(metadata_response, tmp_path, existing, source): response, _ = metadata_response response.content = b"invalid XML" output = tmp_path / "contract.yaml" - output.write_text("existing contract") - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - ROOT, - "--entity-set", - "ref_country", - "--metadata-url", - METADATA, - "--output", - str(output), - ], - ) + if existing: + output.write_text("existing contract") + args = ["--metadata-url", METADATA] + if source == "file": + metadata = tmp_path / "invalid.xml" + metadata.write_bytes(response.content) + args = ["--metadata-file", metadata] + result = run_cli("--entity-set", "ref_country", *args, "--output", output) assert result.exit_code != 0 - assert output.read_text() == "existing contract" + assert isinstance(result.exception, DataContractException) + assert "Invalid or unsafe" in str(result.exception) + if existing: + assert output.read_text() == "existing contract" + else: + assert not output.exists() + + +# Local metadata files @pytest.mark.parametrize("path_type", [str, Path]) @@ -400,57 +371,7 @@ def test_local_who_metadata_matches_url_import(metadata_response, path_type): get.assert_not_called() -@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) -def test_local_metadata_encoding(metadata_response, tmp_path, encoding): - _, get = metadata_response - path = tmp_path / "metadata.xml" - document = metadata_xml('').decode() - path.write_bytes((f'{document}').encode(encoding)) - contract = DataContract.import_from_source( - "odata", source=ROOT, odata_entity_set=["ref_country"], odata_metadata_file=path - ) - assert contract.schema_[0].properties[0].name == "Code" - get.assert_not_called() - - -@pytest.mark.parametrize("kind", ["missing", "directory", "unreadable", "malformed", "dtd"]) -def test_local_metadata_errors_preserve_output(metadata_response, tmp_path, monkeypatch, kind): - _, get = metadata_response - path = tmp_path / "metadata.xml" - if kind == "directory": - path.mkdir() - elif kind == "unreadable": - monkeypatch.setattr(Path, "read_bytes", Mock(side_effect=PermissionError("Permission denied"))) - elif kind == "malformed": - path.write_bytes(b"not XML") - elif kind == "dtd": - path.write_bytes(b']>' + metadata_xml()) - output = tmp_path / "contract.yaml" - output.write_text("existing contract") - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - ROOT, - "--entity-set", - "ref_country", - "--metadata-file", - str(path), - "--output", - str(output), - ], - ) - assert result.exit_code != 0 - assert isinstance(result.exception, DataContractException) - message = "Invalid or unsafe" if kind in ("malformed", "dtd") else "Failed to read OData metadata file" - assert message in str(result.exception) - assert output.read_text() == "existing contract" - get.assert_not_called() - - -SERVICE_ROOT = "https://example.com/odata/" +# CSDL JSON and XML/JSON equivalence @pytest.fixture @@ -461,7 +382,7 @@ def json_metadata(metadata_response): @pytest.mark.parametrize("source", ["file", "url"]) -def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source): +def test_xml_json_equivalent_contracts(metadata_response, tmp_path, odcs_schema, source): response, get = metadata_response response.headers = {"OData-Version": "4.01", "Content-Type": "text/plain"} contracts = [] @@ -479,8 +400,7 @@ def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source): response.content = content contract = import_contract("Products", source=SERVICE_ROOT) actual = yaml.safe_load(contract.to_yaml()) - schema = json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) - jsonschema.validate(actual, schema) + jsonschema.validate(actual, odcs_schema) contracts.append(actual) assert contracts[0] == contracts[1] properties = contracts[0]["schema"][0]["properties"] @@ -503,73 +423,6 @@ def test_xml_json_equivalent_contracts(metadata_response, tmp_path, source): ] -@pytest.mark.parametrize("format", ["xml", "json"]) -def test_cli_synthetic_metadata_file(metadata_response, tmp_path, monkeypatch, format): - _, get = metadata_response - content = (FIXTURES / f"products.{format}").read_bytes() - monkeypatch.chdir(tmp_path) - Path("metadata").write_bytes(content) - args = [ - "import", - "odata", - "--service-root-url", - SERVICE_ROOT, - "--entity-set", - "Products", - "--metadata-file", - "metadata", - "--owner", - "Catalog", - "--id", - "products", - ] - stdout = CliRunner().invoke(app, args) - assert stdout.exit_code == 0, stdout.output - actual = yaml.safe_load(stdout.stdout) - assert actual["name"] == "Products" - assert actual["id"] == "products" and actual["team"]["name"] == "Catalog" - assert {"property": "odataMetadataFile", "value": "metadata"} in actual["servers"][0]["customProperties"] - result = CliRunner().invoke(app, args + ["--output", "contract.yaml"]) - assert result.exit_code == 0, result.output - assert yaml.safe_load(Path("contract.yaml").read_text()) == actual - get.assert_not_called() - - -@pytest.mark.parametrize("source", ["file", "url"]) -@pytest.mark.parametrize("encoding", ["utf-8", "utf-8-sig", "utf-16", "utf-32"]) -def test_json_encoding_and_leading_whitespace(metadata_response, tmp_path, source, encoding): - response, get = metadata_response - content = (" \n\t" * 100 + (FIXTURES / "products.json").read_text()).encode(encoding) - response.headers = {"Content-Type": "application/xml"} - if source == "file": - path = tmp_path / "metadata.xml" - path.write_bytes(content) - contract = DataContract.import_from_source( - "odata", source=SERVICE_ROOT, odata_entity_set=["Products"], odata_metadata_file=path - ) - get.assert_not_called() - else: - response.content = content - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - SERVICE_ROOT, - "--entity-set", - "Products", - "--metadata-url", - METADATA, - ], - ) - assert result.exit_code == 0, result.output - assert yaml.safe_load(result.stdout)["name"] == "Products" - get.assert_called_once_with(METADATA, **get.call_args.kwargs) - return - assert contract.name == "Products" - - def test_json_defaults_and_annotations(metadata_response, json_metadata): response, _ = metadata_response json_metadata["Catalog.Model"]["Product"] = { @@ -609,29 +462,9 @@ def test_json_alias_and_navigation(metadata_response, json_metadata, caplog, typ get.assert_called_once() -@pytest.mark.parametrize("entity_set", ["Products", "products", "PRODUCTS"]) -def test_json_name_matching(metadata_response, json_metadata, entity_set): - response, _ = metadata_response - response.content = json.dumps(json_metadata).encode() - contract = import_contract(entity_set, source=SERVICE_ROOT) - assert contract.name == "Products" and contract.servers[0].location == SERVICE_ROOT - - -def test_json_exact_match_and_ambiguous_names(metadata_response, json_metadata): - response, _ = metadata_response - json_metadata["Catalog.Service"]["Store"]["products"] = {"$Collection": True, "$Type": "Catalog.Product"} - response.content = json.dumps(json_metadata).encode() - assert import_contract("Products").name == "Products" - with pytest.raises(DataContractException, match="ambiguous"): - import_contract("PRODUCTS") - with pytest.raises(DataContractException, match="not found"): - import_contract("Missing") - - @pytest.mark.parametrize( "path,value,error", [ - (("$Version",), 4.01, "Unsupported OData version"), (("$EntityContainer",), [], "namespace-qualified"), (("$EntityContainer",), "Missing.Store", "JSON object"), (("Catalog.Model", "$Alias"), 42, "Invalid .*Alias"), @@ -661,110 +494,36 @@ def test_json_exact_match_and_ambiguous_names(metadata_response, json_metadata): (("Catalog.Model", "Product", "Sku", "$MaxLength"), None, "Invalid MaxLength"), ], ) -def test_json_invalid_csdl(metadata_response, json_metadata, path, value, error): - response, _ = metadata_response +def test_json_invalid_csdl(json_metadata, path, value, error): target = json_metadata for key in path[:-1]: target = target[key] target[path[-1]] = value - response.content = json.dumps(json_metadata).encode() with pytest.raises(DataContractException, match=error): - import_contract("Products") + _odata_4_schema(_read_json(json.dumps(json_metadata).encode()), "Products") @pytest.mark.parametrize("kind", ["ComplexType", "EnumType", "TypeDefinition"]) -def test_json_unsupported_selected_type(metadata_response, json_metadata, kind): - response, _ = metadata_response +def test_json_unsupported_selected_type(json_metadata, kind): json_metadata["Catalog.Model"]["Special"] = {"$Kind": kind} json_metadata["Catalog.Model"]["Product"]["Price"]["$Type"] = "Catalog.Special" - response.content = json.dumps(json_metadata).encode() with pytest.raises(DataContractException, match="Catalog.Special.*Products.Price"): - import_contract("Products") + _odata_4_schema(_read_json(json.dumps(json_metadata).encode()), "Products") -@pytest.mark.parametrize( - "content,error", - [ - (b'{"$Version": "4.01",', "Invalid or unsafe"), - (b'{"$Version":"4.01","$Version":"4.0"}', "Duplicate JSON key"), - (b'{"Schema":{"Field":{},"Field":{}}}', "Duplicate JSON key"), - (b'{"value":NaN}', "Invalid JSON constant"), - (b'{"value":Infinity}', "Invalid JSON constant"), - (b"[]", "JSON object"), - (b"null", "JSON object"), - (b'{"bad":"\xff"}', "Invalid or unsafe"), - ( - b'{"@odata.context":"https://example.com/odata/$metadata","value":[{"name":"Products","kind":"EntitySet","url":"Products"}]}', - "service/data document.*field definitions", - ), - ], -) -def test_json_invalid_documents_preserve_output(metadata_response, tmp_path, content, error): - response, get = metadata_response - response.headers = {} - response.content = content - output = tmp_path / "contract.yaml" - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - SERVICE_ROOT, - "--entity-set", - "Products", - "--metadata-url", - METADATA, - "--output", - str(output), - ], - ) - assert result.exit_code != 0 - assert isinstance(result.exception, DataContractException) - assert re.search(error, str(result.exception)) - assert not output.exists() - get.assert_called_once() +# Version validation and format detection @pytest.mark.parametrize("document_version, header_version", [("4.01", "4.0"), ("4.123", "4.1"), ("4.01", "4.1")]) -def test_json_version_conflict(metadata_response, json_metadata, document_version, header_version): - response, _ = metadata_response - json_metadata["$Version"] = document_version - response.headers = {"OData-Version": header_version} - response.content = json.dumps(json_metadata).encode() +def test_version_conflict(document_version, header_version): with pytest.raises(DataContractException, match="Conflicting OData versions"): - import_contract("Products") + _odata_version(document_version, header_version) -@pytest.mark.parametrize("format", ["xml", "json"]) -@pytest.mark.parametrize("version", [None, "3.0", "5.0", "4", "4.", "4.x", "4.1.0", "4.01beta", "4.١"]) -def test_invalid_document_versions(metadata_response, format, version): - response, _ = metadata_response - response.headers = {"OData-Version": "4.01"} - if format == "json": - document = json.loads((FIXTURES / "products.json").read_text()) - if version is None: - document.pop("$Version") - else: - document["$Version"] = version - response.content = json.dumps(document).encode() - else: - response.content = (FIXTURES / "products.xml").read_bytes() - if version is None: - response.content = response.content.replace(b' Version="4.01"', b"") - else: - response.content = response.content.replace(b'Version="4.01"', f'Version="{version}"'.encode()) +@pytest.mark.parametrize("version", [None, "3.0", "5.0", "4", "4.", "4.x", "4.1.0", "4.01beta", "4.١", 4.01]) +def test_invalid_versions(version): with pytest.raises(DataContractException, match="Unsupported OData version"): - import_contract("Products") - - -@pytest.mark.parametrize("format", ["xml", "json"]) -def test_version_header_whitespace_preserves_version_string(metadata_response, format): - response, _ = metadata_response - response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", b"4.00123") - response.headers = {"OData-Version": " 4.00123 "} - contract = import_contract("Products") - assert {p.property: p.value for p in contract.servers[0].customProperties}["odataVersion"] == "4.00123" + _odata_version(version, None) @pytest.mark.parametrize("prefix", [b" \t\n" * 100, b"\xef\xbb\xbf \t\n"]) @@ -775,12 +534,7 @@ def test_xml_detection_with_whitespace_and_bom(metadata_response, prefix): assert import_contract().schema_[0].properties[0].name == "Label" -def test_xml_encoding_declaration_is_respected(metadata_response): - response, _ = metadata_response - response.headers = {} - document = metadata_xml('').decode() - response.content = ('' + document).encode("iso-8859-1") - assert import_contract().schema_[0].properties[0].name == "Libellé" +# Service documents and selection of multiple EntitySets @pytest.fixture @@ -809,7 +563,9 @@ def service_response(metadata_response): pytest.param("xml", "file", "url", SERVICE_ROOT, id="local-metadata-http-service"), ], ) -def test_import_all_advertised_sets(metadata_response, service_response, format, metadata_source, service_source, root): +def test_import_all_advertised_sets( + metadata_response, service_response, odcs_schema, format, metadata_source, service_source, root +): metadata, _ = metadata_response _, get = service_response metadata.content = (FIXTURES / f"products.{format}").read_bytes() @@ -840,8 +596,7 @@ def test_import_all_advertised_sets(metadata_response, service_response, format, assert server_properties["odataVersion"] == "4.01" if metadata_source != "file": assert server_properties["odataMetadataUrl"] == expected_requests[0] - schema = json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) - jsonschema.validate(yaml.safe_load(contract.to_yaml()), schema) + jsonschema.validate(yaml.safe_load(contract.to_yaml()), odcs_schema) assert [call.args[0] for call in get.call_args_list] == expected_requests if service_source == "url": assert get.call_args.kwargs["headers"] == {"Accept": "application/json"} @@ -851,40 +606,29 @@ def test_import_all_advertised_sets(metadata_response, service_response, format, @pytest.mark.parametrize("format", ["xml", "json"]) -def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_path, format): +def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_path, monkeypatch, format): _, get = metadata_response - path = tmp_path / "metadata" + monkeypatch.chdir(tmp_path) + path = Path("metadata") # Relative path without a format-specific extension. path.write_bytes((FIXTURES / f"products.{format}").read_bytes()) - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - SERVICE_ROOT, - "--metadata-file", - str(path), - "--service-root-file", - str(tmp_path / "does-not-exist.json"), - "--entity-set", - "Orders", - "--entity-set", - "products", - "--entity-set", - "ArchivedProducts", - "--entity-set", - "Products", - "--owner", - "Catalog", - "--id", - "catalog", - "--debug", - ], + result = run_cli( + "--metadata-file", + path, + "--service-root-file", + "does-not-exist.json", + "--entity-set", + "Orders", + "--entity-set", + "products", + "--entity-set", + "ArchivedProducts", + "--entity-set", + "Products", ) assert result.exit_code == 0, result.output contract = yaml.safe_load(result.stdout) - assert contract["name"] == "Store" and contract["id"] == "catalog" - assert contract["team"]["name"] == "Catalog" + assert contract["name"] == "Store" + assert {"property": "odataMetadataFile", "value": "metadata"} in contract["servers"][0]["customProperties"] schemas = contract["schema"] assert [s["name"] for s in schemas] == ["Orders", "Products", "ArchivedProducts"] assert schemas[1]["properties"] == schemas[2]["properties"] @@ -903,14 +647,6 @@ def test_derived_metadata_url_with_explicit_selection(metadata_response, root): assert contract.servers[0].location == root.rstrip("/") + "/" -@pytest.mark.parametrize("selection", [[], "Products", [None], [""], [" "]]) -def test_invalid_explicit_selection_does_not_fetch(metadata_response, selection): - _, get = metadata_response - with pytest.raises(DataContractException, match="non-empty list"): - DataContract.import_from_source("odata", source=SERVICE_ROOT, odata_entity_set=selection) - get.assert_not_called() - - @pytest.mark.parametrize("context_key", ["@odata.context", "@context"]) @pytest.mark.parametrize("context", ["$metadata", "https://example.com/custom/$metadata"]) def test_service_document_url_resolution(service_response, context_key, context): @@ -943,7 +679,7 @@ def test_redirected_service_document_base(service_response, url, root): assert [call.args[0] for call in get.call_args_list] == [SERVICE_ROOT + "$metadata", SERVICE_ROOT] -@pytest.mark.parametrize("version", [None, *ODATA_VERSIONS]) +@pytest.mark.parametrize("version", [None, *ODATA_TEST_VERSIONS]) def test_service_header_does_not_override_csdl_version(service_response, version): service, _ = service_response service.headers = {"OData-Version": version} if version else {} @@ -951,82 +687,6 @@ def test_service_header_does_not_override_csdl_version(service_response, version assert {p.property: p.value for p in contract.servers[0].customProperties}["odataVersion"] == "4.01" -@pytest.mark.parametrize("target", ["metadata", "service"]) -@pytest.mark.parametrize("version", ["", "3.0", "5.0", "4", "4.", "4.x", "4.1.0"]) -def test_invalid_http_versions(metadata_response, service_response, target, version): - metadata, _ = metadata_response - service, _ = service_response - response = metadata if target == "metadata" else service - response.headers = {"OData-Version": version} - with pytest.raises(DataContractException, match="Unsupported OData version"): - DataContract.import_from_source("odata", source=SERVICE_ROOT) - - -@pytest.mark.parametrize( - "document,error", - [ - ({}, "context URL"), - ({"@context": None, "value": []}, "Invalid context URL"), - ({"@context": "http://[", "value": []}, "Invalid context URL"), - ({"@context": "$metadata", "@odata.context": "other", "value": []}, "Conflicting context"), - ({"@context": "$metadata", "value": {}}, "value array"), - ({"@context": "$metadata", "value": []}, "No EntitySets"), - ({"@context": "$metadata", "value": [None]}, "JSON object"), - ({"@context": "$metadata", "value": [{"name": "Products", "url": "Products", "kind": 1}]}, "Invalid kind"), - ({"@context": "$metadata", "value": [{"name": "Products"}]}, "name and url"), - ({"@context": "$metadata", "value": [{"name": 1, "url": "Products"}]}, "name and url"), - ({"@context": "$metadata", "value": [{"name": "Products", "url": "file:///tmp/data"}]}, "Invalid .*URL"), - ({"@context": "$metadata", "value": [{"name": "Products", "url": "http://["}]}, "Invalid service document URL"), - ({"@context": "$metadata", "value": [{"name": "Absent", "url": "Absent"}]}, "not found"), - ({"@context": "$metadata", "value": [{"name": "Products", "url": "Products"}] * 2}, "Duplicate EntitySet"), - ( - { - "@context": "$metadata", - "value": [{"name": "products", "url": "Products"}, {"name": "Products", "url": "Products"}], - }, - "Duplicate EntitySet", - ), - ], -) -def test_invalid_service_documents(service_response, tmp_path, document, error): - service, _ = service_response - service.content = json.dumps(document).encode() - output = tmp_path / "contract.yaml" - output.write_text("existing contract") - result = CliRunner().invoke(app, ["import", "odata", "--service-root-url", SERVICE_ROOT, "--output", str(output)]) - assert result.exit_code != 0 and isinstance(result.exception, DataContractException) - assert re.search(error, str(result.exception)) - assert output.read_text() == "existing contract" - - -@pytest.mark.parametrize("content", [b"", b"{", b'{"value":[],"value":[]}']) -def test_service_document_invalid_json(service_response, content): - service, _ = service_response - service.content = content - with pytest.raises(DataContractException, match="Invalid or unsafe|Duplicate JSON key"): - DataContract.import_from_source("odata", source=SERVICE_ROOT) - - -@pytest.mark.parametrize("failure", ["http", "timeout", "version"]) -def test_service_document_transport_errors(service_response, failure): - service, get = service_response - if failure == "http": - service.raise_for_status.side_effect = requests.HTTPError("HTTP 403") - elif failure == "timeout": - previous = get.side_effect - - def fetch(url, **kwargs): - if url == SERVICE_ROOT: - raise requests.Timeout("timed out") - return previous(url, **kwargs) - - get.side_effect = fetch - else: - service.headers = {"OData-Version": "3.0"} - with pytest.raises(DataContractException, match="Failed to fetch OData service document|Unsupported OData version"): - DataContract.import_from_source("odata", source=SERVICE_ROOT) - - def test_missing_service_file_does_not_fall_back_to_network(metadata_response, tmp_path): _, get = metadata_response with pytest.raises(DataContractException, match="Failed to read OData service document file"): @@ -1042,20 +702,13 @@ def test_missing_service_file_does_not_fall_back_to_network(metadata_response, t def test_cli_offline_all_sets(metadata_response, tmp_path): _, get = metadata_response output = tmp_path / "contract.yaml" - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - SERVICE_ROOT, - "--metadata-file", - str(FIXTURES / "products.xml"), - "--service-root-file", - str(FIXTURES / "service-document.json"), - "--output", - str(output), - ], + result = run_cli( + "--metadata-file", + FIXTURES / "products.xml", + "--service-root-file", + FIXTURES / "service-document.json", + "--output", + output, ) assert result.exit_code == 0, result.output assert [s["name"] for s in yaml.safe_load(output.read_text())["schema"]] == ["Products", "Orders"] @@ -1068,22 +721,8 @@ def test_failure_in_later_selected_schema_preserves_output(metadata_response, js metadata.content = json.dumps(json_metadata).encode() output = tmp_path / "contract.yaml" output.write_text("existing contract") - result = CliRunner().invoke( - app, - [ - "import", - "odata", - "--service-root-url", - SERVICE_ROOT, - "--metadata-url", - METADATA, - "--entity-set", - "Products", - "--entity-set", - "Orders", - "--output", - str(output), - ], + result = run_cli( + "--metadata-url", METADATA, "--entity-set", "Products", "--entity-set", "Orders", "--output", output ) assert result.exit_code != 0 and isinstance(result.exception, DataContractException) assert "Orders.Total" in str(result.exception) @@ -1091,12 +730,12 @@ def test_failure_in_later_selected_schema_preserves_output(metadata_response, js get.assert_called_once() -def test_xml_requires_unambiguous_container(metadata_response): - metadata, _ = metadata_response - metadata.headers = {} - metadata.content = metadata_xml(extra_schema='') +def test_xml_requires_unambiguous_container(): with pytest.raises(DataContractException, match="EntityContainer.*ambiguous"): - import_contract() + xml_schema(extra_schema='') + + +# Authentication and configuration @pytest.mark.parametrize( @@ -1150,23 +789,7 @@ def test_cli_authentication_from_config_file(service_response, tmp_path, monkeyp assert "overridden-token" not in result.output -@pytest.mark.parametrize("selection", [True, False]) -def test_authenticated_offline_import(metadata_response, selection): - _, get = metadata_response - options = ( - {"odata_entity_set": ["Products"]} - if selection - else {"odata_service_root_file": FIXTURES / "service-document.json"} - ) - contract = DataContract.import_from_source( - "odata", - source=SERVICE_ROOT, - odata_metadata_file=FIXTURES / "products.xml", - config=Config(api_header_authorization="Bearer offline-token"), - **options, - ) - get.assert_not_called() - assert "offline-token" not in contract.to_yaml() +# HTTP transport, redirects and protection of credentials @pytest.fixture @@ -1252,10 +875,10 @@ def test_authentication_errors_do_not_expose_secrets(http_transport, tmp_path, m monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", secret) output = tmp_path / "contract.yaml" output.write_text("existing contract") - args = ["import", "odata", "--service-root-url", SERVICE_ROOT, "--output", str(output), "--debug"] + args = ["--output", output, "--debug"] if target == "service": args += ["--metadata-file", str(FIXTURES / "products.json")] - result = CliRunner().invoke(app, args) + result = run_cli(*args) assert result.exit_code != 0 assert isinstance(result.exception, DataContractException) assert result.exception.type == "connection" From 067c3831cc99f45abefe95e32fba0659e6a1a9ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 24 Sep 2026 10:49:12 +0200 Subject: [PATCH 5/7] test: remove duplicate OData test fixtures --- docs/static/img/icons/odata.svg | 13 ++ tests/fixtures/odata/README.md | 16 --- tests/fixtures/odata/who-contract.yaml | 162 ------------------------- tests/test_import_odata.py | 22 +--- 4 files changed, 17 insertions(+), 196 deletions(-) create mode 100644 docs/static/img/icons/odata.svg delete mode 100644 tests/fixtures/odata/README.md delete mode 100644 tests/fixtures/odata/who-contract.yaml diff --git a/docs/static/img/icons/odata.svg b/docs/static/img/icons/odata.svg new file mode 100644 index 000000000..3f12a9ea5 --- /dev/null +++ b/docs/static/img/icons/odata.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/tests/fixtures/odata/README.md b/tests/fixtures/odata/README.md deleted file mode 100644 index 8a9eefbff..000000000 --- a/tests/fixtures/odata/README.md +++ /dev/null @@ -1,16 +0,0 @@ -`who-metadata.xml` was retrieved from -https://xmart-api-public-uat.who.int/refmart/$metadata on 2026-09-18. -The response declared `OData-Version: 4.0`. XML whitespace was formatted for readability. - -`who-contract.yaml` describes the expected import of -https://xmart-api-public-uat.who.int/refmart/ref_country. -All network access is mocked in the tests. - -`products.xml` and `products.json` are independently authored synthetic CSDL examples -of the same catalog model. Entity types and the service container live in separate -schemas and use an alias. They cover composite keys, ordered properties and facets; -they contain no data or schema from a private service. - -`service-document.json` advertises Products and Orders from the synthetic model, -with relative and absolute addresses, plus other resource kinds to ignore. -ArchivedProducts shares the Product type but is available only through explicit selection. diff --git a/tests/fixtures/odata/who-contract.yaml b/tests/fixtures/odata/who-contract.yaml deleted file mode 100644 index 2e17cd2c9..000000000 --- a/tests/fixtures/odata/who-contract.yaml +++ /dev/null @@ -1,162 +0,0 @@ -apiVersion: v3.2.0 -kind: DataContract -id: my-data-contract -name: REF_COUNTRY -version: 1.0.0 -status: draft -servers: -- server: source - type: api - location: https://xmart-api-public-uat.who.int/refmart/ - customProperties: - - property: apiType - value: odata - - property: odataVersion - value: '4.0' - - property: odataMetadataUrl - value: https://xmart-api-public-uat.who.int/refmart/$metadata - - property: format - value: json -schema: -- name: REF_COUNTRY - physicalName: REF_COUNTRY - logicalType: object - physicalType: object - customProperties: - - property: odataEntitySet - value: REF_COUNTRY - - property: odataEntitySetUrl - value: https://xmart-api-public-uat.who.int/refmart/REF_COUNTRY - properties: - - name: GEO_M49_CODE - logicalType: string - physicalType: Edm.String - required: false - - name: CODE_ISO_2 - logicalType: string - physicalType: Edm.String - required: false - - name: CODE_ISO_3 - logicalType: string - physicalType: Edm.String - required: false - - name: CODE_WHO - logicalType: string - physicalType: Edm.String - required: false - - name: CODE_ISO_NUMERIC - logicalType: integer - physicalType: Edm.Int32 - required: false - - name: NAME_SHORT_EN - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_FORMAL_EN - logicalType: string - physicalType: Edm.String - required: false - - name: CAPITAL_CITY - logicalType: string - physicalType: Edm.String - required: false - - name: ADJECTIVE_PEOPLE - logicalType: string - physicalType: Edm.String - required: false - - name: GEO_SMALL_POP_FLAG - logicalType: boolean - physicalType: Edm.Boolean - required: false - - name: GEO_SOVEREIGN - logicalType: string - physicalType: Edm.String - required: false - - name: SOVEREIGN_ISO_3 - logicalType: string - physicalType: Edm.String - required: false - - name: GRP_WB_INCOME - logicalType: string - physicalType: Edm.String - required: false - - name: GRP_WHO_REGION - logicalType: string - physicalType: Edm.String - required: false - - name: WHO_LEGAL_STATUS - logicalType: string - physicalType: Edm.String - required: false - - name: WHO_LEGAL_STATUS_TITLE - logicalType: string - physicalType: Edm.String - required: false - - name: DATE_START - logicalType: date - physicalType: Edm.Date - required: false - - name: NAME_CHANGE - logicalType: string - physicalType: Edm.String - required: false - - name: ISO_CHANGE - logicalType: string - physicalType: Edm.String - required: false - - name: CAPITAL_CHANGE - logicalType: string - physicalType: Edm.String - required: false - - name: STATISTICAL_CHANGE - logicalType: string - physicalType: Edm.String - required: false - - name: GEO_PRECEDED_BY - logicalType: string - physicalType: Edm.String - required: false - - name: GEO_SUCCEEDED_BY - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_SHORT_AR - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_FORMAL_AR - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_SHORT_ES - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_FORMAL_ES - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_SHORT_FR - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_FORMAL_FR - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_SHORT_RU - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_FORMAL_RU - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_SHORT_ZH - logicalType: string - physicalType: Edm.String - required: false - - name: NAME_FORMAL_ZH - logicalType: string - physicalType: Edm.String - required: false diff --git a/tests/test_import_odata.py b/tests/test_import_odata.py index b5f5fa170..84bd9223a 100644 --- a/tests/test_import_odata.py +++ b/tests/test_import_odata.py @@ -27,6 +27,7 @@ ROOT = "https://xmart-api-public-uat.who.int/refmart/" METADATA = "https://xmart-api-public-uat.who.int/refmart/$metadata" FIXTURES = Path(__file__).parent / "fixtures/odata" +EXPECTED_CONTRACT = Path(__file__).parents[1] / "examples/imports/odata/datacontract.odcs.yaml" ODATA_TEST_VERSIONS = ["4.0", "4.01", "4.123"] SERVICE_ROOT = "https://example.com/odata/" @@ -91,7 +92,7 @@ def test_who_contract_and_transport(metadata_response, odcs_schema): response, get = metadata_response contract = import_contract() actual = yaml.safe_load(contract.to_yaml()) - assert actual == yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) + assert actual == yaml.safe_load(EXPECTED_CONTRACT.read_text()) jsonschema.validate(actual, odcs_schema) get.assert_called_once() assert get.call_args.args == (METADATA,) @@ -322,7 +323,7 @@ def test_cli_stdout_and_output(metadata_response, tmp_path): args = ["--entity-set", "ref_country", "--metadata-url", METADATA] stdout = run_cli(*args, source=ROOT) assert stdout.exit_code == 0, stdout.output - assert yaml.safe_load(stdout.stdout) == yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) + assert yaml.safe_load(stdout.stdout) == yaml.safe_load(EXPECTED_CONTRACT.read_text()) output = tmp_path / "contract.yaml" result = run_cli(*args, "--output", output, "--owner", "WHO", "--id", "country", "--debug", source=ROOT) assert result.exit_code == 0, result.output @@ -365,7 +366,7 @@ def test_local_who_metadata_matches_url_import(metadata_response, path_type): "odata", source=ROOT, odata_entity_set=["ref_country"], odata_metadata_file=path ) actual = yaml.safe_load(contract.to_yaml()) - expected = yaml.safe_load((FIXTURES / "who-contract.yaml").read_text()) + expected = yaml.safe_load(EXPECTED_CONTRACT.read_text()) expected["servers"][0]["customProperties"][2] = {"property": "odataMetadataFile", "value": str(path)} assert actual == expected get.assert_not_called() @@ -715,21 +716,6 @@ def test_cli_offline_all_sets(metadata_response, tmp_path): get.assert_not_called() -def test_failure_in_later_selected_schema_preserves_output(metadata_response, json_metadata, tmp_path): - metadata, get = metadata_response - json_metadata["Catalog.Model"]["Order"]["Total"]["$Type"] = "Edm.Binary" - metadata.content = json.dumps(json_metadata).encode() - output = tmp_path / "contract.yaml" - output.write_text("existing contract") - result = run_cli( - "--metadata-url", METADATA, "--entity-set", "Products", "--entity-set", "Orders", "--output", output - ) - assert result.exit_code != 0 and isinstance(result.exception, DataContractException) - assert "Orders.Total" in str(result.exception) - assert output.read_text() == "existing contract" - get.assert_called_once() - - def test_xml_requires_unambiguous_container(): with pytest.raises(DataContractException, match="EntityContainer.*ambiguous"): xml_schema(extra_schema='') From 30576ae436bb126369f46954069d10cff04bd485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20Sch=C3=B6dl?= Date: Thu, 24 Sep 2026 10:49:12 +0200 Subject: [PATCH 6/7] docs: add OData logo icon and update import pages to use it --- datacontract/command_import.py | 4 +++- docs/docs/imports/index.md | 2 +- docs/docs/imports/odata.md | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/datacontract/command_import.py b/datacontract/command_import.py index 720ad9d7c..09965290d 100644 --- a/datacontract/command_import.py +++ b/datacontract/command_import.py @@ -923,7 +923,9 @@ def import_odata( ] = None, entity_set: Annotated[ Optional[List[str]], - typer.Option(help="EntitySet to import (repeat for multiple sets). If omitted, will import all sets from the service document."), + typer.Option( + help="EntitySet to import (repeat for multiple sets). If omitted, will import all sets from the service document." + ), ] = None, metadata_url: Annotated[ Optional[str], diff --git a/docs/docs/imports/index.md b/docs/docs/imports/index.md index 3a6a7a3ae..cdc33f0d4 100644 --- a/docs/docs/imports/index.md +++ b/docs/docs/imports/index.md @@ -89,7 +89,7 @@ Each import page shows a runnable example: a small source file under [`examples/ mysqlA MySQL database. - + odataAn OData 4.x service. diff --git a/docs/docs/imports/odata.md b/docs/docs/imports/odata.md index 6359fa100..c5a8e9dd1 100644 --- a/docs/docs/imports/odata.md +++ b/docs/docs/imports/odata.md @@ -4,7 +4,7 @@ title: "Import: OData" description: "Create a data contract from one or more OData 4.x EntitySets using CSDL XML or JSON." --- -# Import: OData +# Import: OData Creates a data contract from OData 4.x CSDL XML or JSON metadata. Select individual EntitySets, or automatically import all EntitySets advertised in the service document. Both public and authenticated services are supported. From dd95b21cc6cbc63a293ee975542bd83e0ee80710 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20=C3=85hl=C3=A9n?= Date: Thu, 24 Sep 2026 18:48:47 +0200 Subject: [PATCH 7/7] Updated after review. --- datacontract/command_import.py | 8 +- datacontract/imports/odata_importer.py | 49 +++++------- docs/docs/imports/odata.md | 4 +- examples/imports/odata/datacontract.odcs.yaml | 4 +- tests/test_import_odata.py | 76 +++++++++---------- 5 files changed, 64 insertions(+), 77 deletions(-) diff --git a/datacontract/command_import.py b/datacontract/command_import.py index 09965290d..1a22ee335 100644 --- a/datacontract/command_import.py +++ b/datacontract/command_import.py @@ -917,7 +917,7 @@ def import_athena( ) def import_odata( service_root_url: Annotated[str, typer.Option(help="HTTP(S) root URL of the OData service.")], - service_root_file: Annotated[ + service_document_file: Annotated[ Optional[Path], typer.Option(help="Local JSON service document. Ignored when --entity-set is supplied."), ] = None, @@ -930,14 +930,12 @@ def import_odata( metadata_url: Annotated[ Optional[str], typer.Option( - "--metadata-url", help="CSDL XML or JSON URL. Defaults to SERVICE_ROOT_URL/$metadata.", ), ] = None, metadata_file: Annotated[ Optional[Path], typer.Option( - "--metadata-file", help="Path to a local OData CSDL XML or JSON file. Use either --metadata-file or --metadata-url.", ), ] = None, @@ -947,14 +945,12 @@ def import_odata( debug: debug_option = None, ): """Import a data contract from OData 4.x CSDL XML or JSON metadata, using a URL or local file.""" - if metadata_url is not None and metadata_file is not None: - raise typer.BadParameter("--metadata-url and --metadata-file are mutually exclusive.") enable_debug_logging(debug) result = DataContract.import_from_source( config=cli_config(), format="odata", source=service_root_url, - odata_service_root_file=service_root_file, + odata_service_document_file=service_document_file, odata_entity_set=entity_set, odata_metadata_url=metadata_url, odata_metadata_file=metadata_file, diff --git a/datacontract/imports/odata_importer.py b/datacontract/imports/odata_importer.py index 85f6a46d7..db8a6ec27 100644 --- a/datacontract/imports/odata_importer.py +++ b/datacontract/imports/odata_importer.py @@ -13,7 +13,13 @@ from datacontract.config import Config from datacontract.imports.importer import Importer -from datacontract.imports.odcs_helper import create_odcs, create_property, create_schema_object, create_server +from datacontract.imports.odcs_helper import ( + create_odcs, + create_property, + create_schema_object, + create_server, + report_unmapped_types, +) from datacontract.model.exceptions import DataContractException logger = logging.getLogger(__name__) @@ -54,7 +60,7 @@ def import_source(self, source: str, import_args: dict, config: Config | None = config = Config.resolve(config) metadata_url = import_args.get("odata_metadata_url") metadata_file = import_args.get("odata_metadata_file") - service_file = import_args.get("odata_service_root_file") + service_file = import_args.get("odata_service_document_file") selected_names = import_args.get("odata_entity_set") if metadata_url is not None and metadata_file is not None: raise _schema_error("--metadata-url and --metadata-file are mutually exclusive.") @@ -66,7 +72,6 @@ def import_source(self, source: str, import_args: dict, config: Config | None = ): raise _schema_error("--entity-set must be a non-empty list of EntitySet names.") root_url = source.rstrip("/") + "/" - header_version = None if metadata_file is not None: content = _read_file(metadata_file, "metadata") else: @@ -75,12 +80,11 @@ def import_source(self, source: str, import_args: dict, config: Config | None = _validate_http_url(metadata_url, "metadata-url") response = _fetch_document(metadata_url, "metadata", "application/xml, application/json;q=0.9", config) content = response.content - header_version = response.headers.get("OData-Version") # Inspect the first non-whitespace character; leave XML's encoding declaration to its parser. prefix = content.decode(json.detect_encoding(content), errors="ignore").lstrip("\ufeff \t\r\n") document = _read_xml(content) if prefix.startswith("<") else _read_json(content) - version = _odata_version(document["version"], header_version) + version = _odata_version(document["version"]) if selected_names is not None: selections = [(name, None) for name in selected_names] else: @@ -92,9 +96,6 @@ def import_source(self, source: str, import_args: dict, config: Config | None = content = response.content # Redirects can change the base for relative context URLs. document_base = response.url - service_version = response.headers.get("OData-Version") - if service_version is not None: - _odata_version(service_version.strip(), service_version) selections = _read_service_document(content, document_base) if not selections: raise _schema_error("No EntitySets selected: the service document contains no EntitySets.") @@ -120,13 +121,13 @@ def import_source(self, source: str, import_args: dict, config: Config | None = server.customProperties = [ CustomProperty(property="apiType", value="odata"), CustomProperty(property="odataVersion", value=version), - CustomProperty(property="odataMetadataFile", value=str(metadata_file)) - if metadata_file is not None - else CustomProperty(property="odataMetadataUrl", value=metadata_url), # ODCS API servers do not allow a top-level format field. CustomProperty(property="format", value="json"), ] + if metadata_url is not None: + server.customProperties.append(CustomProperty(property="odataMetadataUrl", value=metadata_url)) contract.servers = [server] + report_unmapped_types(contract) return contract @@ -148,10 +149,8 @@ def _validate_http_url(url: str, label: str, *, root: bool = False) -> None: except ValueError: valid = False if not valid: - raise _schema_error( - f"Invalid {label}: expected an HTTP(S) URL without embedded credentials. " - "The service root must not contain a query or fragment." - ) + message = f"Invalid {label}: the URL is invalid or doesn't conform to the expected format." + raise _schema_error(message) def _read_file(path: str | Path, label: str) -> bytes: @@ -244,19 +243,13 @@ def _read_service_document(content: bytes, base: str) -> list[tuple[str, str]]: return selections -def _odata_version(document_version: object, header: str | None) -> str: +def _odata_version(document_version: object) -> str: """Select the supported protocol family independently of its schema mapping.""" - version = header.strip() if header is not None else document_version - if any( - not isinstance(value, str) or re.fullmatch(r"4\.[0-9]+", value) is None for value in (document_version, version) - ): + if not isinstance(document_version, str) or re.fullmatch(r"4\.[0-9]+", document_version) is None: raise _schema_error( - f"Unsupported OData version (header={header!r}, document={document_version!r}); " - "expected an OData 4.x version in the form 4.[0-9]+." + f"Unsupported OData version {document_version!r}; expected an OData 4.x version in the form 4.[0-9]+." ) - if version != document_version: - raise _schema_error(f"Conflicting OData versions: header={version}, document={document_version}.") - return version + return document_version def _read_xml(content: bytes) -> dict: @@ -474,10 +467,10 @@ def _odata_4_schema(document: dict, requested_name: str) -> SchemaObject: raise _schema_error(f"Unsupported $Kind for field {name}.{field_name}: {field['kind']!r}.") if type(field["collection"]) is not bool: raise _schema_error(f"Invalid $Collection for field {name}.{field_name}: expected a boolean.") - if not isinstance(field_type, str) or field_type not in ODATA_4_TYPES or field["collection"]: + if not isinstance(field_type, str) or not field_type.startswith("Edm.") or field["collection"]: raise _schema_error( f"Unsupported OData type {field_type!r} for field {name}.{field_name}. " - "Only supported primitive types can be imported; complex, collection, enum types and type definitions " + "Only primitive types can be imported; complex, collection, enum types and type definitions " "are not supported." ) nullable = field["nullable"] @@ -499,7 +492,7 @@ def _odata_4_schema(document: dict, requested_name: str) -> SchemaObject: properties.append( create_property( name=field_name, - logical_type=ODATA_4_TYPES[field_type], + logical_type=ODATA_4_TYPES.get(field_type), physical_type=field_type, required=not nullable, primary_key=is_key, diff --git a/docs/docs/imports/odata.md b/docs/docs/imports/odata.md index c5a8e9dd1..8e91e60e4 100644 --- a/docs/docs/imports/odata.md +++ b/docs/docs/imports/odata.md @@ -19,11 +19,11 @@ This example imports the WHO [country endpoint](https://xmart-api-public-uat.who Repeat `--entity-set` to select several EntitySets. An explicit selection skips the service document. Without `--entity-set`, the importer reads the JSON service document from the root URL and imports every EntitySet listed there into the same contract. -For offline import, supply a local CSDL XML or JSON file through `--metadata-file`. Also supply `--service-root-file` or `--entity-set`. +For offline import, supply a local CSDL XML or JSON file through `--metadata-file`. Also supply `--service-document-file` or `--entity-set`. For authentication, set `DATACONTRACT_API_HEADER_AUTHORIZATION` to the complete header value, such as `Bearer ` or a precomputed `Basic `. -The importer reads metadata only; no data records are fetched. It preserves supported primitive field types, nullability, declared keys and constraints. Complex types, collection-valued fields, enums, type definitions, inheritance and unsupported primitive types cause errors when used by a selected schema. Navigation properties are omitted, and external metadata references are not downloaded. +The importer reads metadata only; no data records are fetched. It preserves primitive field types, nullability, declared keys and constraints. Primitive types without a logical type mapping are imported with their original `physicalType` and produce a warning. Complex types, collection-valued fields, enums, type definitions and inheritance cause errors when used by a selected schema. Navigation properties are omitted, and external metadata references are not downloaded. OData support currently covers import only. Support for `datacontract test` is planned to compare the contract's schema with `$metadata`, without fetching data records. diff --git a/examples/imports/odata/datacontract.odcs.yaml b/examples/imports/odata/datacontract.odcs.yaml index 7a83eb39f..e3d450366 100644 --- a/examples/imports/odata/datacontract.odcs.yaml +++ b/examples/imports/odata/datacontract.odcs.yaml @@ -12,10 +12,10 @@ servers: value: odata - property: odataVersion value: '4.0' - - property: odataMetadataUrl - value: https://xmart-api-public-uat.who.int/refmart/$metadata - property: format value: json + - property: odataMetadataUrl + value: https://xmart-api-public-uat.who.int/refmart/$metadata location: https://xmart-api-public-uat.who.int/refmart/ schema: - name: REF_COUNTRY diff --git a/tests/test_import_odata.py b/tests/test_import_odata.py index 84bd9223a..186548db4 100644 --- a/tests/test_import_odata.py +++ b/tests/test_import_odata.py @@ -107,12 +107,11 @@ def test_who_contract_and_transport(metadata_response, odcs_schema): @pytest.mark.parametrize("version", ODATA_TEST_VERSIONS) -@pytest.mark.parametrize("header", [True, False]) @pytest.mark.parametrize("format", ["xml", "json"]) -def test_versions_and_header_fallback(metadata_response, version, header, format): +def test_document_versions(metadata_response, version, format): response, _ = metadata_response response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode()) - response.headers = {"OData-Version": version} if header else {} + response.headers = {} contract = import_contract("Products") assert contract.servers[0].customProperties[1].value == version assert contract.schema_[0].properties[0].name == "Sku" @@ -161,6 +160,19 @@ def test_primitive_types(edm_type, logical_type, format): assert "scale" not in facets +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_unmapped_primitive_type_warns(metadata_response, caplog, format): + response, _ = metadata_response + response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"Edm.Guid", b"Edm.Binary") + contract = import_contract("Products") + properties = yaml.safe_load(contract.to_yaml())["schema"][0]["properties"] + prop = next(prop for prop in properties if prop["name"] == "TrackingId") + assert prop["physicalType"] == "Edm.Binary" + assert "logicalType" not in prop + assert "without a logicalType" in caplog.text + assert "TrackingId (Edm.Binary)" in caplog.text + + def test_keys_nullability_and_facets(): schema = xml_schema( """ @@ -216,7 +228,7 @@ def test_navigation_and_unrelated_types_are_omitted(caplog): @pytest.mark.parametrize( "field_type", - ["D.Address", "Collection(Edm.String)", "Collection(D.Address)", "D.Status", "Edm.Binary", "Edm.Unknown"], + ["D.Address", "Collection(Edm.String)", "Collection(D.Address)", "D.Status"], ) def test_unsupported_types_fail_with_field_and_type(field_type): with pytest.raises(DataContractException) as error: @@ -226,25 +238,24 @@ def test_unsupported_types_fail_with_field_and_type(field_type): @pytest.mark.parametrize( - "document, header, error", + "document, error", [ - (metadata_xml(version="4.0"), "4.01", "Conflicting OData versions"), - (metadata_xml(entity_attributes='BaseType="Demo.Base"'), None, "inheritance"), - (metadata_xml().replace(b'Name="Service"', b'Name="Service" Extends="Demo.Base"'), None, "inheritance"), - (metadata_xml().replace(b"D.Country", b"Other.Country"), None, "must resolve uniquely"), - (metadata_xml().replace(b"REF_COUNTRY", b"OTHER"), None, "not found"), - (metadata_xml().replace(b'http://docs.oasis-open.org/odata/ns/edm"', b'urn:wrong"'), None, "not found"), - (b"", None, "Expected an OData 4 Edmx"), - (b"'), None, "Invalid Nullable"), - (metadata_xml(''), None, "Invalid MaxLength"), - (metadata_xml(keys=''), None, "key references"), + (metadata_xml(entity_attributes='BaseType="Demo.Base"'), "inheritance"), + (metadata_xml().replace(b'Name="Service"', b'Name="Service" Extends="Demo.Base"'), "inheritance"), + (metadata_xml().replace(b"D.Country", b"Other.Country"), "must resolve uniquely"), + (metadata_xml().replace(b"REF_COUNTRY", b"OTHER"), "not found"), + (metadata_xml().replace(b'http://docs.oasis-open.org/odata/ns/edm"', b'urn:wrong"'), "not found"), + (b"", "Expected an OData 4 Edmx"), + (b"'), "Invalid Nullable"), + (metadata_xml(''), "Invalid MaxLength"), + (metadata_xml(keys=''), "key references"), ], ) -def test_metadata_errors(metadata_response, document, header, error): +def test_metadata_errors(metadata_response, document, error): response, _ = metadata_response response.content = document - response.headers = {"OData-Version": header} if header else {} + response.headers = {} with pytest.raises(DataContractException, match=error): import_contract() @@ -367,7 +378,9 @@ def test_local_who_metadata_matches_url_import(metadata_response, path_type): ) actual = yaml.safe_load(contract.to_yaml()) expected = yaml.safe_load(EXPECTED_CONTRACT.read_text()) - expected["servers"][0]["customProperties"][2] = {"property": "odataMetadataFile", "value": str(path)} + expected["servers"][0]["customProperties"] = [ + prop for prop in expected["servers"][0]["customProperties"] if prop["property"] != "odataMetadataUrl" + ] assert actual == expected get.assert_not_called() @@ -515,16 +528,10 @@ def test_json_unsupported_selected_type(json_metadata, kind): # Version validation and format detection -@pytest.mark.parametrize("document_version, header_version", [("4.01", "4.0"), ("4.123", "4.1"), ("4.01", "4.1")]) -def test_version_conflict(document_version, header_version): - with pytest.raises(DataContractException, match="Conflicting OData versions"): - _odata_version(document_version, header_version) - - @pytest.mark.parametrize("version", [None, "3.0", "5.0", "4", "4.", "4.x", "4.1.0", "4.01beta", "4.١", 4.01]) def test_invalid_versions(version): with pytest.raises(DataContractException, match="Unsupported OData version"): - _odata_version(version, None) + _odata_version(version) @pytest.mark.parametrize("prefix", [b" \t\n" * 100, b"\xef\xbb\xbf \t\n"]) @@ -579,7 +586,7 @@ def test_import_all_advertised_sets( else: expected_requests.append(SERVICE_ROOT + "$metadata") if service_source == "file": - options["odata_service_root_file"] = FIXTURES / "service-document.json" + options["odata_service_document_file"] = FIXTURES / "service-document.json" else: expected_requests.append(SERVICE_ROOT) contract = DataContract.import_from_source("odata", source=root, **options) @@ -615,7 +622,7 @@ def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_pat result = run_cli( "--metadata-file", path, - "--service-root-file", + "--service-document-file", "does-not-exist.json", "--entity-set", "Orders", @@ -629,7 +636,6 @@ def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_pat assert result.exit_code == 0, result.output contract = yaml.safe_load(result.stdout) assert contract["name"] == "Store" - assert {"property": "odataMetadataFile", "value": "metadata"} in contract["servers"][0]["customProperties"] schemas = contract["schema"] assert [s["name"] for s in schemas] == ["Orders", "Products", "ArchivedProducts"] assert schemas[1]["properties"] == schemas[2]["properties"] @@ -680,14 +686,6 @@ def test_redirected_service_document_base(service_response, url, root): assert [call.args[0] for call in get.call_args_list] == [SERVICE_ROOT + "$metadata", SERVICE_ROOT] -@pytest.mark.parametrize("version", [None, *ODATA_TEST_VERSIONS]) -def test_service_header_does_not_override_csdl_version(service_response, version): - service, _ = service_response - service.headers = {"OData-Version": version} if version else {} - contract = DataContract.import_from_source("odata", source=SERVICE_ROOT) - assert {p.property: p.value for p in contract.servers[0].customProperties}["odataVersion"] == "4.01" - - def test_missing_service_file_does_not_fall_back_to_network(metadata_response, tmp_path): _, get = metadata_response with pytest.raises(DataContractException, match="Failed to read OData service document file"): @@ -695,7 +693,7 @@ def test_missing_service_file_does_not_fall_back_to_network(metadata_response, t "odata", source=SERVICE_ROOT, odata_metadata_file=FIXTURES / "products.xml", - odata_service_root_file=tmp_path / "missing.json", + odata_service_document_file=tmp_path / "missing.json", ) get.assert_not_called() @@ -706,7 +704,7 @@ def test_cli_offline_all_sets(metadata_response, tmp_path): result = run_cli( "--metadata-file", FIXTURES / "products.xml", - "--service-root-file", + "--service-document-file", FIXTURES / "service-document.json", "--output", output,