diff --git a/CHANGELOG.md b/CHANGELOG.md index f0ad67584..ae9083286 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - `datacontract api`: `--contract-variables` and `--allow-local-files` options, as alternatives to their environment variables +- `datacontract import odata` creates a datacontract from OData 4 metadata at an URL or from a local file. - `datacontract test` checks constraints and quality rules of nested properties on servers read through DuckDB (#1278) ### Changed diff --git a/datacontract/command_import.py b/datacontract/command_import.py index df145de1a..1a22ee335 100644 --- a/datacontract/command_import.py +++ b/datacontract/command_import.py @@ -909,3 +909,52 @@ def import_athena( id=id, ) _write_result(result, output) + + +@import_app.command( + name="odata", + epilog="Example: datacontract import odata --service-root-url https://example.com/odata/ --entity-set Products --output datacontract.yaml", +) +def import_odata( + service_root_url: Annotated[str, typer.Option(help="HTTP(S) root URL of the OData service.")], + service_document_file: Annotated[ + Optional[Path], + typer.Option(help="Local JSON service document. Ignored when --entity-set is supplied."), + ] = None, + entity_set: Annotated[ + Optional[List[str]], + typer.Option( + help="EntitySet to import (repeat for multiple sets). If omitted, will import all sets from the service document." + ), + ] = None, + metadata_url: Annotated[ + Optional[str], + typer.Option( + help="CSDL XML or JSON URL. Defaults to SERVICE_ROOT_URL/$metadata.", + ), + ] = None, + metadata_file: Annotated[ + Optional[Path], + typer.Option( + help="Path to a local OData CSDL XML or JSON file. Use either --metadata-file or --metadata-url.", + ), + ] = None, + output: output_option = None, + owner: owner_option = None, + id: id_option = None, + debug: debug_option = None, +): + """Import a data contract from OData 4.x CSDL XML or JSON metadata, using a URL or local file.""" + enable_debug_logging(debug) + result = DataContract.import_from_source( + config=cli_config(), + format="odata", + source=service_root_url, + odata_service_document_file=service_document_file, + odata_entity_set=entity_set, + odata_metadata_url=metadata_url, + odata_metadata_file=metadata_file, + owner=owner, + id=id, + ) + _write_result(result, output) diff --git a/datacontract/imports/importer.py b/datacontract/imports/importer.py index 415c24908..5c20f47d5 100644 --- a/datacontract/imports/importer.py +++ b/datacontract/imports/importer.py @@ -40,6 +40,7 @@ class ImportFormat(str, Enum): json = "json" bigquery = "bigquery" odcs = "odcs" + odata = "odata" unity = "unity" databricks = "databricks" spark = "spark" diff --git a/datacontract/imports/importer_factory.py b/datacontract/imports/importer_factory.py index adb3e673c..a4bb12cc9 100644 --- a/datacontract/imports/importer_factory.py +++ b/datacontract/imports/importer_factory.py @@ -187,3 +187,8 @@ def load_module_class(module_path, class_name): module_path="datacontract.imports.json_importer", class_name="JsonImporter", ) +importer_factory.register_lazy_importer( + name=ImportFormat.odata, + module_path="datacontract.imports.odata_importer", + class_name="ODataImporter", +) diff --git a/datacontract/imports/odata_importer.py b/datacontract/imports/odata_importer.py new file mode 100644 index 000000000..db8a6ec27 --- /dev/null +++ b/datacontract/imports/odata_importer.py @@ -0,0 +1,506 @@ +"""Import structural properties of OData 4.x EntitySets from CSDL XML or JSON.""" + +import json +import logging +import re +from pathlib import Path +from urllib.parse import quote, urljoin, urlsplit +from xml.etree import ElementTree +from xml.etree.ElementTree import ParseError, TreeBuilder + +import requests +from open_data_contract_standard.model import CustomProperty, OpenDataContractStandard, SchemaObject + +from datacontract.config import Config +from datacontract.imports.importer import Importer +from datacontract.imports.odcs_helper import ( + create_odcs, + create_property, + create_schema_object, + create_server, + report_unmapped_types, +) +from datacontract.model.exceptions import DataContractException + +logger = logging.getLogger(__name__) + +EDMX = "{http://docs.oasis-open.org/odata/ns/edmx}" +EDM = "{http://docs.oasis-open.org/odata/ns/edm}" +ODATA_4_TYPES = { + "Edm.String": "string", + "Edm.Guid": "string", + "Edm.Byte": "integer", + "Edm.SByte": "integer", + "Edm.Int16": "integer", + "Edm.Int32": "integer", + "Edm.Int64": "integer", + "Edm.Decimal": "number", + "Edm.Single": "number", + "Edm.Double": "number", + "Edm.Boolean": "boolean", + "Edm.Date": "date", + "Edm.DateTimeOffset": "timestamp", + "Edm.TimeOfDay": "time", +} + + +def _schema_error(reason: str) -> DataContractException: + return DataContractException(type="schema", name="Import OData metadata", reason=reason) + + +class _MetadataTreeBuilder(TreeBuilder): + """Reject DTDs before the parser processes their entity declarations.""" + + def doctype(self, name: str, pubid: str | None, system: str | None) -> None: + raise ParseError("DTD declarations are not allowed in OData metadata.") + + +class ODataImporter(Importer): + def import_source(self, source: str, import_args: dict, config: Config | None = None) -> OpenDataContractStandard: + config = Config.resolve(config) + metadata_url = import_args.get("odata_metadata_url") + metadata_file = import_args.get("odata_metadata_file") + service_file = import_args.get("odata_service_document_file") + selected_names = import_args.get("odata_entity_set") + if metadata_url is not None and metadata_file is not None: + raise _schema_error("--metadata-url and --metadata-file are mutually exclusive.") + _validate_http_url(source, "service-root-url", root=True) + if selected_names is not None and ( + not isinstance(selected_names, list) + or not selected_names + or any(not isinstance(name, str) or not name.strip() for name in selected_names) + ): + raise _schema_error("--entity-set must be a non-empty list of EntitySet names.") + root_url = source.rstrip("/") + "/" + if metadata_file is not None: + content = _read_file(metadata_file, "metadata") + else: + if metadata_url is None: + metadata_url = root_url + "$metadata" + _validate_http_url(metadata_url, "metadata-url") + response = _fetch_document(metadata_url, "metadata", "application/xml, application/json;q=0.9", config) + content = response.content + + # Inspect the first non-whitespace character; leave XML's encoding declaration to its parser. + prefix = content.decode(json.detect_encoding(content), errors="ignore").lstrip("\ufeff \t\r\n") + document = _read_xml(content) if prefix.startswith("<") else _read_json(content) + version = _odata_version(document["version"]) + if selected_names is not None: + selections = [(name, None) for name in selected_names] + else: + document_base = root_url + if service_file is not None: + content = _read_file(service_file, "service document") + else: + response = _fetch_document(root_url, "service document", "application/json", config) + content = response.content + # Redirects can change the base for relative context URLs. + document_base = response.url + selections = _read_service_document(content, document_base) + if not selections: + raise _schema_error("No EntitySets selected: the service document contains no EntitySets.") + schemas = [] + seen = set() + for name, entity_url in selections: + schema = _odata_4_schema(document, name) + if schema.name in seen: + if selected_names is not None: + continue + raise _schema_error(f"Duplicate EntitySet {schema.name!r} in the service document.") + seen.add(schema.name) + schema.customProperties = [ + CustomProperty(property="odataEntitySet", value=schema.name), + CustomProperty( + property="odataEntitySetUrl", value=entity_url or root_url + quote(schema.name, safe="") + ), + ] + schemas.append(schema) + contract = create_odcs(name=schemas[0].name if len(schemas) == 1 else document["container_name"]) + contract.schema_ = schemas + server = create_server(name="source", server_type="api", location=root_url) + server.customProperties = [ + CustomProperty(property="apiType", value="odata"), + CustomProperty(property="odataVersion", value=version), + # ODCS API servers do not allow a top-level format field. + CustomProperty(property="format", value="json"), + ] + if metadata_url is not None: + server.customProperties.append(CustomProperty(property="odataMetadataUrl", value=metadata_url)) + contract.servers = [server] + report_unmapped_types(contract) + return contract + + +def _validate_http_url(url: str, label: str, *, root: bool = False) -> None: + try: + parsed = urlsplit(url) if isinstance(url, str) else None + valid = ( + parsed is not None + and parsed.scheme in ("http", "https") + and bool(parsed.hostname) + and parsed.username is None + and parsed.password is None + and not any(char.isspace() for char in url) + ) + if valid: + parsed.port # Validate a supplied port as well as the hostname. + if root and valid and ("?" in url or "#" in url): + valid = False + except ValueError: + valid = False + if not valid: + message = f"Invalid {label}: the URL is invalid or doesn't conform to the expected format." + raise _schema_error(message) + + +def _read_file(path: str | Path, label: str) -> bytes: + try: + return Path(path).read_bytes() + except (OSError, ValueError) as exc: + raise _schema_error(f"Failed to read OData {label} file {path}: {exc}") from exc + + +class _ODataSession(requests.Session): + """Keep Requests' redirect policy without introducing .netrc credentials.""" + + def rebuild_auth(self, prepared_request: requests.PreparedRequest, response: requests.Response) -> None: + if self.should_strip_auth(response.request.url, prepared_request.url): + prepared_request.headers.pop("Authorization", None) + + +def _fetch_document(url: str, label: str, accept: str, config: Config) -> requests.Response: + headers = {"Accept": accept} + authorization = config.get_api_header_authorization() + if authorization is not None: + headers["Authorization"] = authorization + try: + with _ODataSession() as session: + response = session.get( + url, + headers=headers, + timeout=30, + # Prevent .netrc from replacing the header on the initial request. + auth=lambda request: request, + ) + response.raise_for_status() + return response + except requests.RequestException as exc: + detail = type(exc).__name__ + if exc.response is not None: + detail += f" (HTTP {exc.response.status_code})" + # InvalidHeader and other transport errors may include credentials in their text. + raise DataContractException( + type="connection", + name=f"Fetch OData {label}", + reason=f"Failed to fetch OData {label}: {detail}.", + ) from None + + +def _context_base(obj: dict, base: str) -> str: + context = obj.get("@odata.context", obj.get("@context")) + if "@odata.context" not in obj and "@context" not in obj: + return base + if not isinstance(context, str) or not context: + raise _schema_error("Invalid context URL in OData service document.") + if "@odata.context" in obj and "@context" in obj and obj["@odata.context"] != obj["@context"]: + raise _schema_error("Conflicting context URLs in OData service document.") + try: + result = urljoin(base, context) + except ValueError as exc: + raise _schema_error(f"Invalid context URL in OData service document: {exc}") from exc + _validate_http_url(result, "service document context") + return result + + +def _read_service_document(content: bytes, base: str) -> list[tuple[str, str]]: + document = _json_object(_load_json(content, "service document"), "the service document") + if "@odata.context" not in document and "@context" not in document: + raise _schema_error("Expected an OData service document with a context URL and a value array.") + base = _context_base(document, base) + entries = document.get("value") + if not isinstance(entries, list): + raise _schema_error("Expected a value array in the OData service document.") + selections, names = [], set() + for entry in entries: + entry = _json_object(entry, "service document entry") + kind = entry.get("kind", "EntitySet") + if not isinstance(kind, str): + raise _schema_error("Invalid kind in OData service document: expected a string.") + if kind != "EntitySet": + continue # Singletons, operations and linked services are not imported. + name, url = entry.get("name"), entry.get("url") + if not isinstance(name, str) or not name.strip() or not isinstance(url, str) or not url.strip(): + raise _schema_error("Each service document EntitySet must have a non-empty string name and url.") + if name in names: + raise _schema_error(f"Duplicate EntitySet {name!r} in the service document.") + names.add(name) + try: + entity_url = urljoin(_context_base(entry, base), url) + except ValueError as exc: + raise _schema_error(f"Invalid service document URL for EntitySet {name!r}: {exc}") from exc + _validate_http_url(entity_url, f"EntitySet {name!r} URL") + selections.append((name, entity_url)) + return selections + + +def _odata_version(document_version: object) -> str: + """Select the supported protocol family independently of its schema mapping.""" + if not isinstance(document_version, str) or re.fullmatch(r"4\.[0-9]+", document_version) is None: + raise _schema_error( + f"Unsupported OData version {document_version!r}; expected an OData 4.x version in the form 4.[0-9]+." + ) + return document_version + + +def _read_xml(content: bytes) -> dict: + try: + root = ElementTree.fromstring(content, parser=ElementTree.XMLParser(target=_MetadataTreeBuilder())) + except (ParseError, LookupError, ValueError) as exc: + raise _schema_error(f"Invalid or unsafe OData XML metadata: {exc}") from exc + if root.tag != f"{EDMX}Edmx": + raise _schema_error("Expected an OData 4 Edmx document with the OData 4 XML namespace.") + containers = root.findall(f"{EDMX}DataServices/{EDM}Schema/{EDM}EntityContainer") + if len(containers) != 1 or not containers[0].get("Name"): + raise _schema_error( + "The XML service EntityContainer is not found or ambiguous; expected exactly one named container." + ) + document = { + "version": root.get("Version"), + "container_name": containers[0].get("Name"), + "entity_sets": [], + "types": [], + } + for schema in root.findall(f"{EDMX}DataServices/{EDM}Schema"): + for container in schema.findall(f"{EDM}EntityContainer"): + for entity_set in container.findall(f"{EDM}EntitySet"): + document["entity_sets"].append( + { + "name": entity_set.get("Name"), + "type": entity_set.get("EntityType"), + "extends": container.get("Extends"), + } + ) + for entity_type in schema.findall(f"{EDM}EntityType"): + fields = [] + for field in entity_type.findall(f"{EDM}Property"): + facets = {} + for key in ("MaxLength", "Precision", "Scale"): + value = field.get(key) + if value is not None: + try: + value = int(value) + except ValueError: + pass # Preserve symbolic values for validation during mapping. + facets[key] = value + if field.get("Type") == "Edm.Decimal": + facets.setdefault("Scale", 0) + fields.append( + { + "name": field.get("Name"), + "type": field.get("Type"), + "nullable": {"true": True, "false": False}.get( + field.get("Nullable", "true"), field.get("Nullable") + ), + "collection": False, + "facets": facets, + } + ) + document["types"].append( + { + "names": { + f"{prefix}.{entity_type.get('Name')}" + for prefix in (schema.get("Namespace"), schema.get("Alias")) + if prefix + }, + "base_type": entity_type.get("BaseType"), + "keys": [key.get("Name") for key in entity_type.findall(f"{EDM}Key/{EDM}PropertyRef")], + "fields": fields, + "navigation": [prop.get("Name") for prop in entity_type.findall(f"{EDM}NavigationProperty")], + } + ) + return document + + +def _json_pairs(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise _schema_error(f"Duplicate JSON key {key!r} in OData JSON document.") + result[key] = value + return result + + +def _json_object(value: object, context: str) -> dict: + if not isinstance(value, dict): + raise _schema_error(f"Expected a JSON object for {context} in OData metadata.") + return value + + +def _load_json(content: bytes, label: str) -> object: + def reject_constant(value: str): + raise _schema_error(f"Invalid JSON constant {value!r} in OData metadata.") + + try: + return json.loads(content, object_pairs_hook=_json_pairs, parse_constant=reject_constant) + except (ValueError, UnicodeError) as exc: + raise _schema_error(f"Invalid or unsafe OData JSON {label}: {exc}") from exc + + +def _read_json(content: bytes) -> dict: + root = _json_object(_load_json(content, "metadata"), "the CSDL document") + if "$Version" not in root and "value" in root and ("@odata.context" in root or "@context" in root): + raise _schema_error( + "This is an OData service/data document, not CSDL metadata. Supply the $metadata document with field definitions." + ) + container_name = root.get("$EntityContainer") + if not isinstance(container_name, str) or "." not in container_name: + raise _schema_error("CSDL JSON metadata requires a namespace-qualified $EntityContainer.") + namespace, _, local_name = container_name.rpartition(".") + container_schema = _json_object(root.get(namespace), f"schema {namespace!r}") + container = _json_object(container_schema.get(local_name), f"$EntityContainer {container_name!r}") + if container.get("$Kind") != "EntityContainer": + raise _schema_error(f"{container_name!r} must have $Kind EntityContainer.") + if "$Extends" in container: + raise _schema_error(f"EntityContainer {container_name!r} uses unsupported inheritance ($Extends).") + document = {"version": root.get("$Version"), "container_name": local_name, "entity_sets": [], "types": []} + for name, member in container.items(): + if name.startswith("$") or "@" in name: + continue + member = _json_object(member, f"container member {name!r}") + collection = member.get("$Collection", False) + if type(collection) is not bool: + raise _schema_error(f"Invalid $Collection for container member {name!r}: expected a boolean.") + if collection: + document["entity_sets"].append({"name": name, "type": member.get("$Type"), "extends": None}) + for namespace, schema in root.items(): + if namespace.startswith("$") or "@" in namespace: + continue + schema = _json_object(schema, f"schema {namespace!r}") + alias = schema.get("$Alias") + if "$Alias" in schema and not isinstance(alias, str): + raise _schema_error(f"Invalid $Alias in schema {namespace!r}: expected a string.") + for type_name, entity_type in schema.items(): + if type_name.startswith("$") or "@" in type_name: + continue + # Action/function overloads are arrays; only entity types are needed. + if not isinstance(entity_type, dict) or entity_type.get("$Kind") != "EntityType": + continue + if "$BaseType" in entity_type and not isinstance(entity_type["$BaseType"], str): + raise _schema_error(f"Invalid $BaseType for {namespace}.{type_name}: expected a qualified type name.") + fields, navigation = [], [] + for field_name, field in entity_type.items(): + if field_name.startswith("$") or "@" in field_name: + continue + field = _json_object(field, f"field {namespace}.{type_name}.{field_name}") + kind = field.get("$Kind", "Property") + if kind == "NavigationProperty": + navigation.append(field_name) + continue + facets = {key: field[f"${key}"] for key in ("MaxLength", "Precision", "Scale") if f"${key}" in field} + if field.get("$Type") == "Edm.Decimal": + facets.setdefault("Scale", "variable") + fields.append( + { + "name": field_name, + "type": field.get("$Type", "Edm.String"), + "nullable": field.get("$Nullable", False), + "collection": field.get("$Collection", False), + "facets": facets, + "kind": kind, + } + ) + document["types"].append( + { + "names": {f"{prefix}.{type_name}" for prefix in (namespace, alias) if prefix}, + "base_type": entity_type.get("$BaseType"), + "keys": entity_type.get("$Key", []), + "fields": fields, + "navigation": navigation, + } + ) + return document + + +def _odata_4_schema(document: dict, requested_name: str) -> SchemaObject: + entity_sets = document["entity_sets"] + matches = [entity_set for entity_set in entity_sets if entity_set["name"] == requested_name] + if not matches: + matches = [ + entity_set + for entity_set in entity_sets + if (entity_set["name"] or "").casefold() == requested_name.casefold() + ] + if len(matches) != 1: + raise _schema_error( + f"EntitySet {requested_name!r} is {'ambiguous' if matches else 'not found'} in the metadata." + ) + entity_set = matches[0] + name, type_name = entity_set["name"], entity_set["type"] + if entity_set["extends"]: + raise _schema_error(f"EntitySet {name!r} uses unsupported container inheritance (Extends).") + if not isinstance(type_name, str): + raise _schema_error(f"Invalid EntityType for EntitySet {name!r}: expected a qualified type name.") + types = [entity_type for entity_type in document["types"] if type_name in entity_type["names"]] + if len(types) != 1: + raise _schema_error( + f"EntityType {type_name!r} for EntitySet {name!r} must resolve uniquely within this metadata document. " + "External metadata references are not fetched." + ) + entity_type = types[0] + if entity_type["base_type"] is not None: + raise _schema_error(f"EntityType {type_name!r} uses unsupported inheritance (BaseType).") + key_names, fields = entity_type["keys"], entity_type["fields"] + field_names = [field["name"] for field in fields] + if any(not name for name in field_names) or len(set(field_names)) != len(field_names): + raise _schema_error(f"EntityType {type_name!r} has missing or duplicate property names.") + if ( + not isinstance(key_names, list) + or any(not isinstance(key, str) for key in key_names) + or len(set(key_names)) != len(key_names) + or any(key not in field_names for key in key_names) + ): + raise _schema_error(f"EntityType {type_name!r} has invalid or unsupported key references: {key_names}.") + properties = [] + for field in fields: + field_name, field_type = field["name"], field["type"] + if field.get("kind", "Property") != "Property": + raise _schema_error(f"Unsupported $Kind for field {name}.{field_name}: {field['kind']!r}.") + if type(field["collection"]) is not bool: + raise _schema_error(f"Invalid $Collection for field {name}.{field_name}: expected a boolean.") + if not isinstance(field_type, str) or not field_type.startswith("Edm.") or field["collection"]: + raise _schema_error( + f"Unsupported OData type {field_type!r} for field {name}.{field_name}. " + "Only primitive types can be imported; complex, collection, enum types and type definitions " + "are not supported." + ) + nullable = field["nullable"] + if type(nullable) is not bool: + raise _schema_error(f"Invalid Nullable value {nullable!r} for field {name}.{field_name}.") + is_key = field_name in key_names + if is_key and nullable: + raise _schema_error(f"Key field {name}.{field_name} must declare Nullable=false.") + facets = {} + for facet, value in field["facets"].items(): + if facet == "MaxLength" and value == "max": + continue + if facet == "Scale" and isinstance(value, str) and value.lower() in ("variable", "floating"): + facets["custom_properties"] = {"scale": value.lower()} + continue + if type(value) is not int or value < 0 or (facet == "MaxLength" and value == 0): + raise _schema_error(f"Invalid {facet} value {value!r} for field {name}.{field_name}.") + facets[{"MaxLength": "max_length", "Precision": "precision", "Scale": "scale"}[facet]] = value + properties.append( + create_property( + name=field_name, + logical_type=ODATA_4_TYPES.get(field_type), + physical_type=field_type, + required=not nullable, + primary_key=is_key, + primary_key_position=key_names.index(field_name) + 1 if is_key else None, + format="uuid" if field_type == "Edm.Guid" else None, + **facets, + ) + ) + for navigation in entity_type["navigation"]: + logger.warning("Omitting OData navigation property %s.%s from the imported schema.", name, navigation) + return create_schema_object(name=name, physical_type="object", properties=properties) diff --git a/docs/docs/imports/index.md b/docs/docs/imports/index.md index f10838560..cdc33f0d4 100644 --- a/docs/docs/imports/index.md +++ b/docs/docs/imports/index.md @@ -88,6 +88,10 @@ Each import page shows a runnable example: a small source file under [`examples/ mysqlA MySQL database. + + + odataAn OData 4.x service. + odcsAn ODCS data contract file. diff --git a/docs/docs/imports/odata.md b/docs/docs/imports/odata.md new file mode 100644 index 000000000..8e91e60e4 --- /dev/null +++ b/docs/docs/imports/odata.md @@ -0,0 +1,30 @@ +--- +sidebar_position: 18 +title: "Import: OData" +description: "Create a data contract from one or more OData 4.x EntitySets using CSDL XML or JSON." +--- + +# Import: OData + +Creates a data contract from OData 4.x CSDL XML or JSON metadata. Select individual EntitySets, or automatically import all EntitySets advertised in the service document. Both public and authenticated services are supported. + +```bash +datacontract import odata \ + --service-root-url 'https://xmart-api-public-uat.who.int/refmart/' \ + --entity-set ref_country \ + --output datacontract.yaml +``` + +This example imports the WHO [country endpoint](https://xmart-api-public-uat.who.int/refmart/ref_country). `--service-root-url` is always required and is stored with a trailing `/` as the server's `location`. Metadata is read from `$metadata` under that root unless you supply `--metadata-url` or `--metadata-file`. + +Repeat `--entity-set` to select several EntitySets. An explicit selection skips the service document. Without `--entity-set`, the importer reads the JSON service document from the root URL and imports every EntitySet listed there into the same contract. + +For offline import, supply a local CSDL XML or JSON file through `--metadata-file`. Also supply `--service-document-file` or `--entity-set`. + +For authentication, set `DATACONTRACT_API_HEADER_AUTHORIZATION` to the complete header value, such as `Bearer ` or a precomputed `Basic `. + +The importer reads metadata only; no data records are fetched. It preserves primitive field types, nullability, declared keys and constraints. Primitive types without a logical type mapping are imported with their original `physicalType` and produce a warning. Complex types, collection-valued fields, enums, type definitions and inheritance cause errors when used by a selected schema. Navigation properties are omitted, and external metadata references are not downloaded. + +OData support currently covers import only. Support for `datacontract test` is planned to compare the contract's schema with `$metadata`, without fetching data records. + +All options: **[`datacontract import odata`](../commands/import/odata.md)**. diff --git a/docs/docs/imports/odcs.md b/docs/docs/imports/odcs.md index f0327d463..75567dcff 100644 --- a/docs/docs/imports/odcs.md +++ b/docs/docs/imports/odcs.md @@ -1,5 +1,5 @@ --- -sidebar_position: 18 +sidebar_position: 19 title: "Import: ODCS" description: "Create a data contract from an existing ODCS file." --- diff --git a/docs/docs/imports/oracle.md b/docs/docs/imports/oracle.md index d01dc0c5d..8a4f40568 100644 --- a/docs/docs/imports/oracle.md +++ b/docs/docs/imports/oracle.md @@ -1,5 +1,5 @@ --- -sidebar_position: 19 +sidebar_position: 20 title: "Import: Oracle" description: "Create a data contract from an Oracle database." --- diff --git a/docs/docs/imports/parquet.md b/docs/docs/imports/parquet.md index 14e7e3e28..db496a481 100644 --- a/docs/docs/imports/parquet.md +++ b/docs/docs/imports/parquet.md @@ -1,5 +1,5 @@ --- -sidebar_position: 20 +sidebar_position: 21 title: "Import: Parquet" description: "Create a data contract from a Parquet file." --- diff --git a/docs/docs/imports/postgres.md b/docs/docs/imports/postgres.md index e0ecbb97e..ac41bbd19 100644 --- a/docs/docs/imports/postgres.md +++ b/docs/docs/imports/postgres.md @@ -1,5 +1,5 @@ --- -sidebar_position: 21 +sidebar_position: 22 title: "Import: Postgres" description: "Create a data contract from a Postgres schema." --- diff --git a/docs/docs/imports/powerbi.md b/docs/docs/imports/powerbi.md index 81db13d25..8c852bc8b 100644 --- a/docs/docs/imports/powerbi.md +++ b/docs/docs/imports/powerbi.md @@ -1,5 +1,5 @@ --- -sidebar_position: 22 +sidebar_position: 23 title: "Import: Power BI" description: "Create a data contract from a Power BI semantic model (.pbit, .bim, or .json)." --- diff --git a/docs/docs/imports/protobuf.md b/docs/docs/imports/protobuf.md index cb4716017..40fd9b5ec 100644 --- a/docs/docs/imports/protobuf.md +++ b/docs/docs/imports/protobuf.md @@ -1,5 +1,5 @@ --- -sidebar_position: 23 +sidebar_position: 24 title: "Import: Protobuf" description: "Create a data contract from a Protobuf schema file." --- diff --git a/docs/docs/imports/pydantic-model.md b/docs/docs/imports/pydantic-model.md index 38384de23..a45dd55a5 100644 --- a/docs/docs/imports/pydantic-model.md +++ b/docs/docs/imports/pydantic-model.md @@ -1,5 +1,5 @@ --- -sidebar_position: 24 +sidebar_position: 25 title: "Import: Pydantic Model" description: "Create a data contract from Pydantic models." --- diff --git a/docs/docs/imports/snowflake.md b/docs/docs/imports/snowflake.md index ebcbf4509..4259ffb28 100644 --- a/docs/docs/imports/snowflake.md +++ b/docs/docs/imports/snowflake.md @@ -1,5 +1,5 @@ --- -sidebar_position: 25 +sidebar_position: 26 title: "Import: Snowflake" description: "Create a data contract from a Snowflake workspace." --- diff --git a/docs/docs/imports/spark.md b/docs/docs/imports/spark.md index 2c379a59d..c5dcaf7bc 100644 --- a/docs/docs/imports/spark.md +++ b/docs/docs/imports/spark.md @@ -1,5 +1,5 @@ --- -sidebar_position: 26 +sidebar_position: 27 title: "Import: Spark" description: "Create a data contract from Spark tables or DataFrames (programmatic)." --- diff --git a/docs/docs/imports/sql.md b/docs/docs/imports/sql.md index 6895d5b78..dc8e5f619 100644 --- a/docs/docs/imports/sql.md +++ b/docs/docs/imports/sql.md @@ -1,5 +1,5 @@ --- -sidebar_position: 27 +sidebar_position: 28 title: "Import: SQL DDL" description: "Create a data contract from a SQL DDL file." --- diff --git a/docs/docs/imports/sqlserver.md b/docs/docs/imports/sqlserver.md index 3df1b5415..a1a0e71dc 100644 --- a/docs/docs/imports/sqlserver.md +++ b/docs/docs/imports/sqlserver.md @@ -1,5 +1,5 @@ --- -sidebar_position: 28 +sidebar_position: 29 title: "Import: SQL Server" description: "Create a data contract from a SQL Server database." --- diff --git a/docs/docs/imports/trino.md b/docs/docs/imports/trino.md index 228247bb4..dedf04c81 100644 --- a/docs/docs/imports/trino.md +++ b/docs/docs/imports/trino.md @@ -1,5 +1,5 @@ --- -sidebar_position: 29 +sidebar_position: 30 title: "Import: Trino" description: "Create a data contract from a Trino catalog." --- diff --git a/docs/static/img/icons/odata.svg b/docs/static/img/icons/odata.svg new file mode 100644 index 000000000..3f12a9ea5 --- /dev/null +++ b/docs/static/img/icons/odata.svg @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/examples/imports/odata/datacontract.odcs.yaml b/examples/imports/odata/datacontract.odcs.yaml new file mode 100644 index 000000000..e3d450366 --- /dev/null +++ b/examples/imports/odata/datacontract.odcs.yaml @@ -0,0 +1,162 @@ +version: 1.0.0 +kind: DataContract +apiVersion: v3.2.0 +id: my-data-contract +name: REF_COUNTRY +status: draft +servers: +- server: source + type: api + customProperties: + - property: apiType + value: odata + - property: odataVersion + value: '4.0' + - property: format + value: json + - property: odataMetadataUrl + value: https://xmart-api-public-uat.who.int/refmart/$metadata + location: https://xmart-api-public-uat.who.int/refmart/ +schema: +- name: REF_COUNTRY + physicalType: object + customProperties: + - property: odataEntitySet + value: REF_COUNTRY + - property: odataEntitySetUrl + value: https://xmart-api-public-uat.who.int/refmart/REF_COUNTRY + logicalType: object + physicalName: REF_COUNTRY + properties: + - name: GEO_M49_CODE + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_ISO_2 + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_ISO_3 + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_WHO + physicalType: Edm.String + logicalType: string + required: false + - name: CODE_ISO_NUMERIC + physicalType: Edm.Int32 + logicalType: integer + required: false + - name: NAME_SHORT_EN + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_EN + physicalType: Edm.String + logicalType: string + required: false + - name: CAPITAL_CITY + physicalType: Edm.String + logicalType: string + required: false + - name: ADJECTIVE_PEOPLE + physicalType: Edm.String + logicalType: string + required: false + - name: GEO_SMALL_POP_FLAG + physicalType: Edm.Boolean + logicalType: boolean + required: false + - name: GEO_SOVEREIGN + physicalType: Edm.String + logicalType: string + required: false + - name: SOVEREIGN_ISO_3 + physicalType: Edm.String + logicalType: string + required: false + - name: GRP_WB_INCOME + physicalType: Edm.String + logicalType: string + required: false + - name: GRP_WHO_REGION + physicalType: Edm.String + logicalType: string + required: false + - name: WHO_LEGAL_STATUS + physicalType: Edm.String + logicalType: string + required: false + - name: WHO_LEGAL_STATUS_TITLE + physicalType: Edm.String + logicalType: string + required: false + - name: DATE_START + physicalType: Edm.Date + logicalType: date + required: false + - name: NAME_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: ISO_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: CAPITAL_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: STATISTICAL_CHANGE + physicalType: Edm.String + logicalType: string + required: false + - name: GEO_PRECEDED_BY + physicalType: Edm.String + logicalType: string + required: false + - name: GEO_SUCCEEDED_BY + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_AR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_AR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_ES + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_ES + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_FR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_FR + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_RU + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_RU + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_SHORT_ZH + physicalType: Edm.String + logicalType: string + required: false + - name: NAME_FORMAL_ZH + physicalType: Edm.String + logicalType: string + required: false diff --git a/tests/fixtures/odata/products.json b/tests/fixtures/odata/products.json new file mode 100644 index 000000000..a102a7988 --- /dev/null +++ b/tests/fixtures/odata/products.json @@ -0,0 +1,32 @@ +{ + "$Version": "4.01", + "$EntityContainer": "Catalog.Service.Store", + "Catalog.Model": { + "$Alias": "Catalog", + "Product": { + "$Kind": "EntityType", + "$Key": ["Revision", "Sku"], + "Sku": {"$MaxLength": 32}, + "Revision": {"$Type": "Edm.Int32"}, + "Price": {"$Type": "Edm.Decimal", "$Nullable": true, "$Precision": 10, "$Scale": 2}, + "PublishedAt": {"$Type": "Edm.DateTimeOffset", "$Nullable": true, "$Precision": 3}, + "Active": {"$Type": "Edm.Boolean"}, + "TrackingId": {"$Type": "Edm.Guid", "$Nullable": true}, + "Description": {"$Kind": "Property", "$Nullable": true, "$MaxLength": "max"} + }, + "Order": { + "$Kind": "EntityType", + "$Key": ["OrderId"], + "OrderId": {"$Type": "Edm.Guid"}, + "Total": {"$Type": "Edm.Decimal", "$Precision": 12, "$Scale": 2} + } + }, + "Catalog.Service": { + "Store": { + "$Kind": "EntityContainer", + "Products": {"$Collection": true, "$Type": "Catalog.Product"}, + "Orders": {"$Collection": true, "$Type": "Catalog.Order"}, + "ArchivedProducts": {"$Collection": true, "$Type": "Catalog.Product", "$IncludeInServiceDocument": false} + } + } +} diff --git a/tests/fixtures/odata/products.xml b/tests/fixtures/odata/products.xml new file mode 100644 index 000000000..a173fd6d3 --- /dev/null +++ b/tests/fixtures/odata/products.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/fixtures/odata/service-document.json b/tests/fixtures/odata/service-document.json new file mode 100644 index 000000000..d144f069e --- /dev/null +++ b/tests/fixtures/odata/service-document.json @@ -0,0 +1,10 @@ +{ + "@odata.context": "$metadata", + "value": [ + {"name": "Products", "kind": "EntitySet", "url": "products"}, + {"name": "Orders", "url": "https://example.com/fulfilment/orders"}, + {"name": "FeaturedProduct", "kind": "Singleton", "url": "FeaturedProduct"}, + {"name": "Search", "kind": "FunctionImport", "url": "Search"}, + {"name": "RelatedService", "kind": "ServiceDocument", "url": "https://example.com/other/"} + ] +} diff --git a/tests/fixtures/odata/who-metadata.xml b/tests/fixtures/odata/who-metadata.xml new file mode 100644 index 000000000..2fa4cced1 --- /dev/null +++ b/tests/fixtures/odata/who-metadata.xml @@ -0,0 +1,330 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/tests/test_import_odata.py b/tests/test_import_odata.py new file mode 100644 index 000000000..186548db4 --- /dev/null +++ b/tests/test_import_odata.py @@ -0,0 +1,873 @@ +"""Tests for the OData importer, using local CSDL fixtures and mocked HTTP responses.""" + +import json +import traceback +from io import BytesIO +from pathlib import Path +from unittest.mock import Mock + +import jsonschema +import pytest +import requests +import yaml +from typer.testing import CliRunner + +from datacontract import Config +from datacontract.cli import app +from datacontract.config import set_cli_config +from datacontract.data_contract import DataContract +from datacontract.imports.odata_importer import ( + _odata_4_schema, + _odata_version, + _read_json, + _read_xml, +) +from datacontract.model.exceptions import DataContractException + +ROOT = "https://xmart-api-public-uat.who.int/refmart/" +METADATA = "https://xmart-api-public-uat.who.int/refmart/$metadata" +FIXTURES = Path(__file__).parent / "fixtures/odata" +EXPECTED_CONTRACT = Path(__file__).parents[1] / "examples/imports/odata/datacontract.odcs.yaml" +ODATA_TEST_VERSIONS = ["4.0", "4.01", "4.123"] +SERVICE_ROOT = "https://example.com/odata/" + + +# Helper functions and shared fixtures + + +def run_cli(*args, source=SERVICE_ROOT): + return CliRunner().invoke(app, ["import", "odata", "--service-root-url", source, *map(str, args)]) + + +@pytest.fixture(scope="module") +def odcs_schema(): + return json.loads((Path(__file__).parents[1] / "datacontract/schemas/odcs-3.2.0.schema.json").read_text()) + + +@pytest.fixture(autouse=True) +def isolate_authentication(monkeypatch): + monkeypatch.delenv("DATACONTRACT_API_HEADER_AUTHORIZATION", raising=False) + set_cli_config(None) + yield + set_cli_config(None) + + +@pytest.fixture +def metadata_response(monkeypatch): + response = Mock() + response.content = (FIXTURES / "who-metadata.xml").read_bytes() + response.headers = {"OData-Version": "4.0"} + get = Mock(return_value=response) + monkeypatch.setattr("datacontract.imports.odata_importer._ODataSession.get", get) + return response, get + + +def import_contract(entity_set="ref_country", *, source=ROOT, **kwargs): + return DataContract.import_from_source( + "odata", source=source, odata_entity_set=[entity_set], odata_metadata_url=METADATA, **kwargs + ) + + +def metadata_xml(properties="", *, version="4.01", entity_attributes="", extra_schema="", extra_sets="", keys=""): + return f''' + + + {keys}{properties} + + {extra_sets} + {extra_schema} + + + '''.encode() + + +def xml_schema(properties="", **kwargs): + return _odata_4_schema(_read_xml(metadata_xml(properties, **kwargs)), "REF_COUNTRY") + + +# Contract generation, type mapping and metadata validation + + +def test_who_contract_and_transport(metadata_response, odcs_schema): + response, get = metadata_response + contract = import_contract() + actual = yaml.safe_load(contract.to_yaml()) + assert actual == yaml.safe_load(EXPECTED_CONTRACT.read_text()) + jsonschema.validate(actual, odcs_schema) + get.assert_called_once() + assert get.call_args.args == (METADATA,) + assert get.call_args.kwargs["headers"] == { + "Accept": "application/xml, application/json;q=0.9", + } + assert get.call_args.kwargs["timeout"] == 30 + response.raise_for_status.assert_called_once() + # The explicit auth handler must prevent implicit .netrc credentials. + request = requests.Request("GET", METADATA).prepare() + assert get.call_args.kwargs["auth"](request).headers.get("Authorization") is None + + +@pytest.mark.parametrize("version", ODATA_TEST_VERSIONS) +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_document_versions(metadata_response, version, format): + response, _ = metadata_response + response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"4.01", version.encode()) + response.headers = {} + contract = import_contract("Products") + assert contract.servers[0].customProperties[1].value == version + assert contract.schema_[0].properties[0].name == "Sku" + + +@pytest.mark.parametrize( + "edm_type, logical_type", + [ + ("String", "string"), + ("Guid", "string"), + ("Byte", "integer"), + ("SByte", "integer"), + ("Int16", "integer"), + ("Int32", "integer"), + ("Int64", "integer"), + ("Decimal", "number"), + ("Single", "number"), + ("Double", "number"), + ("Boolean", "boolean"), + ("Date", "date"), + ("DateTimeOffset", "timestamp"), + ("TimeOfDay", "time"), + ], +) +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_primitive_types(edm_type, logical_type, format): + if format == "xml": + schema = xml_schema(f'') + else: + document = json.loads((FIXTURES / "products.json").read_text()) + document["Catalog.Model"]["Product"] = { + "$Kind": "EntityType", + "Value": {"$Type": f"Edm.{edm_type}", "$Nullable": True}, + } + schema = _odata_4_schema(_read_json(json.dumps(document).encode()), "Products") + prop = schema.properties[0] + assert prop.logicalType == logical_type + assert prop.physicalType == f"Edm.{edm_type}" + assert prop.required is False + if edm_type == "Guid": + assert prop.logicalTypeOptions == {"format": "uuid"} + facets = {p.property: p.value for p in prop.customProperties or []} + if edm_type == "Decimal": + assert facets == {"scale": 0 if format == "xml" else "variable"} + else: + assert "scale" not in facets + + +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_unmapped_primitive_type_warns(metadata_response, caplog, format): + response, _ = metadata_response + response.content = (FIXTURES / f"products.{format}").read_bytes().replace(b"Edm.Guid", b"Edm.Binary") + contract = import_contract("Products") + properties = yaml.safe_load(contract.to_yaml())["schema"][0]["properties"] + prop = next(prop for prop in properties if prop["name"] == "TrackingId") + assert prop["physicalType"] == "Edm.Binary" + assert "logicalType" not in prop + assert "without a logicalType" in caplog.text + assert "TrackingId (Edm.Binary)" in caplog.text + + +def test_keys_nullability_and_facets(): + schema = xml_schema( + """ + + + + + """, + keys='', + ) + code, year, amount, variable, floating, text = schema.properties + assert code.required is True and year.required is True + assert code.primaryKey is True and year.primaryKey is True + assert (code.primaryKeyPosition, year.primaryKeyPosition) == (2, 1) + assert not code.unique and not year.unique + assert code.logicalTypeOptions == {"maxLength": 3} + assert amount.required is False and not amount.primaryKey + assert {p.property: p.value for p in amount.customProperties} == {"precision": 12, "scale": 2} + assert variable.customProperties[0].value == "variable" + assert floating.customProperties[0].value == "floating" + assert not text.logicalTypeOptions + + +@pytest.mark.parametrize("entity_set", ["REF_COUNTRY", "ref_country", "Ref_Country"]) +@pytest.mark.parametrize("root", ["https://example.com", "https://example.com/service", "https://example.com/service/"]) +def test_entity_set_matching_normalizes_root(metadata_response, entity_set, root): + contract = import_contract(entity_set, source=root) + assert contract.schema_[0].name == "REF_COUNTRY" + assert contract.servers[0].location == root.rstrip("/") + "/" + assert {p.property: p.value for p in contract.schema_[0].customProperties} == { + "odataEntitySet": "REF_COUNTRY", + "odataEntitySetUrl": root.rstrip("/") + "/REF_COUNTRY", + } + + +def test_exact_match_wins_and_casefold_ambiguity_fails(metadata_response): + response, _ = metadata_response + response.headers = {} + response.content = metadata_xml(extra_sets='') + assert import_contract().schema_[0].name == "ref_country" + with pytest.raises(DataContractException, match="ambiguous"): + import_contract("Ref_Country") + + +def test_navigation_and_unrelated_types_are_omitted(caplog): + schema = xml_schema( + '', + extra_schema='', + ) + assert [p.name for p in schema.properties] == ["Code"] + assert "Omitting OData navigation property REF_COUNTRY.Related" in caplog.text + + +@pytest.mark.parametrize( + "field_type", + ["D.Address", "Collection(Edm.String)", "Collection(D.Address)", "D.Status"], +) +def test_unsupported_types_fail_with_field_and_type(field_type): + with pytest.raises(DataContractException) as error: + xml_schema(f'') + assert "REF_COUNTRY.Value" in str(error.value) + assert field_type in str(error.value) + + +@pytest.mark.parametrize( + "document, error", + [ + (metadata_xml(entity_attributes='BaseType="Demo.Base"'), "inheritance"), + (metadata_xml().replace(b'Name="Service"', b'Name="Service" Extends="Demo.Base"'), "inheritance"), + (metadata_xml().replace(b"D.Country", b"Other.Country"), "must resolve uniquely"), + (metadata_xml().replace(b"REF_COUNTRY", b"OTHER"), "not found"), + (metadata_xml().replace(b'http://docs.oasis-open.org/odata/ns/edm"', b'urn:wrong"'), "not found"), + (b"", "Expected an OData 4 Edmx"), + (b"'), "Invalid Nullable"), + (metadata_xml(''), "Invalid MaxLength"), + (metadata_xml(keys=''), "key references"), + ], +) +def test_metadata_errors(metadata_response, document, error): + response, _ = metadata_response + response.content = document + response.headers = {} + with pytest.raises(DataContractException, match=error): + import_contract() + + +@pytest.mark.parametrize("encoding", ["utf-8", "utf-16"]) +@pytest.mark.parametrize( + "doctype", + [ + "", + ']>', + ']>', + '', + ], +) +def test_metadata_rejects_dtd(encoding, doctype): + document = metadata_xml().decode().replace('Name="REF_COUNTRY"', 'Name="&country;"') + content = (f'{doctype}{document}').encode(encoding) + with pytest.raises(DataContractException, match="DTD declarations are not allowed"): + _read_xml(content) + + +@pytest.mark.parametrize( + "location", + [ + None, + "", + "file:///tmp/source", + "https://example.com:bad/", + ROOT + "?$select=CODE_ISO_3", + ROOT + "#fragment", + "https://user:secret@example.com/Countries", + ], +) +def test_invalid_root_does_not_fetch_metadata(metadata_response, location): + _, get = metadata_response + with pytest.raises(DataContractException, match="Invalid service-root-url"): + import_contract(source=location) + get.assert_not_called() + + +def test_metadata_sources_are_mutually_exclusive_for_python_api(metadata_response): + _, get = metadata_response + with pytest.raises(DataContractException, match="mutually exclusive"): + DataContract.import_from_source( + "odata", + source=ROOT, + odata_entity_set=["ref_country"], + odata_metadata_url=METADATA, + odata_metadata_file=FIXTURES / "who-metadata.xml", + ) + get.assert_not_called() + + +@pytest.mark.parametrize("error", [requests.Timeout("timed out"), requests.ConnectionError("connection failed")]) +def test_network_errors(metadata_response, error): + _, get = metadata_response + get.side_effect = error + with pytest.raises(DataContractException, match="Failed to fetch OData metadata") as exc: + import_contract() + assert exc.value.type == "connection" + + +@pytest.mark.parametrize("status", [401, 403, 404, 500]) +def test_http_errors(metadata_response, status): + response, _ = metadata_response + response.status_code = status + response.raise_for_status.side_effect = requests.HTTPError(f"HTTP {status}", response=response) + with pytest.raises(DataContractException, match=f"HTTP {status}"): + import_contract() + + +# CLI options and output handling + + +def test_cli_stdout_and_output(metadata_response, tmp_path): + args = ["--entity-set", "ref_country", "--metadata-url", METADATA] + stdout = run_cli(*args, source=ROOT) + assert stdout.exit_code == 0, stdout.output + assert yaml.safe_load(stdout.stdout) == yaml.safe_load(EXPECTED_CONTRACT.read_text()) + output = tmp_path / "contract.yaml" + result = run_cli(*args, "--output", output, "--owner", "WHO", "--id", "country", "--debug", source=ROOT) + assert result.exit_code == 0, result.output + contract = yaml.safe_load(output.read_text()) + assert contract["id"] == "country" + assert contract["team"]["name"] == "WHO" + + +@pytest.mark.parametrize("existing", [False, True], ids=["no-new-file", "preserve-existing-file"]) +@pytest.mark.parametrize("source", ["file", "url"]) +def test_failed_import_preserves_output(metadata_response, tmp_path, existing, source): + response, _ = metadata_response + response.content = b"invalid XML" + output = tmp_path / "contract.yaml" + if existing: + output.write_text("existing contract") + args = ["--metadata-url", METADATA] + if source == "file": + metadata = tmp_path / "invalid.xml" + metadata.write_bytes(response.content) + args = ["--metadata-file", metadata] + result = run_cli("--entity-set", "ref_country", *args, "--output", output) + assert result.exit_code != 0 + assert isinstance(result.exception, DataContractException) + assert "Invalid or unsafe" in str(result.exception) + if existing: + assert output.read_text() == "existing contract" + else: + assert not output.exists() + + +# Local metadata files + + +@pytest.mark.parametrize("path_type", [str, Path]) +def test_local_who_metadata_matches_url_import(metadata_response, path_type): + _, get = metadata_response + path = path_type(FIXTURES / "who-metadata.xml") + contract = DataContract.import_from_source( + "odata", source=ROOT, odata_entity_set=["ref_country"], odata_metadata_file=path + ) + actual = yaml.safe_load(contract.to_yaml()) + expected = yaml.safe_load(EXPECTED_CONTRACT.read_text()) + expected["servers"][0]["customProperties"] = [ + prop for prop in expected["servers"][0]["customProperties"] if prop["property"] != "odataMetadataUrl" + ] + assert actual == expected + get.assert_not_called() + + +# CSDL JSON and XML/JSON equivalence + + +@pytest.fixture +def json_metadata(metadata_response): + response, _ = metadata_response + response.headers = {} + return json.loads((FIXTURES / "products.json").read_text()) + + +@pytest.mark.parametrize("source", ["file", "url"]) +def test_xml_json_equivalent_contracts(metadata_response, tmp_path, odcs_schema, source): + response, get = metadata_response + response.headers = {"OData-Version": "4.01", "Content-Type": "text/plain"} + contracts = [] + for format in ("xml", "json"): + content = (FIXTURES / f"products.{format}").read_bytes() + if source == "file": + # Deliberately use an unrelated extension: content determines the format. + path = tmp_path / "metadata.txt" + path.write_bytes(content) + contract = DataContract.import_from_source( + "odata", source=SERVICE_ROOT, odata_entity_set=["Products"], odata_metadata_file=path + ) + get.assert_not_called() + else: + response.content = content + contract = import_contract("Products", source=SERVICE_ROOT) + actual = yaml.safe_load(contract.to_yaml()) + jsonschema.validate(actual, odcs_schema) + contracts.append(actual) + assert contracts[0] == contracts[1] + properties = contracts[0]["schema"][0]["properties"] + assert [p["name"] for p in properties] == [ + "Sku", + "Revision", + "Price", + "PublishedAt", + "Active", + "TrackingId", + "Description", + ] + assert [p["primaryKeyPosition"] for p in properties[:2]] == [2, 1] + assert not any(p.get("unique") for p in properties) + assert properties[0]["required"] and not properties[2]["required"] + assert properties[0]["logicalTypeOptions"] == {"maxLength": 32} + assert properties[2]["customProperties"] == [ + {"property": "precision", "value": 10}, + {"property": "scale", "value": 2}, + ] + + +def test_json_defaults_and_annotations(metadata_response, json_metadata): + response, _ = metadata_response + json_metadata["Catalog.Model"]["Product"] = { + "$Kind": "EntityType", + "Label": {}, + "Amount": {"$Type": "Edm.Decimal"}, + "Optional": {"$Kind": "Property", "$Nullable": True}, + "@Example.Note": "ignored", + "Label@Example.Note": "also ignored", + } + response.content = json.dumps(json_metadata).encode() + label, amount, optional = import_contract("Products").schema_[0].properties + assert label.physicalType == "Edm.String" and label.required + assert amount.required + assert {p.property: p.value for p in amount.customProperties} == {"scale": "variable"} + assert len(amount.customProperties) == 1 # No inferred precision. + assert not optional.required + assert not any(p.primaryKey for p in (label, amount, optional)) + + +@pytest.mark.parametrize("type_name", ["Catalog.Product", "Catalog.Model.Product"]) +def test_json_alias_and_navigation(metadata_response, json_metadata, caplog, type_name): + response, get = metadata_response + json_metadata["Catalog.Service"]["Store"]["Products"]["$Type"] = type_name + json_metadata["Catalog.Model"]["Product"]["Orders"] = { + "$Kind": "NavigationProperty", + "$Type": "External.Order", + "$Collection": True, + } + json_metadata["Catalog.Model"]["Unused"] = {"$Kind": "ComplexType", "Nested": {"$Type": "External.Unknown"}} + json_metadata["$Reference"] = {"https://example.com/external/$metadata": {"$Include": [{"$Namespace": "External"}]}} + json_metadata["Catalog.Service"]["Store"]["Featured"] = {"$Type": "Catalog.Product"} # Singleton. + response.content = json.dumps(json_metadata).encode() + contract = import_contract("Products") + assert len(contract.schema_[0].properties) == 7 + assert "Omitting OData navigation property Products.Orders" in caplog.text + get.assert_called_once() + + +@pytest.mark.parametrize( + "path,value,error", + [ + (("$EntityContainer",), [], "namespace-qualified"), + (("$EntityContainer",), "Missing.Store", "JSON object"), + (("Catalog.Model", "$Alias"), 42, "Invalid .*Alias"), + (("Catalog.Model", "$Alias"), None, "Invalid .*Alias"), + (("Catalog.Service", "Store", "$Kind"), "EntityType", "EntityContainer"), + (("Catalog.Service", "Store", "$Extends"), "Other.Store", "inheritance"), + (("Catalog.Service", "Store", "Products", "$Collection"), "true", "Invalid .*Collection"), + (("Catalog.Service", "Store", "Products", "$Collection"), False, "not found"), + (("Catalog.Service", "Store", "Products", "$Type"), [], "qualified type name"), + (("Catalog.Service", "Store", "Products", "$Type"), "External.Product", "External metadata references"), + (("Catalog.Model", "Product", "$BaseType"), "Catalog.Base", "inheritance"), + (("Catalog.Model", "Product", "$BaseType"), None, "Invalid .*BaseType"), + (("Catalog.Model", "Product", "$Key"), "Sku", "key references"), + (("Catalog.Model", "Product", "$Key"), ["Sku", "Sku"], "key references"), + (("Catalog.Model", "Product", "$Key"), ["Missing"], "key references"), + (("Catalog.Model", "Product", "$Key"), [{"Alias": "Address/Code"}], "key references"), + (("Catalog.Model", "Product", "Sku", "$Nullable"), True, "Key field"), + (("Catalog.Model", "Product", "Price"), [], "JSON object"), + (("Catalog.Model", "Product", "Price", "$Kind"), "Unexpected", "Unsupported .*Kind"), + (("Catalog.Model", "Product", "Price", "$Type"), None, "Unsupported OData type"), + (("Catalog.Model", "Product", "Price", "$Nullable"), "false", "Invalid Nullable"), + (("Catalog.Model", "Product", "Price", "$Collection"), 1, "Invalid .*Collection"), + (("Catalog.Model", "Product", "Price", "$Collection"), True, "collection"), + (("Catalog.Model", "Product", "Price", "$Precision"), "10", "Invalid Precision"), + (("Catalog.Model", "Product", "Price", "$Scale"), True, "Invalid Scale"), + (("Catalog.Model", "Product", "Sku", "$MaxLength"), -1, "Invalid MaxLength"), + (("Catalog.Model", "Product", "Sku", "$MaxLength"), None, "Invalid MaxLength"), + ], +) +def test_json_invalid_csdl(json_metadata, path, value, error): + target = json_metadata + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with pytest.raises(DataContractException, match=error): + _odata_4_schema(_read_json(json.dumps(json_metadata).encode()), "Products") + + +@pytest.mark.parametrize("kind", ["ComplexType", "EnumType", "TypeDefinition"]) +def test_json_unsupported_selected_type(json_metadata, kind): + json_metadata["Catalog.Model"]["Special"] = {"$Kind": kind} + json_metadata["Catalog.Model"]["Product"]["Price"]["$Type"] = "Catalog.Special" + with pytest.raises(DataContractException, match="Catalog.Special.*Products.Price"): + _odata_4_schema(_read_json(json.dumps(json_metadata).encode()), "Products") + + +# Version validation and format detection + + +@pytest.mark.parametrize("version", [None, "3.0", "5.0", "4", "4.", "4.x", "4.1.0", "4.01beta", "4.١", 4.01]) +def test_invalid_versions(version): + with pytest.raises(DataContractException, match="Unsupported OData version"): + _odata_version(version) + + +@pytest.mark.parametrize("prefix", [b" \t\n" * 100, b"\xef\xbb\xbf \t\n"]) +def test_xml_detection_with_whitespace_and_bom(metadata_response, prefix): + response, _ = metadata_response + response.headers = {"Content-Type": "application/json"} + response.content = prefix + metadata_xml('') + assert import_contract().schema_[0].properties[0].name == "Label" + + +# Service documents and selection of multiple EntitySets + + +@pytest.fixture +def service_response(metadata_response): + metadata, get = metadata_response + metadata.content = (FIXTURES / "products.json").read_bytes() + metadata.headers = {"OData-Version": "4.01"} + service = Mock( + content=(FIXTURES / "service-document.json").read_bytes(), + headers={"OData-Version": "4.01"}, + url=SERVICE_ROOT, + ) + documents = {SERVICE_ROOT + "$metadata": metadata, METADATA: metadata, SERVICE_ROOT: service} + get.side_effect = lambda url, **kwargs: documents[url] + return service, get + + +@pytest.mark.parametrize( + "format, metadata_source, service_source, root", + [ + pytest.param("xml", "file", "file", SERVICE_ROOT, id="offline-xml"), + pytest.param("json", "file", "file", SERVICE_ROOT, id="offline-json"), + pytest.param("xml", "derived_url", "url", SERVICE_ROOT, id="http-xml"), + pytest.param("json", "derived_url", "url", SERVICE_ROOT.rstrip("/"), id="http-json-normalized-root"), + pytest.param("json", "explicit_url", "file", SERVICE_ROOT, id="explicit-metadata-url-local-service"), + pytest.param("xml", "file", "url", SERVICE_ROOT, id="local-metadata-http-service"), + ], +) +def test_import_all_advertised_sets( + metadata_response, service_response, odcs_schema, format, metadata_source, service_source, root +): + metadata, _ = metadata_response + _, get = service_response + metadata.content = (FIXTURES / f"products.{format}").read_bytes() + options, expected_requests = {}, [] + if metadata_source == "file": + options["odata_metadata_file"] = FIXTURES / f"products.{format}" + elif metadata_source == "explicit_url": + options["odata_metadata_url"] = METADATA + expected_requests.append(METADATA) + else: + expected_requests.append(SERVICE_ROOT + "$metadata") + if service_source == "file": + options["odata_service_document_file"] = FIXTURES / "service-document.json" + else: + expected_requests.append(SERVICE_ROOT) + contract = DataContract.import_from_source("odata", source=root, **options) + assert contract.name == "Store" + assert contract.servers[0].location == SERVICE_ROOT + assert [s.name for s in contract.schema_] == ["Products", "Orders"] + assert [s.physicalName for s in contract.schema_] == ["Products", "Orders"] + assert all(s.logicalType == s.physicalType == "object" for s in contract.schema_) + assert [{p.property: p.value for p in s.customProperties} for s in contract.schema_] == [ + {"odataEntitySet": "Products", "odataEntitySetUrl": SERVICE_ROOT + "products"}, + {"odataEntitySet": "Orders", "odataEntitySetUrl": "https://example.com/fulfilment/orders"}, + ] + server_properties = {p.property: p.value for p in contract.servers[0].customProperties} + assert "odataEntitySet" not in server_properties + assert server_properties["odataVersion"] == "4.01" + if metadata_source != "file": + assert server_properties["odataMetadataUrl"] == expected_requests[0] + jsonschema.validate(yaml.safe_load(contract.to_yaml()), odcs_schema) + assert [call.args[0] for call in get.call_args_list] == expected_requests + if service_source == "url": + assert get.call_args.kwargs["headers"] == {"Accept": "application/json"} + assert get.call_args.kwargs["timeout"] == 30 + request = requests.Request("GET", SERVICE_ROOT).prepare() + assert get.call_args.kwargs["auth"](request).headers.get("Authorization") is None + + +@pytest.mark.parametrize("format", ["xml", "json"]) +def test_cli_multiple_sets_offline_skips_service_file(metadata_response, tmp_path, monkeypatch, format): + _, get = metadata_response + monkeypatch.chdir(tmp_path) + path = Path("metadata") # Relative path without a format-specific extension. + path.write_bytes((FIXTURES / f"products.{format}").read_bytes()) + result = run_cli( + "--metadata-file", + path, + "--service-document-file", + "does-not-exist.json", + "--entity-set", + "Orders", + "--entity-set", + "products", + "--entity-set", + "ArchivedProducts", + "--entity-set", + "Products", + ) + assert result.exit_code == 0, result.output + contract = yaml.safe_load(result.stdout) + assert contract["name"] == "Store" + schemas = contract["schema"] + assert [s["name"] for s in schemas] == ["Orders", "Products", "ArchivedProducts"] + assert schemas[1]["properties"] == schemas[2]["properties"] + assert schemas[1]["customProperties"] != schemas[2]["customProperties"] + get.assert_not_called() + + +@pytest.mark.parametrize("root", [SERVICE_ROOT, SERVICE_ROOT.rstrip("/")]) +def test_derived_metadata_url_with_explicit_selection(metadata_response, root): + metadata, get = metadata_response + metadata.content = (FIXTURES / "products.json").read_bytes() + metadata.headers = {} + contract = DataContract.import_from_source("odata", source=root, odata_entity_set=["Products", "Orders"]) + get.assert_called_once() + assert get.call_args.args == (SERVICE_ROOT + "$metadata",) + assert contract.servers[0].location == root.rstrip("/") + "/" + + +@pytest.mark.parametrize("context_key", ["@odata.context", "@context"]) +@pytest.mark.parametrize("context", ["$metadata", "https://example.com/custom/$metadata"]) +def test_service_document_url_resolution(service_response, context_key, context): + service, get = service_response + service.content = json.dumps( + { + context_key: context, + "value": [ + {"name": "Products", "url": "items/products"}, + {"name": "Orders", "@context": "../shipping/$metadata", "url": "orders"}, + {"name": "Unknown", "kind": "FutureResource", "url": "ignored"}, + ], + } + ).encode() + contract = DataContract.import_from_source("odata", source=SERVICE_ROOT) + urls = [{p.property: p.value for p in s.customProperties}["odataEntitySetUrl"] for s in contract.schema_] + directory = "odata" if context == "$metadata" else "custom" + assert urls == [f"https://example.com/{directory}/items/products", "https://example.com/shipping/orders"] + assert [call.args[0] for call in get.call_args_list] == [SERVICE_ROOT + "$metadata", SERVICE_ROOT] + + +@pytest.mark.parametrize("url", ["https://example.com/redirected/", "https://example.com/redirected/service.json"]) +@pytest.mark.parametrize("root", [SERVICE_ROOT, SERVICE_ROOT.rstrip("/")]) +def test_redirected_service_document_base(service_response, url, root): + service, get = service_response + service.url = url + contract = DataContract.import_from_source("odata", source=root) + assert contract.schema_[0].customProperties[1].value == "https://example.com/redirected/products" + assert contract.servers[0].location == SERVICE_ROOT + assert [call.args[0] for call in get.call_args_list] == [SERVICE_ROOT + "$metadata", SERVICE_ROOT] + + +def test_missing_service_file_does_not_fall_back_to_network(metadata_response, tmp_path): + _, get = metadata_response + with pytest.raises(DataContractException, match="Failed to read OData service document file"): + DataContract.import_from_source( + "odata", + source=SERVICE_ROOT, + odata_metadata_file=FIXTURES / "products.xml", + odata_service_document_file=tmp_path / "missing.json", + ) + get.assert_not_called() + + +def test_cli_offline_all_sets(metadata_response, tmp_path): + _, get = metadata_response + output = tmp_path / "contract.yaml" + result = run_cli( + "--metadata-file", + FIXTURES / "products.xml", + "--service-document-file", + FIXTURES / "service-document.json", + "--output", + output, + ) + assert result.exit_code == 0, result.output + assert [s["name"] for s in yaml.safe_load(output.read_text())["schema"]] == ["Products", "Orders"] + get.assert_not_called() + + +def test_xml_requires_unambiguous_container(): + with pytest.raises(DataContractException, match="EntityContainer.*ambiguous"): + xml_schema(extra_schema='') + + +# Authentication and configuration + + +@pytest.mark.parametrize( + "authorization, config_source", + [ + pytest.param(None, "environment", id="anonymous"), + pytest.param("Bearer synthetic-token", "environment", id="bearer-environment"), + pytest.param("Bearer synthetic-token", "config", id="bearer-config"), + pytest.param("Basic dXNlcjpwYXNz", "config", id="basic-config"), + pytest.param("Bearer synthetic-token", "dict", id="bearer-dict"), + ], +) +def test_authentication_on_both_documents(service_response, monkeypatch, authorization, config_source): + _, get = service_response + config = None + if config_source == "environment": + if authorization is not None: + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", authorization) + elif authorization is not None: + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", "Bearer overridden-token") + config = ( + Config(api_header_authorization=authorization) + if config_source == "config" + else {"DATACONTRACT_API_HEADER_AUTHORIZATION": authorization} + ) + contract = DataContract.import_from_source("odata", source=SERVICE_ROOT, odata_metadata_url=METADATA, config=config) + assert [call.args[0] for call in get.call_args_list] == [METADATA, SERVICE_ROOT] + for call in get.call_args_list: + headers = call.kwargs["headers"] + assert headers.get("Authorization") == authorization + if authorization is None: + assert "Authorization" not in headers + if authorization is not None: + assert authorization not in contract.to_yaml() + + +def test_cli_authentication_from_config_file(service_response, tmp_path, monkeypatch): + _, get = service_response + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", "Bearer overridden-token") + monkeypatch.setenv("ODATA_TEST_AUTH", "Bearer config-file-token") + config_file = tmp_path / "config.yaml" + config_file.write_text('api_header_authorization: "${ODATA_TEST_AUTH}"\n') + result = CliRunner().invoke( + app, + ["--config-file", str(config_file), "import", "odata", "--service-root-url", SERVICE_ROOT], + ) + assert result.exit_code == 0, result.output + assert len(get.call_args_list) == 2 + assert all(call.kwargs["headers"]["Authorization"] == "Bearer config-file-token" for call in get.call_args_list) + assert "config-file-token" not in result.output + assert "overridden-token" not in result.output + + +# HTTP transport, redirects and protection of credentials + + +@pytest.fixture +def http_transport(monkeypatch): + # Exercise real request preparation and redirects; only the adapter's I/O is mocked. + routes = { + SERVICE_ROOT + "$metadata": (200, {}, (FIXTURES / "products.json").read_bytes()), + SERVICE_ROOT: (200, {}, (FIXTURES / "service-document.json").read_bytes()), + } + sent = [] + + def send(adapter, request, **kwargs): + sent.append((request, kwargs)) + status, headers, content = routes[request.url] + response = requests.Response() + response.status_code = status + response.headers.update(headers) + response._content = content + response.raw = BytesIO(content) + response.request = request + response.url = request.url + return response + + monkeypatch.setattr(requests.adapters.HTTPAdapter, "send", send) + return routes, sent + + +@pytest.mark.parametrize("authorization", [None, "Bearer redirect-token"]) +@pytest.mark.parametrize("target", ["metadata", "service"]) +@pytest.mark.parametrize( + "redirect_url, retained", + [ + ("https://example.com/redirected/document", True), + ("https://other.example.com/document", False), + ("http://example.com/document", False), + ], +) +def test_redirect_authentication(http_transport, monkeypatch, authorization, target, redirect_url, retained): + routes, sent = http_transport + netrc = Mock(return_value=("unexpected-user", "unexpected-password")) + monkeypatch.setattr("requests.sessions.get_netrc_auth", netrc) + initial_url = SERVICE_ROOT + "$metadata" if target == "metadata" else SERVICE_ROOT + final_url = initial_url + "final" + document = routes[initial_url] + routes[initial_url] = (302, {"Location": redirect_url}, b"") + routes[redirect_url] = (302, {"Location": final_url}, b"") + routes[final_url] = document + DataContract.import_from_source("odata", source=SERVICE_ROOT, config=Config(api_header_authorization=authorization)) + requests_by_url = {request.url: request for request, _ in sent} + assert requests_by_url[initial_url].headers.get("Authorization") == authorization + expected = authorization if retained else None + assert requests_by_url[redirect_url].headers.get("Authorization") == expected + # Returning to the initial host must not restore credentials removed earlier. + assert requests_by_url[final_url].headers.get("Authorization") == expected + netrc.assert_not_called() + + +def test_authentication_preserves_proxy_and_certificate_environment(http_transport, monkeypatch): + _, sent = http_transport + monkeypatch.setenv("HTTPS_PROXY", "http://proxy.example.com:8080") + monkeypatch.setenv("NO_PROXY", "") + monkeypatch.setenv("REQUESTS_CA_BUNDLE", "/synthetic/ca.pem") + DataContract.import_from_source( + "odata", source=SERVICE_ROOT, config=Config(api_header_authorization="Bearer proxy-token") + ) + assert len(sent) == 2 + for request, options in sent: + assert request.headers["Authorization"] == "Bearer proxy-token" + assert options["proxies"]["https"] == "http://proxy.example.com:8080" + assert options["verify"] == "/synthetic/ca.pem" + + +@pytest.mark.parametrize("failure", [401, 403, "invalid_header"]) +@pytest.mark.parametrize("target", ["metadata", "service"]) +def test_authentication_errors_do_not_expose_secrets(http_transport, tmp_path, monkeypatch, caplog, failure, target): + routes, _ = http_transport + secret = "Bearer confidential-test-token" + if failure == "invalid_header": + secret += "\ninvalid" + else: + url = SERVICE_ROOT + "$metadata" if target == "metadata" else SERVICE_ROOT + routes[url] = (failure, {}, secret.encode()) + monkeypatch.setenv("DATACONTRACT_API_HEADER_AUTHORIZATION", secret) + output = tmp_path / "contract.yaml" + output.write_text("existing contract") + args = ["--output", output, "--debug"] + if target == "service": + args += ["--metadata-file", str(FIXTURES / "products.json")] + result = run_cli(*args) + assert result.exit_code != 0 + assert isinstance(result.exception, DataContractException) + assert result.exception.type == "connection" + assert ("InvalidHeader" if failure == "invalid_header" else f"HTTP {failure}") in str(result.exception) + assert result.exception.original_exception is None + rendered = result.output + "".join(traceback.format_exception(result.exception)) + caplog.text + assert "confidential-test-token" not in rendered + assert output.read_text() == "existing contract"