diff --git a/opengin/core-api/engine/attribute_resolver.go b/opengin/core-api/engine/attribute_resolver.go index 988c6894..c9e5c623 100644 --- a/opengin/core-api/engine/attribute_resolver.go +++ b/opengin/core-api/engine/attribute_resolver.go @@ -131,8 +131,44 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // Create or update graph metadata BEFORE processing the attribute // NOTE: for the attribute the timestamp is always the value carried at the attribute level // not the entity level. The entity level timestamp is used for the entity itself. - attributeStartTime, _ := time.Parse(time.RFC3339, value.StartTime) - if err := p.handleAttributeLookUp(ctx, entity.Id, attrName, storageType, operation, attributeStartTime); err != nil { + + // start time is required to create the attribute + if operation != "read" && value.StartTime == "" { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("StartTime is required for attribute: %s", attrName), + } + continue + } + var attributeStartTime time.Time + + if value.StartTime != "" { + attributeStartTime, err = time.Parse(time.RFC3339, value.StartTime) + if err != nil { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("invalid StartTime format for attribute %s: %v", attrName, err), + } + continue + } + } + + var attributeEndTime *time.Time + + if value.EndTime != "" { + t, err := time.Parse(time.RFC3339, value.EndTime) + if err != nil { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("invalid EndTime format for attribute %s: %v", attrName, err), + } + continue + } + attributeEndTime = &t + } + + + if err := p.handleAttributeLookUp(ctx, entity.Id, attrName, storageType, operation, attributeStartTime, attributeEndTime); err != nil { attributeResults[attrName] = &Result{ Success: false, Data: nil, @@ -196,7 +232,7 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // It creates the attribute look up metadata and the attribute node in the graph. // It also creates the IS_ATTRIBUTE relationship between the entity and the attribute. // It also creates the attribute metadata in the document database. -func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, entityID, attrName string, storageType storageinference.StorageType, operation string, startTime time.Time) error { +func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, entityID, attrName string, storageType storageinference.StorageType, operation string, startTime time.Time, endTime *time.Time) error { // Generate attribute metadata fmt.Printf("DEBUG: Handling graph metadata for attribute %s\n", attrName) attributeID := GenerateAttributeID(entityID, attrName) @@ -209,11 +245,13 @@ func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, en StorageType: storageType, StoragePath: storagePath, Created: startTime, + Terminated: endTime, Updated: time.Now(), Schema: make(map[string]interface{}), // TODO: Extract schema from value } // Note: endTime parameter is optional and available for future use if needed + log.Printf("DEBUG: Handling graph metadata for attribute %s: [endTime: %v] [startTime: %v]", attrName, endTime, startTime) switch operation { case "create": diff --git a/opengin/core-api/engine/graph_metadata_manager.go b/opengin/core-api/engine/graph_metadata_manager.go index 62305b7c..668373ad 100644 --- a/opengin/core-api/engine/graph_metadata_manager.go +++ b/opengin/core-api/engine/graph_metadata_manager.go @@ -56,7 +56,7 @@ type AttributeMetadata struct { StoragePath string // Path/location in the specific storage system Created time.Time Updated time.Time - EndTime time.Time + Terminated *time.Time Schema map[string]interface{} // Schema information } @@ -126,6 +126,14 @@ func (g *GraphMetadataManager) createAttributeLookUpGraph(ctx context.Context, m // create the attribute node in the graph // stored parameters: id, kind, name, created + log.Printf("DEBUG: Creating attribute node for attribute %s: [endTime: %v] [startTime: %v]", metadata.AttributeName, metadata.Terminated, metadata.Created) + + var terminated string + + if metadata.Terminated != nil { + terminated = metadata.Terminated.Format(time.RFC3339) + } + attributeNode := &pb.Entity{ Id: metadata.AttributeID, Kind: &pb.Kind{ @@ -134,7 +142,7 @@ func (g *GraphMetadataManager) createAttributeLookUpGraph(ctx context.Context, m }, Name: commons.CreateTimeBasedValue(metadata.Created.Format(time.RFC3339), "", metadata.AttributeName), Created: metadata.Created.Format(time.RFC3339), // contains the data object's time relation with the world - Terminated: "", // TODO: Implement invalidating a dataset for a specific time range + Terminated: terminated, Metadata: MakeMetadataOfAttributeMetadata(metadata), Attributes: make(map[string]*pb.TimeBasedValueList), Relationships: make(map[string]*pb.Relationship), @@ -234,12 +242,16 @@ func MakeMetadataOfAttributeMetadata(metadata *AttributeMetadata) map[string]*an // MakeRelationshipProto creates a Relationship protobuf object for IS_ATTRIBUTE relationship func MakeRelationshipFromAttributeMetadata(metadata *AttributeMetadata) *pb.Relationship { + var endTime string + if metadata.Terminated != nil { + endTime = metadata.Terminated.Format(time.RFC3339) + } return &pb.Relationship{ Id: GenerateAttributeRelationshipID(metadata.EntityID, metadata.AttributeName), RelatedEntityId: metadata.AttributeID, Name: IS_ATTRIBUTE_RELATIONSHIP, StartTime: metadata.Created.Format(time.RFC3339), - EndTime: "", // TODO: Implement invalidating a relationship for a specific time range + EndTime: endTime, Direction: IS_ATTRIBUTE_RELATIONSHIP_DIRECTION, } } diff --git a/opengin/ingestion-api/tests/service_test.bal b/opengin/ingestion-api/tests/service_test.bal index 62935946..6bb5a3cf 100644 --- a/opengin/ingestion-api/tests/service_test.bal +++ b/opengin/ingestion-api/tests/service_test.bal @@ -1,11 +1,11 @@ // Copyright 2025 Lanka Data Foundation // SPDX-License-Identifier: Apache-2.0 -import ballerina/io; -import ballerina/test; -import ballerina/protobuf.types.'any as pbAny; import ballerina/http; +import ballerina/io; import ballerina/os; +import ballerina/protobuf.types.'any as pbAny; +import ballerina/test; // Get environment variables without fallback values string testIngestionHostname = os:getEnv("INGESTION_SERVICE_HOST"); @@ -45,7 +45,7 @@ function unwrapAnyToJson(pbAny:Any anyValue) returns json|error { if stringValue is string { return stringValue; } - + // If string unpacking fails, return the string representation as a fallback return anyValue.toString(); } @@ -106,14 +106,14 @@ function jsonToAny(json data) returns pbAny:Any|error { function testMetadataHandling() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "test-entity-1"; string expectedValue1 = "value1"; string expectedValue2 = "value2"; - + // Create the metadata array - record {| string key; pbAny:Any value; |}[] metadataArray = []; + record {|string key; pbAny:Any value;|}[] metadataArray = []; // Pack string values into protobuf.Any directly pbAny:Any packedValue1 = check pbAny:pack(expectedValue1); @@ -144,7 +144,7 @@ function testMetadataHandling() returns error? { Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); io:println("Created entity metadata: ", createEntityResponse.metadata); - + // Read entity ReadEntityRequest readEntityRequest = { entity: { @@ -168,7 +168,7 @@ function testMetadataHandling() returns error? { io:println("Entity retrieved, verifying data..."); io:println("Retrieved entity: ", readEntityResponse); io:println("Retrieved entity metadata: ", readEntityResponse.metadata); - + // Verify metadata values map actualValues = {}; foreach var item in readEntityResponse.metadata { @@ -179,15 +179,15 @@ function testMetadataHandling() returns error? { test:assertFail("Failed to unpack metadata value for key: " + item.key); } } - + // Assert the values match test:assertEquals(actualValues["key1"], expectedValue1, "Metadata value for key1 doesn't match"); test:assertEquals(actualValues["key2"], expectedValue2, "Metadata value for key2 doesn't match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity deleted"); - + return; } @@ -199,7 +199,7 @@ function testMetadataHandling() returns error? { function testMetadataUnpackError() returns error? { // Test case to verify handling of non-existent entities COREServiceClient ep = check new (testCoreServiceUrl); - + // Try to read a non-existent entity ReadEntityRequest readEntityRequest = { entity: { @@ -219,10 +219,10 @@ function testMetadataUnpackError() returns error? { output: ["metadata"] }; Entity|error response = ep->ReadEntity(readEntityRequest); - + // Assert that we get an error for non-existent entity test:assertTrue(response is error, "Expected error for non-existent entity"); - + return; } @@ -230,21 +230,21 @@ function testMetadataUnpackError() returns error? { function testMetadataUpdating() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "test-entity-update"; - + // Initial metadata values string initialValue1 = "initial-value1"; string initialValue2 = "initial-value2"; - + // Updated metadata values string updatedValue1 = "updated-value1"; string updatedValue2 = "updated-value2"; string newValue3 = "new-value3"; - + // Create the initial metadata array - record {| string key; pbAny:Any value; |}[] initialMetadataArray = []; + record {|string key; pbAny:Any value;|}[] initialMetadataArray = []; pbAny:Any packedInitialValue1 = check pbAny:pack(initialValue1); pbAny:Any packedInitialValue2 = check pbAny:pack(initialValue2); initialMetadataArray.push({key: "key1", value: packedInitialValue1}); @@ -270,7 +270,7 @@ function testMetadataUpdating() returns error? { // Create entity Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Verify initial metadata ReadEntityRequest readEntityRequest = { entity: { @@ -292,9 +292,9 @@ function testMetadataUpdating() returns error? { Entity initialReadResponse = check ep->ReadEntity(readEntityRequest); verifyMetadata(initialReadResponse.metadata, {"key1": initialValue1, "key2": initialValue2}); io:println("Initial metadata verified"); - + // Create updated metadata array - record {| string key; pbAny:Any value; |}[] updatedMetadataArray = []; + record {|string key; pbAny:Any value;|}[] updatedMetadataArray = []; pbAny:Any packedUpdatedValue1 = check pbAny:pack(updatedValue1); pbAny:Any packedUpdatedValue2 = check pbAny:pack(updatedValue2); pbAny:Any packedNewValue3 = check pbAny:pack(newValue3); @@ -321,7 +321,7 @@ function testMetadataUpdating() returns error? { attributes: [], relationships: [] }; - + // Update entity UpdateEntityRequest updateRequest = { id: testId, @@ -329,7 +329,7 @@ function testMetadataUpdating() returns error? { }; Entity updateEntityResponse = check ep->UpdateEntity(updateRequest); io:println("Entity updated with ID: " + updateEntityResponse.id); - + // Verify updated metadata ReadEntityRequest updatedReadRequest = { entity: { @@ -350,21 +350,21 @@ function testMetadataUpdating() returns error? { }; Entity updatedReadResponse = check ep->ReadEntity(updatedReadRequest); verifyMetadata(updatedReadResponse.metadata, { - "key1": updatedValue1, - "key2": updatedValue2, - "key3": newValue3 - }); + "key1": updatedValue1, + "key2": updatedValue2, + "key3": newValue3 + }); io:println("Updated metadata verified"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity deleted"); - + return; } // Helper function to verify metadata -function verifyMetadata(record {| string key; pbAny:Any value; |}[] metadata, map expected) { +function verifyMetadata(record {|string key; pbAny:Any value;|}[] metadata, map expected) { map actual = {}; foreach var item in metadata { string|error unwrapped = unwrapAny(item.value); @@ -372,12 +372,12 @@ function verifyMetadata(record {| string key; pbAny:Any value; |}[] metadata, ma actual[item.key] = unwrapped.trim(); } } - + // Verify all expected key-value pairs exist in the actual metadata foreach var [key, expectedValue] in expected.entries() { test:assertTrue(actual.hasKey(key), "Metadata key not found: " + key); - test:assertEquals(actual[key] ?: "", expectedValue, - string `Metadata value for ${key} doesn't match: expected ${expectedValue}, got ${actual[key] ?: ""}`); + test:assertEquals(actual[key] ?: "", expectedValue, + string `Metadata value for ${key} doesn't match: expected ${expectedValue}, got ${actual[key] ?: ""}`); } } @@ -385,17 +385,17 @@ function verifyMetadata(record {| string key; pbAny:Any value; |}[] metadata, ma function testEntityReading() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "test-entity-read"; string metadataKey = "readTest"; string metadataValue = "read-test-value"; - + // Create a test entity first - record {| string key; pbAny:Any value; |}[] metadataArray = []; + record {|string key; pbAny:Any value;|}[] metadataArray = []; pbAny:Any packedValue = check pbAny:pack(metadataValue); metadataArray.push({key: metadataKey, value: packedValue}); - + Entity createEntityRequest = { id: testId, kind: { @@ -413,12 +413,12 @@ function testEntityReading() returns error? { attributes: [], relationships: [] }; - + // Create entity Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Test entity created with ID: " + createEntityResponse.id); io:println("Created entity metadata: ", createEntityResponse.metadata); - + // Read the entity ReadEntityRequest readEntityRequest = { entity: { @@ -442,25 +442,25 @@ function testEntityReading() returns error? { io:println("Entity retrieved, verifying data..."); io:println("Retrieved entity: ", readEntityResponse); io:println("Retrieved entity metadata: ", readEntityResponse.metadata); - + // Verify entity fields test:assertEquals(readEntityResponse.id, testId, "Entity ID mismatch"); - + // Verify metadata boolean foundMetadata = false; foreach var item in readEntityResponse.metadata { if item.key == metadataKey { string|error unwrapped = unwrapAny(item.value); if unwrapped is string { - test:assertEquals(unwrapped.trim(), metadataValue, - string `Metadata value mismatch: expected ${metadataValue}, got ${unwrapped}`); + test:assertEquals(unwrapped.trim(), metadataValue, + string `Metadata value mismatch: expected ${metadataValue}, got ${unwrapped}`); foundMetadata = true; } } } - + test:assertTrue(foundMetadata, "Expected metadata key not found"); - + // Test reading non-existent entity string nonExistentId = "non-existent-entity-" + testId; ReadEntityRequest nonExistentRequest = { @@ -481,23 +481,23 @@ function testEntityReading() returns error? { output: ["metadata"] }; Entity|error nonExistentResult = ep->ReadEntity(nonExistentRequest); - + // For now, expect an error for non-existent entities test:assertTrue(nonExistentResult is error, "Expected error for non-existent entity"); if nonExistentResult is error { io:println("Non-existent entity correctly returned error: " + nonExistentResult.message()); } - + // Assert that we get an error for non-existent entity // For non-existence entities, we send a response with an empty data // But once the Result API is integrated this can be tested. // FIXME: https://github.com/LDFLK/nexoan/issues/23 // test:assertTrue(nonExistentResponse is error, "Expected error for non-existent entity ID"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity deleted"); - + return; } @@ -505,10 +505,10 @@ function testEntityReading() returns error? { function testCreateMinimalGraphEntity() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup - minimal entity with just required fields string testId = "test-minimal-entity"; - + // Create entity request with only required fields - no metadata, attributes, or relationships Entity createEntityRequest = { id: testId, @@ -531,7 +531,7 @@ function testCreateMinimalGraphEntity() returns error? { // Create entity Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Minimal entity created with ID: " + createEntityResponse.id); - + // Verify entity was created correctly ReadEntityRequest readEntityRequest = { entity: { @@ -551,21 +551,21 @@ function testCreateMinimalGraphEntity() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Basic entity verification test:assertEquals(readEntityResponse.id, testId, "Entity ID doesn't match"); test:assertEquals(readEntityResponse.kind.major, "test", "Entity kind.major doesn't match"); test:assertEquals(readEntityResponse.kind.minor, "minimal", "Entity kind.minor doesn't match"); - + // Verify empty collections test:assertEquals(readEntityResponse.metadata.length(), 0, "Metadata should be empty"); test:assertEquals(readEntityResponse.attributes.length(), 0, "Attributes default value should be empty"); test:assertEquals(readEntityResponse.relationships.length(), 0, "Relationships should be empty"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test minimal entity deleted"); - + return; } @@ -576,10 +576,10 @@ function testCreateMinimalGraphEntityViaRest() returns error? { httpVersion: "2.0" // Enable HTTP/2 }; http:Client restClient = check new (testIngestionServiceUrl, httpConfig); - + // Test data setup - minimal JSON entity string testId = "test-minimal-json-entity"; - + // Minimal JSON payload with required fields matching the Entity structure json minimalEntityJson = { "id": testId, @@ -601,22 +601,22 @@ function testCreateMinimalGraphEntityViaRest() returns error? { // Create entity via REST API http:Response|error response = restClient->post("/entities", minimalEntityJson); - + // Verify HTTP request was successful if response is error { test:assertFail("Failed to create entity via REST API: " + response.message()); } - + http:Response httpResponse = response; test:assertEquals(httpResponse.statusCode, 201, "Expected 201 OK status code"); - + // Parse response JSON json responseJson = check httpResponse.getJsonPayload(); test:assertEquals(check responseJson.id, testId, "Entity ID in response doesn't match"); - + // Initialize the gRPC client to verify entity was properly created COREServiceClient ep = check new (testCoreServiceUrl); - + // Verify entity data ReadEntityRequest readEntityRequest = { entity: { @@ -633,24 +633,24 @@ function testCreateMinimalGraphEntityViaRest() returns error? { attributes: [], relationships: [] }, - output: ["metadata","attributes", "relationships"] + output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Basic entity verification test:assertEquals(readEntityResponse.id, testId, "Entity ID doesn't match"); test:assertEquals(readEntityResponse.kind.major, "test", "Entity kind.major doesn't match"); test:assertEquals(readEntityResponse.kind.minor, "minimal-json", "Entity kind.minor doesn't match"); - + // Verify empty collections test:assertEquals(readEntityResponse.metadata.length(), 0, "Metadata should be empty"); test:assertEquals(readEntityResponse.attributes.length(), 0, "Attributes default value should be empty"); test:assertEquals(readEntityResponse.relationships.length(), 0, "Relationships should be empty"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test minimal JSON entity deleted"); - + return; } @@ -661,13 +661,13 @@ function testEntityWithRelationship() returns error? { // Test IDs for entities string sourceEntityId = "test-entity-with-relationship-source"; string targetEntityId = "test-entity-with-relationship-target"; - + // Initialize REST client with HTTP/2 support http:ClientConfiguration httpConfig = { httpVersion: "2.0" // Enable HTTP/2 }; http:Client restClient = check new (testIngestionServiceUrl, httpConfig); - + // Create source entity json sourceEntityJson = { "id": sourceEntityId, @@ -686,7 +686,7 @@ function testEntityWithRelationship() returns error? { "attributes": [], "relationships": [] }; - + // Create target entity json targetEntityJson = { "id": targetEntityId, @@ -694,7 +694,7 @@ function testEntityWithRelationship() returns error? { "major": "test", "minor": "relationship-target" }, - "created": "2023-01-01", + "created": "2023-01-01", "terminated": "", "name": { "startTime": "2023-01-01", @@ -705,11 +705,11 @@ function testEntityWithRelationship() returns error? { "attributes": [], "relationships": [] }; - + // Create both entities via REST API http:Response|error sourceResponse = restClient->post("/entities", sourceEntityJson); http:Response|error targetResponse = restClient->post("/entities", targetEntityJson); - + // Verify HTTP requests were successful if sourceResponse is error { test:assertFail("Failed to create source entity: " + sourceResponse.message()); @@ -717,12 +717,12 @@ function testEntityWithRelationship() returns error? { if targetResponse is error { test:assertFail("Failed to create target entity: " + targetResponse.message()); } - + http:Response sourceHttpResponse = sourceResponse; http:Response targetHttpResponse = targetResponse; test:assertEquals(sourceHttpResponse.statusCode, 201, "Expected 201 status code for source entity"); test:assertEquals(targetHttpResponse.statusCode, 201, "Expected 201 status code for target entity"); - + // Create relationship between entities - include full entity structure string relationshipId = "rel-" + sourceEntityId + "-" + targetEntityId; json relationshipJson = { @@ -745,21 +745,21 @@ function testEntityWithRelationship() returns error? { } } }; - + // Update source entity with relationship http:Response|error updateResponse = restClient->put("/entities/" + sourceEntityId, relationshipJson); - + // Verify update was successful if updateResponse is error { test:assertFail("Failed to update entity with relationship: " + updateResponse.message()); } - + http:Response updateHttpResponse = updateResponse; test:assertEquals(updateHttpResponse.statusCode, 200, "Expected 200 status code for relationship update"); - + // Initialize the gRPC client to verify relationship was properly created COREServiceClient ep = check new (testCoreServiceUrl); - + // Read source entity to verify relationship ReadEntityRequest readEntityRequest = { entity: { @@ -779,10 +779,10 @@ function testEntityWithRelationship() returns error? { output: ["relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify relationship data test:assertEquals(readEntityResponse.relationships.length(), 1, "Entity should have one relationship"); - + // Find the relationship by iterating through the array Relationship? targetRelationship = (); foreach var rel in readEntityResponse.relationships { @@ -791,7 +791,7 @@ function testEntityWithRelationship() returns error? { break; } } - + io:println("Target relationship: " + targetRelationship.toJsonString()); test:assertFalse(targetRelationship is (), "Relationship with key 'CONNECTS_TO' not found"); Relationship relationship = targetRelationship; @@ -799,12 +799,12 @@ function testEntityWithRelationship() returns error? { test:assertEquals(relationship.name, "CONNECTS_TO", "Relationship name doesn't match"); test:assertEquals(relationship.startTime, "2023-01-01T00:00:00Z", "Relationship start time doesn't match"); test:assertEquals(relationship.id, relationshipId, "Relationship ID doesn't match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: sourceEntityId}); Empty _ = check ep->DeleteEntity({id: targetEntityId}); io:println("Test entities with relationship deleted"); - + return; } @@ -816,10 +816,10 @@ function testEntityWithSimpleOnlyNodesGraphAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-simple-only-nodes-graph"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with tabular data in attributes json socialNetworkGraph = { "nodes": [ @@ -873,7 +873,7 @@ function testEntityWithSimpleOnlyNodesGraphAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -893,16 +893,16 @@ function testEntityWithSimpleOnlyNodesGraphAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with graph attributes deleted"); - + return; } @@ -914,10 +914,10 @@ function testEntityWithSimpleGraphAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-simple-entity-graph"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with tabular data in attributes json socialNetworkGraph = { "nodes": [ @@ -978,7 +978,7 @@ function testEntityWithSimpleGraphAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -998,16 +998,16 @@ function testEntityWithSimpleGraphAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with graph attributes deleted"); - + return; } @@ -1019,26 +1019,26 @@ function testEntityWithMultiGraphAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-graph"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with tabular data in attributes json salaryGraph = { "nodes": [ - {"id": "user1", "type": "user", "properties": {"name": "Alice", "age": 30, "location": "NY"}}, - {"id": "user2", "type": "user", "properties": {"name": "Bob", "age": 25, "location": "SF"}}, - {"id": "user3", "type": "user", "properties": {"name": "Charlie", "age": 35, "location": "LA"}}, - {"id": "post1", "type": "post", "properties": {"title": "Hello", "content": "World", "created": "2024-03-20"}}, - {"id": "post2", "type": "post", "properties": {"title": "Graph", "content": "DB", "created": "2024-03-21"}} - ], - "edges": [ - {"source": "user1", "target": "user2", "type": "follows", "properties": {"since": "2024-01-01"}}, - {"source": "user2", "target": "user3", "type": "follows", "properties": {"since": "2024-02-01"}}, - {"source": "user1", "target": "post1", "type": "created", "properties": {"timestamp": "2024-03-20T10:00:00Z"}}, - {"source": "user2", "target": "post1", "type": "likes", "properties": {"timestamp": "2024-03-20T11:00:00Z"}}, - {"source": "user3", "target": "post2", "type": "created", "properties": {"timestamp": "2024-03-21T09:00:00Z"}} - ] + {"id": "user1", "type": "user", "properties": {"name": "Alice", "age": 30, "location": "NY"}}, + {"id": "user2", "type": "user", "properties": {"name": "Bob", "age": 25, "location": "SF"}}, + {"id": "user3", "type": "user", "properties": {"name": "Charlie", "age": 35, "location": "LA"}}, + {"id": "post1", "type": "post", "properties": {"title": "Hello", "content": "World", "created": "2024-03-20"}}, + {"id": "post2", "type": "post", "properties": {"title": "Graph", "content": "DB", "created": "2024-03-21"}} + ], + "edges": [ + {"source": "user1", "target": "user2", "type": "follows", "properties": {"since": "2024-01-01"}}, + {"source": "user2", "target": "user3", "type": "follows", "properties": {"since": "2024-02-01"}}, + {"source": "user1", "target": "post1", "type": "created", "properties": {"timestamp": "2024-03-20T10:00:00Z"}}, + {"source": "user2", "target": "post1", "type": "likes", "properties": {"timestamp": "2024-03-20T11:00:00Z"}}, + {"source": "user3", "target": "post2", "type": "created", "properties": {"timestamp": "2024-03-21T09:00:00Z"}} + ] }; json projectGraph = { @@ -1113,7 +1113,7 @@ function testEntityWithMultiGraphAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1133,16 +1133,16 @@ function testEntityWithMultiGraphAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up - + Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with graph attributes deleted"); - + return; } @@ -1154,10 +1154,10 @@ function testEntityWithSimpleListAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-list"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with list data in attributes json salaryList = { "values": [ @@ -1209,11 +1209,11 @@ function testEntityWithSimpleListAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with list attributes deleted"); - + return; } @@ -1225,10 +1225,10 @@ function testEntityWithMixedTypeListAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-mixed-list"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with mixed type list data in attributes json mixedTypeList = { "values": [ @@ -1283,7 +1283,7 @@ function testEntityWithMixedTypeListAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1303,15 +1303,15 @@ function testEntityWithMixedTypeListAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with mixed type list attributes deleted"); - + return; } @@ -1323,10 +1323,10 @@ function testEntityWithEmptyListAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-empty-list"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with empty list data in attributes json emptyList = { "values": [] @@ -1374,7 +1374,7 @@ function testEntityWithEmptyListAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1394,15 +1394,15 @@ function testEntityWithEmptyListAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with empty list attributes deleted"); - + return; } @@ -1414,17 +1414,17 @@ function testEntityWithMapAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-map"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with map data in attributes json userProfileMap = { - "properties": { - "name": "John", - "age": 30, - "active": true - } + "properties": { + "name": "John", + "age": 30, + "active": true + } }; // Convert JSON to protobuf Any values @@ -1469,7 +1469,7 @@ function testEntityWithMapAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1489,15 +1489,15 @@ function testEntityWithMapAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with map attributes deleted"); - + return; } @@ -1509,10 +1509,10 @@ function testEntityWithNestedMapAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-nested-map"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with nested map data in attributes json nestedMap = { "organization": { @@ -1598,7 +1598,7 @@ function testEntityWithNestedMapAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1618,15 +1618,15 @@ function testEntityWithNestedMapAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with nested map attributes deleted"); - + return; } @@ -1638,10 +1638,10 @@ function testEntityWithEmptyMapValues() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-empty-map-values"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with map data containing empty values // FIXME: https://github.com/LDFLK/nexoan/issues/137 json emptyValuesMap = { @@ -1694,7 +1694,7 @@ function testEntityWithEmptyMapValues() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1714,15 +1714,15 @@ function testEntityWithEmptyMapValues() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with empty map values deleted"); - + return; } @@ -1734,10 +1734,10 @@ function testEntityWithNestedMapValues() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-nested-map-values"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with deeply nested map data json nestedMap = { "properties": { @@ -1832,7 +1832,7 @@ function testEntityWithNestedMapValues() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1852,27 +1852,26 @@ function testEntityWithNestedMapValues() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with nested map values deleted"); - + return; } - @test:Config {} function testEntityWithTabularAttributes() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "ID-MIN-A"; - + // Create tabular data structure // TODO: https://github.com/LDFLK/nexoan/issues/284 json tabularData = { @@ -1937,7 +1936,7 @@ function testEntityWithTabularAttributes() returns error? { }; pbAny:Any tabularDataFilterAny = check jsonToAny(tabularDataFilter); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1957,7 +1956,7 @@ function testEntityWithTabularAttributes() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-04-01T00:00:00Z", endTime: "", value: tabularDataFilterAny } @@ -1970,7 +1969,7 @@ function testEntityWithTabularAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); @@ -2001,15 +2000,14 @@ function testEntityWithTabularAttributes() returns error? { return; } - @test:Config {} function testEntityWithTabularAttributesMultiRels() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "ID-MIN-A-MULTI-RELS"; - + // Create tabular data structure // TODO: https://github.com/LDFLK/nexoan/issues/284 json employeeData = { @@ -2022,7 +2020,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { [5, "Charlie Davis", 32, "Finance", 80000] ] }; - + json budgetData = { "columns": ["id", "category", "amount", "quarter", "status"], "rows": [ @@ -2104,7 +2102,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { pbAny:Any employeeDataFilterAny = check jsonToAny(employeeDataFilter); pbAny:Any budgetDataFilterAny = check jsonToAny(budgetDataFilter); - + // Read entity to verify attributes ReadEntityRequest readEntityWithBudgetDataRequest = { entity: { @@ -2124,7 +2122,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-06-01T00:00:00Z", endTime: "", value: budgetDataFilterAny } @@ -2155,7 +2153,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-04-01T00:00:00Z", endTime: "", value: employeeDataFilterAny } @@ -2215,15 +2213,14 @@ function testEntityWithTabularAttributesMultiRels() returns error? { return; } - @test:Config {} function testEntityWithTabularAttributesUpdate() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "ID-MIN-A-UPDATE"; - + // Create tabular data structure json tabularData = { "columns": ["id", "name", "age", "department", "salary"], @@ -2332,7 +2329,7 @@ function testEntityWithTabularAttributesUpdate() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-12-01T00:00:00Z", endTime: "", value: tabularDataFilterAny } @@ -2509,7 +2506,7 @@ function testTabularAttributeIdempotency() returns error? { key: attrName, value: { values: [ - { startTime: "", endTime: "", value: allRowsFilterAny } + { startTime: "2025-02-01T00:00:00Z", endTime: "", value: allRowsFilterAny } ] } } @@ -2727,4 +2724,4 @@ function testTabularDuplicatePrimaryKey() returns error? { Empty _ = check ep->DeleteEntity({ id: testId }); io:println("[testTabularDuplicatePrimaryKey] Cleaned up test entity"); return; -} \ No newline at end of file +} diff --git a/opengin/read-api/tests/read_api_service_test.bal b/opengin/read-api/tests/read_api_service_test.bal index 3ed3ad0d..d8067995 100644 --- a/opengin/read-api/tests/read_api_service_test.bal +++ b/opengin/read-api/tests/read_api_service_test.bal @@ -123,6 +123,220 @@ function verifyTabularData(json actual, json expected) { test:assertEquals(actual.toString(), expected.toString(), "Data JSON should match"); } + +// Test entity creation with attribute missing startTime and endTime (should fail with error) +@test:Config { + groups: ["entity", "attribute"], + enable: true +} +function testEntityAttributeCreationWithMissingStartTimeAndEndTime() returns error? { + io:println("[read_api_service_test.bal][testEntityAttributeCreationWithMissingStartTimeAndEndTime]"); + string|error coreUrl = getCoreServiceUrl(); + if coreUrl is error { + return coreUrl; + } + COREServiceClient ep = check new (coreUrl); + + string testId = "ATTR_MISSING_TIME_001"; + json attributeValue = { + "columns": ["emp_id", "name", "salary"], + "rows": [ + [1001, "John Doe", 75000.50] + ] + }; + + pbAny:Any attributeValueAny = check convertJsonToAny(attributeValue); + + Entity createEntityRequest = { + id: testId, + kind: { + major: "Organization", + minor: "Private Limited" + }, + created: "2025-02-01T00:00:00Z", + terminated: "", + name: { + startTime: "2025-02-01T00:00:00Z", + endTime: "", + value: check pbAny:pack("Attr Missing Time Pvt Ltd") + }, + metadata: [ + { + key: "attr_missing_time_metadata", + value: check pbAny:pack("attr_missing_time_test_value") + } + ], + attributes: [ + { + key: "employee_data", + value: { + values: [ + { + startTime: "", + endTime: "", + value: attributeValueAny + } + ] + } + } + ], + relationships: [] + }; + + Entity|error createResponse = ep->CreateEntity(createEntityRequest); + + test:assertTrue(createResponse is error, + "Creating an entity with attribute missing both startTime and endTime should return an error (HTTP 500)"); + + if createResponse is error { + io:println("Received expected error for missing startTime/endTime: " + createResponse.message()); + } + + return; +} + +// Test entity creation with attribute having an invalid startTime and empty endTime (should fail with error) +@test:Config { + groups: ["entity", "attribute"], + enable: true +} +function testEntityAttributeCreationWithInvalidStartTime() returns error? { + io:println("[read_api_service_test.bal][testEntityAttributeCreationWithInvalidStartTime]"); + string|error coreUrl = getCoreServiceUrl(); + if coreUrl is error { + return coreUrl; + } + COREServiceClient ep = check new (coreUrl); + + string testId = "ATTR_INVALID_START_002"; + json attributeValue = { + "columns": ["emp_id", "name", "salary"], + "rows": [ + [1001, "John Doe", 75000.50] + ] + }; + + pbAny:Any attributeValueAny = check convertJsonToAny(attributeValue); + + Entity createEntityRequest = { + id: testId, + kind: { + major: "Organization", + minor: "Private Limited" + }, + created: "2025-02-01T00:00:00Z", + terminated: "", + name: { + startTime: "2025-02-01T00:00:00Z", + endTime: "", + value: check pbAny:pack("Attr Invalid Start Pvt Ltd") + }, + metadata: [ + { + key: "attr_invalid_start_metadata", + value: check pbAny:pack("attr_invalid_start_test_value") + } + ], + attributes: [ + { + key: "employee_data", + value: { + values: [ + { + startTime: "not-a-valid-date", + endTime: "", + value: attributeValueAny + } + ] + } + } + ], + relationships: [] + }; + + Entity|error createResponse = ep->CreateEntity(createEntityRequest); + + test:assertTrue(createResponse is error, + "Creating an entity with an invalid startTime and empty endTime should return an error (HTTP 500)"); + + if createResponse is error { + io:println("Received expected error for invalid startTime: " + createResponse.message()); + } + + return; +} + +// Test entity creation with attribute having a valid startTime and invalid endTime (should fail with error) +@test:Config { + groups: ["entity", "attribute"], + enable: true +} +function testEntityAttributeCreationWithInvalidEndTime() returns error? { + io:println("[read_api_service_test.bal][testEntityAttributeCreationWithInvalidEndTime]"); + string|error coreUrl = getCoreServiceUrl(); + if coreUrl is error { + return coreUrl; + } + COREServiceClient ep = check new (coreUrl); + + string testId = "ATTR_INVALID_END_003"; + json attributeValue = { + "columns": ["emp_id", "name", "salary"], + "rows": [ + [1001, "John Doe", 75000.50] + ] + }; + + pbAny:Any attributeValueAny = check convertJsonToAny(attributeValue); + + Entity createEntityRequest = { + id: testId, + kind: { + major: "Organization", + minor: "Private Limited" + }, + created: "2025-02-01T00:00:00Z", + terminated: "", + name: { + startTime: "2025-02-01T00:00:00Z", + endTime: "", + value: check pbAny:pack("Attr Invalid End Pvt Ltd") + }, + metadata: [ + { + key: "attr_invalid_end_metadata", + value: check pbAny:pack("attr_invalid_end_test_value") + } + ], + attributes: [ + { + key: "employee_data", + value: { + values: [ + { + startTime: "2025-04-01T00:00:00Z", + endTime: "not-a-valid-date", + value: attributeValueAny + } + ] + } + } + ], + relationships: [] + }; + + Entity|error createResponse = ep->CreateEntity(createEntityRequest); + + test:assertTrue(createResponse is error, + "Creating an entity with a valid startTime and an invalid endTime should return an error (HTTP 500)"); + + if createResponse is error { + io:println("Received expected error for invalid endTime: " + createResponse.message()); + } + + return; +} + // Test entity attribute retrieval @test:Config { groups: ["entity", "attribute"], @@ -220,7 +434,7 @@ function testEntityAttributeRetrieval() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-04-01T00:00:00Z", endTime: "", value: attributeValueFilterAny } diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index 6ea4ee42..cb319e3a 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -35,50 +35,225 @@ """ -class CoreTestUtils: +class CoreTestUtils: @staticmethod def decode_protobuf_any_value(any_value): - """Decode a protobuf Any value to get the actual string value""" + """Decode a protobuf Any value to get the actual value""" if isinstance(any_value, dict) and 'typeUrl' in any_value and 'value' in any_value: - if 'StringValue' in any_value['typeUrl']: + type_url = any_value['typeUrl'] + value = any_value['value'] + + if 'StringValue' in type_url: + try: + # If it's hex encoded (which appears to be the case) + hex_value = value + binary_data = bytes.fromhex(hex_value) + # For StringValue in hex format, typically the structure is: + # 0A (field tag) + 03 (length) + actual string bytes + # Skip the first 2 bytes (field tag and length) + if len(binary_data) > 2: + return binary_data[2:].decode('utf-8') + except Exception as e: + print(f"Failed to decode StringValue: {e}") + return value + + elif 'Struct' in type_url: + try: + # For Struct type, the value is hex-encoded protobuf data + binary_data = bytes.fromhex(value) + + # Try to use protobuf library first + try: + from google.protobuf import struct_pb2 + from google.protobuf.json_format import MessageToDict + + struct_msg = struct_pb2.Struct() + struct_msg.ParseFromString(binary_data) + result = MessageToDict(struct_msg) + + # If the result has a 'data' field that's a string, try to parse it as JSON + if isinstance(result, dict) and 'data' in result and isinstance(result['data'], str): + try: + data_json = json.loads(result['data']) + return data_json + except json.JSONDecodeError: + pass + + return result + + except ImportError: + print("protobuf library not available, trying manual extraction") + except Exception as e: + print(f"Failed to decode with protobuf library: {e}") + + # Manual extraction fallback - look for JSON patterns in the binary data + try: + # Convert binary data to string and look for JSON + text_data = binary_data.decode('utf-8', errors='ignore') + print(f"Debug: Extracted text from binary: {repr(text_data[:200])}...") + + # Look for the actual start of JSON by searching for common patterns + # like {"columns" or {"rows" which indicate the start of our data + json_patterns = ['{"columns"', '{"rows"', '{"data"'] + start_idx = -1 + for pattern in json_patterns: + idx = text_data.find(pattern) + if idx != -1: + start_idx = idx + break + + if start_idx != -1: + # Find the matching closing brace + end_idx = text_data.rfind('}') + if end_idx != -1 and end_idx > start_idx: + json_str = text_data[start_idx:end_idx + 1] + print(f"Debug: Extracted JSON string: {repr(json_str[:100])}...") + return json.loads(json_str) + + except Exception as e: + print(f"Failed to extract JSON from binary data: {e}") + + # Try a different approach - look for specific patterns + try: + # The hex data might contain the JSON in a different format + # Let's try to find the actual JSON content + text_data = binary_data.decode('utf-8', errors='ignore') + + # Look for common JSON patterns + patterns = ['"columns"', '"rows"', '"data"'] + for pattern in patterns: + if pattern in text_data: + # Find the start of the JSON object containing this pattern + start_idx = text_data.find('{', text_data.find(pattern) - 100) + if start_idx != -1: + # Find the matching closing brace + brace_count = 0 + end_idx = start_idx + for i, char in enumerate(text_data[start_idx:], start_idx): + if char == '{': + brace_count += 1 + elif char == '}': + brace_count -= 1 + if brace_count == 0: + end_idx = i + break + + if end_idx > start_idx: + json_str = text_data[start_idx:end_idx + 1] + print(f"Debug: Found JSON with pattern {pattern}: {repr(json_str[:100])}...") + try: + return json.loads(json_str) + except json.JSONDecodeError as e: + print(f"Debug: JSON decode failed: {e}") + continue + + except Exception as e: + print(f"Failed to extract JSON with pattern matching: {e}") + + # If all else fails, try to decode as base64 + try: + import base64 + # The hex might actually be base64 encoded + decoded_bytes = base64.b64decode(value) + json_str = decoded_bytes.decode('utf-8') + return json.loads(json_str) + except Exception as e: + print(f"Failed to decode as base64: {e}") + + # Final fallback: return the hex value + return value + + except Exception as e: + print(f"Failed to decode Struct: {e}") + return value + + elif 'Int32Value' in type_url or 'Int64Value' in type_url: + try: + # For integer values + hex_value = value + binary_data = bytes.fromhex(hex_value) + # Skip field tag and length bytes + if len(binary_data) > 2: + return int.from_bytes(binary_data[2:], byteorder='little') + except Exception as e: + print(f"Failed to decode integer value: {e}") + return value + + elif 'DoubleValue' in type_url or 'FloatValue' in type_url: + try: + # For float/double values + hex_value = value + binary_data = bytes.fromhex(hex_value) + # Skip field tag and length bytes + if len(binary_data) > 2: + import struct + return struct.unpack(' 2: - return binary_data[2:].decode('utf-8') + return binary_data[2:].decode("utf-8") except Exception as e: print(f"Failed to decode protobuf value: {e}") # Return the original value if decoding fails return any_value.strip() + class TestCOREAPI(unittest.TestCase): def setUp(self): print("šŸŸ¢ Setting up test environment...") - ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', f"http://0.0.0.0:8080") + ingestion_service_url = os.getenv( + "INGESTION_SERVICE_URL", "http://0.0.0.0:8080" + ) print("šŸŸ¢ INGESTION_SERVICE_URL: ", ingestion_service_url) self.base_url = f"{ingestion_service_url}/entities" print("šŸŸ¢ BASE_URL: ", self.base_url) -class BasicCORETests: +class BasicCORETests: def __init__(self, entity_id): self.entity_id = entity_id self.base_url = get_base_url() - self.headers = { - 'Content-Type': 'application/json' - } + self.base_read_url = get_base_read_url() + self.headers = {"Content-Type": "application/json"} self.payload = self.create_payload() def create_payload(self): @@ -92,38 +267,38 @@ def create_payload(self): "name": { "startTime": "2024-03-17T10:00:00Z", "endTime": "", - "value": "entity-name" + "value": "entity-name", }, "metadata": [ {"key": "owner", "value": "test-user"}, {"key": "version", "value": "1.0"}, - {"key": "developer", "value": "V8A"} + {"key": "developer", "value": "V8A"}, ], "attributes": [], - "relationships": [] + "relationships": [], }, "update": { "id": self.entity_id, "created": "2024-03-18T00:00:00Z", - "name": { - "startTime": "2024-03-18T00:00:00Z", - "value": "entity-name" - }, - "metadata": [{"key": "version", "value": "5.0"}] - } + "name": {"startTime": "2024-03-18T00:00:00Z", "value": "entity-name"}, + "metadata": [{"key": "version", "value": "5.0"}], + }, } class MetadataValidationTests(BasicCORETests): - def __init__(self, entity_id): super().__init__(entity_id) def create_entity(self): """Creates an entity and validates the response.""" print("\nšŸŸ¢ Creating entity...") - response = requests.post(self.base_url, json=self.payload["create"], headers={"Content-Type": "application/json"}) - + response = requests.post( + self.base_url, + json=self.payload["create"], + headers={"Content-Type": "application/json"}, + ) + if response.status_code == 201: print("āœ… Entity created:", json.dumps(response.json(), indent=2)) else: @@ -134,7 +309,7 @@ def read_entity(self): """Reads and validates the created entity.""" print("\nšŸŸ¢ Reading entity...") response = requests.get(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 200: data = response.json() assert data["id"] == self.entity_id, "Read entity ID mismatch" @@ -146,11 +321,17 @@ def read_entity(self): def update_entity(self): """Updates the entity and validates the response.""" print("\nšŸŸ¢ Updating entity...") - response = requests.put(f"{self.base_url}/{self.entity_id}", json=self.payload["update"], headers={"Content-Type": "application/json"}) - + response = requests.put( + f"{self.base_url}/{self.entity_id}", + json=self.payload["update"], + headers={"Content-Type": "application/json"}, + ) + if response.status_code == 200: updated_entity = response.json() - decoded_value = CoreTestUtils.decode_protobuf_any_value(updated_entity["metadata"][0]["value"]) + decoded_value = CoreTestUtils.decode_protobuf_string_value( + updated_entity["metadata"][0]["value"] + ) print("decoded value: ", decoded_value) assert decoded_value == "5.0", "Update did not modify metadata" print("āœ… Entity updated:", json.dumps(updated_entity, indent=2)) @@ -162,10 +343,12 @@ def validate_update(self): """Validates that the update has been applied correctly.""" print("\nšŸŸ¢ Validating update...") response = requests.get(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 200: updated_data = response.json() - decoded_value = CoreTestUtils.decode_protobuf_any_value(updated_data["metadata"][0]["value"]) + decoded_value = CoreTestUtils.decode_protobuf_string_value( + updated_data["metadata"][0]["value"] + ) assert decoded_value == "5.0", "Updated entity does not reflect changes" print("āœ… Update Validation Passed:", json.dumps(updated_data, indent=2)) else: @@ -176,7 +359,7 @@ def delete_entity(self): """Deletes the entity.""" print("\nšŸŸ¢ Deleting entity...") response = requests.delete(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 204: print("āœ… Entity deleted successfully.") else: @@ -187,31 +370,34 @@ def verify_deletion(self): """Verifies that the entity has been deleted.""" print("\nšŸŸ¢ Verifying deletion...") response = requests.get(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 500: print("āŒ Server error occurred:", response.text) sys.exit(1) else: - print(f"\nšŸŸ¢ Entity was not deleted properly: {response.status_code} {response.text}") + print( + f"\nšŸŸ¢ Entity was not deleted properly: {response.status_code} {response.text}" + ) class GraphEntityTests(BasicCORETests): - def __init__(self): super().__init__(None) self.MINISTER_ID = "minister_education" self.DEPARTMENTS = [ {"id": "dept_exams", "name": "Department of Exams"}, {"id": "dept_nie", "name": "National Institute of Education"}, - {"id": "dept_ed_publications", "name": "Department of Educational Publications"} + { + "id": "dept_ed_publications", + "name": "Department of Educational Publications", + }, ] self.START_DATE = "2015-04-11T00:00:00Z" - def create_minister(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister entity...") - + payload = { "id": self.MINISTER_ID, "kind": {"major": "Organization", "minor": "Minister"}, @@ -220,13 +406,13 @@ def create_minister(self): "name": { "startTime": self.START_DATE, "endTime": "", - "value": "Minister of Education" + "value": "Minister of Education", }, "metadata": [], "attributes": [], - "relationships": [] + "relationships": [], } - + res = requests.post(self.base_url, json=payload) print(res.status_code, res.json()) assert res.status_code in [201], f"Failed to create Minister: {res.text}" @@ -234,31 +420,39 @@ def create_minister(self): print(f"Response: {res.status_code} - {res.text}") print("āœ… Created Minister entity.") - def read_minister(self): """Read the Minister entity.""" print("\nšŸŸ¢ Reading Minister entity...") res = requests.get(f"{self.base_url}/{self.MINISTER_ID}") print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to read Minister: {res.text}" - + # Verify the response data response_data = res.json() - assert response_data["id"] == self.MINISTER_ID, f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" - assert response_data["kind"]["major"] == "Organization", f"Expected major kind 'Organization', got {response_data['kind']['major']}" - assert response_data["kind"]["minor"] == "Minister", f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" - assert response_data["created"] == self.START_DATE, f"Expected created date {self.START_DATE}, got {response_data['created']}" + assert response_data["id"] == self.MINISTER_ID, ( + f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" + ) + assert response_data["kind"]["major"] == "Organization", ( + f"Expected major kind 'Organization', got {response_data['kind']['major']}" + ) + assert response_data["kind"]["minor"] == "Minister", ( + f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" + ) + assert response_data["created"] == self.START_DATE, ( + f"Expected created date {self.START_DATE}, got {response_data['created']}" + ) # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) - assert decoded_name == "Minister of Education", f"Expected name 'Minister of Education', got {name_value}" + decoded_name = CoreTestUtils.decode_protobuf_string_value(name_value) + assert decoded_name == "Minister of Education", ( + f"Expected name 'Minister of Education', got {name_value}" + ) print(f"āœ… Validated {decoded_name} entity.") - def create_departments(self): """Create Department entities.""" print("\nšŸŸ¢ Creating Department entities...") - + for dept in self.DEPARTMENTS: payload = { "id": dept["id"], @@ -268,44 +462,54 @@ def create_departments(self): "name": { "startTime": self.START_DATE, "endTime": "", - "value": dept["name"] + "value": dept["name"], }, - "metadata": [] + "metadata": [], } - + res = requests.post(self.base_url, json=payload) - assert res.status_code in [200, 201], f"Failed to create {dept['name']}: {res.text}" + assert res.status_code in [200, 201], ( + f"Failed to create {dept['name']}: {res.text}" + ) print(f"Response: {res.status_code} - {res.text}") print(f"āœ… Created {dept['name']} entity.") - def read_departments(self): """Validate the Department entities in Neo4j.""" print("\nšŸŸ¢ Validating Department entities in Neo4j...") - + for dept in self.DEPARTMENTS: res = requests.get(f"{self.base_url}/{dept['id']}") assert res.status_code == 200, f"Failed to read {dept['name']}: {res.text}" - + # Verify the response data response_data = res.json() - assert response_data["id"] == dept["id"], f"Expected ID {dept['id']}, got {response_data['id']}" - assert response_data["kind"]["major"] == "Organization", f"Expected major kind 'Organization', got {response_data['kind']['major']}" - assert response_data["kind"]["minor"] == "Department", f"Expected minor kind 'Department', got {response_data['kind']['minor']}" - assert response_data["created"] == self.START_DATE, f"Expected created date {self.START_DATE}, got {response_data['created']}" - + assert response_data["id"] == dept["id"], ( + f"Expected ID {dept['id']}, got {response_data['id']}" + ) + assert response_data["kind"]["major"] == "Organization", ( + f"Expected major kind 'Organization', got {response_data['kind']['major']}" + ) + assert response_data["kind"]["minor"] == "Department", ( + f"Expected minor kind 'Department', got {response_data['kind']['minor']}" + ) + assert response_data["created"] == self.START_DATE, ( + f"Expected created date {self.START_DATE}, got {response_data['created']}" + ) + # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) - assert decoded_name == dept["name"], f"Expected name '{dept['name']}', got {decoded_name}" - + decoded_name = CoreTestUtils.decode_protobuf_string_value(name_value) + assert decoded_name == dept["name"], ( + f"Expected name '{dept['name']}', got {decoded_name}" + ) + print(f"āœ… Validated {dept['name']} entity.") - - + def create_relationships(self): """Create HAS_DEPARTMENT relationships from Minister to Departments.""" print("\nšŸ”— Creating relationships...") - + for dept in self.DEPARTMENTS: rel_id = f"rel_{dept['id']}" payload = { @@ -313,8 +517,7 @@ def create_relationships(self): "kind": {}, "created": "", "terminated": "", - "name": { - }, + "name": {}, "metadata": [], "attributes": [], "relationships": [ @@ -325,28 +528,30 @@ def create_relationships(self): "startTime": self.START_DATE, "endTime": "", "id": rel_id, - "name": "HAS_DEPARTMENT" - } + "name": "HAS_DEPARTMENT", + }, } - ] + ], } - + url = f"{self.base_url}/{self.MINISTER_ID}" res = requests.put(url, json=payload) if res.status_code in [200]: print(f"āœ… Created relationship between Minister and {dept['name']}.") else: - print(f"āŒ Failed to create relationship for {dept['name']}: {res.status_code} - {res.text}") + print( + f"āŒ Failed to create relationship for {dept['name']}: {res.status_code} - {res.text}" + ) sys.exit(1) def update_relationships(self): """Update HAS_DEPARTMENT relationships to add termination dates.""" print("\nšŸ”— Updating relationships...") - + # Set a termination date for all relationships termination_date = "2024-12-31T00:00:00Z" - + for dept in self.DEPARTMENTS: rel_id = f"rel_{dept['id']}" payload = { @@ -361,19 +566,23 @@ def update_relationships(self): "id": rel_id, "relatedEntityId": "", "startTime": "", - "name": "" - } + "name": "", + }, } - ] + ], } - + url = f"{self.base_url}/{self.MINISTER_ID}" res = requests.put(url, json=payload) if res.status_code in [200]: - print(f"āœ… Updated relationship between Minister and {dept['name']} with termination date {termination_date}.") + print( + f"āœ… Updated relationship between Minister and {dept['name']} with termination date {termination_date}." + ) else: - print(f"āŒ Failed to update relationship for {dept['name']}: {res.status_code} - {res.text}") + print( + f"āŒ Failed to update relationship for {dept['name']}: {res.status_code} - {res.text}" + ) sys.exit(1) # Verify the updates @@ -384,31 +593,43 @@ def update_relationships(self): relationships = data.get("relationships", []) for rel in relationships: if rel.get("name") == "HAS_DEPARTMENT": - assert rel.get("endTime") == termination_date, f"Expected termination date {termination_date}, got {rel.get('endTime')}" + assert rel.get("endTime") == termination_date, ( + f"Expected termination date {termination_date}, got {rel.get('endTime')}" + ) print("āœ… Successfully verified relationship updates.") else: - print(f"āŒ Failed to verify relationship updates: {res.status_code} - {res.text}") + print( + f"āŒ Failed to verify relationship updates: {res.status_code} - {res.text}" + ) sys.exit(1) class AttributeValidationTests(BasicCORETests): - def __init__(self): super().__init__(None) self.MINISTER_ID = "minister_of_finance_and_economy" self.DEPARTMENTS = [ {"id": "dept_finance", "name": "Department of Finance"}, - {"id": "dept_economy", "name": "Department of Economy"} + {"id": "dept_economy", "name": "Department of Economy"}, ] self.START_DATE = "2025-11-01T00:00:00Z" self.DATA_START_DATE = "2025-12-01T00:00:00Z" - self.ATTRIBUTE_NAME = "employee_data" + self.DATA_END_DATE = "2025-12-31T00:00:00Z" + self.INVALID_DATE = "2025-12-31a" + self.INVALID_DATE_2 = "2025-12-33T00:00:00Z" + self.INVALID_DATE_3 = "test-123" + + self.ATTRIBUTE_NAME = "employee_data" + self.ATTRIBUTE_NAME_2 = "employee_data_2" + self.ATTRIBUTE_NAME_3 = "employee_data_3" + self.ATTRIBUTE_NAME_4 = "employee_data_4" + self.ATTRIBUTE_NAME_5 = "employee_data_5" def create_minister_with_attributes(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister entity...") - + payload = { "id": self.MINISTER_ID, "kind": {"major": "Organization", "minor": "Minister"}, @@ -417,7 +638,7 @@ def create_minister_with_attributes(self): "name": { "startTime": self.START_DATE, "endTime": "", - "value": "Minister of Finance and Economy" + "value": "Minister of Finance and Economy", }, "metadata": [], "attributes": [ @@ -429,23 +650,29 @@ def create_minister_with_attributes(self): "startTime": self.DATA_START_DATE, "endTime": "", "value": { - "columns": ["id", "name", "age", "department", "salary"], + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ [1, "John Doe", 30, "Engineering", 75000.50], [2, "Jane Smith", 25, "Marketing", 65000], [3, "Bob Wilson", 35, "Sales", 85000.75], [4, "Alice Brown", 28, "Engineering", 70000.25], - [5, "Charlie Davis", 32, "Finance", 80000] - ] - } + [5, "Charlie Davis", 32, "Finance", 80000], + ], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } - + res = requests.post(self.base_url, json=payload) print(res.status_code, res.json()) assert res.status_code in [201], f"Failed to create Minister: {res.text}" @@ -453,99 +680,403 @@ def create_minister_with_attributes(self): print(f"Response: {res.status_code} - {res.text}") print("āœ… Created Minister entity with attributes.") - def read_minister(self): """Read the Minister entity.""" print("\nšŸŸ¢ Reading Minister entity...") - res = requests.get(f"{self.base_url}/{self.MINISTER_ID}") + payload = { + "id": self.MINISTER_ID + } + res = requests.post(f"{self.base_read_url}/search", json=payload) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to read Minister: {res.text}" - + # Verify the response data response_data = res.json() + response_data = response_data["body"][0] + print(response_data) - assert response_data["id"] == self.MINISTER_ID, f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" - assert response_data["kind"]["major"] == "Organization", f"Expected major kind 'Organization', got {response_data['kind']['major']}" - assert response_data["kind"]["minor"] == "Minister", f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" - assert response_data["created"] == self.START_DATE, f"Expected created date {self.START_DATE}, got {response_data['created']}" + assert response_data["id"] == self.MINISTER_ID, ( + f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" + ) + assert response_data["kind"]["major"] == "Organization", ( + f"Expected major kind 'Organization', got {response_data['kind']['major']}" + ) + assert response_data["kind"]["minor"] == "Minister", ( + f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" + ) + assert response_data["created"] == self.START_DATE, ( + f"Expected created date {self.START_DATE}, got {response_data['created']}" + ) + + print("im here befire you bitchhhhh....") + # The name value is a protobuf Any that needs to be decoded - name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) - assert decoded_name == "Minister of Finance and Economy", f"Expected name 'Minister of Finance and Economy', got {decoded_name}" + print("i am here..............") + name_obj = json.loads(response_data["name"]) + print(name_obj) + hex_value = name_obj["value"] + print(hex_value) + decoded_name = bytes.fromhex(hex_value).decode('utf-8') + assert decoded_name == "Minister of Finance and Economy", ( + f"Expected name 'Minister of Finance and Economy', got {decoded_name}" + ) def update_attributes_stage_1(self): """Update the attributes of the Minister entity.""" print("\nšŸŸ¢ Updating attributes stage 1...") update_payload = { - "id": self.MINISTER_ID, - "attributes": [ - { - "key": self.ATTRIBUTE_NAME, - "value": { - "values": [ - { - "startTime": "2024-08-01T00:00:00Z", - "endTime": "", - "value": { - "columns": ["id", "name", "age", "department", "salary"], + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME, + "value": { + "values": [ + { + "startTime": "2024-08-01T00:00:00Z", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ - [6, "Peter Parker", 30, "Engineering", 75000.50], + [ + 6, + "Peter Parker", + 30, + "Engineering", + 75000.50, + ], [7, "Clark Kent", 25, "Media", 65000], - ] + ], + }, } - } - ] + ] + }, } - } - ] - } - res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=update_payload, headers={"Content-Type": "application/json"}) + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to update attributes: {res.text}" print("āœ… Attributes updated.") - def update_attributes_stage_2(self): """Update the attributes of the Minister entity.""" print("\nšŸŸ¢ Updating attributes stage 2...") update_payload = { - "id": self.MINISTER_ID, - "attributes": [ - { - "key": self.ATTRIBUTE_NAME, - "value": { - "values": [ - { - "startTime": "2024-08-01T00:00:00Z", - "endTime": "", - "value": { - "columns": ["id", "name", "age", "department", "salary"], + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME, + "value": { + "values": [ + { + "startTime": "2024-08-01T00:00:00Z", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ [8, "Iris West", 30, "Marketing", 12300.50], [9, "Barry Allen", 25, "Sales", 22300.50], - ] + ], + }, } - } - ] + ] + }, } - } - ] - } - res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=update_payload, headers={"Content-Type": "application/json"}) + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to update attributes: {res.text}" print("āœ… Attributes updated.") + # normalize the attribute response and compare + def normalize(self, data): + columns = data["columns"] + rows = data["rows"] + return [ + dict(zip(columns, row)) + for row in rows + ] + + def update_minister_with_attributes_with_startDate_and_endDate(self): + """Create a Minister entity.""" + print("\nšŸŸ¢ Creating Minister with attribute with startDate and endDate...") + + payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_2, + "value": { + "values": [ + { + "startTime": self.DATA_START_DATE, + "endTime": self.DATA_END_DATE, + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [1, "John Doe", 30, "Engineering", 75000.50], + [2, "Jane Smith", 25, "Marketing", 65000], + [3, "Bob Wilson", 35, "Sales", 85000.75], + [4, "Alice Brown", 28, "Engineering", 70000.25], + [5, "Charlie Davis", 32, "Finance", 80000], + ], + }, + } + ] + }, + } + ], + } + + res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to update attributes: {res.text}" + + print("\nšŸŸ¢ checking Minister with attribute with startDate and endDate...") + + url = f"{self.base_read_url}/{self.MINISTER_ID}/attributes/{self.ATTRIBUTE_NAME_2}" + payload = {} + + res = requests.post(url, json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to read Minister: {res.text}" + + response_data = res.json() + print(response_data) + value = response_data["value"] + print(f"value {value}") + decoded_value = CoreTestUtils.decode_protobuf_any_value(value) + + expected_data = { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [1, "John Doe", 30, "Engineering", 75000.50], + [2, "Jane Smith", 25, "Marketing", 65000], + [3, "Bob Wilson", 35, "Sales", 85000.75], + [4, "Alice Brown", 28, "Engineering", 70000.25], + [5, "Charlie Davis", 32, "Finance", 80000], + ], + } + + assert self.normalize(expected_data) == self.normalize(decoded_value), f"Expected {expected_data}, got {decoded_value}" + + print("Decoded value: ", decoded_value) + + url = f"{self.base_read_url}/{self.MINISTER_ID}/relations" + + payload = { + "name": "IS_ATTRIBUTE", + "startTime": self.DATA_START_DATE, + "endTime": self.DATA_END_DATE + } + + res = requests.post(url, json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to read Minister relationships: {res.text}" + + relationships = res.json() + + assert relationships[0]["startTime"] == self.DATA_START_DATE + assert relationships[0]["endTime"] == self.DATA_END_DATE + + print("āœ… Updated Minister entity with attributes with startDate + endDate.") + + def update_minister_with_attributes_without_startDate(self): + """Create a Minister entity.""" + print("\nšŸŸ¢ Updating Minister entity with an attribute without startDate...") + + payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_3, + "value": { + "values": [ + { + "startTime": "", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [1, "John Doe", 30, "Engineering", 75000.50], + [2, "Jane Smith", 25, "Marketing", 65000], + [3, "Bob Wilson", 35, "Sales", 85000.75], + [4, "Alice Brown", 28, "Engineering", 70000.25], + [5, "Charlie Davis", 32, "Finance", 80000], + ], + }, + } + ] + }, + } + ], + "relationships": [], + } + + res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=payload) + print(res.status_code, res.json()) + assert res.status_code == 500, ( + f"Expected 500, got {res.status_code}: {res.text}" + ) + + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Received expected error for Minister creation.") + + def add_attributes_to_minister_with_invalid_starttime_format(self): + print("\nšŸŸ¢ Updating attributes stage 3...") + update_payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_4, + "value": { + "values": [ + { + "startTime": self.INVALID_DATE, + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [ + 6, + "Peter Parker", + 30, + "Engineering", + 75000.50, + ], + [7, "Clark Kent", 25, "Media", 65000], + ], + }, + } + ] + }, + } + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) + print(res.status_code, res.json()) + assert res.status_code == 500, ( + f"Expected 500, got {res.status_code}: {res.text}" + ) + + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Received expected error for Minister creation.") + + def add_attributes_to_minister_with_invalid_endtime_format(self): + print("\nšŸŸ¢ Updating attributes stage 3...") + update_payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_5, + "value": { + "values": [ + { + "startTime": self.DATA_START_DATE, + "endTime": self.INVALID_DATE, + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [ + 6, + "Peter Parker", + 30, + "Engineering", + 75000.50, + ], + [7, "Clark Kent", 25, "Media", 65000], + ], + }, + } + ] + }, + } + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) + print(res.status_code, res.json()) + assert res.status_code == 500, ( + f"Expected 500, got {res.status_code}: {res.text}" + ) + + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Received expected error for Minister creation.") + + def get_base_url(): print("šŸŸ¢ Setting up test environment...") - ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', f"http://0.0.0.0:8080") + ingestion_service_url = os.getenv("INGESTION_SERVICE_URL", "http://0.0.0.0:8080") print("šŸŸ¢ INGESTION_SERVICE_URL: ", ingestion_service_url) return f"{ingestion_service_url}/entities" +def get_base_read_url(): + print("šŸŸ¢ Setting up test environment...") + read_service_url = os.getenv("READ_SERVICE_URL", "http://read:8081") + print("šŸŸ¢ READ_SERVICE_URL: ", read_service_url) + return f"{read_service_url}/v1/entities" -# Tabular Attribute Integrity Tests +# Tabular Attribute Integrity Tests class TabularIntegrityTests(BasicCORETests): """Tests that verify safety constraints for tabular attribute ingestion. @@ -562,7 +1093,6 @@ def __init__(self): super().__init__(None) self.ATTR_NAME = "test_data" - # Test 1 ā€“ Idempotency def test_idempotency(self): @@ -576,7 +1106,11 @@ def test_idempotency(self): "kind": {"major": "test", "minor": "tabular-integrity"}, "created": "2025-01-01T00:00:00Z", "terminated": "", - "name": {"startTime": "2025-01-01T00:00:00Z", "endTime": "", "value": "Idempotency Test"}, + "name": { + "startTime": "2025-01-01T00:00:00Z", + "endTime": "", + "value": "Idempotency Test", + }, "metadata": [], "attributes": [ { @@ -590,15 +1124,15 @@ def test_idempotency(self): "columns": ["id", "name", "department"], "rows": [ [1, "Alice", "Engineering"], - [2, "Bob", "Marketing"] - ] - } + [2, "Bob", "Marketing"], + ], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } # Create entity + first batch @@ -620,14 +1154,14 @@ def test_idempotency(self): "columns": ["id", "name", "department"], "rows": [ [3, "Charlie", "Finance"], - [4, "Diana", "Sales"] - ] - } + [4, "Diana", "Sales"], + ], + }, } ] - } + }, } - ] + ], } # Append second batch ā€“ same attribute name @@ -647,18 +1181,18 @@ def test_idempotency(self): "endTime": "", "value": { "columns": ["id", "name", "department"], - "rows": [[]] - } + "rows": [[]], + }, } ] - } + }, } ] } res = requests.post( f"{self.base_url}/{entity_id}/attributes/test_data", json=read_payload, - headers={"Content-Type": "application/json"} + headers={"Content-Type": "application/json"}, ) # The read API returns 200 or 404 if not supported yet ā€“ skip row count # verification if the read path is not available; still assert the PUT succeeded. @@ -666,7 +1200,9 @@ def test_idempotency(self): data = res.json() print(f" ā„¹ļø Read API returned: {json.dumps(data)[:200]}") else: - print(f" ā„¹ļø Read API returned {res.status_code} ā€“ row-count verification skipped") + print( + f" ā„¹ļø Read API returned {res.status_code} ā€“ row-count verification skipped" + ) print(" āœ… [Test 1] Idempotency PASSED") @@ -683,7 +1219,11 @@ def test_schema_mismatch(self): "kind": {"major": "test", "minor": "tabular-integrity"}, "created": "2025-01-01T00:00:00Z", "terminated": "", - "name": {"startTime": "2025-01-01T00:00:00Z", "endTime": "", "value": "Schema Mismatch Test"}, + "name": { + "startTime": "2025-01-01T00:00:00Z", + "endTime": "", + "value": "Schema Mismatch Test", + }, "metadata": [], "attributes": [ { @@ -695,17 +1235,14 @@ def test_schema_mismatch(self): "endTime": "", "value": { "columns": ["id", "score"], - "rows": [ - [1, 99], - [2, 85] - ] - } + "rows": [[1, 99], [2, 85]], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } res = requests.post(self.base_url, json=initial_payload) assert res.status_code == 201, f"Initial create failed: {res.text}" @@ -724,23 +1261,22 @@ def test_schema_mismatch(self): "endTime": "", "value": { "columns": ["id", "score"], - "rows": [ - [3, "not-a-number"], - [4, "also-wrong"] - ] - } + "rows": [[3, "not-a-number"], [4, "also-wrong"]], + }, } ] - } + }, } - ] + ], } res = requests.put(f"{self.base_url}/{entity_id}", json=bad_payload) assert res.status_code != 200, ( f"Expected an error response when inserting schema-incompatible data, " f"but got HTTP {res.status_code}: {res.text}" ) - print(f" āœ… Schema mismatch correctly rejected: HTTP {res.status_code} ā€“ {res.text}") + print( + f" āœ… Schema mismatch correctly rejected: HTTP {res.status_code} ā€“ {res.text}" + ) print(" āœ… [Test 2] Schema Mismatch PASSED") # Test 3 ā€“ Duplicate Primary Key @@ -756,7 +1292,11 @@ def test_duplicate_primary_key(self): "kind": {"major": "test", "minor": "tabular-integrity"}, "created": "2025-01-01T00:00:00Z", "terminated": "", - "name": {"startTime": "2025-01-01T00:00:00Z", "endTime": "", "value": "Duplicate PK Test"}, + "name": { + "startTime": "2025-01-01T00:00:00Z", + "endTime": "", + "value": "Duplicate PK Test", + }, "metadata": [], "attributes": [ { @@ -768,17 +1308,14 @@ def test_duplicate_primary_key(self): "endTime": "", "value": { "columns": ["id", "value"], - "rows": [ - [1, "first"], - [2, "second"] - ] - } + "rows": [[1, "first"], [2, "second"]], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } res = requests.post(self.base_url, json=initial_payload) assert res.status_code == 201, f"Initial create failed: {res.text}" @@ -797,28 +1334,28 @@ def test_duplicate_primary_key(self): "endTime": "", "value": { "columns": ["id", "value"], - "rows": [ - [1, "duplicate!"] - ] - } + "rows": [[1, "duplicate!"]], + }, } ] - } + }, } - ] + ], } res = requests.put(f"{self.base_url}/{entity_id}", json=dup_payload) assert res.status_code != 200, ( f"Expected an error when inserting a duplicate primary key, " f"but got HTTP {res.status_code}: {res.text}" ) - print(f" āœ… Duplicate PK correctly rejected: HTTP {res.status_code} ā€“ {res.text}") + print( + f" āœ… Duplicate PK correctly rejected: HTTP {res.status_code} ā€“ {res.text}" + ) print(" āœ… [Test 3] Duplicate PK PASSED") if __name__ == "__main__": print("šŸš€ Running End-to-End API Test Suite...") - + try: print("šŸŸ¢ Running Metadata Validation Tests...") metadata_validation_tests = MetadataValidationTests(entity_id="123") @@ -846,6 +1383,10 @@ def test_duplicate_primary_key(self): attribute_validation_tests.read_minister() attribute_validation_tests.update_attributes_stage_1() attribute_validation_tests.update_attributes_stage_2() + attribute_validation_tests.update_minister_with_attributes_with_startDate_and_endDate() + attribute_validation_tests.add_attributes_to_minister_with_invalid_starttime_format() + attribute_validation_tests.add_attributes_to_minister_with_invalid_endtime_format() + attribute_validation_tests.update_minister_with_attributes_without_startDate() print("\nšŸŸ¢ Running Attribute Validation Tests... Done") print("\nšŸŸ¢ Running Tabular Integrity Tests...") @@ -856,7 +1397,7 @@ def test_duplicate_primary_key(self): print("\nšŸŸ¢ Running Tabular Integrity Tests... Done") print("\nšŸŽ‰ All tests passed successfully!") - + except AssertionError as e: print(f"\nāŒ Test failed: {e}") sys.exit(1) diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index 2f5b73b3..fe3e8465 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -1,3 +1,4 @@ +from requests import request import requests import json import sys @@ -633,7 +634,7 @@ def create_entity_for_read(): "values": [ { "startTime": "2024-11-01T00:00:00Z", - "endTime": "", + "endTime": "2024-11-30T00:00:00Z", "value": { "columns": ["e_id", "name", "age", "department", "salary"], "rows": [ @@ -697,6 +698,37 @@ def create_entity_for_read(): "relationships": [] } + # first entity with a new attribute + payload_child_4 = { + "id": RELATED_ID_1, + "attributes": [ + { + "key": "test_data", + "value": { + "values": [ + { + "startTime": "", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [1, "Iris West", 30, "Marketing", 12300.50], + [2, "Barry Allen", 25, "Sales", 22300.50], + ], + }, + } + ] + }, + } + ], + } + payload_source = { "id": ENTITY_ID, "kind": {"major": "test", "minor": "parent"}, @@ -785,10 +817,37 @@ def create_entity_for_read(): assert res.status_code == 201 or res.status_code == 200, f"Failed to create entity: {res.text}" print("āœ… Created third related entity.") + res = requests.put(INGESTION_API_URL + "/" + RELATED_ID_1, json=payload_child_4) + assert res.status_code == 500, f"Failed to update entity: {res.text}" + print("āœ… Failed to update entity. Working as expected.") + res = requests.post(INGESTION_API_URL, json=payload_source) assert res.status_code == 201 or res.status_code == 200, f"Failed to create entity: {res.text}" print("āœ… Created base entity for read tests.") + res = requests.get(INGESTION_API_URL + "/" + RELATED_ID_1) + assert res.status_code == 200, f"Failed to fetch entity: {res.text}" + + url = READ_API_URL + "/" + RELATED_ID_1 + "/relations" + + payload = { + "name": "IS_ATTRIBUTE", + "startTime": "2024-11-01T00:00:00Z", + "endTime": "2024-11-30T00:00:00Z" + } + + res = requests.post(url, json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to read relationships: {res.text}" + print("āœ… Fetched relationship_1...") + + relationship = res.json() + + assert relationship[0]["startTime"] == "2024-11-01T00:00:00Z" + assert relationship[0]["endTime"] == "2024-11-30T00:00:00Z" + + print("āœ… Verified start and end times of entity's attributes.") + def test_attribute_fields_combinations(): """Test different field combinations for attribute retrieval.""" print("\nšŸ” Testing attribute field combinations...")