From 3ade3a5ffccdb62c034ba8a47adf4b76ea6d15ba Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Wed, 25 Mar 2026 16:00:49 +0530 Subject: [PATCH 01/23] fix : invalidating attributes after the given startTime and endTime period --- opengin/core-api/engine/attribute_resolver.go | 6 ++++-- opengin/core-api/engine/graph_metadata_manager.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/opengin/core-api/engine/attribute_resolver.go b/opengin/core-api/engine/attribute_resolver.go index 988c6894..35e92133 100644 --- a/opengin/core-api/engine/attribute_resolver.go +++ b/opengin/core-api/engine/attribute_resolver.go @@ -132,7 +132,8 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // NOTE: for the attribute the timestamp is always the value carried at the attribute level // not the entity level. The entity level timestamp is used for the entity itself. attributeStartTime, _ := time.Parse(time.RFC3339, value.StartTime) - if err := p.handleAttributeLookUp(ctx, entity.Id, attrName, storageType, operation, attributeStartTime); err != nil { + attributeEndTime, _ := time.Parse(time.RFC3339, value.EndTime) + if err := p.handleAttributeLookUp(ctx, entity.Id, attrName, storageType, operation, attributeStartTime, attributeEndTime); err != nil { attributeResults[attrName] = &Result{ Success: false, Data: nil, @@ -196,7 +197,7 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // It creates the attribute look up metadata and the attribute node in the graph. // It also creates the IS_ATTRIBUTE relationship between the entity and the attribute. // It also creates the attribute metadata in the document database. -func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, entityID, attrName string, storageType storageinference.StorageType, operation string, startTime time.Time) error { +func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, entityID, attrName string, storageType storageinference.StorageType, operation string, startTime time.Time, endTime time.Time) error { // Generate attribute metadata fmt.Printf("DEBUG: Handling graph metadata for attribute %s\n", attrName) attributeID := GenerateAttributeID(entityID, attrName) @@ -209,6 +210,7 @@ func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, en StorageType: storageType, StoragePath: storagePath, Created: startTime, + EndTime: endTime, Updated: time.Now(), Schema: make(map[string]interface{}), // TODO: Extract schema from value } diff --git a/opengin/core-api/engine/graph_metadata_manager.go b/opengin/core-api/engine/graph_metadata_manager.go index 62305b7c..6eb40976 100644 --- a/opengin/core-api/engine/graph_metadata_manager.go +++ b/opengin/core-api/engine/graph_metadata_manager.go @@ -134,7 +134,7 @@ func (g *GraphMetadataManager) createAttributeLookUpGraph(ctx context.Context, m }, Name: commons.CreateTimeBasedValue(metadata.Created.Format(time.RFC3339), "", metadata.AttributeName), Created: metadata.Created.Format(time.RFC3339), // contains the data object's time relation with the world - Terminated: "", // TODO: Implement invalidating a dataset for a specific time range + Terminated: metadata.EndTime.Format(time.RFC3339), Metadata: MakeMetadataOfAttributeMetadata(metadata), Attributes: make(map[string]*pb.TimeBasedValueList), Relationships: make(map[string]*pb.Relationship), @@ -239,7 +239,7 @@ func MakeRelationshipFromAttributeMetadata(metadata *AttributeMetadata) *pb.Rela RelatedEntityId: metadata.AttributeID, Name: IS_ATTRIBUTE_RELATIONSHIP, StartTime: metadata.Created.Format(time.RFC3339), - EndTime: "", // TODO: Implement invalidating a relationship for a specific time range + EndTime: metadata.EndTime.Format(time.RFC3339), Direction: IS_ATTRIBUTE_RELATIONSHIP_DIRECTION, } } From 2e5a34acdced05fb14d6f8a17b865b1f95590d4d Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 26 Mar 2026 01:44:07 +0530 Subject: [PATCH 02/23] fix : fix handling zero time values and make it accepts "" vlaues --- opengin/core-api/engine/attribute_resolver.go | 10 ++++++++-- .../core-api/engine/graph_metadata_manager.go | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/opengin/core-api/engine/attribute_resolver.go b/opengin/core-api/engine/attribute_resolver.go index 35e92133..11e05e12 100644 --- a/opengin/core-api/engine/attribute_resolver.go +++ b/opengin/core-api/engine/attribute_resolver.go @@ -132,7 +132,12 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // NOTE: for the attribute the timestamp is always the value carried at the attribute level // not the entity level. The entity level timestamp is used for the entity itself. attributeStartTime, _ := time.Parse(time.RFC3339, value.StartTime) - attributeEndTime, _ := time.Parse(time.RFC3339, value.EndTime) + var attributeEndTime *time.Time + if value.EndTime != "" { + t, _ := time.Parse(time.RFC3339, value.EndTime) + attributeEndTime = &t + } + if err := p.handleAttributeLookUp(ctx, entity.Id, attrName, storageType, operation, attributeStartTime, attributeEndTime); err != nil { attributeResults[attrName] = &Result{ Success: false, @@ -197,7 +202,7 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // It creates the attribute look up metadata and the attribute node in the graph. // It also creates the IS_ATTRIBUTE relationship between the entity and the attribute. // It also creates the attribute metadata in the document database. -func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, entityID, attrName string, storageType storageinference.StorageType, operation string, startTime time.Time, endTime time.Time) error { +func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, entityID, attrName string, storageType storageinference.StorageType, operation string, startTime time.Time, endTime *time.Time) error { // Generate attribute metadata fmt.Printf("DEBUG: Handling graph metadata for attribute %s\n", attrName) attributeID := GenerateAttributeID(entityID, attrName) @@ -216,6 +221,7 @@ func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, en } // Note: endTime parameter is optional and available for future use if needed + log.Printf("DEBUG: Handling graph metadata for attribute %s: [endTime: %v] [startTime: %v]", attrName, endTime, startTime) switch operation { case "create": diff --git a/opengin/core-api/engine/graph_metadata_manager.go b/opengin/core-api/engine/graph_metadata_manager.go index 6eb40976..220203af 100644 --- a/opengin/core-api/engine/graph_metadata_manager.go +++ b/opengin/core-api/engine/graph_metadata_manager.go @@ -56,7 +56,7 @@ type AttributeMetadata struct { StoragePath string // Path/location in the specific storage system Created time.Time Updated time.Time - EndTime time.Time + EndTime *time.Time Schema map[string]interface{} // Schema information } @@ -126,6 +126,14 @@ func (g *GraphMetadataManager) createAttributeLookUpGraph(ctx context.Context, m // create the attribute node in the graph // stored parameters: id, kind, name, created + log.Printf("DEBUG: Creating attribute node for attribute %s: [endTime: %v] [startTime: %v]", metadata.AttributeName, metadata.EndTime, metadata.Created) + + var terminated string + + if metadata.EndTime != nil { + terminated = metadata.EndTime.Format(time.RFC3339) + } + attributeNode := &pb.Entity{ Id: metadata.AttributeID, Kind: &pb.Kind{ @@ -134,7 +142,7 @@ func (g *GraphMetadataManager) createAttributeLookUpGraph(ctx context.Context, m }, Name: commons.CreateTimeBasedValue(metadata.Created.Format(time.RFC3339), "", metadata.AttributeName), Created: metadata.Created.Format(time.RFC3339), // contains the data object's time relation with the world - Terminated: metadata.EndTime.Format(time.RFC3339), + Terminated: terminated, Metadata: MakeMetadataOfAttributeMetadata(metadata), Attributes: make(map[string]*pb.TimeBasedValueList), Relationships: make(map[string]*pb.Relationship), @@ -234,12 +242,16 @@ func MakeMetadataOfAttributeMetadata(metadata *AttributeMetadata) map[string]*an // MakeRelationshipProto creates a Relationship protobuf object for IS_ATTRIBUTE relationship func MakeRelationshipFromAttributeMetadata(metadata *AttributeMetadata) *pb.Relationship { + var endTime string + if metadata.EndTime != nil { + endTime = metadata.EndTime.Format(time.RFC3339) + } return &pb.Relationship{ Id: GenerateAttributeRelationshipID(metadata.EntityID, metadata.AttributeName), RelatedEntityId: metadata.AttributeID, Name: IS_ATTRIBUTE_RELATIONSHIP, StartTime: metadata.Created.Format(time.RFC3339), - EndTime: metadata.EndTime.Format(time.RFC3339), + EndTime: endTime, Direction: IS_ATTRIBUTE_RELATIONSHIP_DIRECTION, } } From 2485263a32768af752dbadaf3fa84d81138714fb Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Fri, 27 Mar 2026 15:44:07 +0530 Subject: [PATCH 03/23] fix: fixing time parse issue --- opengin/core-api/engine/attribute_resolver.go | 33 +++++++++++++++++-- .../core-api/engine/graph_metadata_manager.go | 12 +++---- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/opengin/core-api/engine/attribute_resolver.go b/opengin/core-api/engine/attribute_resolver.go index 11e05e12..5cbac57a 100644 --- a/opengin/core-api/engine/attribute_resolver.go +++ b/opengin/core-api/engine/attribute_resolver.go @@ -131,13 +131,40 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // Create or update graph metadata BEFORE processing the attribute // NOTE: for the attribute the timestamp is always the value carried at the attribute level // not the entity level. The entity level timestamp is used for the entity itself. - attributeStartTime, _ := time.Parse(time.RFC3339, value.StartTime) + + // start time is required to create the attribute + if value.StartTime == "" { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("StartTime is required for attribute: %s", attrName), + } + continue + } + + attributeStartTime, err := time.Parse(time.RFC3339, value.StartTime) + if err != nil { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("invalid StartTime format for attribute %s: %v", attrName, err), + } + continue + } + var attributeEndTime *time.Time + if value.EndTime != "" { - t, _ := time.Parse(time.RFC3339, value.EndTime) + t, err := time.Parse(time.RFC3339, value.EndTime) + if err != nil { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("invalid EndTime format for attribute %s: %v", attrName, err), + } + continue + } attributeEndTime = &t } + if err := p.handleAttributeLookUp(ctx, entity.Id, attrName, storageType, operation, attributeStartTime, attributeEndTime); err != nil { attributeResults[attrName] = &Result{ Success: false, @@ -215,7 +242,7 @@ func (p *EntityAttributeProcessor) handleAttributeLookUp(ctx context.Context, en StorageType: storageType, StoragePath: storagePath, Created: startTime, - EndTime: endTime, + Terminated: endTime, Updated: time.Now(), Schema: make(map[string]interface{}), // TODO: Extract schema from value } diff --git a/opengin/core-api/engine/graph_metadata_manager.go b/opengin/core-api/engine/graph_metadata_manager.go index 220203af..668373ad 100644 --- a/opengin/core-api/engine/graph_metadata_manager.go +++ b/opengin/core-api/engine/graph_metadata_manager.go @@ -56,7 +56,7 @@ type AttributeMetadata struct { StoragePath string // Path/location in the specific storage system Created time.Time Updated time.Time - EndTime *time.Time + Terminated *time.Time Schema map[string]interface{} // Schema information } @@ -126,12 +126,12 @@ func (g *GraphMetadataManager) createAttributeLookUpGraph(ctx context.Context, m // create the attribute node in the graph // stored parameters: id, kind, name, created - log.Printf("DEBUG: Creating attribute node for attribute %s: [endTime: %v] [startTime: %v]", metadata.AttributeName, metadata.EndTime, metadata.Created) + log.Printf("DEBUG: Creating attribute node for attribute %s: [endTime: %v] [startTime: %v]", metadata.AttributeName, metadata.Terminated, metadata.Created) var terminated string - if metadata.EndTime != nil { - terminated = metadata.EndTime.Format(time.RFC3339) + if metadata.Terminated != nil { + terminated = metadata.Terminated.Format(time.RFC3339) } attributeNode := &pb.Entity{ @@ -243,8 +243,8 @@ func MakeMetadataOfAttributeMetadata(metadata *AttributeMetadata) map[string]*an // MakeRelationshipProto creates a Relationship protobuf object for IS_ATTRIBUTE relationship func MakeRelationshipFromAttributeMetadata(metadata *AttributeMetadata) *pb.Relationship { var endTime string - if metadata.EndTime != nil { - endTime = metadata.EndTime.Format(time.RFC3339) + if metadata.Terminated != nil { + endTime = metadata.Terminated.Format(time.RFC3339) } return &pb.Relationship{ Id: GenerateAttributeRelationshipID(metadata.EntityID, metadata.AttributeName), From 310f89ed6f3af3c3a957afc0861811d3a1624f12 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Mon, 30 Mar 2026 17:09:15 +0530 Subject: [PATCH 04/23] fix : update the test results comparison --- opengin/ingestion-api/tests/service_test.bal | 365 +++++++++--------- .../read-api/tests/read_api_service_test.bal | 2 +- 2 files changed, 182 insertions(+), 185 deletions(-) diff --git a/opengin/ingestion-api/tests/service_test.bal b/opengin/ingestion-api/tests/service_test.bal index 62935946..b8aa23bc 100644 --- a/opengin/ingestion-api/tests/service_test.bal +++ b/opengin/ingestion-api/tests/service_test.bal @@ -1,11 +1,11 @@ // Copyright 2025 Lanka Data Foundation // SPDX-License-Identifier: Apache-2.0 -import ballerina/io; -import ballerina/test; -import ballerina/protobuf.types.'any as pbAny; import ballerina/http; +import ballerina/io; import ballerina/os; +import ballerina/protobuf.types.'any as pbAny; +import ballerina/test; // Get environment variables without fallback values string testIngestionHostname = os:getEnv("INGESTION_SERVICE_HOST"); @@ -45,7 +45,7 @@ function unwrapAnyToJson(pbAny:Any anyValue) returns json|error { if stringValue is string { return stringValue; } - + // If string unpacking fails, return the string representation as a fallback return anyValue.toString(); } @@ -106,14 +106,14 @@ function jsonToAny(json data) returns pbAny:Any|error { function testMetadataHandling() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "test-entity-1"; string expectedValue1 = "value1"; string expectedValue2 = "value2"; - + // Create the metadata array - record {| string key; pbAny:Any value; |}[] metadataArray = []; + record {|string key; pbAny:Any value;|}[] metadataArray = []; // Pack string values into protobuf.Any directly pbAny:Any packedValue1 = check pbAny:pack(expectedValue1); @@ -144,7 +144,7 @@ function testMetadataHandling() returns error? { Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); io:println("Created entity metadata: ", createEntityResponse.metadata); - + // Read entity ReadEntityRequest readEntityRequest = { entity: { @@ -168,7 +168,7 @@ function testMetadataHandling() returns error? { io:println("Entity retrieved, verifying data..."); io:println("Retrieved entity: ", readEntityResponse); io:println("Retrieved entity metadata: ", readEntityResponse.metadata); - + // Verify metadata values map actualValues = {}; foreach var item in readEntityResponse.metadata { @@ -179,15 +179,15 @@ function testMetadataHandling() returns error? { test:assertFail("Failed to unpack metadata value for key: " + item.key); } } - + // Assert the values match test:assertEquals(actualValues["key1"], expectedValue1, "Metadata value for key1 doesn't match"); test:assertEquals(actualValues["key2"], expectedValue2, "Metadata value for key2 doesn't match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity deleted"); - + return; } @@ -199,7 +199,7 @@ function testMetadataHandling() returns error? { function testMetadataUnpackError() returns error? { // Test case to verify handling of non-existent entities COREServiceClient ep = check new (testCoreServiceUrl); - + // Try to read a non-existent entity ReadEntityRequest readEntityRequest = { entity: { @@ -219,10 +219,10 @@ function testMetadataUnpackError() returns error? { output: ["metadata"] }; Entity|error response = ep->ReadEntity(readEntityRequest); - + // Assert that we get an error for non-existent entity test:assertTrue(response is error, "Expected error for non-existent entity"); - + return; } @@ -230,21 +230,21 @@ function testMetadataUnpackError() returns error? { function testMetadataUpdating() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "test-entity-update"; - + // Initial metadata values string initialValue1 = "initial-value1"; string initialValue2 = "initial-value2"; - + // Updated metadata values string updatedValue1 = "updated-value1"; string updatedValue2 = "updated-value2"; string newValue3 = "new-value3"; - + // Create the initial metadata array - record {| string key; pbAny:Any value; |}[] initialMetadataArray = []; + record {|string key; pbAny:Any value;|}[] initialMetadataArray = []; pbAny:Any packedInitialValue1 = check pbAny:pack(initialValue1); pbAny:Any packedInitialValue2 = check pbAny:pack(initialValue2); initialMetadataArray.push({key: "key1", value: packedInitialValue1}); @@ -270,7 +270,7 @@ function testMetadataUpdating() returns error? { // Create entity Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Verify initial metadata ReadEntityRequest readEntityRequest = { entity: { @@ -292,9 +292,9 @@ function testMetadataUpdating() returns error? { Entity initialReadResponse = check ep->ReadEntity(readEntityRequest); verifyMetadata(initialReadResponse.metadata, {"key1": initialValue1, "key2": initialValue2}); io:println("Initial metadata verified"); - + // Create updated metadata array - record {| string key; pbAny:Any value; |}[] updatedMetadataArray = []; + record {|string key; pbAny:Any value;|}[] updatedMetadataArray = []; pbAny:Any packedUpdatedValue1 = check pbAny:pack(updatedValue1); pbAny:Any packedUpdatedValue2 = check pbAny:pack(updatedValue2); pbAny:Any packedNewValue3 = check pbAny:pack(newValue3); @@ -321,7 +321,7 @@ function testMetadataUpdating() returns error? { attributes: [], relationships: [] }; - + // Update entity UpdateEntityRequest updateRequest = { id: testId, @@ -329,7 +329,7 @@ function testMetadataUpdating() returns error? { }; Entity updateEntityResponse = check ep->UpdateEntity(updateRequest); io:println("Entity updated with ID: " + updateEntityResponse.id); - + // Verify updated metadata ReadEntityRequest updatedReadRequest = { entity: { @@ -350,21 +350,21 @@ function testMetadataUpdating() returns error? { }; Entity updatedReadResponse = check ep->ReadEntity(updatedReadRequest); verifyMetadata(updatedReadResponse.metadata, { - "key1": updatedValue1, - "key2": updatedValue2, - "key3": newValue3 - }); + "key1": updatedValue1, + "key2": updatedValue2, + "key3": newValue3 + }); io:println("Updated metadata verified"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity deleted"); - + return; } // Helper function to verify metadata -function verifyMetadata(record {| string key; pbAny:Any value; |}[] metadata, map expected) { +function verifyMetadata(record {|string key; pbAny:Any value;|}[] metadata, map expected) { map actual = {}; foreach var item in metadata { string|error unwrapped = unwrapAny(item.value); @@ -372,12 +372,12 @@ function verifyMetadata(record {| string key; pbAny:Any value; |}[] metadata, ma actual[item.key] = unwrapped.trim(); } } - + // Verify all expected key-value pairs exist in the actual metadata foreach var [key, expectedValue] in expected.entries() { test:assertTrue(actual.hasKey(key), "Metadata key not found: " + key); - test:assertEquals(actual[key] ?: "", expectedValue, - string `Metadata value for ${key} doesn't match: expected ${expectedValue}, got ${actual[key] ?: ""}`); + test:assertEquals(actual[key] ?: "", expectedValue, + string `Metadata value for ${key} doesn't match: expected ${expectedValue}, got ${actual[key] ?: ""}`); } } @@ -385,17 +385,17 @@ function verifyMetadata(record {| string key; pbAny:Any value; |}[] metadata, ma function testEntityReading() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "test-entity-read"; string metadataKey = "readTest"; string metadataValue = "read-test-value"; - + // Create a test entity first - record {| string key; pbAny:Any value; |}[] metadataArray = []; + record {|string key; pbAny:Any value;|}[] metadataArray = []; pbAny:Any packedValue = check pbAny:pack(metadataValue); metadataArray.push({key: metadataKey, value: packedValue}); - + Entity createEntityRequest = { id: testId, kind: { @@ -413,12 +413,12 @@ function testEntityReading() returns error? { attributes: [], relationships: [] }; - + // Create entity Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Test entity created with ID: " + createEntityResponse.id); io:println("Created entity metadata: ", createEntityResponse.metadata); - + // Read the entity ReadEntityRequest readEntityRequest = { entity: { @@ -442,25 +442,25 @@ function testEntityReading() returns error? { io:println("Entity retrieved, verifying data..."); io:println("Retrieved entity: ", readEntityResponse); io:println("Retrieved entity metadata: ", readEntityResponse.metadata); - + // Verify entity fields test:assertEquals(readEntityResponse.id, testId, "Entity ID mismatch"); - + // Verify metadata boolean foundMetadata = false; foreach var item in readEntityResponse.metadata { if item.key == metadataKey { string|error unwrapped = unwrapAny(item.value); if unwrapped is string { - test:assertEquals(unwrapped.trim(), metadataValue, - string `Metadata value mismatch: expected ${metadataValue}, got ${unwrapped}`); + test:assertEquals(unwrapped.trim(), metadataValue, + string `Metadata value mismatch: expected ${metadataValue}, got ${unwrapped}`); foundMetadata = true; } } } - + test:assertTrue(foundMetadata, "Expected metadata key not found"); - + // Test reading non-existent entity string nonExistentId = "non-existent-entity-" + testId; ReadEntityRequest nonExistentRequest = { @@ -481,23 +481,23 @@ function testEntityReading() returns error? { output: ["metadata"] }; Entity|error nonExistentResult = ep->ReadEntity(nonExistentRequest); - + // For now, expect an error for non-existent entities test:assertTrue(nonExistentResult is error, "Expected error for non-existent entity"); if nonExistentResult is error { io:println("Non-existent entity correctly returned error: " + nonExistentResult.message()); } - + // Assert that we get an error for non-existent entity // For non-existence entities, we send a response with an empty data // But once the Result API is integrated this can be tested. // FIXME: https://github.com/LDFLK/nexoan/issues/23 // test:assertTrue(nonExistentResponse is error, "Expected error for non-existent entity ID"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity deleted"); - + return; } @@ -505,10 +505,10 @@ function testEntityReading() returns error? { function testCreateMinimalGraphEntity() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup - minimal entity with just required fields string testId = "test-minimal-entity"; - + // Create entity request with only required fields - no metadata, attributes, or relationships Entity createEntityRequest = { id: testId, @@ -531,7 +531,7 @@ function testCreateMinimalGraphEntity() returns error? { // Create entity Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Minimal entity created with ID: " + createEntityResponse.id); - + // Verify entity was created correctly ReadEntityRequest readEntityRequest = { entity: { @@ -551,21 +551,21 @@ function testCreateMinimalGraphEntity() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Basic entity verification test:assertEquals(readEntityResponse.id, testId, "Entity ID doesn't match"); test:assertEquals(readEntityResponse.kind.major, "test", "Entity kind.major doesn't match"); test:assertEquals(readEntityResponse.kind.minor, "minimal", "Entity kind.minor doesn't match"); - + // Verify empty collections test:assertEquals(readEntityResponse.metadata.length(), 0, "Metadata should be empty"); test:assertEquals(readEntityResponse.attributes.length(), 0, "Attributes default value should be empty"); test:assertEquals(readEntityResponse.relationships.length(), 0, "Relationships should be empty"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test minimal entity deleted"); - + return; } @@ -576,10 +576,10 @@ function testCreateMinimalGraphEntityViaRest() returns error? { httpVersion: "2.0" // Enable HTTP/2 }; http:Client restClient = check new (testIngestionServiceUrl, httpConfig); - + // Test data setup - minimal JSON entity string testId = "test-minimal-json-entity"; - + // Minimal JSON payload with required fields matching the Entity structure json minimalEntityJson = { "id": testId, @@ -601,22 +601,22 @@ function testCreateMinimalGraphEntityViaRest() returns error? { // Create entity via REST API http:Response|error response = restClient->post("/entities", minimalEntityJson); - + // Verify HTTP request was successful if response is error { test:assertFail("Failed to create entity via REST API: " + response.message()); } - + http:Response httpResponse = response; test:assertEquals(httpResponse.statusCode, 201, "Expected 201 OK status code"); - + // Parse response JSON json responseJson = check httpResponse.getJsonPayload(); test:assertEquals(check responseJson.id, testId, "Entity ID in response doesn't match"); - + // Initialize the gRPC client to verify entity was properly created COREServiceClient ep = check new (testCoreServiceUrl); - + // Verify entity data ReadEntityRequest readEntityRequest = { entity: { @@ -633,24 +633,24 @@ function testCreateMinimalGraphEntityViaRest() returns error? { attributes: [], relationships: [] }, - output: ["metadata","attributes", "relationships"] + output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Basic entity verification test:assertEquals(readEntityResponse.id, testId, "Entity ID doesn't match"); test:assertEquals(readEntityResponse.kind.major, "test", "Entity kind.major doesn't match"); test:assertEquals(readEntityResponse.kind.minor, "minimal-json", "Entity kind.minor doesn't match"); - + // Verify empty collections test:assertEquals(readEntityResponse.metadata.length(), 0, "Metadata should be empty"); test:assertEquals(readEntityResponse.attributes.length(), 0, "Attributes default value should be empty"); test:assertEquals(readEntityResponse.relationships.length(), 0, "Relationships should be empty"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test minimal JSON entity deleted"); - + return; } @@ -661,13 +661,13 @@ function testEntityWithRelationship() returns error? { // Test IDs for entities string sourceEntityId = "test-entity-with-relationship-source"; string targetEntityId = "test-entity-with-relationship-target"; - + // Initialize REST client with HTTP/2 support http:ClientConfiguration httpConfig = { httpVersion: "2.0" // Enable HTTP/2 }; http:Client restClient = check new (testIngestionServiceUrl, httpConfig); - + // Create source entity json sourceEntityJson = { "id": sourceEntityId, @@ -686,7 +686,7 @@ function testEntityWithRelationship() returns error? { "attributes": [], "relationships": [] }; - + // Create target entity json targetEntityJson = { "id": targetEntityId, @@ -694,7 +694,7 @@ function testEntityWithRelationship() returns error? { "major": "test", "minor": "relationship-target" }, - "created": "2023-01-01", + "created": "2023-01-01", "terminated": "", "name": { "startTime": "2023-01-01", @@ -705,11 +705,11 @@ function testEntityWithRelationship() returns error? { "attributes": [], "relationships": [] }; - + // Create both entities via REST API http:Response|error sourceResponse = restClient->post("/entities", sourceEntityJson); http:Response|error targetResponse = restClient->post("/entities", targetEntityJson); - + // Verify HTTP requests were successful if sourceResponse is error { test:assertFail("Failed to create source entity: " + sourceResponse.message()); @@ -717,12 +717,12 @@ function testEntityWithRelationship() returns error? { if targetResponse is error { test:assertFail("Failed to create target entity: " + targetResponse.message()); } - + http:Response sourceHttpResponse = sourceResponse; http:Response targetHttpResponse = targetResponse; test:assertEquals(sourceHttpResponse.statusCode, 201, "Expected 201 status code for source entity"); test:assertEquals(targetHttpResponse.statusCode, 201, "Expected 201 status code for target entity"); - + // Create relationship between entities - include full entity structure string relationshipId = "rel-" + sourceEntityId + "-" + targetEntityId; json relationshipJson = { @@ -745,21 +745,21 @@ function testEntityWithRelationship() returns error? { } } }; - + // Update source entity with relationship http:Response|error updateResponse = restClient->put("/entities/" + sourceEntityId, relationshipJson); - + // Verify update was successful if updateResponse is error { test:assertFail("Failed to update entity with relationship: " + updateResponse.message()); } - + http:Response updateHttpResponse = updateResponse; test:assertEquals(updateHttpResponse.statusCode, 200, "Expected 200 status code for relationship update"); - + // Initialize the gRPC client to verify relationship was properly created COREServiceClient ep = check new (testCoreServiceUrl); - + // Read source entity to verify relationship ReadEntityRequest readEntityRequest = { entity: { @@ -779,10 +779,10 @@ function testEntityWithRelationship() returns error? { output: ["relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify relationship data test:assertEquals(readEntityResponse.relationships.length(), 1, "Entity should have one relationship"); - + // Find the relationship by iterating through the array Relationship? targetRelationship = (); foreach var rel in readEntityResponse.relationships { @@ -791,7 +791,7 @@ function testEntityWithRelationship() returns error? { break; } } - + io:println("Target relationship: " + targetRelationship.toJsonString()); test:assertFalse(targetRelationship is (), "Relationship with key 'CONNECTS_TO' not found"); Relationship relationship = targetRelationship; @@ -799,12 +799,12 @@ function testEntityWithRelationship() returns error? { test:assertEquals(relationship.name, "CONNECTS_TO", "Relationship name doesn't match"); test:assertEquals(relationship.startTime, "2023-01-01T00:00:00Z", "Relationship start time doesn't match"); test:assertEquals(relationship.id, relationshipId, "Relationship ID doesn't match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: sourceEntityId}); Empty _ = check ep->DeleteEntity({id: targetEntityId}); io:println("Test entities with relationship deleted"); - + return; } @@ -816,10 +816,10 @@ function testEntityWithSimpleOnlyNodesGraphAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-simple-only-nodes-graph"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with tabular data in attributes json socialNetworkGraph = { "nodes": [ @@ -873,7 +873,7 @@ function testEntityWithSimpleOnlyNodesGraphAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -893,16 +893,16 @@ function testEntityWithSimpleOnlyNodesGraphAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with graph attributes deleted"); - + return; } @@ -914,10 +914,10 @@ function testEntityWithSimpleGraphAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-simple-entity-graph"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with tabular data in attributes json socialNetworkGraph = { "nodes": [ @@ -978,7 +978,7 @@ function testEntityWithSimpleGraphAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -998,16 +998,16 @@ function testEntityWithSimpleGraphAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with graph attributes deleted"); - + return; } @@ -1019,26 +1019,26 @@ function testEntityWithMultiGraphAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-graph"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with tabular data in attributes json salaryGraph = { "nodes": [ - {"id": "user1", "type": "user", "properties": {"name": "Alice", "age": 30, "location": "NY"}}, - {"id": "user2", "type": "user", "properties": {"name": "Bob", "age": 25, "location": "SF"}}, - {"id": "user3", "type": "user", "properties": {"name": "Charlie", "age": 35, "location": "LA"}}, - {"id": "post1", "type": "post", "properties": {"title": "Hello", "content": "World", "created": "2024-03-20"}}, - {"id": "post2", "type": "post", "properties": {"title": "Graph", "content": "DB", "created": "2024-03-21"}} - ], - "edges": [ - {"source": "user1", "target": "user2", "type": "follows", "properties": {"since": "2024-01-01"}}, - {"source": "user2", "target": "user3", "type": "follows", "properties": {"since": "2024-02-01"}}, - {"source": "user1", "target": "post1", "type": "created", "properties": {"timestamp": "2024-03-20T10:00:00Z"}}, - {"source": "user2", "target": "post1", "type": "likes", "properties": {"timestamp": "2024-03-20T11:00:00Z"}}, - {"source": "user3", "target": "post2", "type": "created", "properties": {"timestamp": "2024-03-21T09:00:00Z"}} - ] + {"id": "user1", "type": "user", "properties": {"name": "Alice", "age": 30, "location": "NY"}}, + {"id": "user2", "type": "user", "properties": {"name": "Bob", "age": 25, "location": "SF"}}, + {"id": "user3", "type": "user", "properties": {"name": "Charlie", "age": 35, "location": "LA"}}, + {"id": "post1", "type": "post", "properties": {"title": "Hello", "content": "World", "created": "2024-03-20"}}, + {"id": "post2", "type": "post", "properties": {"title": "Graph", "content": "DB", "created": "2024-03-21"}} + ], + "edges": [ + {"source": "user1", "target": "user2", "type": "follows", "properties": {"since": "2024-01-01"}}, + {"source": "user2", "target": "user3", "type": "follows", "properties": {"since": "2024-02-01"}}, + {"source": "user1", "target": "post1", "type": "created", "properties": {"timestamp": "2024-03-20T10:00:00Z"}}, + {"source": "user2", "target": "post1", "type": "likes", "properties": {"timestamp": "2024-03-20T11:00:00Z"}}, + {"source": "user3", "target": "post2", "type": "created", "properties": {"timestamp": "2024-03-21T09:00:00Z"}} + ] }; json projectGraph = { @@ -1113,7 +1113,7 @@ function testEntityWithMultiGraphAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1133,16 +1133,16 @@ function testEntityWithMultiGraphAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up - + Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with graph attributes deleted"); - + return; } @@ -1154,10 +1154,10 @@ function testEntityWithSimpleListAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-list"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with list data in attributes json salaryList = { "values": [ @@ -1209,11 +1209,11 @@ function testEntityWithSimpleListAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with list attributes deleted"); - + return; } @@ -1225,10 +1225,10 @@ function testEntityWithMixedTypeListAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-mixed-list"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with mixed type list data in attributes json mixedTypeList = { "values": [ @@ -1283,7 +1283,7 @@ function testEntityWithMixedTypeListAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1303,15 +1303,15 @@ function testEntityWithMixedTypeListAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with mixed type list attributes deleted"); - + return; } @@ -1323,10 +1323,10 @@ function testEntityWithEmptyListAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-empty-list"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with empty list data in attributes json emptyList = { "values": [] @@ -1374,7 +1374,7 @@ function testEntityWithEmptyListAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1394,15 +1394,15 @@ function testEntityWithEmptyListAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with empty list attributes deleted"); - + return; } @@ -1414,17 +1414,17 @@ function testEntityWithMapAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-map"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with map data in attributes json userProfileMap = { - "properties": { - "name": "John", - "age": 30, - "active": true - } + "properties": { + "name": "John", + "age": 30, + "active": true + } }; // Convert JSON to protobuf Any values @@ -1469,7 +1469,7 @@ function testEntityWithMapAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1489,15 +1489,15 @@ function testEntityWithMapAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with map attributes deleted"); - + return; } @@ -1509,10 +1509,10 @@ function testEntityWithNestedMapAttributes() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-nested-map"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with nested map data in attributes json nestedMap = { "organization": { @@ -1598,7 +1598,7 @@ function testEntityWithNestedMapAttributes() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1618,15 +1618,15 @@ function testEntityWithNestedMapAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with nested map attributes deleted"); - + return; } @@ -1638,10 +1638,10 @@ function testEntityWithEmptyMapValues() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-empty-map-values"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with map data containing empty values // FIXME: https://github.com/LDFLK/nexoan/issues/137 json emptyValuesMap = { @@ -1694,7 +1694,7 @@ function testEntityWithEmptyMapValues() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1714,15 +1714,15 @@ function testEntityWithEmptyMapValues() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with empty map values deleted"); - + return; } @@ -1734,10 +1734,10 @@ function testEntityWithNestedMapValues() returns error? { // TODO: Complete Test Case https://github.com/LDFLK/nexoan/issues/143 // Test ID for entity string testId = "test-entity-nested-map-values"; - + // Initialize the gRPC client to verify entity COREServiceClient ep = check new (testCoreServiceUrl); - + // Create entity with deeply nested map data json nestedMap = { "properties": { @@ -1832,7 +1832,7 @@ function testEntityWithNestedMapValues() returns error? { // Create entity via gRPC Entity createEntityResponse = check ep->CreateEntity(createEntityRequest); io:println("Entity created with ID: " + createEntityResponse.id); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1852,27 +1852,26 @@ function testEntityWithNestedMapValues() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); - + // Clean up Empty _ = check ep->DeleteEntity({id: testId}); io:println("Test entity with nested map values deleted"); - + return; } - @test:Config {} function testEntityWithTabularAttributes() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "ID-MIN-A"; - + // Create tabular data structure // TODO: https://github.com/LDFLK/nexoan/issues/284 json tabularData = { @@ -1937,7 +1936,7 @@ function testEntityWithTabularAttributes() returns error? { }; pbAny:Any tabularDataFilterAny = check jsonToAny(tabularDataFilter); - + // Read entity to verify attributes ReadEntityRequest readEntityRequest = { entity: { @@ -1957,7 +1956,7 @@ function testEntityWithTabularAttributes() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-04-01T00:00:00Z", endTime: "", value: tabularDataFilterAny } @@ -1970,7 +1969,7 @@ function testEntityWithTabularAttributes() returns error? { output: ["metadata", "attributes", "relationships"] }; Entity readEntityResponse = check ep->ReadEntity(readEntityRequest); - + // Verify the response test:assertTrue(readEntityResponse.id != "", "Entity should be found"); test:assertEquals(readEntityResponse.id, testId, "Entity ID should match"); @@ -2001,15 +2000,14 @@ function testEntityWithTabularAttributes() returns error? { return; } - @test:Config {} function testEntityWithTabularAttributesMultiRels() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "ID-MIN-A-MULTI-RELS"; - + // Create tabular data structure // TODO: https://github.com/LDFLK/nexoan/issues/284 json employeeData = { @@ -2022,7 +2020,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { [5, "Charlie Davis", 32, "Finance", 80000] ] }; - + json budgetData = { "columns": ["id", "category", "amount", "quarter", "status"], "rows": [ @@ -2104,7 +2102,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { pbAny:Any employeeDataFilterAny = check jsonToAny(employeeDataFilter); pbAny:Any budgetDataFilterAny = check jsonToAny(budgetDataFilter); - + // Read entity to verify attributes ReadEntityRequest readEntityWithBudgetDataRequest = { entity: { @@ -2124,7 +2122,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-06-01T00:00:00Z", endTime: "", value: budgetDataFilterAny } @@ -2155,7 +2153,7 @@ function testEntityWithTabularAttributesMultiRels() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-04-01T00:00:00Z", endTime: "", value: employeeDataFilterAny } @@ -2215,15 +2213,14 @@ function testEntityWithTabularAttributesMultiRels() returns error? { return; } - @test:Config {} function testEntityWithTabularAttributesUpdate() returns error? { // Initialize the client COREServiceClient ep = check new (testCoreServiceUrl); - + // Test data setup string testId = "ID-MIN-A-UPDATE"; - + // Create tabular data structure json tabularData = { "columns": ["id", "name", "age", "department", "salary"], @@ -2332,7 +2329,7 @@ function testEntityWithTabularAttributesUpdate() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-12-01T00:00:00Z", endTime: "", value: tabularDataFilterAny } @@ -2509,7 +2506,7 @@ function testTabularAttributeIdempotency() returns error? { key: attrName, value: { values: [ - { startTime: "", endTime: "", value: allRowsFilterAny } + { startTime: "2025-02-01T00:00:00Z", endTime: "", value: allRowsFilterAny } ] } } @@ -2727,4 +2724,4 @@ function testTabularDuplicatePrimaryKey() returns error? { Empty _ = check ep->DeleteEntity({ id: testId }); io:println("[testTabularDuplicatePrimaryKey] Cleaned up test entity"); return; -} \ No newline at end of file +} diff --git a/opengin/read-api/tests/read_api_service_test.bal b/opengin/read-api/tests/read_api_service_test.bal index 3ed3ad0d..81bf9bcb 100644 --- a/opengin/read-api/tests/read_api_service_test.bal +++ b/opengin/read-api/tests/read_api_service_test.bal @@ -220,7 +220,7 @@ function testEntityAttributeRetrieval() returns error? { value: { values: [ { - startTime: "", + startTime: "2025-04-01T00:00:00Z", endTime: "", value: attributeValueFilterAny } From 4abf2ecc4123eeac850cca6e23fa00eeb4cbdf4a Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:27:06 +0530 Subject: [PATCH 05/23] fix : fallback when operation changes. --- opengin/core-api/engine/attribute_resolver.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/opengin/core-api/engine/attribute_resolver.go b/opengin/core-api/engine/attribute_resolver.go index 5cbac57a..c9e5c623 100644 --- a/opengin/core-api/engine/attribute_resolver.go +++ b/opengin/core-api/engine/attribute_resolver.go @@ -133,21 +133,24 @@ func (p *EntityAttributeProcessor) ProcessEntityAttributes(ctx context.Context, // not the entity level. The entity level timestamp is used for the entity itself. // start time is required to create the attribute - if value.StartTime == "" { + if operation != "read" && value.StartTime == "" { attributeResults[attrName] = &Result{ Success: false, Error: fmt.Errorf("StartTime is required for attribute: %s", attrName), } continue } + var attributeStartTime time.Time - attributeStartTime, err := time.Parse(time.RFC3339, value.StartTime) - if err != nil { - attributeResults[attrName] = &Result{ - Success: false, - Error: fmt.Errorf("invalid StartTime format for attribute %s: %v", attrName, err), + if value.StartTime != "" { + attributeStartTime, err = time.Parse(time.RFC3339, value.StartTime) + if err != nil { + attributeResults[attrName] = &Result{ + Success: false, + Error: fmt.Errorf("invalid StartTime format for attribute %s: %v", attrName, err), + } + continue } - continue } var attributeEndTime *time.Time From d38601311fd60ddb39d40e02b3bc08aafb770989 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 31 Mar 2026 16:30:49 +0530 Subject: [PATCH 06/23] test : adding e2e tests --- opengin/tests/e2e/basic_core_tests.py | 89 +++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index 6ea4ee42..b7c992ab 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -66,7 +66,7 @@ def decode_protobuf_any_value(any_value): class TestCOREAPI(unittest.TestCase): def setUp(self): print("šŸŸ¢ Setting up test environment...") - ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', f"http://0.0.0.0:8080") + ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', "http://0.0.0.0:8080") print("šŸŸ¢ INGESTION_SERVICE_URL: ", ingestion_service_url) self.base_url = f"{ingestion_service_url}/entities" print("šŸŸ¢ BASE_URL: ", self.base_url) @@ -402,8 +402,10 @@ def __init__(self): ] self.START_DATE = "2025-11-01T00:00:00Z" self.DATA_START_DATE = "2025-12-01T00:00:00Z" + self.DATA_END_DATE = "2025-12-31T00:00:00Z" self.ATTRIBUTE_NAME = "employee_data" - + self.ATTRIBUTE_NAME_2 = "employee_data_2" + self.ATTRIBUTE_NAME_3 = "employee_data_3" def create_minister_with_attributes(self): """Create a Minister entity.""" @@ -453,7 +455,6 @@ def create_minister_with_attributes(self): print(f"Response: {res.status_code} - {res.text}") print("āœ… Created Minister entity with attributes.") - def read_minister(self): """Read the Minister entity.""" print("\nšŸŸ¢ Reading Minister entity...") @@ -504,7 +505,6 @@ def update_attributes_stage_1(self): assert res.status_code in [200], f"Failed to update attributes: {res.text}" print("āœ… Attributes updated.") - def update_attributes_stage_2(self): """Update the attributes of the Minister entity.""" print("\nšŸŸ¢ Updating attributes stage 2...") @@ -536,10 +536,87 @@ def update_attributes_stage_2(self): assert res.status_code in [200], f"Failed to update attributes: {res.text}" print("āœ… Attributes updated.") + def create_minister_with_attributes_with_startDate_and_endDate(self): + """Create a Minister entity.""" + print("\nšŸŸ¢ Creating Minister with attribute with startDate and endDate...") + + payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_2, + "value": { + "values": [ + { + "startTime": self.DATA_START_DATE, + "endTime": self.DATA_END_DATE, + "value": { + "columns": ["id", "name", "age", "department", "salary"], + "rows": [ + [1, "John Doe", 30, "Engineering", 75000.50], + [2, "Jane Smith", 25, "Marketing", 65000], + [3, "Bob Wilson", 35, "Sales", 85000.75], + [4, "Alice Brown", 28, "Engineering", 70000.25], + [5, "Charlie Davis", 32, "Finance", 80000] + ] + } + } + ] + } + } + ] + } + + res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to update attributes: {res.text}" + + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Updated Minister entity with attributes with startDate + endDate.") + + def create_minister_with_attributes_without_startDate(self): + """Create a Minister entity.""" + print("\nšŸŸ¢ Creating Minister entity + attribute without startDate...") + + payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_3, + "value": { + "values": [ + { + "startTime": "", + "endTime": "", + "value": { + "columns": ["id", "name", "age", "department", "salary"], + "rows": [ + [1, "John Doe", 30, "Engineering", 75000.50], + [2, "Jane Smith", 25, "Marketing", 65000], + [3, "Bob Wilson", 35, "Sales", 85000.75], + [4, "Alice Brown", 28, "Engineering", 70000.25], + [5, "Charlie Davis", 32, "Finance", 80000] + ] + } + } + ] + } + } + ], + "relationships": [] + } + + res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=payload) + print(res.status_code, res.json()) + assert res.status_code == 500, f"Expected 500, got {res.status_code}: {res.text}" + + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Received expected error for Minister creation.") + def get_base_url(): print("šŸŸ¢ Setting up test environment...") - ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', f"http://0.0.0.0:8080") + ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', "http://0.0.0.0:8080") print("šŸŸ¢ INGESTION_SERVICE_URL: ", ingestion_service_url) return f"{ingestion_service_url}/entities" @@ -846,6 +923,8 @@ def test_duplicate_primary_key(self): attribute_validation_tests.read_minister() attribute_validation_tests.update_attributes_stage_1() attribute_validation_tests.update_attributes_stage_2() + attribute_validation_tests.create_minister_with_attributes_with_startDate_and_endDate() + attribute_validation_tests.create_minister_with_attributes_without_startDate() print("\nšŸŸ¢ Running Attribute Validation Tests... Done") print("\nšŸŸ¢ Running Tabular Integrity Tests...") From 90d23f703574453d6bebcddde95d9abc0e3072ae Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Tue, 31 Mar 2026 18:52:09 +0530 Subject: [PATCH 07/23] test : adding read tests e2e --- opengin/tests/e2e/basic_read_tests.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index 2f5b73b3..77a5e9c9 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -633,7 +633,7 @@ def create_entity_for_read(): "values": [ { "startTime": "2024-11-01T00:00:00Z", - "endTime": "", + "endTime": "2024-11-30T00:00:00Z", "value": { "columns": ["e_id", "name", "age", "department", "salary"], "rows": [ @@ -789,6 +789,23 @@ def create_entity_for_read(): assert res.status_code == 201 or res.status_code == 200, f"Failed to create entity: {res.text}" print("āœ… Created base entity for read tests.") + +def check_relationship_timeStamps(): + res = requests.get(INGESTION_API_URL + "/" + RELATED_ID_1) + assert res.status_code == 200, f"Failed to fetch entity: {res.text}" + print("āœ… Fetched entity.") + + # Verify the response data + response_data = res.json() + relationship = response_data["relationships"][0] + start_time = relationship["value"]["startTime"] + end_time = relationship["value"]["endTime"] + print("start_time: ", start_time) + print("end_time: ", end_time) + assert start_time == "2024-11-01T00:00:00Z", f"Expected start time '2024-11-01T00:00:00Z', got '{start_time}'" + assert end_time == "2024-11-30T00:00:00Z", f"Expected end time '2024-11-30T00:00:00Z', got '{end_time}'" + + def test_attribute_fields_combinations(): """Test different field combinations for attribute retrieval.""" print("\nšŸ” Testing attribute field combinations...") @@ -2383,6 +2400,7 @@ def test_minister_relations_filter_by_active_at_only(): test_protobuf_decoding() print("Creating entity for read tests...") create_entity_for_read() + check_relationship_timeStamps() print("Testing generic validation examples...") test_generic_validation_examples() print("Testing attribute field combinations...") From 9374b3efd4aeeb8985499eba5054d6b97080ca25 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 9 Apr 2026 12:39:42 +0530 Subject: [PATCH 08/23] review : resolve review comments --- opengin/ingestion-api/tests/service_test.bal | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/opengin/ingestion-api/tests/service_test.bal b/opengin/ingestion-api/tests/service_test.bal index b8aa23bc..6bb5a3cf 100644 --- a/opengin/ingestion-api/tests/service_test.bal +++ b/opengin/ingestion-api/tests/service_test.bal @@ -350,10 +350,10 @@ function testMetadataUpdating() returns error? { }; Entity updatedReadResponse = check ep->ReadEntity(updatedReadRequest); verifyMetadata(updatedReadResponse.metadata, { - "key1": updatedValue1, - "key2": updatedValue2, - "key3": newValue3 - }); + "key1": updatedValue1, + "key2": updatedValue2, + "key3": newValue3 + }); io:println("Updated metadata verified"); // Clean up From 16b547a97cc3ab4a6f33d4b2a8ee6cea504a3195 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:23:11 +0530 Subject: [PATCH 09/23] fix : fixing and tuning the test cases --- opengin/tests/e2e/basic_core_tests.py | 867 ++++++++++++++++++++------ 1 file changed, 661 insertions(+), 206 deletions(-) diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index b7c992ab..bd147692 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -35,50 +35,226 @@ """ + class CoreTestUtils: + # DEBUG: Attribute protobuf response cannot be decode by the decode_protobuf_any_value() function , so i am using this here. + @staticmethod + def decode_protobuf_any_value_2(any_value): + """Decode a protobuf Any value to get the actual value""" + if isinstance(any_value, dict) and 'typeUrl' in any_value and 'value' in any_value: + type_url = any_value['typeUrl'] + value = any_value['value'] + + if 'StringValue' in type_url: + try: + # If it's hex encoded (which appears to be the case) + hex_value = value + binary_data = bytes.fromhex(hex_value) + # For StringValue in hex format, typically the structure is: + # 0A (field tag) + 03 (length) + actual string bytes + # Skip the first 2 bytes (field tag and length) + if len(binary_data) > 2: + return binary_data[2:].decode('utf-8') + except Exception as e: + print(f"Failed to decode StringValue: {e}") + return value + + elif 'Struct' in type_url: + try: + # For Struct type, the value is hex-encoded protobuf data + binary_data = bytes.fromhex(value) + + # Try to use protobuf library first + try: + from google.protobuf import struct_pb2 + from google.protobuf.json_format import MessageToDict + + struct_msg = struct_pb2.Struct() + struct_msg.ParseFromString(binary_data) + result = MessageToDict(struct_msg) + + # If the result has a 'data' field that's a string, try to parse it as JSON + if isinstance(result, dict) and 'data' in result and isinstance(result['data'], str): + try: + data_json = json.loads(result['data']) + return data_json + except json.JSONDecodeError: + pass + + return result + + except ImportError: + print("protobuf library not available, trying manual extraction") + except Exception as e: + print(f"Failed to decode with protobuf library: {e}") + + # Manual extraction fallback - look for JSON patterns in the binary data + try: + # Convert binary data to string and look for JSON + text_data = binary_data.decode('utf-8', errors='ignore') + print(f"Debug: Extracted text from binary: {repr(text_data[:200])}...") + + # Look for the actual start of JSON by searching for common patterns + # like {"columns" or {"rows" which indicate the start of our data + json_patterns = ['{"columns"', '{"rows"', '{"data"'] + start_idx = -1 + for pattern in json_patterns: + idx = text_data.find(pattern) + if idx != -1: + start_idx = idx + break + + if start_idx != -1: + # Find the matching closing brace + end_idx = text_data.rfind('}') + if end_idx != -1 and end_idx > start_idx: + json_str = text_data[start_idx:end_idx + 1] + print(f"Debug: Extracted JSON string: {repr(json_str[:100])}...") + return json.loads(json_str) + + except Exception as e: + print(f"Failed to extract JSON from binary data: {e}") + + # Try a different approach - look for specific patterns + try: + # The hex data might contain the JSON in a different format + # Let's try to find the actual JSON content + text_data = binary_data.decode('utf-8', errors='ignore') + + # Look for common JSON patterns + patterns = ['"columns"', '"rows"', '"data"'] + for pattern in patterns: + if pattern in text_data: + # Find the start of the JSON object containing this pattern + start_idx = text_data.find('{', text_data.find(pattern) - 100) + if start_idx != -1: + # Find the matching closing brace + brace_count = 0 + end_idx = start_idx + for i, char in enumerate(text_data[start_idx:], start_idx): + if char == '{': + brace_count += 1 + elif char == '}': + brace_count -= 1 + if brace_count == 0: + end_idx = i + break + + if end_idx > start_idx: + json_str = text_data[start_idx:end_idx + 1] + print(f"Debug: Found JSON with pattern {pattern}: {repr(json_str[:100])}...") + try: + return json.loads(json_str) + except json.JSONDecodeError as e: + print(f"Debug: JSON decode failed: {e}") + continue + + except Exception as e: + print(f"Failed to extract JSON with pattern matching: {e}") + + # If all else fails, try to decode as base64 + try: + import base64 + # The hex might actually be base64 encoded + decoded_bytes = base64.b64decode(value) + json_str = decoded_bytes.decode('utf-8') + return json.loads(json_str) + except Exception as e: + print(f"Failed to decode as base64: {e}") + + # Final fallback: return the hex value + return value + + except Exception as e: + print(f"Failed to decode Struct: {e}") + return value + + elif 'Int32Value' in type_url or 'Int64Value' in type_url: + try: + # For integer values + hex_value = value + binary_data = bytes.fromhex(hex_value) + # Skip field tag and length bytes + if len(binary_data) > 2: + return int.from_bytes(binary_data[2:], byteorder='little') + except Exception as e: + print(f"Failed to decode integer value: {e}") + return value + + elif 'DoubleValue' in type_url or 'FloatValue' in type_url: + try: + # For float/double values + hex_value = value + binary_data = bytes.fromhex(hex_value) + # Skip field tag and length bytes + if len(binary_data) > 2: + import struct + return struct.unpack(' 2: - return binary_data[2:].decode('utf-8') + return binary_data[2:].decode("utf-8") except Exception as e: print(f"Failed to decode protobuf value: {e}") # Return the original value if decoding fails return any_value.strip() + class TestCOREAPI(unittest.TestCase): def setUp(self): print("šŸŸ¢ Setting up test environment...") - ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', "http://0.0.0.0:8080") + ingestion_service_url = os.getenv( + "INGESTION_SERVICE_URL", "http://0.0.0.0:8080" + ) print("šŸŸ¢ INGESTION_SERVICE_URL: ", ingestion_service_url) self.base_url = f"{ingestion_service_url}/entities" print("šŸŸ¢ BASE_URL: ", self.base_url) -class BasicCORETests: +class BasicCORETests: def __init__(self, entity_id): self.entity_id = entity_id self.base_url = get_base_url() - self.headers = { - 'Content-Type': 'application/json' - } + self.base_read_url = get_base_read_url() + self.headers = {"Content-Type": "application/json"} self.payload = self.create_payload() def create_payload(self): @@ -92,38 +268,38 @@ def create_payload(self): "name": { "startTime": "2024-03-17T10:00:00Z", "endTime": "", - "value": "entity-name" + "value": "entity-name", }, "metadata": [ {"key": "owner", "value": "test-user"}, {"key": "version", "value": "1.0"}, - {"key": "developer", "value": "V8A"} + {"key": "developer", "value": "V8A"}, ], "attributes": [], - "relationships": [] + "relationships": [], }, "update": { "id": self.entity_id, "created": "2024-03-18T00:00:00Z", - "name": { - "startTime": "2024-03-18T00:00:00Z", - "value": "entity-name" - }, - "metadata": [{"key": "version", "value": "5.0"}] - } + "name": {"startTime": "2024-03-18T00:00:00Z", "value": "entity-name"}, + "metadata": [{"key": "version", "value": "5.0"}], + }, } class MetadataValidationTests(BasicCORETests): - def __init__(self, entity_id): super().__init__(entity_id) def create_entity(self): """Creates an entity and validates the response.""" print("\nšŸŸ¢ Creating entity...") - response = requests.post(self.base_url, json=self.payload["create"], headers={"Content-Type": "application/json"}) - + response = requests.post( + self.base_url, + json=self.payload["create"], + headers={"Content-Type": "application/json"}, + ) + if response.status_code == 201: print("āœ… Entity created:", json.dumps(response.json(), indent=2)) else: @@ -134,7 +310,7 @@ def read_entity(self): """Reads and validates the created entity.""" print("\nšŸŸ¢ Reading entity...") response = requests.get(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 200: data = response.json() assert data["id"] == self.entity_id, "Read entity ID mismatch" @@ -146,11 +322,17 @@ def read_entity(self): def update_entity(self): """Updates the entity and validates the response.""" print("\nšŸŸ¢ Updating entity...") - response = requests.put(f"{self.base_url}/{self.entity_id}", json=self.payload["update"], headers={"Content-Type": "application/json"}) - + response = requests.put( + f"{self.base_url}/{self.entity_id}", + json=self.payload["update"], + headers={"Content-Type": "application/json"}, + ) + if response.status_code == 200: updated_entity = response.json() - decoded_value = CoreTestUtils.decode_protobuf_any_value(updated_entity["metadata"][0]["value"]) + decoded_value = CoreTestUtils.decode_protobuf_any_value( + updated_entity["metadata"][0]["value"] + ) print("decoded value: ", decoded_value) assert decoded_value == "5.0", "Update did not modify metadata" print("āœ… Entity updated:", json.dumps(updated_entity, indent=2)) @@ -162,10 +344,12 @@ def validate_update(self): """Validates that the update has been applied correctly.""" print("\nšŸŸ¢ Validating update...") response = requests.get(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 200: updated_data = response.json() - decoded_value = CoreTestUtils.decode_protobuf_any_value(updated_data["metadata"][0]["value"]) + decoded_value = CoreTestUtils.decode_protobuf_any_value( + updated_data["metadata"][0]["value"] + ) assert decoded_value == "5.0", "Updated entity does not reflect changes" print("āœ… Update Validation Passed:", json.dumps(updated_data, indent=2)) else: @@ -176,7 +360,7 @@ def delete_entity(self): """Deletes the entity.""" print("\nšŸŸ¢ Deleting entity...") response = requests.delete(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 204: print("āœ… Entity deleted successfully.") else: @@ -187,31 +371,34 @@ def verify_deletion(self): """Verifies that the entity has been deleted.""" print("\nšŸŸ¢ Verifying deletion...") response = requests.get(f"{self.base_url}/{self.entity_id}") - + if response.status_code == 500: print("āŒ Server error occurred:", response.text) sys.exit(1) else: - print(f"\nšŸŸ¢ Entity was not deleted properly: {response.status_code} {response.text}") + print( + f"\nšŸŸ¢ Entity was not deleted properly: {response.status_code} {response.text}" + ) class GraphEntityTests(BasicCORETests): - def __init__(self): super().__init__(None) self.MINISTER_ID = "minister_education" self.DEPARTMENTS = [ {"id": "dept_exams", "name": "Department of Exams"}, {"id": "dept_nie", "name": "National Institute of Education"}, - {"id": "dept_ed_publications", "name": "Department of Educational Publications"} + { + "id": "dept_ed_publications", + "name": "Department of Educational Publications", + }, ] self.START_DATE = "2015-04-11T00:00:00Z" - def create_minister(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister entity...") - + payload = { "id": self.MINISTER_ID, "kind": {"major": "Organization", "minor": "Minister"}, @@ -220,13 +407,13 @@ def create_minister(self): "name": { "startTime": self.START_DATE, "endTime": "", - "value": "Minister of Education" + "value": "Minister of Education", }, "metadata": [], "attributes": [], - "relationships": [] + "relationships": [], } - + res = requests.post(self.base_url, json=payload) print(res.status_code, res.json()) assert res.status_code in [201], f"Failed to create Minister: {res.text}" @@ -234,31 +421,39 @@ def create_minister(self): print(f"Response: {res.status_code} - {res.text}") print("āœ… Created Minister entity.") - def read_minister(self): """Read the Minister entity.""" print("\nšŸŸ¢ Reading Minister entity...") res = requests.get(f"{self.base_url}/{self.MINISTER_ID}") print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to read Minister: {res.text}" - + # Verify the response data response_data = res.json() - assert response_data["id"] == self.MINISTER_ID, f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" - assert response_data["kind"]["major"] == "Organization", f"Expected major kind 'Organization', got {response_data['kind']['major']}" - assert response_data["kind"]["minor"] == "Minister", f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" - assert response_data["created"] == self.START_DATE, f"Expected created date {self.START_DATE}, got {response_data['created']}" + assert response_data["id"] == self.MINISTER_ID, ( + f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" + ) + assert response_data["kind"]["major"] == "Organization", ( + f"Expected major kind 'Organization', got {response_data['kind']['major']}" + ) + assert response_data["kind"]["minor"] == "Minister", ( + f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" + ) + assert response_data["created"] == self.START_DATE, ( + f"Expected created date {self.START_DATE}, got {response_data['created']}" + ) # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) - assert decoded_name == "Minister of Education", f"Expected name 'Minister of Education', got {name_value}" + assert decoded_name == "Minister of Education", ( + f"Expected name 'Minister of Education', got {name_value}" + ) print(f"āœ… Validated {decoded_name} entity.") - def create_departments(self): """Create Department entities.""" print("\nšŸŸ¢ Creating Department entities...") - + for dept in self.DEPARTMENTS: payload = { "id": dept["id"], @@ -268,44 +463,54 @@ def create_departments(self): "name": { "startTime": self.START_DATE, "endTime": "", - "value": dept["name"] + "value": dept["name"], }, - "metadata": [] + "metadata": [], } - + res = requests.post(self.base_url, json=payload) - assert res.status_code in [200, 201], f"Failed to create {dept['name']}: {res.text}" + assert res.status_code in [200, 201], ( + f"Failed to create {dept['name']}: {res.text}" + ) print(f"Response: {res.status_code} - {res.text}") print(f"āœ… Created {dept['name']} entity.") - def read_departments(self): """Validate the Department entities in Neo4j.""" print("\nšŸŸ¢ Validating Department entities in Neo4j...") - + for dept in self.DEPARTMENTS: res = requests.get(f"{self.base_url}/{dept['id']}") assert res.status_code == 200, f"Failed to read {dept['name']}: {res.text}" - + # Verify the response data response_data = res.json() - assert response_data["id"] == dept["id"], f"Expected ID {dept['id']}, got {response_data['id']}" - assert response_data["kind"]["major"] == "Organization", f"Expected major kind 'Organization', got {response_data['kind']['major']}" - assert response_data["kind"]["minor"] == "Department", f"Expected minor kind 'Department', got {response_data['kind']['minor']}" - assert response_data["created"] == self.START_DATE, f"Expected created date {self.START_DATE}, got {response_data['created']}" - + assert response_data["id"] == dept["id"], ( + f"Expected ID {dept['id']}, got {response_data['id']}" + ) + assert response_data["kind"]["major"] == "Organization", ( + f"Expected major kind 'Organization', got {response_data['kind']['major']}" + ) + assert response_data["kind"]["minor"] == "Department", ( + f"Expected minor kind 'Department', got {response_data['kind']['minor']}" + ) + assert response_data["created"] == self.START_DATE, ( + f"Expected created date {self.START_DATE}, got {response_data['created']}" + ) + # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) - assert decoded_name == dept["name"], f"Expected name '{dept['name']}', got {decoded_name}" - + assert decoded_name == dept["name"], ( + f"Expected name '{dept['name']}', got {decoded_name}" + ) + print(f"āœ… Validated {dept['name']} entity.") - - + def create_relationships(self): """Create HAS_DEPARTMENT relationships from Minister to Departments.""" print("\nšŸ”— Creating relationships...") - + for dept in self.DEPARTMENTS: rel_id = f"rel_{dept['id']}" payload = { @@ -313,8 +518,7 @@ def create_relationships(self): "kind": {}, "created": "", "terminated": "", - "name": { - }, + "name": {}, "metadata": [], "attributes": [], "relationships": [ @@ -325,28 +529,30 @@ def create_relationships(self): "startTime": self.START_DATE, "endTime": "", "id": rel_id, - "name": "HAS_DEPARTMENT" - } + "name": "HAS_DEPARTMENT", + }, } - ] + ], } - + url = f"{self.base_url}/{self.MINISTER_ID}" res = requests.put(url, json=payload) if res.status_code in [200]: print(f"āœ… Created relationship between Minister and {dept['name']}.") else: - print(f"āŒ Failed to create relationship for {dept['name']}: {res.status_code} - {res.text}") + print( + f"āŒ Failed to create relationship for {dept['name']}: {res.status_code} - {res.text}" + ) sys.exit(1) def update_relationships(self): """Update HAS_DEPARTMENT relationships to add termination dates.""" print("\nšŸ”— Updating relationships...") - + # Set a termination date for all relationships termination_date = "2024-12-31T00:00:00Z" - + for dept in self.DEPARTMENTS: rel_id = f"rel_{dept['id']}" payload = { @@ -361,19 +567,23 @@ def update_relationships(self): "id": rel_id, "relatedEntityId": "", "startTime": "", - "name": "" - } + "name": "", + }, } - ] + ], } - + url = f"{self.base_url}/{self.MINISTER_ID}" res = requests.put(url, json=payload) if res.status_code in [200]: - print(f"āœ… Updated relationship between Minister and {dept['name']} with termination date {termination_date}.") + print( + f"āœ… Updated relationship between Minister and {dept['name']} with termination date {termination_date}." + ) else: - print(f"āŒ Failed to update relationship for {dept['name']}: {res.status_code} - {res.text}") + print( + f"āŒ Failed to update relationship for {dept['name']}: {res.status_code} - {res.text}" + ) sys.exit(1) # Verify the updates @@ -384,33 +594,43 @@ def update_relationships(self): relationships = data.get("relationships", []) for rel in relationships: if rel.get("name") == "HAS_DEPARTMENT": - assert rel.get("endTime") == termination_date, f"Expected termination date {termination_date}, got {rel.get('endTime')}" + assert rel.get("endTime") == termination_date, ( + f"Expected termination date {termination_date}, got {rel.get('endTime')}" + ) print("āœ… Successfully verified relationship updates.") else: - print(f"āŒ Failed to verify relationship updates: {res.status_code} - {res.text}") + print( + f"āŒ Failed to verify relationship updates: {res.status_code} - {res.text}" + ) sys.exit(1) class AttributeValidationTests(BasicCORETests): - def __init__(self): super().__init__(None) self.MINISTER_ID = "minister_of_finance_and_economy" self.DEPARTMENTS = [ {"id": "dept_finance", "name": "Department of Finance"}, - {"id": "dept_economy", "name": "Department of Economy"} + {"id": "dept_economy", "name": "Department of Economy"}, ] self.START_DATE = "2025-11-01T00:00:00Z" self.DATA_START_DATE = "2025-12-01T00:00:00Z" self.DATA_END_DATE = "2025-12-31T00:00:00Z" + + self.INVALID_DATE = "2025-12-31a" + self.INVALID_DATE_2 = "2025-12-33T00:00:00Z" + self.INVALID_DATE_3 = "test-123" + self.ATTRIBUTE_NAME = "employee_data" self.ATTRIBUTE_NAME_2 = "employee_data_2" self.ATTRIBUTE_NAME_3 = "employee_data_3" + self.ATTRIBUTE_NAME_4 = "employee_data_4" + self.ATTRIBUTE_NAME_5 = "employee_data_5" def create_minister_with_attributes(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister entity...") - + payload = { "id": self.MINISTER_ID, "kind": {"major": "Organization", "minor": "Minister"}, @@ -419,7 +639,7 @@ def create_minister_with_attributes(self): "name": { "startTime": self.START_DATE, "endTime": "", - "value": "Minister of Finance and Economy" + "value": "Minister of Finance and Economy", }, "metadata": [], "attributes": [ @@ -431,23 +651,29 @@ def create_minister_with_attributes(self): "startTime": self.DATA_START_DATE, "endTime": "", "value": { - "columns": ["id", "name", "age", "department", "salary"], + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ [1, "John Doe", 30, "Engineering", 75000.50], [2, "Jane Smith", 25, "Marketing", 65000], [3, "Bob Wilson", 35, "Sales", 85000.75], [4, "Alice Brown", 28, "Engineering", 70000.25], - [5, "Charlie Davis", 32, "Finance", 80000] - ] - } + [5, "Charlie Davis", 32, "Finance", 80000], + ], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } - + res = requests.post(self.base_url, json=payload) print(res.status_code, res.json()) assert res.status_code in [201], f"Failed to create Minister: {res.text}" @@ -461,46 +687,72 @@ def read_minister(self): res = requests.get(f"{self.base_url}/{self.MINISTER_ID}") print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to read Minister: {res.text}" - + # Verify the response data response_data = res.json() print(response_data) - assert response_data["id"] == self.MINISTER_ID, f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" - assert response_data["kind"]["major"] == "Organization", f"Expected major kind 'Organization', got {response_data['kind']['major']}" - assert response_data["kind"]["minor"] == "Minister", f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" - assert response_data["created"] == self.START_DATE, f"Expected created date {self.START_DATE}, got {response_data['created']}" + assert response_data["id"] == self.MINISTER_ID, ( + f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" + ) + assert response_data["kind"]["major"] == "Organization", ( + f"Expected major kind 'Organization', got {response_data['kind']['major']}" + ) + assert response_data["kind"]["minor"] == "Minister", ( + f"Expected minor kind 'Minister', got {response_data['kind']['minor']}" + ) + assert response_data["created"] == self.START_DATE, ( + f"Expected created date {self.START_DATE}, got {response_data['created']}" + ) # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) - assert decoded_name == "Minister of Finance and Economy", f"Expected name 'Minister of Finance and Economy', got {decoded_name}" + assert decoded_name == "Minister of Finance and Economy", ( + f"Expected name 'Minister of Finance and Economy', got {decoded_name}" + ) def update_attributes_stage_1(self): """Update the attributes of the Minister entity.""" print("\nšŸŸ¢ Updating attributes stage 1...") update_payload = { - "id": self.MINISTER_ID, - "attributes": [ - { - "key": self.ATTRIBUTE_NAME, - "value": { - "values": [ - { - "startTime": "2024-08-01T00:00:00Z", - "endTime": "", - "value": { - "columns": ["id", "name", "age", "department", "salary"], + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME, + "value": { + "values": [ + { + "startTime": "2024-08-01T00:00:00Z", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ - [6, "Peter Parker", 30, "Engineering", 75000.50], + [ + 6, + "Peter Parker", + 30, + "Engineering", + 75000.50, + ], [7, "Clark Kent", 25, "Media", 65000], - ] + ], + }, } - } - ] + ] + }, } - } - ] - } - res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=update_payload, headers={"Content-Type": "application/json"}) + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to update attributes: {res.text}" print("āœ… Attributes updated.") @@ -509,37 +761,57 @@ def update_attributes_stage_2(self): """Update the attributes of the Minister entity.""" print("\nšŸŸ¢ Updating attributes stage 2...") update_payload = { - "id": self.MINISTER_ID, - "attributes": [ - { - "key": self.ATTRIBUTE_NAME, - "value": { - "values": [ - { - "startTime": "2024-08-01T00:00:00Z", - "endTime": "", - "value": { - "columns": ["id", "name", "age", "department", "salary"], + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME, + "value": { + "values": [ + { + "startTime": "2024-08-01T00:00:00Z", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ [8, "Iris West", 30, "Marketing", 12300.50], [9, "Barry Allen", 25, "Sales", 22300.50], - ] + ], + }, } - } - ] + ] + }, } - } - ] - } - res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=update_payload, headers={"Content-Type": "application/json"}) + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to update attributes: {res.text}" print("āœ… Attributes updated.") + # normalize the attribute response and compare + def normalize(self, data): + columns = data["columns"] + rows = data["rows"] + + return [ + dict(zip(columns, row)) + for row in rows + ] + def create_minister_with_attributes_with_startDate_and_endDate(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister with attribute with startDate and endDate...") - + payload = { "id": self.MINISTER_ID, "attributes": [ @@ -551,33 +823,95 @@ def create_minister_with_attributes_with_startDate_and_endDate(self): "startTime": self.DATA_START_DATE, "endTime": self.DATA_END_DATE, "value": { - "columns": ["id", "name", "age", "department", "salary"], + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ [1, "John Doe", 30, "Engineering", 75000.50], [2, "Jane Smith", 25, "Marketing", 65000], [3, "Bob Wilson", 35, "Sales", 85000.75], [4, "Alice Brown", 28, "Engineering", 70000.25], - [5, "Charlie Davis", 32, "Finance", 80000] - ] - } + [5, "Charlie Davis", 32, "Finance", 80000], + ], + }, } ] - } + }, } - ] + ], } - + res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=payload) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to update attributes: {res.text}" - print(f"Response: {res.status_code} - {res.text}") + print("\nšŸŸ¢ checking Minister with attribute with startDate and endDate...") + + url = f"{self.base_read_url}/v1/entities/{self.MINISTER_ID}/attributes/{self.ATTRIBUTE_NAME_2}" + payload = {} + + res = requests.post(url, json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to read Minister: {res.text}" + + response_data = res.json() + print(response_data) + value = response_data["value"] + print(f"value {value}") + decoded_value = CoreTestUtils.decode_protobuf_any_value_2(value) + + expected_data = { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [1, "John Doe", 30, "Engineering", 75000.50], + [2, "Jane Smith", 25, "Marketing", 65000], + [3, "Bob Wilson", 35, "Sales", 85000.75], + [4, "Alice Brown", 28, "Engineering", 70000.25], + [5, "Charlie Davis", 32, "Finance", 80000], + ], + } + + assert self.normalize(expected_data) == self.normalize(decoded_value), f"Expected {expected_data}, got {decoded_value}" + + print("Decoded value: ", decoded_value) + + url = f"{self.base_url}/{self.MINISTER_ID}" + + res = requests.get(url) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to read Minister: {res.text}" + + response_data = res.json() + print(response_data) + + relationships = response_data["relationships"] + + print("relationships", relationships) + + # check for the start data and the end date + for relationship in relationships: + if relationship["value"]["startTime"] != "" and relationship["value"]["endTime"] != "": + assert relationship["value"]["startTime"] == self.DATA_START_DATE + assert relationship["value"]["endTime"] == self.DATA_END_DATE + else: + pass + print("āœ… Updated Minister entity with attributes with startDate + endDate.") def create_minister_with_attributes_without_startDate(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister entity + attribute without startDate...") - + payload = { "id": self.MINISTER_ID, "attributes": [ @@ -589,40 +923,153 @@ def create_minister_with_attributes_without_startDate(self): "startTime": "", "endTime": "", "value": { - "columns": ["id", "name", "age", "department", "salary"], + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], "rows": [ [1, "John Doe", 30, "Engineering", 75000.50], [2, "Jane Smith", 25, "Marketing", 65000], [3, "Bob Wilson", 35, "Sales", 85000.75], [4, "Alice Brown", 28, "Engineering", 70000.25], - [5, "Charlie Davis", 32, "Finance", 80000] - ] - } + [5, "Charlie Davis", 32, "Finance", 80000], + ], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } - + res = requests.put(f"{self.base_url}/{self.MINISTER_ID}", json=payload) print(res.status_code, res.json()) - assert res.status_code == 500, f"Expected 500, got {res.status_code}: {res.text}" + assert res.status_code == 500, ( + f"Expected 500, got {res.status_code}: {res.text}" + ) print(f"Response: {res.status_code} - {res.text}") print("āœ… Received expected error for Minister creation.") + def add_attributes_to_minister_with_invalid_time_format(self): + print("\nšŸŸ¢ Updating attributes stage 3...") + update_payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_4, + "value": { + "values": [ + { + "startTime": self.INVALID_DATE, + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [ + 6, + "Peter Parker", + 30, + "Engineering", + 75000.50, + ], + [7, "Clark Kent", 25, "Media", 65000], + ], + }, + } + ] + }, + } + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) + print(res.status_code, res.json()) + assert res.status_code == 500, ( + f"Expected 500, got {res.status_code}: {res.text}" + ) + + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Received expected error for Minister creation.") + + def add_attributes_to_minister_with_invalid_time_format_2(self): + print("\nšŸŸ¢ Updating attributes stage 3...") + update_payload = { + "id": self.MINISTER_ID, + "attributes": [ + { + "key": self.ATTRIBUTE_NAME_5, + "value": { + "values": [ + { + "startTime": self.DATA_START_DATE, + "endTime": self.INVALID_DATE, + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [ + 6, + "Peter Parker", + 30, + "Engineering", + 75000.50, + ], + [7, "Clark Kent", 25, "Media", 65000], + ], + }, + } + ] + }, + } + ], + } + res = requests.put( + f"{self.base_url}/{self.MINISTER_ID}", + json=update_payload, + headers={"Content-Type": "application/json"}, + ) + print(res.status_code, res.json()) + assert res.status_code == 500, ( + f"Expected 500, got {res.status_code}: {res.text}" + ) + print(f"Response: {res.status_code} - {res.text}") + print("āœ… Received expected error for Minister creation.") + + def get_base_url(): print("šŸŸ¢ Setting up test environment...") - ingestion_service_url = os.getenv('INGESTION_SERVICE_URL', "http://0.0.0.0:8080") + ingestion_service_url = os.getenv("INGESTION_SERVICE_URL", "http://0.0.0.0:8080") print("šŸŸ¢ INGESTION_SERVICE_URL: ", ingestion_service_url) return f"{ingestion_service_url}/entities" +def get_base_read_url(): + print("šŸŸ¢ Setting up test environment...") + read_service_url = os.getenv("READ_SERVICE_URL", "http://read:8081") + print("šŸŸ¢ READ_SERVICE_URL: ", read_service_url) + return read_service_url -# Tabular Attribute Integrity Tests +# Tabular Attribute Integrity Tests class TabularIntegrityTests(BasicCORETests): """Tests that verify safety constraints for tabular attribute ingestion. @@ -639,7 +1086,6 @@ def __init__(self): super().__init__(None) self.ATTR_NAME = "test_data" - # Test 1 ā€“ Idempotency def test_idempotency(self): @@ -653,7 +1099,11 @@ def test_idempotency(self): "kind": {"major": "test", "minor": "tabular-integrity"}, "created": "2025-01-01T00:00:00Z", "terminated": "", - "name": {"startTime": "2025-01-01T00:00:00Z", "endTime": "", "value": "Idempotency Test"}, + "name": { + "startTime": "2025-01-01T00:00:00Z", + "endTime": "", + "value": "Idempotency Test", + }, "metadata": [], "attributes": [ { @@ -667,15 +1117,15 @@ def test_idempotency(self): "columns": ["id", "name", "department"], "rows": [ [1, "Alice", "Engineering"], - [2, "Bob", "Marketing"] - ] - } + [2, "Bob", "Marketing"], + ], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } # Create entity + first batch @@ -697,14 +1147,14 @@ def test_idempotency(self): "columns": ["id", "name", "department"], "rows": [ [3, "Charlie", "Finance"], - [4, "Diana", "Sales"] - ] - } + [4, "Diana", "Sales"], + ], + }, } ] - } + }, } - ] + ], } # Append second batch ā€“ same attribute name @@ -724,18 +1174,18 @@ def test_idempotency(self): "endTime": "", "value": { "columns": ["id", "name", "department"], - "rows": [[]] - } + "rows": [[]], + }, } ] - } + }, } ] } res = requests.post( f"{self.base_url}/{entity_id}/attributes/test_data", json=read_payload, - headers={"Content-Type": "application/json"} + headers={"Content-Type": "application/json"}, ) # The read API returns 200 or 404 if not supported yet ā€“ skip row count # verification if the read path is not available; still assert the PUT succeeded. @@ -743,7 +1193,9 @@ def test_idempotency(self): data = res.json() print(f" ā„¹ļø Read API returned: {json.dumps(data)[:200]}") else: - print(f" ā„¹ļø Read API returned {res.status_code} ā€“ row-count verification skipped") + print( + f" ā„¹ļø Read API returned {res.status_code} ā€“ row-count verification skipped" + ) print(" āœ… [Test 1] Idempotency PASSED") @@ -760,7 +1212,11 @@ def test_schema_mismatch(self): "kind": {"major": "test", "minor": "tabular-integrity"}, "created": "2025-01-01T00:00:00Z", "terminated": "", - "name": {"startTime": "2025-01-01T00:00:00Z", "endTime": "", "value": "Schema Mismatch Test"}, + "name": { + "startTime": "2025-01-01T00:00:00Z", + "endTime": "", + "value": "Schema Mismatch Test", + }, "metadata": [], "attributes": [ { @@ -772,17 +1228,14 @@ def test_schema_mismatch(self): "endTime": "", "value": { "columns": ["id", "score"], - "rows": [ - [1, 99], - [2, 85] - ] - } + "rows": [[1, 99], [2, 85]], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } res = requests.post(self.base_url, json=initial_payload) assert res.status_code == 201, f"Initial create failed: {res.text}" @@ -801,23 +1254,22 @@ def test_schema_mismatch(self): "endTime": "", "value": { "columns": ["id", "score"], - "rows": [ - [3, "not-a-number"], - [4, "also-wrong"] - ] - } + "rows": [[3, "not-a-number"], [4, "also-wrong"]], + }, } ] - } + }, } - ] + ], } res = requests.put(f"{self.base_url}/{entity_id}", json=bad_payload) assert res.status_code != 200, ( f"Expected an error response when inserting schema-incompatible data, " f"but got HTTP {res.status_code}: {res.text}" ) - print(f" āœ… Schema mismatch correctly rejected: HTTP {res.status_code} ā€“ {res.text}") + print( + f" āœ… Schema mismatch correctly rejected: HTTP {res.status_code} ā€“ {res.text}" + ) print(" āœ… [Test 2] Schema Mismatch PASSED") # Test 3 ā€“ Duplicate Primary Key @@ -833,7 +1285,11 @@ def test_duplicate_primary_key(self): "kind": {"major": "test", "minor": "tabular-integrity"}, "created": "2025-01-01T00:00:00Z", "terminated": "", - "name": {"startTime": "2025-01-01T00:00:00Z", "endTime": "", "value": "Duplicate PK Test"}, + "name": { + "startTime": "2025-01-01T00:00:00Z", + "endTime": "", + "value": "Duplicate PK Test", + }, "metadata": [], "attributes": [ { @@ -845,17 +1301,14 @@ def test_duplicate_primary_key(self): "endTime": "", "value": { "columns": ["id", "value"], - "rows": [ - [1, "first"], - [2, "second"] - ] - } + "rows": [[1, "first"], [2, "second"]], + }, } ] - } + }, } ], - "relationships": [] + "relationships": [], } res = requests.post(self.base_url, json=initial_payload) assert res.status_code == 201, f"Initial create failed: {res.text}" @@ -874,28 +1327,28 @@ def test_duplicate_primary_key(self): "endTime": "", "value": { "columns": ["id", "value"], - "rows": [ - [1, "duplicate!"] - ] - } + "rows": [[1, "duplicate!"]], + }, } ] - } + }, } - ] + ], } res = requests.put(f"{self.base_url}/{entity_id}", json=dup_payload) assert res.status_code != 200, ( f"Expected an error when inserting a duplicate primary key, " f"but got HTTP {res.status_code}: {res.text}" ) - print(f" āœ… Duplicate PK correctly rejected: HTTP {res.status_code} ā€“ {res.text}") + print( + f" āœ… Duplicate PK correctly rejected: HTTP {res.status_code} ā€“ {res.text}" + ) print(" āœ… [Test 3] Duplicate PK PASSED") if __name__ == "__main__": print("šŸš€ Running End-to-End API Test Suite...") - + try: print("šŸŸ¢ Running Metadata Validation Tests...") metadata_validation_tests = MetadataValidationTests(entity_id="123") @@ -924,6 +1377,8 @@ def test_duplicate_primary_key(self): attribute_validation_tests.update_attributes_stage_1() attribute_validation_tests.update_attributes_stage_2() attribute_validation_tests.create_minister_with_attributes_with_startDate_and_endDate() + attribute_validation_tests.add_attributes_to_minister_with_invalid_time_format() + attribute_validation_tests.add_attributes_to_minister_with_invalid_time_format_2() attribute_validation_tests.create_minister_with_attributes_without_startDate() print("\nšŸŸ¢ Running Attribute Validation Tests... Done") @@ -935,7 +1390,7 @@ def test_duplicate_primary_key(self): print("\nšŸŸ¢ Running Tabular Integrity Tests... Done") print("\nšŸŽ‰ All tests passed successfully!") - + except AssertionError as e: print(f"\nāŒ Test failed: {e}") sys.exit(1) From b91e0be5efea20ee6e428ff3372cd4f0b391b96a Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 9 Apr 2026 13:54:03 +0530 Subject: [PATCH 10/23] fix : fixing test coverage --- opengin/tests/e2e/basic_read_tests.py | 58 ++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 11 deletions(-) diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index 77a5e9c9..b3d509f5 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -1,3 +1,4 @@ +from requests import request import requests import json import sys @@ -697,6 +698,37 @@ def create_entity_for_read(): "relationships": [] } + # first entity with a new attribute + payload_chiild_4 = { + "id": RELATED_ID_1, + "attributes": [ + { + "key": "tet_data", + "value": { + "values": [ + { + "startTime": "", + "endTime": "", + "value": { + "columns": [ + "id", + "name", + "age", + "department", + "salary", + ], + "rows": [ + [1, "Iris West", 30, "Marketing", 12300.50], + [2, "Barry Allen", 25, "Sales", 22300.50], + ], + }, + } + ] + }, + } + ], + } + payload_source = { "id": ENTITY_ID, "kind": {"major": "test", "minor": "parent"}, @@ -785,26 +817,31 @@ def create_entity_for_read(): assert res.status_code == 201 or res.status_code == 200, f"Failed to create entity: {res.text}" print("āœ… Created third related entity.") + res = requests.put(INGESTION_API_URL + "/" + RELATED_ID_1, json=payload_chiild_4) + assert res.status_code == 500, f"Failed to update entity: {res.text}" + print("āœ… Failed to update entity. Working as expected.") + res = requests.post(INGESTION_API_URL, json=payload_source) assert res.status_code == 201 or res.status_code == 200, f"Failed to create entity: {res.text}" print("āœ… Created base entity for read tests.") - -def check_relationship_timeStamps(): res = requests.get(INGESTION_API_URL + "/" + RELATED_ID_1) assert res.status_code == 200, f"Failed to fetch entity: {res.text}" print("āœ… Fetched entity.") # Verify the response data response_data = res.json() - relationship = response_data["relationships"][0] - start_time = relationship["value"]["startTime"] - end_time = relationship["value"]["endTime"] - print("start_time: ", start_time) - print("end_time: ", end_time) - assert start_time == "2024-11-01T00:00:00Z", f"Expected start time '2024-11-01T00:00:00Z', got '{start_time}'" - assert end_time == "2024-11-30T00:00:00Z", f"Expected end time '2024-11-30T00:00:00Z', got '{end_time}'" - + relationships = response_data["relationships"] + print(relationships) + + # check for the start data and the end date + for relationship in relationships: + if relationship["value"]["startTime"] != "" and relationship["value"]["endTime"] != "": + assert relationship["value"]["startTime"] in ["2024-11-01T00:00:00Z", "2024-01-01T00:00:00Z"] + assert relationship["value"]["endTime"] in ["2024-11-30T00:00:00Z","2024-12-31T23:59:59Z"] + else: + pass + print("āœ… Verified start and end times.") def test_attribute_fields_combinations(): """Test different field combinations for attribute retrieval.""" @@ -2400,7 +2437,6 @@ def test_minister_relations_filter_by_active_at_only(): test_protobuf_decoding() print("Creating entity for read tests...") create_entity_for_read() - check_relationship_timeStamps() print("Testing generic validation examples...") test_generic_validation_examples() print("Testing attribute field combinations...") From 0923ff27974a1e7b47bac7fe3195ff150c1401e2 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:46:23 +0530 Subject: [PATCH 11/23] test : adding read layer testing --- .../read-api/tests/read_api_service_test.bal | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/opengin/read-api/tests/read_api_service_test.bal b/opengin/read-api/tests/read_api_service_test.bal index 81bf9bcb..d8067995 100644 --- a/opengin/read-api/tests/read_api_service_test.bal +++ b/opengin/read-api/tests/read_api_service_test.bal @@ -123,6 +123,220 @@ function verifyTabularData(json actual, json expected) { test:assertEquals(actual.toString(), expected.toString(), "Data JSON should match"); } + +// Test entity creation with attribute missing startTime and endTime (should fail with error) +@test:Config { + groups: ["entity", "attribute"], + enable: true +} +function testEntityAttributeCreationWithMissingStartTimeAndEndTime() returns error? { + io:println("[read_api_service_test.bal][testEntityAttributeCreationWithMissingStartTimeAndEndTime]"); + string|error coreUrl = getCoreServiceUrl(); + if coreUrl is error { + return coreUrl; + } + COREServiceClient ep = check new (coreUrl); + + string testId = "ATTR_MISSING_TIME_001"; + json attributeValue = { + "columns": ["emp_id", "name", "salary"], + "rows": [ + [1001, "John Doe", 75000.50] + ] + }; + + pbAny:Any attributeValueAny = check convertJsonToAny(attributeValue); + + Entity createEntityRequest = { + id: testId, + kind: { + major: "Organization", + minor: "Private Limited" + }, + created: "2025-02-01T00:00:00Z", + terminated: "", + name: { + startTime: "2025-02-01T00:00:00Z", + endTime: "", + value: check pbAny:pack("Attr Missing Time Pvt Ltd") + }, + metadata: [ + { + key: "attr_missing_time_metadata", + value: check pbAny:pack("attr_missing_time_test_value") + } + ], + attributes: [ + { + key: "employee_data", + value: { + values: [ + { + startTime: "", + endTime: "", + value: attributeValueAny + } + ] + } + } + ], + relationships: [] + }; + + Entity|error createResponse = ep->CreateEntity(createEntityRequest); + + test:assertTrue(createResponse is error, + "Creating an entity with attribute missing both startTime and endTime should return an error (HTTP 500)"); + + if createResponse is error { + io:println("Received expected error for missing startTime/endTime: " + createResponse.message()); + } + + return; +} + +// Test entity creation with attribute having an invalid startTime and empty endTime (should fail with error) +@test:Config { + groups: ["entity", "attribute"], + enable: true +} +function testEntityAttributeCreationWithInvalidStartTime() returns error? { + io:println("[read_api_service_test.bal][testEntityAttributeCreationWithInvalidStartTime]"); + string|error coreUrl = getCoreServiceUrl(); + if coreUrl is error { + return coreUrl; + } + COREServiceClient ep = check new (coreUrl); + + string testId = "ATTR_INVALID_START_002"; + json attributeValue = { + "columns": ["emp_id", "name", "salary"], + "rows": [ + [1001, "John Doe", 75000.50] + ] + }; + + pbAny:Any attributeValueAny = check convertJsonToAny(attributeValue); + + Entity createEntityRequest = { + id: testId, + kind: { + major: "Organization", + minor: "Private Limited" + }, + created: "2025-02-01T00:00:00Z", + terminated: "", + name: { + startTime: "2025-02-01T00:00:00Z", + endTime: "", + value: check pbAny:pack("Attr Invalid Start Pvt Ltd") + }, + metadata: [ + { + key: "attr_invalid_start_metadata", + value: check pbAny:pack("attr_invalid_start_test_value") + } + ], + attributes: [ + { + key: "employee_data", + value: { + values: [ + { + startTime: "not-a-valid-date", + endTime: "", + value: attributeValueAny + } + ] + } + } + ], + relationships: [] + }; + + Entity|error createResponse = ep->CreateEntity(createEntityRequest); + + test:assertTrue(createResponse is error, + "Creating an entity with an invalid startTime and empty endTime should return an error (HTTP 500)"); + + if createResponse is error { + io:println("Received expected error for invalid startTime: " + createResponse.message()); + } + + return; +} + +// Test entity creation with attribute having a valid startTime and invalid endTime (should fail with error) +@test:Config { + groups: ["entity", "attribute"], + enable: true +} +function testEntityAttributeCreationWithInvalidEndTime() returns error? { + io:println("[read_api_service_test.bal][testEntityAttributeCreationWithInvalidEndTime]"); + string|error coreUrl = getCoreServiceUrl(); + if coreUrl is error { + return coreUrl; + } + COREServiceClient ep = check new (coreUrl); + + string testId = "ATTR_INVALID_END_003"; + json attributeValue = { + "columns": ["emp_id", "name", "salary"], + "rows": [ + [1001, "John Doe", 75000.50] + ] + }; + + pbAny:Any attributeValueAny = check convertJsonToAny(attributeValue); + + Entity createEntityRequest = { + id: testId, + kind: { + major: "Organization", + minor: "Private Limited" + }, + created: "2025-02-01T00:00:00Z", + terminated: "", + name: { + startTime: "2025-02-01T00:00:00Z", + endTime: "", + value: check pbAny:pack("Attr Invalid End Pvt Ltd") + }, + metadata: [ + { + key: "attr_invalid_end_metadata", + value: check pbAny:pack("attr_invalid_end_test_value") + } + ], + attributes: [ + { + key: "employee_data", + value: { + values: [ + { + startTime: "2025-04-01T00:00:00Z", + endTime: "not-a-valid-date", + value: attributeValueAny + } + ] + } + } + ], + relationships: [] + }; + + Entity|error createResponse = ep->CreateEntity(createEntityRequest); + + test:assertTrue(createResponse is error, + "Creating an entity with a valid startTime and an invalid endTime should return an error (HTTP 500)"); + + if createResponse is error { + io:println("Received expected error for invalid endTime: " + createResponse.message()); + } + + return; +} + // Test entity attribute retrieval @test:Config { groups: ["entity", "attribute"], From c7670a3b702c232bb56a161a9293ca2c93a97b32 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 9 Apr 2026 14:54:40 +0530 Subject: [PATCH 12/23] fix : fixing a warning --- opengin/read-api/types.bal | 1 + 1 file changed, 1 insertion(+) diff --git a/opengin/read-api/types.bal b/opengin/read-api/types.bal index d7c059da..8206b3d5 100644 --- a/opengin/read-api/types.bal +++ b/opengin/read-api/types.bal @@ -19,6 +19,7 @@ public type EntitiesEntityIdMetadataResponse record { public type entitiessearch_kind record { # Required if `id` is not provided string major?; + # The minor classification string minor?; }; From 24994023b46d60fa8b13e0e2f9bd934dc7c920ce Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:34:45 +0530 Subject: [PATCH 13/23] review : resolving zaeema's review comments --- opengin/tests/e2e/basic_core_tests.py | 27 +++++++++++++-------------- opengin/tests/e2e/basic_read_tests.py | 6 +++--- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index bd147692..c7afe3a1 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -37,9 +37,8 @@ class CoreTestUtils: - # DEBUG: Attribute protobuf response cannot be decode by the decode_protobuf_any_value() function , so i am using this here. @staticmethod - def decode_protobuf_any_value_2(any_value): + def decode_protobuf_any_value(any_value): """Decode a protobuf Any value to get the actual value""" if isinstance(any_value, dict) and 'typeUrl' in any_value and 'value' in any_value: type_url = any_value['typeUrl'] @@ -200,7 +199,7 @@ def decode_protobuf_any_value_2(any_value): # Try to parse it as JSON obj = json.loads(any_value) # Recursively decode - return CoreTestUtils.decode_protobuf_any_value_2(obj) + return CoreTestUtils.decode_protobuf_any_value(obj) except json.JSONDecodeError: pass @@ -208,7 +207,7 @@ def decode_protobuf_any_value_2(any_value): return any_value @staticmethod - def decode_protobuf_any_value(any_value): + def decode_protobuf_string_value(any_value): """Decode a protobuf Any value to get the actual string value""" if ( isinstance(any_value, dict) @@ -330,7 +329,7 @@ def update_entity(self): if response.status_code == 200: updated_entity = response.json() - decoded_value = CoreTestUtils.decode_protobuf_any_value( + decoded_value = CoreTestUtils.decode_protobuf_string_value( updated_entity["metadata"][0]["value"] ) print("decoded value: ", decoded_value) @@ -347,7 +346,7 @@ def validate_update(self): if response.status_code == 200: updated_data = response.json() - decoded_value = CoreTestUtils.decode_protobuf_any_value( + decoded_value = CoreTestUtils.decode_protobuf_string_value( updated_data["metadata"][0]["value"] ) assert decoded_value == "5.0", "Updated entity does not reflect changes" @@ -444,7 +443,7 @@ def read_minister(self): ) # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) + decoded_name = CoreTestUtils.decode_protobuf_string_value(name_value) assert decoded_name == "Minister of Education", ( f"Expected name 'Minister of Education', got {name_value}" ) @@ -500,7 +499,7 @@ def read_departments(self): # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) + decoded_name = CoreTestUtils.decode_protobuf_string_value(name_value) assert decoded_name == dept["name"], ( f"Expected name '{dept['name']}', got {decoded_name}" ) @@ -705,7 +704,7 @@ def read_minister(self): ) # The name value is a protobuf Any that needs to be decoded name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_any_value(name_value) + decoded_name = CoreTestUtils.decode_protobuf_string_value(name_value) assert decoded_name == "Minister of Finance and Economy", ( f"Expected name 'Minister of Finance and Economy', got {decoded_name}" ) @@ -862,7 +861,7 @@ def create_minister_with_attributes_with_startDate_and_endDate(self): print(response_data) value = response_data["value"] print(f"value {value}") - decoded_value = CoreTestUtils.decode_protobuf_any_value_2(value) + decoded_value = CoreTestUtils.decode_protobuf_any_value(value) expected_data = { "columns": [ @@ -955,7 +954,7 @@ def create_minister_with_attributes_without_startDate(self): print(f"Response: {res.status_code} - {res.text}") print("āœ… Received expected error for Minister creation.") - def add_attributes_to_minister_with_invalid_time_format(self): + def add_attributes_to_minister_with_invalid_starttime_format(self): print("\nšŸŸ¢ Updating attributes stage 3...") update_payload = { "id": self.MINISTER_ID, @@ -1005,7 +1004,7 @@ def add_attributes_to_minister_with_invalid_time_format(self): print(f"Response: {res.status_code} - {res.text}") print("āœ… Received expected error for Minister creation.") - def add_attributes_to_minister_with_invalid_time_format_2(self): + def add_attributes_to_minister_with_invalid_endtime_format(self): print("\nšŸŸ¢ Updating attributes stage 3...") update_payload = { "id": self.MINISTER_ID, @@ -1377,8 +1376,8 @@ def test_duplicate_primary_key(self): attribute_validation_tests.update_attributes_stage_1() attribute_validation_tests.update_attributes_stage_2() attribute_validation_tests.create_minister_with_attributes_with_startDate_and_endDate() - attribute_validation_tests.add_attributes_to_minister_with_invalid_time_format() - attribute_validation_tests.add_attributes_to_minister_with_invalid_time_format_2() + attribute_validation_tests.add_attributes_to_minister_with_invalid_starttime_format() + attribute_validation_tests.add_attributes_to_minister_with_invalid_endtime_format() attribute_validation_tests.create_minister_with_attributes_without_startDate() print("\nšŸŸ¢ Running Attribute Validation Tests... Done") diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index b3d509f5..cb0c455e 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -699,11 +699,11 @@ def create_entity_for_read(): } # first entity with a new attribute - payload_chiild_4 = { + payload_child_4 = { "id": RELATED_ID_1, "attributes": [ { - "key": "tet_data", + "key": "test_data", "value": { "values": [ { @@ -817,7 +817,7 @@ def create_entity_for_read(): assert res.status_code == 201 or res.status_code == 200, f"Failed to create entity: {res.text}" print("āœ… Created third related entity.") - res = requests.put(INGESTION_API_URL + "/" + RELATED_ID_1, json=payload_chiild_4) + res = requests.put(INGESTION_API_URL + "/" + RELATED_ID_1, json=payload_child_4) assert res.status_code == 500, f"Failed to update entity: {res.text}" print("āœ… Failed to update entity. Working as expected.") From 430a7620096cbdefe86aa6d0ace19e54dd05bddb Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:55:37 +0530 Subject: [PATCH 14/23] Trigger CI From a23ca7fdf31d8151120c0f91637d76a2fb3cd25c Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 16 Apr 2026 12:55:57 +0530 Subject: [PATCH 15/23] Trigger CI From 3f3a2320dbf9bf59d4e7ad5180876484d0d83156 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:35:14 +0530 Subject: [PATCH 16/23] test : test commit on mongodb-choreo EOL issue --- .../development/docker/mongodb/Dockerfile | 141 ++++++------------ 1 file changed, 49 insertions(+), 92 deletions(-) diff --git a/deployment/choreo/development/docker/mongodb/Dockerfile b/deployment/choreo/development/docker/mongodb/Dockerfile index 3d36d3d1..061d0a95 100644 --- a/deployment/choreo/development/docker/mongodb/Dockerfile +++ b/deployment/choreo/development/docker/mongodb/Dockerfile @@ -1,25 +1,34 @@ # Copyright 2025 Lanka Data Foundation # SPDX-License-Identifier: Apache-2.0 -# MongoDB Server Dockerfile with GitHub backup restore -FROM ubuntu:20.04 - -# Install prerequisites -RUN apt-get update && apt-get install -y \ - wget gnupg2 curl apt-transport-https sudo \ +# Upgraded to Ubuntu 22.04 (Jammy) - LTS supported until 2027 +FROM ubuntu:22.04 + +# Prevent interactive prompts during package install +ENV DEBIAN_FRONTEND=noninteractive + +# āœ… FIX: Switch to a reliable mirror BEFORE apt-get install +# Using kernel.org mirror which is globally stable +# Also adding --retry logic and --fix-missing as fallback +RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://mirrors.edge.kernel.org/ubuntu|g' /etc/apt/sources.list && \ + sed -i 's|http://security.ubuntu.com/ubuntu|http://mirrors.edge.kernel.org/ubuntu|g' /etc/apt/sources.list && \ + apt-get update -o Acquire::Retries=3 && \ + apt-get install -y --fix-missing \ + wget gnupg2 curl apt-transport-https sudo \ && rm -rf /var/lib/apt/lists/* -# Add MongoDB GPG key and repository for 4.4 -RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \ - && echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" > /etc/apt/sources.list.d/mongodb-org-4.4.list - -# Install MongoDB 4.4 and additional tools -RUN apt-get update && apt-get install -y \ - mongodb-org=4.4.28 \ - mongodb-org-server=4.4.28 \ - mongodb-org-shell=4.4.28 \ - mongodb-org-tools=4.4.28 \ - wget unzip \ +# Add MongoDB GPG key and repository +# āœ… FIX: Upgraded to MongoDB 7.0 (actively supported until 2027) +RUN wget -qO - https://www.mongodb.org/static/pgp/server-7.0.asc | \ + gpg --dearmor -o /usr/share/keyrings/mongodb-server-7.0.gpg && \ + echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \ + > /etc/apt/sources.list.d/mongodb-org-7.0.list + +# Install MongoDB 7.0 and tools +RUN apt-get update -o Acquire::Retries=3 && \ + apt-get install -y --fix-missing \ + mongodb-org \ + wget unzip \ && rm -rf /var/lib/apt/lists/* # Create choreo user and group (required for Choreo platform) @@ -30,7 +39,7 @@ RUN groupadd -g 10014 choreo && \ chmod -R 755 /home/choreouser && \ echo "choreouser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -# Create directories with proper permissions for choreo user +# Create directories with proper permissions RUN mkdir -p /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb \ && chown -R 10014:10014 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb \ && chmod -R 755 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb @@ -46,8 +55,8 @@ ENV GITHUB_BACKUP_REPO=${OPENGIN_GITHUB_BACKUP_REPO:-LDFLK/data-backups} \ BACKUP_ENVIRONMENT=${OPENGIN_CHOREO_ENVIRONMENT:-development} \ RESTORE_FROM_GITHUB=true -# Create MongoDB configuration (without fork for choreo user) -RUN echo "net:\n\ +# Create MongoDB configuration +RUN printf "net:\n\ port: 27017\n\ bindIp: 0.0.0.0\n\ storage:\n\ @@ -57,135 +66,92 @@ systemLog:\n\ logAppend: true\n\ path: /var/log/mongodb/mongod.log\n\ processManagement:\n\ - fork: false" > /etc/mongod.conf - -# Additional directories are already created above with proper permissions + fork: false\n" > /etc/mongod.conf -# Create entrypoint script with GitHub backup restore -RUN echo '#!/bin/bash\n\ +# āœ… FIX: Use printf instead of echo for entrypoint (avoids \n literal issues) +RUN printf '#!/bin/bash\n\ set -e\n\ \n\ -# Logging function\n\ log() {\n\ - echo "[$(date +%Y-%m-%d\ %H:%M:%S)] $1: $2"\n\ + echo "[$(date +%%Y-%%m-%%d\\ %%H:%%M:%%S)] $1: $2"\n\ }\n\ \n\ -# Ensure choreo user has proper permissions (volumes may reset ownership)\n\ log "INFO" "Setting up permissions for choreo user..."\n\ sudo chown -R 10014:10014 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb\n\ sudo chmod -R 755 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb\n\ \n\ -# Function to restore from GitHub backup\n\ restore_from_github() {\n\ local github_repo="${GITHUB_BACKUP_REPO:-LDFLK/data-backups}"\n\ local version="${BACKUP_VERSION:-0.0.1}"\n\ local environment="${BACKUP_ENVIRONMENT:-development}"\n\ - \n\ log "INFO" "Starting MongoDB GitHub backup restore..."\n\ - \n\ - # Create temporary directory for download\n\ local temp_dir=$(mktemp -d)\n\ local archive_url="https://github.com/$github_repo/archive/refs/tags/$version.zip"\n\ local archive_file="$temp_dir/archive.zip"\n\ - \n\ log "INFO" "Downloading backup from: $archive_url"\n\ if ! wget -q "$archive_url" -O "$archive_file"; then\n\ log "ERROR" "Failed to download backup archive"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ - \n\ - # Extract the archive\n\ if ! unzip -q "$archive_file" -d "$temp_dir"; then\n\ log "ERROR" "Failed to extract backup archive"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ - \n\ - # Find the MongoDB backup file\n\ local archive_dir="$temp_dir/data-backups-$version"\n\ local mongodb_backup="$archive_dir/opengin/$environment/mongodb/opengin.tar.gz"\n\ - \n\ if [ ! -f "$mongodb_backup" ]; then\n\ log "ERROR" "MongoDB backup file not found: $mongodb_backup"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ - \n\ - log "INFO" "Found MongoDB backup: $(basename "$mongodb_backup")"\n\ - \n\ - # Extract the backup file\n\ local backup_extract_dir="$temp_dir/mongodb_restore"\n\ mkdir -p "$backup_extract_dir"\n\ - \n\ if ! tar -xzf "$mongodb_backup" -C "$backup_extract_dir"; then\n\ log "ERROR" "Failed to extract MongoDB backup"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ - \n\ - # Check if opengin database exists in the backup\n\ if [ ! -d "$backup_extract_dir/opengin" ]; then\n\ log "ERROR" "opengin database not found in backup"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ - \n\ - # Handle nested backup structure\n\ local backup_source="$backup_extract_dir/opengin"\n\ if [ -d "$backup_extract_dir/opengin/opengin" ]; then\n\ - log "INFO" "Found nested backup structure, using inner directory"\n\ backup_source="$backup_extract_dir/opengin/opengin"\n\ fi\n\ - \n\ - # Copy backup to MongoDB backup directory\n\ - log "INFO" "Preparing backup for mongorestore..."\n\ cp -r "$backup_source" /data/backup/opengin\n\ - \n\ - # Clean up any macOS metadata files\n\ find /data/backup/opengin -name "._*" -delete 2>/dev/null || true\n\ - \n\ - # Wait for MongoDB to be ready\n\ log "INFO" "Waiting for MongoDB to be ready..."\n\ for i in {1..30}; do\n\ - if mongo --eval "db.adminCommand('\''ping'\'')" > /dev/null 2>&1; then\n\ + if mongosh --eval "db.adminCommand({ping:1})" > /dev/null 2>&1; then\n\ log "INFO" "MongoDB is ready!"\n\ break\n\ fi\n\ log "INFO" "Waiting for MongoDB... attempt $i/30"\n\ sleep 2\n\ done\n\ - \n\ - # Restore using mongorestore (following the documentation approach)\n\ - log "INFO" "Restoring MongoDB database using mongorestore..."\n\ - if mongorestore --host=localhost:27017 \\\n\ - --db=opengin \\\n\ - --drop \\\n\ - /data/backup/opengin; then\n\ - \n\ - log "SUCCESS" "MongoDB database restored successfully using mongorestore"\n\ - # Clean up backup files\n\ + log "INFO" "Restoring MongoDB database..."\n\ + if mongorestore --host=localhost:27017 --db=opengin --drop /data/backup/opengin; then\n\ + log "SUCCESS" "MongoDB restored successfully"\n\ rm -rf /data/backup/opengin\n\ else\n\ - log "ERROR" "Failed to restore MongoDB database using mongorestore"\n\ - rm -rf /data/backup/opengin\n\ - rm -rf "$temp_dir"\n\ + log "ERROR" "Failed to restore MongoDB"\n\ + rm -rf /data/backup/opengin "$temp_dir"\n\ return 1\n\ fi\n\ - \n\ rm -rf "$temp_dir"\n\ return 0\n\ }\n\ \n\ -# Start MongoDB in background first\n\ log "INFO" "Starting MongoDB in background..."\n\ mongod --dbpath /data/db --logpath /var/log/mongodb/mongod.log --bind_ip_all &\n\ MONGODB_PID=$!\n\ \n\ -# Wait for MongoDB to start\n\ -log "INFO" "Waiting for MongoDB to start..."\n\ for i in {1..30}; do\n\ - if mongo --eval "db.adminCommand('\''ping'\'')" > /dev/null 2>&1; then\n\ + if mongosh --eval "db.adminCommand({ping:1})" > /dev/null 2>&1; then\n\ log "INFO" "MongoDB is ready!"\n\ break\n\ fi\n\ @@ -193,20 +159,17 @@ for i in {1..30}; do\n\ sleep 2\n\ done\n\ \n\ -# Create admin user if it doesn'\''t exist (similar to official mongo image behavior)\n\ if [ -n "${MONGO_INITDB_ROOT_USERNAME}" ] && [ -n "${MONGO_INITDB_ROOT_PASSWORD}" ]; then\n\ log "INFO" "Checking for admin user..."\n\ - if ! mongo admin --quiet --eval "db.getUser('\''${MONGO_INITDB_ROOT_USERNAME}'\'')" 2>/dev/null | grep -q "user"; then\n\ + if ! mongosh admin --quiet --eval "db.getUser('"'"'${MONGO_INITDB_ROOT_USERNAME}'"'"')" 2>/dev/null | grep -q "user"; then\n\ log "INFO" "Creating admin user..."\n\ - mongo admin --quiet > /dev/null 2>&1 <<< "db.createUser({user: '\''${MONGO_INITDB_ROOT_USERNAME}'\'', pwd: '\''${MONGO_INITDB_ROOT_PASSWORD}'\'', roles: [{role: '\''root'\'', db: '\''admin'\''}]})"\n\ + mongosh admin --quiet --eval "db.createUser({user:'"'"'${MONGO_INITDB_ROOT_USERNAME}'"'"',pwd:'"'"'${MONGO_INITDB_ROOT_PASSWORD}'"'"',roles:[{role:'"'"'root'"'"',db:'"'"'admin'"'"'}]})"\n\ log "SUCCESS" "Admin user created"\n\ fi\n\ fi\n\ \n\ -# Restore from GitHub if enabled and database doesn'\''t exist\n\ if [ "${RESTORE_FROM_GITHUB:-false}" = "true" ]; then\n\ - # Check if opengin database exists\n\ - if ! mongo --eval "db.adminCommand('\''listDatabases'\'')" 2>/dev/null | grep -q "opengin"; then\n\ + if ! mongosh --eval "db.adminCommand({listDatabases:1})" 2>/dev/null | grep -q "opengin"; then\n\ log "INFO" "opengin database not found, starting GitHub restore..."\n\ restore_from_github || log "WARNING" "GitHub restore failed, continuing with empty database"\n\ else\n\ @@ -214,29 +177,23 @@ if [ "${RESTORE_FROM_GITHUB:-false}" = "true" ]; then\n\ fi\n\ fi\n\ \n\ -# Stop the background MongoDB and start in foreground\n\ -log "INFO" "Stopping background MongoDB and starting in foreground..."\n\ +log "INFO" "Restarting MongoDB in foreground..."\n\ kill $MONGODB_PID 2>/dev/null || true\n\ wait $MONGODB_PID 2>/dev/null || true\n\ sleep 3\n\ \n\ -# Start MongoDB in foreground\n\ log "INFO" "Starting MongoDB in foreground mode..."\n\ -exec mongod --dbpath /data/db --logpath /var/log/mongodb/mongod.log --bind_ip_all' > /custom-entrypoint.sh \ - && chmod +x /custom-entrypoint.sh +exec mongod --dbpath /data/db --logpath /var/log/mongodb/mongod.log --bind_ip_all\n\ +' > /custom-entrypoint.sh && chmod +x /custom-entrypoint.sh -# Switch to choreo user (required for Choreo platform) USER 10014 -# Define volumes for data persistence VOLUME ["/data/db", "/data/configdb", "/data/backup"] -# Expose ports EXPOSE 27017 -# Health check +# āœ… FIX: Use mongosh instead of mongo (mongo shell removed in MongoDB 6+) HEALTHCHECK --interval=10s --timeout=10s --start-period=60s --retries=10 \ - CMD mongo --eval "db.adminCommand('ping')" || exit 1 + CMD mongosh --eval "db.adminCommand('ping')" || exit 1 -# Use custom entrypoint -CMD ["/custom-entrypoint.sh"] +CMD ["/custom-entrypoint.sh"] \ No newline at end of file From 9a984db93ac7157fb5c927b71a2668b461da681e Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Thu, 16 Apr 2026 17:47:33 +0530 Subject: [PATCH 17/23] restoring the same dockerfile --- .../development/docker/mongodb/Dockerfile | 141 ++++++++++++------ 1 file changed, 92 insertions(+), 49 deletions(-) diff --git a/deployment/choreo/development/docker/mongodb/Dockerfile b/deployment/choreo/development/docker/mongodb/Dockerfile index 061d0a95..3d36d3d1 100644 --- a/deployment/choreo/development/docker/mongodb/Dockerfile +++ b/deployment/choreo/development/docker/mongodb/Dockerfile @@ -1,34 +1,25 @@ # Copyright 2025 Lanka Data Foundation # SPDX-License-Identifier: Apache-2.0 -# Upgraded to Ubuntu 22.04 (Jammy) - LTS supported until 2027 -FROM ubuntu:22.04 - -# Prevent interactive prompts during package install -ENV DEBIAN_FRONTEND=noninteractive - -# āœ… FIX: Switch to a reliable mirror BEFORE apt-get install -# Using kernel.org mirror which is globally stable -# Also adding --retry logic and --fix-missing as fallback -RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://mirrors.edge.kernel.org/ubuntu|g' /etc/apt/sources.list && \ - sed -i 's|http://security.ubuntu.com/ubuntu|http://mirrors.edge.kernel.org/ubuntu|g' /etc/apt/sources.list && \ - apt-get update -o Acquire::Retries=3 && \ - apt-get install -y --fix-missing \ - wget gnupg2 curl apt-transport-https sudo \ +# MongoDB Server Dockerfile with GitHub backup restore +FROM ubuntu:20.04 + +# Install prerequisites +RUN apt-get update && apt-get install -y \ + wget gnupg2 curl apt-transport-https sudo \ && rm -rf /var/lib/apt/lists/* -# Add MongoDB GPG key and repository -# āœ… FIX: Upgraded to MongoDB 7.0 (actively supported until 2027) -RUN wget -qO - https://www.mongodb.org/static/pgp/server-7.0.asc | \ - gpg --dearmor -o /usr/share/keyrings/mongodb-server-7.0.gpg && \ - echo "deb [ arch=amd64,arm64 signed-by=/usr/share/keyrings/mongodb-server-7.0.gpg ] https://repo.mongodb.org/apt/ubuntu jammy/mongodb-org/7.0 multiverse" \ - > /etc/apt/sources.list.d/mongodb-org-7.0.list - -# Install MongoDB 7.0 and tools -RUN apt-get update -o Acquire::Retries=3 && \ - apt-get install -y --fix-missing \ - mongodb-org \ - wget unzip \ +# Add MongoDB GPG key and repository for 4.4 +RUN wget -qO - https://www.mongodb.org/static/pgp/server-4.4.asc | apt-key add - \ + && echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/4.4 multiverse" > /etc/apt/sources.list.d/mongodb-org-4.4.list + +# Install MongoDB 4.4 and additional tools +RUN apt-get update && apt-get install -y \ + mongodb-org=4.4.28 \ + mongodb-org-server=4.4.28 \ + mongodb-org-shell=4.4.28 \ + mongodb-org-tools=4.4.28 \ + wget unzip \ && rm -rf /var/lib/apt/lists/* # Create choreo user and group (required for Choreo platform) @@ -39,7 +30,7 @@ RUN groupadd -g 10014 choreo && \ chmod -R 755 /home/choreouser && \ echo "choreouser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers -# Create directories with proper permissions +# Create directories with proper permissions for choreo user RUN mkdir -p /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb \ && chown -R 10014:10014 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb \ && chmod -R 755 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb @@ -55,8 +46,8 @@ ENV GITHUB_BACKUP_REPO=${OPENGIN_GITHUB_BACKUP_REPO:-LDFLK/data-backups} \ BACKUP_ENVIRONMENT=${OPENGIN_CHOREO_ENVIRONMENT:-development} \ RESTORE_FROM_GITHUB=true -# Create MongoDB configuration -RUN printf "net:\n\ +# Create MongoDB configuration (without fork for choreo user) +RUN echo "net:\n\ port: 27017\n\ bindIp: 0.0.0.0\n\ storage:\n\ @@ -66,92 +57,135 @@ systemLog:\n\ logAppend: true\n\ path: /var/log/mongodb/mongod.log\n\ processManagement:\n\ - fork: false\n" > /etc/mongod.conf + fork: false" > /etc/mongod.conf + +# Additional directories are already created above with proper permissions -# āœ… FIX: Use printf instead of echo for entrypoint (avoids \n literal issues) -RUN printf '#!/bin/bash\n\ +# Create entrypoint script with GitHub backup restore +RUN echo '#!/bin/bash\n\ set -e\n\ \n\ +# Logging function\n\ log() {\n\ - echo "[$(date +%%Y-%%m-%%d\\ %%H:%%M:%%S)] $1: $2"\n\ + echo "[$(date +%Y-%m-%d\ %H:%M:%S)] $1: $2"\n\ }\n\ \n\ +# Ensure choreo user has proper permissions (volumes may reset ownership)\n\ log "INFO" "Setting up permissions for choreo user..."\n\ sudo chown -R 10014:10014 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb\n\ sudo chmod -R 755 /var/lib/mongodb /var/log/mongodb /data/db /data/configdb /data/backup /var/run/mongodb\n\ \n\ +# Function to restore from GitHub backup\n\ restore_from_github() {\n\ local github_repo="${GITHUB_BACKUP_REPO:-LDFLK/data-backups}"\n\ local version="${BACKUP_VERSION:-0.0.1}"\n\ local environment="${BACKUP_ENVIRONMENT:-development}"\n\ + \n\ log "INFO" "Starting MongoDB GitHub backup restore..."\n\ + \n\ + # Create temporary directory for download\n\ local temp_dir=$(mktemp -d)\n\ local archive_url="https://github.com/$github_repo/archive/refs/tags/$version.zip"\n\ local archive_file="$temp_dir/archive.zip"\n\ + \n\ log "INFO" "Downloading backup from: $archive_url"\n\ if ! wget -q "$archive_url" -O "$archive_file"; then\n\ log "ERROR" "Failed to download backup archive"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ + \n\ + # Extract the archive\n\ if ! unzip -q "$archive_file" -d "$temp_dir"; then\n\ log "ERROR" "Failed to extract backup archive"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ + \n\ + # Find the MongoDB backup file\n\ local archive_dir="$temp_dir/data-backups-$version"\n\ local mongodb_backup="$archive_dir/opengin/$environment/mongodb/opengin.tar.gz"\n\ + \n\ if [ ! -f "$mongodb_backup" ]; then\n\ log "ERROR" "MongoDB backup file not found: $mongodb_backup"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ + \n\ + log "INFO" "Found MongoDB backup: $(basename "$mongodb_backup")"\n\ + \n\ + # Extract the backup file\n\ local backup_extract_dir="$temp_dir/mongodb_restore"\n\ mkdir -p "$backup_extract_dir"\n\ + \n\ if ! tar -xzf "$mongodb_backup" -C "$backup_extract_dir"; then\n\ log "ERROR" "Failed to extract MongoDB backup"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ + \n\ + # Check if opengin database exists in the backup\n\ if [ ! -d "$backup_extract_dir/opengin" ]; then\n\ log "ERROR" "opengin database not found in backup"\n\ rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ + \n\ + # Handle nested backup structure\n\ local backup_source="$backup_extract_dir/opengin"\n\ if [ -d "$backup_extract_dir/opengin/opengin" ]; then\n\ + log "INFO" "Found nested backup structure, using inner directory"\n\ backup_source="$backup_extract_dir/opengin/opengin"\n\ fi\n\ + \n\ + # Copy backup to MongoDB backup directory\n\ + log "INFO" "Preparing backup for mongorestore..."\n\ cp -r "$backup_source" /data/backup/opengin\n\ + \n\ + # Clean up any macOS metadata files\n\ find /data/backup/opengin -name "._*" -delete 2>/dev/null || true\n\ + \n\ + # Wait for MongoDB to be ready\n\ log "INFO" "Waiting for MongoDB to be ready..."\n\ for i in {1..30}; do\n\ - if mongosh --eval "db.adminCommand({ping:1})" > /dev/null 2>&1; then\n\ + if mongo --eval "db.adminCommand('\''ping'\'')" > /dev/null 2>&1; then\n\ log "INFO" "MongoDB is ready!"\n\ break\n\ fi\n\ log "INFO" "Waiting for MongoDB... attempt $i/30"\n\ sleep 2\n\ done\n\ - log "INFO" "Restoring MongoDB database..."\n\ - if mongorestore --host=localhost:27017 --db=opengin --drop /data/backup/opengin; then\n\ - log "SUCCESS" "MongoDB restored successfully"\n\ + \n\ + # Restore using mongorestore (following the documentation approach)\n\ + log "INFO" "Restoring MongoDB database using mongorestore..."\n\ + if mongorestore --host=localhost:27017 \\\n\ + --db=opengin \\\n\ + --drop \\\n\ + /data/backup/opengin; then\n\ + \n\ + log "SUCCESS" "MongoDB database restored successfully using mongorestore"\n\ + # Clean up backup files\n\ rm -rf /data/backup/opengin\n\ else\n\ - log "ERROR" "Failed to restore MongoDB"\n\ - rm -rf /data/backup/opengin "$temp_dir"\n\ + log "ERROR" "Failed to restore MongoDB database using mongorestore"\n\ + rm -rf /data/backup/opengin\n\ + rm -rf "$temp_dir"\n\ return 1\n\ fi\n\ + \n\ rm -rf "$temp_dir"\n\ return 0\n\ }\n\ \n\ +# Start MongoDB in background first\n\ log "INFO" "Starting MongoDB in background..."\n\ mongod --dbpath /data/db --logpath /var/log/mongodb/mongod.log --bind_ip_all &\n\ MONGODB_PID=$!\n\ \n\ +# Wait for MongoDB to start\n\ +log "INFO" "Waiting for MongoDB to start..."\n\ for i in {1..30}; do\n\ - if mongosh --eval "db.adminCommand({ping:1})" > /dev/null 2>&1; then\n\ + if mongo --eval "db.adminCommand('\''ping'\'')" > /dev/null 2>&1; then\n\ log "INFO" "MongoDB is ready!"\n\ break\n\ fi\n\ @@ -159,17 +193,20 @@ for i in {1..30}; do\n\ sleep 2\n\ done\n\ \n\ +# Create admin user if it doesn'\''t exist (similar to official mongo image behavior)\n\ if [ -n "${MONGO_INITDB_ROOT_USERNAME}" ] && [ -n "${MONGO_INITDB_ROOT_PASSWORD}" ]; then\n\ log "INFO" "Checking for admin user..."\n\ - if ! mongosh admin --quiet --eval "db.getUser('"'"'${MONGO_INITDB_ROOT_USERNAME}'"'"')" 2>/dev/null | grep -q "user"; then\n\ + if ! mongo admin --quiet --eval "db.getUser('\''${MONGO_INITDB_ROOT_USERNAME}'\'')" 2>/dev/null | grep -q "user"; then\n\ log "INFO" "Creating admin user..."\n\ - mongosh admin --quiet --eval "db.createUser({user:'"'"'${MONGO_INITDB_ROOT_USERNAME}'"'"',pwd:'"'"'${MONGO_INITDB_ROOT_PASSWORD}'"'"',roles:[{role:'"'"'root'"'"',db:'"'"'admin'"'"'}]})"\n\ + mongo admin --quiet > /dev/null 2>&1 <<< "db.createUser({user: '\''${MONGO_INITDB_ROOT_USERNAME}'\'', pwd: '\''${MONGO_INITDB_ROOT_PASSWORD}'\'', roles: [{role: '\''root'\'', db: '\''admin'\''}]})"\n\ log "SUCCESS" "Admin user created"\n\ fi\n\ fi\n\ \n\ +# Restore from GitHub if enabled and database doesn'\''t exist\n\ if [ "${RESTORE_FROM_GITHUB:-false}" = "true" ]; then\n\ - if ! mongosh --eval "db.adminCommand({listDatabases:1})" 2>/dev/null | grep -q "opengin"; then\n\ + # Check if opengin database exists\n\ + if ! mongo --eval "db.adminCommand('\''listDatabases'\'')" 2>/dev/null | grep -q "opengin"; then\n\ log "INFO" "opengin database not found, starting GitHub restore..."\n\ restore_from_github || log "WARNING" "GitHub restore failed, continuing with empty database"\n\ else\n\ @@ -177,23 +214,29 @@ if [ "${RESTORE_FROM_GITHUB:-false}" = "true" ]; then\n\ fi\n\ fi\n\ \n\ -log "INFO" "Restarting MongoDB in foreground..."\n\ +# Stop the background MongoDB and start in foreground\n\ +log "INFO" "Stopping background MongoDB and starting in foreground..."\n\ kill $MONGODB_PID 2>/dev/null || true\n\ wait $MONGODB_PID 2>/dev/null || true\n\ sleep 3\n\ \n\ +# Start MongoDB in foreground\n\ log "INFO" "Starting MongoDB in foreground mode..."\n\ -exec mongod --dbpath /data/db --logpath /var/log/mongodb/mongod.log --bind_ip_all\n\ -' > /custom-entrypoint.sh && chmod +x /custom-entrypoint.sh +exec mongod --dbpath /data/db --logpath /var/log/mongodb/mongod.log --bind_ip_all' > /custom-entrypoint.sh \ + && chmod +x /custom-entrypoint.sh +# Switch to choreo user (required for Choreo platform) USER 10014 +# Define volumes for data persistence VOLUME ["/data/db", "/data/configdb", "/data/backup"] +# Expose ports EXPOSE 27017 -# āœ… FIX: Use mongosh instead of mongo (mongo shell removed in MongoDB 6+) +# Health check HEALTHCHECK --interval=10s --timeout=10s --start-period=60s --retries=10 \ - CMD mongosh --eval "db.adminCommand('ping')" || exit 1 + CMD mongo --eval "db.adminCommand('ping')" || exit 1 -CMD ["/custom-entrypoint.sh"] \ No newline at end of file +# Use custom entrypoint +CMD ["/custom-entrypoint.sh"] From a14a8be017c1bf79fc4a621adae8e354248de51c Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Mon, 20 Apr 2026 09:38:44 +0530 Subject: [PATCH 18/23] chore: trigger CI From 8a5da95a05d19d3791f32c3b9fc7cab9577a8ea6 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:31:43 +0530 Subject: [PATCH 19/23] revert : revert and auto generated file --- opengin/read-api/types.bal | 1 - 1 file changed, 1 deletion(-) diff --git a/opengin/read-api/types.bal b/opengin/read-api/types.bal index 8206b3d5..d7c059da 100644 --- a/opengin/read-api/types.bal +++ b/opengin/read-api/types.bal @@ -19,7 +19,6 @@ public type EntitiesEntityIdMetadataResponse record { public type entitiessearch_kind record { # Required if `id` is not provided string major?; - # The minor classification string minor?; }; From 156be666d19e0ba3ed30e609006c070bbd1eefd4 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Wed, 22 Apr 2026 10:32:52 +0530 Subject: [PATCH 20/23] fix : fixing typo --- opengin/tests/e2e/basic_read_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index cb0c455e..7698a313 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -834,7 +834,7 @@ def create_entity_for_read(): relationships = response_data["relationships"] print(relationships) - # check for the start data and the end date + # check for the start date and the end date for relationship in relationships: if relationship["value"]["startTime"] != "" and relationship["value"]["endTime"] != "": assert relationship["value"]["startTime"] in ["2024-11-01T00:00:00Z", "2024-01-01T00:00:00Z"] From 92eed42ff44d5575de2f9b36f191705d2e4ac172 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Wed, 22 Apr 2026 14:03:41 +0530 Subject: [PATCH 21/23] review : resolving review comments by zaeema --- opengin/tests/e2e/basic_core_tests.py | 41 +++++++++++++++++---------- opengin/tests/e2e/basic_read_tests.py | 21 +++++++++----- 2 files changed, 40 insertions(+), 22 deletions(-) diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index c7afe3a1..611b3cf8 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -683,12 +683,17 @@ def create_minister_with_attributes(self): def read_minister(self): """Read the Minister entity.""" print("\nšŸŸ¢ Reading Minister entity...") - res = requests.get(f"{self.base_url}/{self.MINISTER_ID}") + payload = { + "id": self.MINISTER_ID + } + res = requests.post(f"{self.base_read_url}/search", json=payload) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to read Minister: {res.text}" # Verify the response data response_data = res.json() + response_data = response_data["body"][0] + print(response_data) assert response_data["id"] == self.MINISTER_ID, ( f"Expected ID {self.MINISTER_ID}, got {response_data['id']}" @@ -702,9 +707,16 @@ def read_minister(self): assert response_data["created"] == self.START_DATE, ( f"Expected created date {self.START_DATE}, got {response_data['created']}" ) + + print("im here befire you bitchhhhh....") + # The name value is a protobuf Any that needs to be decoded - name_value = response_data["name"]["value"] - decoded_name = CoreTestUtils.decode_protobuf_string_value(name_value) + print("i am here..............") + name_obj = json.loads(response_data["name"]) + print(name_obj) + hex_value = name_obj["value"] + print(hex_value) + decoded_name = bytes.fromhex(hex_value).decode('utf-8') assert decoded_name == "Minister of Finance and Economy", ( f"Expected name 'Minister of Finance and Economy', got {decoded_name}" ) @@ -850,7 +862,7 @@ def create_minister_with_attributes_with_startDate_and_endDate(self): print("\nšŸŸ¢ checking Minister with attribute with startDate and endDate...") - url = f"{self.base_read_url}/v1/entities/{self.MINISTER_ID}/attributes/{self.ATTRIBUTE_NAME_2}" + url = f"{self.base_read_url}/{self.MINISTER_ID}/attributes/{self.ATTRIBUTE_NAME_2}" payload = {} res = requests.post(url, json=payload) @@ -884,24 +896,23 @@ def create_minister_with_attributes_with_startDate_and_endDate(self): print("Decoded value: ", decoded_value) - url = f"{self.base_url}/{self.MINISTER_ID}" + url = f"{self.base_read_url}/{self.MINISTER_ID}/relations" + payload = {} - res = requests.get(url) + res = requests.post(url, json=payload) print(res.status_code, res.json()) - assert res.status_code in [200], f"Failed to read Minister: {res.text}" - - response_data = res.json() - print(response_data) + assert res.status_code in [200], f"Failed to read Minister relationships: {res.text}" - relationships = response_data["relationships"] + relationships = res.json() + print(relationships) print("relationships", relationships) # check for the start data and the end date for relationship in relationships: - if relationship["value"]["startTime"] != "" and relationship["value"]["endTime"] != "": - assert relationship["value"]["startTime"] == self.DATA_START_DATE - assert relationship["value"]["endTime"] == self.DATA_END_DATE + if relationship["startTime"] != "" and relationship["endTime"] != "": + assert relationship["startTime"] == self.DATA_START_DATE + assert relationship["endTime"] == self.DATA_END_DATE else: pass @@ -1065,7 +1076,7 @@ def get_base_read_url(): print("šŸŸ¢ Setting up test environment...") read_service_url = os.getenv("READ_SERVICE_URL", "http://read:8081") print("šŸŸ¢ READ_SERVICE_URL: ", read_service_url) - return read_service_url + return f"{read_service_url}/v1/entities" # Tabular Attribute Integrity Tests diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index 7698a313..f52181cf 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -834,14 +834,21 @@ def create_entity_for_read(): relationships = response_data["relationships"] print(relationships) - # check for the start date and the end date + found = False + for relationship in relationships: - if relationship["value"]["startTime"] != "" and relationship["value"]["endTime"] != "": - assert relationship["value"]["startTime"] in ["2024-11-01T00:00:00Z", "2024-01-01T00:00:00Z"] - assert relationship["value"]["endTime"] in ["2024-11-30T00:00:00Z","2024-12-31T23:59:59Z"] - else: - pass - print("āœ… Verified start and end times.") + start = relationship["value"].get("startTime") + end = relationship["value"].get("endTime") + + if start and end: + found = True + + assert start in ["2024-11-01T00:00:00Z", "2024-01-01T00:00:00Z"] + assert end in ["2024-11-30T00:00:00Z", "2024-12-31T23:59:59Z"] + + assert found, "No relationship found with both startTime and endTime" + + print("āœ… Verified start and end times of entity's attributes.") def test_attribute_fields_combinations(): """Test different field combinations for attribute retrieval.""" From baa77a8c0c7a474772ab45ab9e9c99899db109d1 Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Fri, 24 Apr 2026 09:57:19 +0530 Subject: [PATCH 22/23] fix : minor fix --- opengin/tests/e2e/basic_core_tests.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index 611b3cf8..3035ffdc 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -819,7 +819,7 @@ def normalize(self, data): for row in rows ] - def create_minister_with_attributes_with_startDate_and_endDate(self): + def update_minister_with_attributes_with_startDate_and_endDate(self): """Create a Minister entity.""" print("\nšŸŸ¢ Creating Minister with attribute with startDate and endDate...") @@ -918,9 +918,9 @@ def create_minister_with_attributes_with_startDate_and_endDate(self): print("āœ… Updated Minister entity with attributes with startDate + endDate.") - def create_minister_with_attributes_without_startDate(self): + def update_minister_with_attributes_without_startDate(self): """Create a Minister entity.""" - print("\nšŸŸ¢ Creating Minister entity + attribute without startDate...") + print("\nšŸŸ¢ Updating Minister entity with an attribute without startDate...") payload = { "id": self.MINISTER_ID, @@ -1386,10 +1386,10 @@ def test_duplicate_primary_key(self): attribute_validation_tests.read_minister() attribute_validation_tests.update_attributes_stage_1() attribute_validation_tests.update_attributes_stage_2() - attribute_validation_tests.create_minister_with_attributes_with_startDate_and_endDate() + attribute_validation_tests.update_minister_with_attributes_with_startDate_and_endDate() attribute_validation_tests.add_attributes_to_minister_with_invalid_starttime_format() attribute_validation_tests.add_attributes_to_minister_with_invalid_endtime_format() - attribute_validation_tests.create_minister_with_attributes_without_startDate() + attribute_validation_tests.update_minister_with_attributes_without_startDate() print("\nšŸŸ¢ Running Attribute Validation Tests... Done") print("\nšŸŸ¢ Running Tabular Integrity Tests...") From 9fc79d80accca01a7998d22582b76145395fe9cb Mon Sep 17 00:00:00 2001 From: Yasandu Imanjith <89386702+yasandu0505@users.noreply.github.com> Date: Fri, 24 Apr 2026 13:39:29 +0530 Subject: [PATCH 23/23] fix : fix minor test case assertions issue --- opengin/tests/e2e/basic_core_tests.py | 19 ++++++++---------- opengin/tests/e2e/basic_read_tests.py | 28 +++++++++++++-------------- 2 files changed, 21 insertions(+), 26 deletions(-) diff --git a/opengin/tests/e2e/basic_core_tests.py b/opengin/tests/e2e/basic_core_tests.py index 3035ffdc..cb319e3a 100644 --- a/opengin/tests/e2e/basic_core_tests.py +++ b/opengin/tests/e2e/basic_core_tests.py @@ -897,24 +897,21 @@ def update_minister_with_attributes_with_startDate_and_endDate(self): print("Decoded value: ", decoded_value) url = f"{self.base_read_url}/{self.MINISTER_ID}/relations" - payload = {} + + payload = { + "name": "IS_ATTRIBUTE", + "startTime": self.DATA_START_DATE, + "endTime": self.DATA_END_DATE + } res = requests.post(url, json=payload) print(res.status_code, res.json()) assert res.status_code in [200], f"Failed to read Minister relationships: {res.text}" relationships = res.json() - print(relationships) - print("relationships", relationships) - - # check for the start data and the end date - for relationship in relationships: - if relationship["startTime"] != "" and relationship["endTime"] != "": - assert relationship["startTime"] == self.DATA_START_DATE - assert relationship["endTime"] == self.DATA_END_DATE - else: - pass + assert relationships[0]["startTime"] == self.DATA_START_DATE + assert relationships[0]["endTime"] == self.DATA_END_DATE print("āœ… Updated Minister entity with attributes with startDate + endDate.") diff --git a/opengin/tests/e2e/basic_read_tests.py b/opengin/tests/e2e/basic_read_tests.py index f52181cf..fe3e8465 100644 --- a/opengin/tests/e2e/basic_read_tests.py +++ b/opengin/tests/e2e/basic_read_tests.py @@ -827,26 +827,24 @@ def create_entity_for_read(): res = requests.get(INGESTION_API_URL + "/" + RELATED_ID_1) assert res.status_code == 200, f"Failed to fetch entity: {res.text}" - print("āœ… Fetched entity.") - # Verify the response data - response_data = res.json() - relationships = response_data["relationships"] - print(relationships) + url = READ_API_URL + "/" + RELATED_ID_1 + "/relations" - found = False - - for relationship in relationships: - start = relationship["value"].get("startTime") - end = relationship["value"].get("endTime") + payload = { + "name": "IS_ATTRIBUTE", + "startTime": "2024-11-01T00:00:00Z", + "endTime": "2024-11-30T00:00:00Z" + } - if start and end: - found = True + res = requests.post(url, json=payload) + print(res.status_code, res.json()) + assert res.status_code in [200], f"Failed to read relationships: {res.text}" + print("āœ… Fetched relationship_1...") - assert start in ["2024-11-01T00:00:00Z", "2024-01-01T00:00:00Z"] - assert end in ["2024-11-30T00:00:00Z", "2024-12-31T23:59:59Z"] + relationship = res.json() - assert found, "No relationship found with both startTime and endTime" + assert relationship[0]["startTime"] == "2024-11-01T00:00:00Z" + assert relationship[0]["endTime"] == "2024-11-30T00:00:00Z" print("āœ… Verified start and end times of entity's attributes.")