now supports geocoordinates from csv

Author: alvaradojl

README.md

@@ -1,6 +1,6 @@
 # gocroissant
 
-[![Version](https://img.shields.io/badge/version-v0.2.4-blue)](https://github.com/beyondcivic/gocroissant/releases/tag/v0.2.4)
+[![Version](https://img.shields.io/badge/version-v0.2.5-blue)](https://github.com/beyondcivic/gocroissant/releases/tag/v0.2.5)
 [![Go Version](https://img.shields.io/badge/Go-1.24+-00ADD8?logo=go)](https://golang.org/doc/devel/release.html)
 [![License](https://img.shields.io/badge/license-TBD-red)](LICENSE)
 

pkg/croissant/croissant.go

@@ -71,16 +71,17 @@ func InferDataType(value string) string {
 func IsValidDataType(dataType string) bool {
 	validTypes := map[string]bool{
 		// Schema.org types
-		"sc:Text":        true,
-		"sc:Boolean":     true,
-		"sc:Integer":     true,
-		"sc:Number":      true,
-		"sc:DateTime":    true,
-		"sc:URL":         true,
-		"sc:ImageObject": true,
-		"sc:VideoObject": true,
-		"sc:Enumeration": true,
-		"sc:GeoShape":    true,
+		"sc:Text":           true,
+		"sc:Boolean":        true,
+		"sc:Integer":        true,
+		"sc:Number":         true,
+		"sc:DateTime":       true,
+		"sc:URL":            true,
+		"sc:ImageObject":    true,
+		"sc:VideoObject":    true,
+		"sc:Enumeration":    true,
+		"sc:GeoShape":       true,
+		"sc:GeoCoordinates": true,
 
 		// Croissant-specific types
 		"cr:Label":            true,

pkg/croissant/metadata_node.go

@@ -102,59 +102,23 @@ func FromMetadata(metadata Metadata) *MetadataNode {
 			},
 			Type:        rs.Type,
 			Description: rs.Description,
-			DataType:    *rs.DataType,
 			Key:         rs.Key,
 			Data:        rs.Data,
 		}
+
+		// Handle DataType safely - check for nil
+		if rs.DataType != nil {
+			rsNode.DataType = *rs.DataType
+		} else {
+			// Set empty DataType if nil
+			rsNode.DataType = DataType{}
+		}
+
 		rsNode.SetParent(node)
 
 		// Convert fields
 		for _, field := range rs.Fields {
-			fieldNode := &FieldNode{
-				BaseNode: BaseNode{
-					ID:   field.ID,
-					Name: field.Name,
-				},
-				Type:        field.Type,
-				Description: field.Description,
-				DataType:    field.DataType,
-				Source: SourceNode{
-					Extract: ExtractNode{
-						Column:       field.Source.Extract.Column,
-						JSONPath:     field.Source.Extract.JSONPath,
-						FileProperty: field.Source.Extract.FileProperty,
-					},
-					FileObject: FileObjectRef{
-						ID: field.Source.FileObject.ID,
-					},
-					FileSet: FileObjectRef{
-						ID: field.Source.FileSet.ID,
-					},
-					Transform: field.Source.Transform,
-					Format:    field.Source.Format,
-				},
-				Repeated:   field.Repeated,
-				Examples:   field.Examples,
-				References: field.References,
-			}
-			fieldNode.SetParent(rsNode)
-
-			// Convert subfields if they exist
-			for _, subField := range field.SubField {
-				subFieldNode := &FieldNode{
-					BaseNode: BaseNode{
-						ID:   subField.ID,
-						Name: subField.Name,
-					},
-					Type:        subField.Type,
-					Description: subField.Description,
-					DataType:    subField.DataType,
-					// Note: subfields may have their own sources, but for now we'll handle basic conversion
-				}
-				subFieldNode.SetParent(fieldNode)
-				fieldNode.SubField = append(fieldNode.SubField, subFieldNode)
-			}
-
+			fieldNode := convertFieldToNode(field, rsNode)
 			rsNode.Fields = append(rsNode.Fields, fieldNode)
 		}
 
@@ -164,6 +128,47 @@ func FromMetadata(metadata Metadata) *MetadataNode {
 	return node
 }
 
+// convertFieldToNode converts a Field to a FieldNode with proper nil handling
+func convertFieldToNode(field Field, parent Node) *FieldNode {
+	fieldNode := &FieldNode{
+		BaseNode: BaseNode{
+			ID:   field.ID,
+			Name: field.Name,
+		},
+		Type:        field.Type,
+		Description: field.Description,
+		DataType:    field.DataType,
+		Source: SourceNode{
+			Extract: ExtractNode{
+				Column:       field.Source.Extract.Column,
+				JSONPath:     field.Source.Extract.JSONPath,
+				Regex:        field.Source.Extract.Regex,
+				FileProperty: field.Source.Extract.FileProperty,
+			},
+			FileObject: FileObjectRef{
+				ID: field.Source.FileObject.ID,
+			},
+			FileSet: FileObjectRef{
+				ID: field.Source.FileSet.ID,
+			},
+			Transform: field.Source.Transform,
+			Format:    field.Source.Format,
+		},
+		Repeated:   field.Repeated,
+		Examples:   field.Examples,
+		References: field.References,
+	}
+	fieldNode.SetParent(parent)
+
+	// Convert subfields if they exist
+	for _, subField := range field.SubField {
+		subFieldNode := convertFieldToNode(subField, fieldNode)
+		fieldNode.SubField = append(fieldNode.SubField, subFieldNode)
+	}
+
+	return fieldNode
+}
+
 // DistributionNode represents a file distribution
 type DistributionNode struct {
 	BaseNode
@@ -410,6 +415,7 @@ func (s *SourceNode) ValidateSource() bool {
 
 // ExtractNode represents extraction details
 type ExtractNode struct {
+	Regex        string `json:"regex,omitempty"`
 	Column       string `json:"column,omitempty"`
 	JSONPath     string `json:"jsonPath,omitempty"`
 	FileProperty string `json:"fileProperty,omitempty"`

pkg/croissant/structs.go

@@ -333,3 +333,13 @@ func NewArrayDataType(dataTypes ...string) DataType {
 		ArrayType: dataTypes,
 	}
 }
+
+// ValidateSource validates the source configuration
+func (fs FieldSource) ValidateSource() bool {
+	// If no source is configured, it's invalid unless it's a parent field with subfields
+	hasFileObject := fs.FileObject.ID != "" || fs.FileSet.ID != ""
+	hasExtract := fs.Extract.Column != "" || fs.Extract.JSONPath != "" || fs.Extract.FileProperty != "" || fs.Extract.Regex != ""
+
+	// A valid source needs either a file object reference with extraction info, or other valid configurations
+	return hasFileObject && (hasExtract || fs.Format != "")
+}

pkg/croissant/validation.go

@@ -249,6 +249,10 @@ func validateRecordSetKey(rs *RecordSetNode, issues *Issues) {
 
 // ValidateFieldNode validates a field node
 func ValidateFieldNode(field *FieldNode, issues *Issues, options ValidationOptions) {
+	if field == nil {
+		return
+	}
+
 	if field.Name == "" {
 		issues.AddError("Property \"https://schema.org/name\" is mandatory, but does not exist.", field)
 	}
@@ -257,29 +261,64 @@ func ValidateFieldNode(field *FieldNode, issues *Issues, options ValidationOptio
 		issues.AddError(fmt.Sprintf("\"%s\" should have an attribute \"@type\": \"http://mlcommons.org/croissant/Field\". Got %s instead.", field.Name, field.Type), field)
 	}
 
+	// More explicit dataType validation
 	if field.DataType.GetFirstType() == "" {
-		issues.AddError("The field does not specify a valid http://mlcommons.org/croissant/dataType, neither does any of its predecessor.", field)
+		issues.AddError(fmt.Sprintf("Field \"%s\" is missing required \"dataType\" property.", field.Name), field)
 	} else if options.CheckDataTypes {
-		allValid, invalidTypes := validateDataTypes(field.DataType)
-		if !allValid {
+		if isValid, invalidTypes := validateDataTypes(field.DataType); !isValid {
 			for _, invalidType := range invalidTypes {
-				issues.AddWarning(fmt.Sprintf("DataType \"%s\" is not a recognized schema.org type.", invalidType), field)
+				issues.AddError(fmt.Sprintf("Field \"%s\" has invalid dataType \"%s\". Valid types include: sc:Text, sc:Number, sc:Boolean, sc:DateTime, sc:URL, sc:GeoCoordinates, sc:ImageObject, cr:BoundingBox, etc.", field.Name, invalidType), field)
 			}
 		}
 	}
 
 	// Strict mode validations for fields
 	if options.StrictMode {
 		if field.Description == "" {
-			issues.AddWarning(fmt.Sprintf("Field \"%s\" is missing a description.", field.Name), field)
+			issues.AddWarning(fmt.Sprintf("Field \"%s\" is missing recommended \"description\" property.", field.Name), field)
+		}
+	}
+
+	// Check if field has subfields - use len check that's safe even if SubFields is nil
+	hasSubFields := field.SubField != nil && len(field.SubField) > 0
+
+	// Only validate source for leaf fields (fields without subfields)
+	if !hasSubFields {
+		// Check if source is properly configured
+		if !hasValidFieldSource(field) {
+			issues.AddError(fmt.Sprintf("Field \"%s\" has invalid or missing source configuration.", field.Name), field)
 		}
 	}
 
-	if !field.Source.ValidateSource() {
-		issues.AddError(fmt.Sprintf("Node \"%s\" is a field and has no source. Please, use http://mlcommons.org/croissant/source to specify the source.", field.ID), field)
+	// Validate subfields recursively
+	if field.SubField != nil {
+		for _, subField := range field.SubField {
+			if subField != nil {
+				subField.SetParent(field)
+				ValidateFieldNode(subField, issues, options)
+			}
+		}
 	}
 }
 
+// hasValidFieldSource checks if a field node has valid source configuration
+func hasValidFieldSource(field *FieldNode) bool {
+	if field == nil {
+		return false
+	}
+
+	// Check if the source has file object reference and extraction method
+	hasFileObject := field.Source.FileObject.ID != ""
+	hasExtract := field.Source.Extract.Column != "" ||
+		field.Source.Extract.JSONPath != "" ||
+		field.Source.Extract.FileProperty != "" ||
+		field.Source.Extract.Regex != ""
+	hasFormat := field.Source.Format != ""
+
+	// A valid source needs either a file object reference with extraction info, or format
+	return hasFileObject && (hasExtract || hasFormat)
+}
+
 // ValidateCrossReferences validates that all references are valid
 func ValidateCrossReferences(node *MetadataNode, issues *Issues) {
 	// Build a map of all available IDs
@@ -443,42 +482,6 @@ func isValidEncodingFormat(format string) bool {
 	return validFormats[format] || strings.HasPrefix(format, "text/") || strings.HasPrefix(format, "application/") || strings.HasPrefix(format, "image/") || strings.HasPrefix(format, "audio/") || strings.HasPrefix(format, "video/")
 }
 
-func isValidDataType(dataType string) bool {
-	validTypes := map[string]bool{
-		"sc:Text":        true,
-		"sc:Integer":     true,
-		"sc:Number":      true,
-		"sc:Float":       true,
-		"sc:Boolean":     true,
-		"sc:Date":        true,
-		"sc:DateTime":    true,
-		"sc:Time":        true,
-		"sc:URL":         true,
-		"sc:ImageObject": true,
-		"sc:name":        true, // Added for categorical data enumerations
-		"sc:Enumeration": true, // Added for enumeration RecordSet dataType
-		"cr:BoundingBox": true,
-		// Wikidata entities are also valid (they start with "wd:")
-	}
-
-	// Check if it's a known schema.org or croissant type
-	if validTypes[dataType] {
-		return true
-	}
-
-	// Allow Wikidata entity IDs (format: wd:Q followed by numbers)
-	if len(dataType) > 3 && dataType[:3] == "wd:" {
-		return true
-	}
-
-	// Allow full URLs (https://schema.org/... or https://www.wikidata.org/...)
-	if len(dataType) > 8 && (dataType[:8] == "https://" || dataType[:7] == "http://") {
-		return true
-	}
-
-	return false
-}
-
 // validateDataTypes validates all data types in a DataType (single or array)
 func validateDataTypes(dt DataType) (bool, []string) {
 	types := dt.GetTypes()
@@ -490,7 +493,7 @@ func validateDataTypes(dt DataType) (bool, []string) {
 	allValid := true
 
 	for _, dataType := range types {
-		if !isValidDataType(dataType) {
+		if !IsValidDataType(dataType) {
 			allValid = false
 			invalidTypes = append(invalidTypes, dataType)
 		}